#!/usr/bin/env python3 # # Copyright 2018 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Project information.""" from __future__ import absolute_import import logging import os import time from aidegen import constant from aidegen.lib import aidegen_metrics from aidegen.lib import common_util from aidegen.lib import errors from aidegen.lib import module_info from aidegen.lib import project_config from aidegen.lib import source_locator from aidegen.idea import iml from atest import atest_utils _CONVERT_MK_URL = ('https://android.googlesource.com/platform/build/soong/' '#convert-android_mk-files') _ROBOLECTRIC_MODULE = 'Robolectric_all' _NOT_TARGET = ('The module %s does not contain any Java or Kotlin file, ' 'therefore we skip this module in the project.') # The module fake-framework have the same package name with framework but empty # content. It will impact the dependency for framework when referencing the # package from fake-framework in IntelliJ. _EXCLUDE_MODULES = ['fake-framework'] # When we use atest_utils.build(), there is a command length limit on # soong_ui.bash. We reserve 5000 characters for rewriting the command line # in soong_ui.bash. _CMD_LENGTH_BUFFER = 5000 # For each argument, it needs a space to separate following argument. _BLANK_SIZE = 1 _CORE_MODULES = [constant.FRAMEWORK_ALL, constant.CORE_ALL, 'org.apache.http.legacy.stubs.system'] _BUILD_BP_JSON_ENV_ON = { constant.GEN_JAVA_DEPS: 'true', constant.GEN_CC_DEPS: 'true', constant.GEN_COMPDB: 'true', constant.GEN_RUST: 'true' } class ProjectInfo: """Project information. Users should call config_project first before starting using ProjectInfo. Class attributes: modules_info: An AidegenModuleInfo instance whose name_to_module_info is combining module-info.json with module_bp_java_deps.json. projects: A list of instances of ProjectInfo that are generated in an AIDEGen command. Attributes: project_absolute_path: The absolute path of the project. project_relative_path: The relative path of the project to common_util.get_android_root_dir(). project_module_names: A set of module names under project_absolute_path directory or it's subdirectories. dep_modules: A dict has recursively dependent modules of project_module_names. iml_path: The project's iml file path. source_path: A dictionary to keep following data: source_folder_path: A set contains the source folder relative paths. test_folder_path: A set contains the test folder relative paths. jar_path: A set contains the jar file paths. jar_module_path: A dictionary contains the jar file and the module's path mapping, only used in Eclipse. r_java_path: A set contains the relative path to the R.java files, only used in Eclipse. srcjar_path: A source content descriptor only used in IntelliJ. e.g. out/.../aapt2.srcjar!/ The "!/" is a content descriptor for compressed files in IntelliJ. is_main_project: A boolean to verify the project is main project. dependencies: A list of dependency projects' iml file names, e.g. base, framework-all. iml_name: The iml project file name of this project. rel_out_soong_jar_path: A string of relative project path in the 'out/soong/.intermediates' directory, e.g., if self.project_relative_path = 'frameworks/base' the rel_out_soong_jar_path should be 'out/soong/.intermediates/frameworks/base/'. """ modules_info = None projects = [] def __init__(self, target=None, is_main_project=False): """ProjectInfo initialize. Args: target: Includes target module or project path from user input, when locating the target, project with matching module name of the given target has a higher priority than project path. is_main_project: A boolean, default is False. True if the target is the main project, otherwise False. """ rel_path, abs_path = common_util.get_related_paths( self.modules_info, target) self.module_name = self.get_target_name(target, abs_path) self.is_main_project = is_main_project self.project_module_names = set( self.modules_info.get_module_names(rel_path)) self.project_relative_path = rel_path self.project_absolute_path = abs_path self.iml_path = '' self._set_default_modules() self._init_source_path() if target == constant.FRAMEWORK_ALL: self.dep_modules = self.get_dep_modules([target]) else: self.dep_modules = self.get_dep_modules() self._filter_out_modules() self.dependencies = [] self.iml_name = iml.IMLGenerator.get_unique_iml_name(abs_path) self.rel_out_soong_jar_path = self._get_rel_project_out_soong_jar_path() def _set_default_modules(self): """Append default hard-code modules, source paths and jar files. 1. framework: Framework module is always needed for dependencies, but it might not always be located by module dependency. 2. org.apache.http.legacy.stubs.system: The module can't be located through module dependency. Without it, a lot of java files will have error of "cannot resolve symbol" in IntelliJ since they import packages android.Manifest and com.android.internal.R. """ # Set the default modules framework-all and core-all as the core # dependency modules. self.project_module_names.update(_CORE_MODULES) def _init_source_path(self): """Initialize source_path dictionary.""" self.source_path = { 'source_folder_path': set(), 'test_folder_path': set(), 'jar_path': set(), 'jar_module_path': {}, 'r_java_path': set(), 'srcjar_path': set() } def _search_android_make_files(self): """Search project and dependency modules contain Android.mk files. If there is only Android.mk but no Android.bp, we'll show the warning message, otherwise we won't. Yields: A string: the relative path of Android.mk. """ if (common_util.exist_android_mk(self.project_absolute_path) and not common_util.exist_android_bp(self.project_absolute_path)): yield '\t' + os.path.join(self.project_relative_path, constant.ANDROID_MK) for mod_name in self.dep_modules: rel_path, abs_path = common_util.get_related_paths( self.modules_info, mod_name) if rel_path and abs_path: if (common_util.exist_android_mk(abs_path) and not common_util.exist_android_bp(abs_path)): yield '\t' + os.path.join(rel_path, constant.ANDROID_MK) def _get_modules_under_project_path(self, rel_path): """Find qualified modules under the rel_path. Find modules which contain any Java or Kotlin file as a target module. If it's the whole source tree project, add all modules into it. Args: rel_path: A string, the project's relative path. Returns: A set of module names. """ logging.info('Find modules contain any Java or Kotlin file under %s.', rel_path) if rel_path == '': return self.modules_info.name_to_module_info.keys() modules = set() root_dir = common_util.get_android_root_dir() for name, data in self.modules_info.name_to_module_info.items(): if module_info.AidegenModuleInfo.is_project_path_relative_module( data, rel_path): if common_util.check_java_or_kotlin_file_exists( os.path.join(root_dir, data[constant.KEY_PATH][0])): modules.add(name) else: logging.debug(_NOT_TARGET, name) return modules def _get_robolectric_dep_module(self, modules): """Return the robolectric module set as dependency if any module is a robolectric test. Args: modules: A set of modules. Returns: A set with a robolectric_all module name if one of the modules needs the robolectric test module. Otherwise return empty list. """ for module in modules: if self.modules_info.is_robolectric_test(module): return {_ROBOLECTRIC_MODULE} return set() def _filter_out_modules(self): """Filter out unnecessary modules.""" for module in _EXCLUDE_MODULES: self.dep_modules.pop(module, None) def get_dep_modules(self, module_names=None, depth=0): """Recursively find dependent modules of the project. Find dependent modules by dependencies parameter of each module. For example: The module_names is ['m1']. The modules_info is { 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']}, 'm2': {'path': ['path_to_m4']}, 'm3': {'path': ['path_to_m1']} 'm4': {'path': []} } The result dependent modules are: { 'm1': {'dependencies': ['m2'], 'path': ['path_to_m1'] 'depth': 0}, 'm2': {'path': ['path_to_m4'], 'depth': 1}, 'm3': {'path': ['path_to_m1'], 'depth': 0} } Note that: 1. m4 is not in the result as it's not among dependent modules. 2. m3 is in the result as it has the same path to m1. Args: module_names: A set of module names. depth: An integer shows the depth of module dependency referenced by source. Zero means the max module depth. Returns: deps: A dict contains all dependent modules data of given modules. """ dep = {} children = set() if not module_names: module_names = self.project_module_names module_names.update( self._get_modules_under_project_path( self.project_relative_path)) module_names.update(self._get_robolectric_dep_module(module_names)) self.project_module_names = set() for name in module_names: if (name in self.modules_info.name_to_module_info and name not in self.project_module_names): dep[name] = self.modules_info.name_to_module_info[name] dep[name][constant.KEY_DEPTH] = depth self.project_module_names.add(name) if (constant.KEY_DEPENDENCIES in dep[name] and dep[name][constant.KEY_DEPENDENCIES]): children.update(dep[name][constant.KEY_DEPENDENCIES]) if children: dep.update(self.get_dep_modules(children, depth + 1)) return dep @staticmethod def generate_projects(targets): """Generate a list of projects in one time by a list of module names. Args: targets: A list of target modules or project paths from user input, when locating the target, project with matched module name of the target has a higher priority than project path. Returns: List: A list of ProjectInfo instances. """ return [ProjectInfo(target, i == 0) for i, target in enumerate(targets)] @staticmethod def get_target_name(target, abs_path): """Get target name from target's absolute path. If the project is for entire Android source tree, change the target to source tree's root folder name. In this way, we give IDE project file a more specific name. e.g, master.iml. Args: target: Includes target module or project path from user input, when locating the target, project with matching module name of the given target has a higher priority than project path. abs_path: A string, target's absolute path. Returns: A string, the target name. """ if abs_path == common_util.get_android_root_dir(): return os.path.basename(abs_path) return target def locate_source(self, build=True): """Locate the paths of dependent source folders and jar files. Try to reference source folder path as dependent module unless the dependent module should be referenced to a jar file, such as modules have jars and jarjar_rules parameter. For example: Module: asm-6.0 java_import { name: 'asm-6.0', host_supported: true, jars: ['asm-6.0.jar'], } Module: bouncycastle java_library { name: 'bouncycastle', ... target: { android: { jarjar_rules: 'jarjar-rules.txt', }, }, } Args: build: A boolean default to true. If false, skip building jar and srcjar files, otherwise build them. Example usage: project.source_path = project.locate_source() E.g. project.source_path = { 'source_folder_path': ['path/to/source/folder1', 'path/to/source/folder2', ...], 'test_folder_path': ['path/to/test/folder', ...], 'jar_path': ['path/to/jar/file1', 'path/to/jar/file2', ...] } """ if not hasattr(self, 'dep_modules') or not self.dep_modules: raise errors.EmptyModuleDependencyError( 'Dependent modules dictionary is empty.') rebuild_targets = set() for module_name, module_data in self.dep_modules.items(): module = self._generate_moduledata(module_name, module_data) module.locate_sources_path() self.source_path['source_folder_path'].update(set(module.src_dirs)) self.source_path['test_folder_path'].update(set(module.test_dirs)) self.source_path['r_java_path'].update(set(module.r_java_paths)) self.source_path['srcjar_path'].update(set(module.srcjar_paths)) self._append_jars_as_dependencies(module) rebuild_targets.update(module.build_targets) config = project_config.ProjectConfig.get_instance() if config.is_skip_build: return if rebuild_targets: if build: logging.info('\nThe batch_build_dependencies function is ' 'called by ProjectInfo\'s locate_source method.') batch_build_dependencies(rebuild_targets) self.locate_source(build=False) else: logging.warning('Jar or srcjar files build skipped:\n\t%s.', '\n\t'.join(rebuild_targets)) def _generate_moduledata(self, module_name, module_data): """Generate a module class to collect dependencies in IDE. The rules of initialize a module data instance: if ide_object isn't None and its ide_name is 'eclipse', we'll create an EclipseModuleData instance otherwise create a ModuleData instance. Args: module_name: Name of the module. module_data: A dictionary holding a module information. Returns: A ModuleData class. """ ide_name = project_config.ProjectConfig.get_instance().ide_name if ide_name == constant.IDE_ECLIPSE: return source_locator.EclipseModuleData( module_name, module_data, self.project_relative_path) depth = project_config.ProjectConfig.get_instance().depth return source_locator.ModuleData(module_name, module_data, depth) def _append_jars_as_dependencies(self, module): """Add given module's jar files into dependent_data as dependencies. Args: module: A ModuleData instance. """ if module.jar_files: self.source_path['jar_path'].update(module.jar_files) for jar in list(module.jar_files): self.source_path['jar_module_path'].update({ jar: module.module_path }) # Collecting the jar files of default core modules as dependencies. if constant.KEY_DEPENDENCIES in module.module_data: self.source_path['jar_path'].update([ x for x in module.module_data[constant.KEY_DEPENDENCIES] if common_util.is_target(x, constant.TARGET_LIBS) ]) def _get_rel_project_out_soong_jar_path(self): """Gets the projects' jar path in 'out/soong/.intermediates' folder. Gets the relative project's jar path in the 'out/soong/.intermediates' directory. For example, if the self.project_relative_path is 'frameworks/base', the returned value should be 'out/soong/.intermediates/frameworks/base/'. Returns: A string of relative project path in out/soong/.intermediates/ directory, e.g. 'out/soong/.intermediates/frameworks/base/'. """ rdir = os.path.relpath(common_util.get_soong_out_path(), common_util.get_android_root_dir()) return os.sep.join( [rdir, constant.INTERMEDIATES, self.project_relative_path]) + os.sep @classmethod def multi_projects_locate_source(cls, projects): """Locate the paths of dependent source folders and jar files. Args: projects: A list of ProjectInfo instances. Information of a project such as project relative path, project real path, project dependencies. """ cls.projects = projects for project in projects: project.locate_source() _update_iml_dep_modules(project) class MultiProjectsInfo(ProjectInfo): """Multiple projects info. Usage example: if folder_base: project = MultiProjectsInfo(['module_name']) project.collect_all_dep_modules() project.gen_folder_base_dependencies() else: ProjectInfo.generate_projects(['module_name']) Attributes: _targets: A list of module names or project paths. path_to_sources: A dictionary of modules' sources, the module's path as key and the sources as value. e.g. { 'frameworks/base': { 'src_dirs': [], 'test_dirs': [], 'r_java_paths': [], 'srcjar_paths': [], 'jar_files': [], 'dep_paths': [], } } """ def __init__(self, targets=None): """MultiProjectsInfo initialize. Args: targets: A list of module names or project paths from user's input. """ super().__init__(targets[0], True) self._targets = targets self.path_to_sources = {} @staticmethod def _clear_srcjar_paths(module): """Clears the srcjar_paths. Args: module: A ModuleData instance. """ module.srcjar_paths = [] def _collect_framework_srcjar_info(self, module): """Clears the framework's srcjars. Args: module: A ModuleData instance. """ if module.module_path == constant.FRAMEWORK_PATH: framework_srcjar_path = os.path.join(constant.FRAMEWORK_PATH, constant.FRAMEWORK_SRCJARS) if module.module_name == constant.FRAMEWORK_ALL: self.path_to_sources[framework_srcjar_path] = { 'src_dirs': [], 'test_dirs': [], 'r_java_paths': [], 'srcjar_paths': module.srcjar_paths, 'jar_files': [], 'dep_paths': [constant.FRAMEWORK_PATH], } # In the folder base case, AIDEGen has to ignore all module's srcjar # files under the frameworks/base except the framework-all. Because # there are too many duplicate srcjars of modules under the # frameworks/base. So that AIDEGen keeps the srcjar files only from # the framework-all module. Other modules' srcjar files will be # removed. However, when users choose the module base case, srcjar # files will be collected by the ProjectInfo class, so that the # removing srcjar_paths in this class does not impact the # srcjar_paths collection of modules in the ProjectInfo class. self._clear_srcjar_paths(module) def collect_all_dep_modules(self): """Collects all dependency modules for the projects.""" self.project_module_names.clear() module_names = set(_CORE_MODULES) for target in self._targets: relpath, _ = common_util.get_related_paths(self.modules_info, target) module_names.update(self._get_modules_under_project_path(relpath)) module_names.update(self._get_robolectric_dep_module(module_names)) self.dep_modules = self.get_dep_modules(module_names) def gen_folder_base_dependencies(self, module): """Generates the folder base dependencies dictionary. Args: module: A ModuleData instance. """ mod_path = module.module_path if not mod_path: logging.debug('The %s\'s path is empty.', module.module_name) return self._collect_framework_srcjar_info(module) if mod_path not in self.path_to_sources: self.path_to_sources[mod_path] = { 'src_dirs': module.src_dirs, 'test_dirs': module.test_dirs, 'r_java_paths': module.r_java_paths, 'srcjar_paths': module.srcjar_paths, 'jar_files': module.jar_files, 'dep_paths': module.dep_paths, } else: for key, val in self.path_to_sources[mod_path].items(): val.extend([v for v in getattr(module, key) if v not in val]) def batch_build_dependencies(rebuild_targets): """Batch build the jar or srcjar files of the modules if they don't exist. Command line has the max length limit, MAX_ARG_STRLEN, and MAX_ARG_STRLEN = (PAGE_SIZE * 32). If the build command is longer than MAX_ARG_STRLEN, this function will separate the rebuild_targets into chunks with size less or equal to MAX_ARG_STRLEN to make sure it can be built successfully. Args: rebuild_targets: A set of jar or srcjar files which do not exist. """ start_time = time.time() logging.info('Ready to build the jar or srcjar files. Files count = %s', str(len(rebuild_targets))) arg_max = os.sysconf('SC_PAGE_SIZE') * 32 - _CMD_LENGTH_BUFFER rebuild_targets = list(rebuild_targets) for start, end in iter(_separate_build_targets(rebuild_targets, arg_max)): _build_target(rebuild_targets[start:end]) duration = time.time() - start_time logging.debug('Build Time, duration = %s', str(duration)) aidegen_metrics.performance_metrics(constant.TYPE_AIDEGEN_BUILD_TIME, duration) def _build_target(targets): """Build the jar or srcjar files. Use -k to keep going when some targets can't be built or build failed. Use -j to speed up building. Args: targets: A list of jar or srcjar files which need to build. """ build_cmd = ['-k', '-j'] build_cmd.extend(list(targets)) atest_utils.update_build_env(_BUILD_BP_JSON_ENV_ON) if not atest_utils.build(build_cmd): message = ('Build failed!\n{}\nAIDEGen will proceed but dependency ' 'correctness is not guaranteed if not all targets being ' 'built successfully.'.format('\n'.join(targets))) print('\n{} {}\n'.format(common_util.COLORED_INFO('Warning:'), message)) def _separate_build_targets(build_targets, max_length): """Separate the build_targets by limit the command size to max command length. Args: build_targets: A list to be separated. max_length: The max number of each build command length. Yields: The start index and end index of build_targets. """ arg_len = 0 first_item_index = 0 for i, item in enumerate(build_targets): arg_len = arg_len + len(item) + _BLANK_SIZE if arg_len > max_length: yield first_item_index, i first_item_index = i arg_len = len(item) + _BLANK_SIZE if first_item_index < len(build_targets): yield first_item_index, len(build_targets) def _update_iml_dep_modules(project): """Gets the dependent modules in the project's iml file. The jar files which have the same source codes as cls.projects' source files should be removed from the dependencies.iml file's jar paths. The codes are written in aidegen.project.project_splitter.py. We should also add the jar project's unique iml name into self.dependencies which later will be written into its own iml project file. If we don't remove these files in dependencies.iml, it will cause the duplicated codes in IDE and raise issues. For example, when users do 'refactor' and rename a class in the IDE, it will search all sources and dependencies' jar paths and lead to the error. """ keys = ('source_folder_path', 'test_folder_path', 'r_java_path', 'srcjar_path', 'jar_path') for key in keys: for jar in project.source_path[key]: for prj in ProjectInfo.projects: if prj is project: continue if (prj.rel_out_soong_jar_path in jar and jar.endswith(constant.JAR_EXT)): if prj.iml_name not in project.dependencies: project.dependencies.append(prj.iml_name) break