# Copyright (C) 2023 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Generates an xml manifest for the `repo` tool. It enumerates all git submodules recursively, and creates a `project` entry in the manifest, pointing at the repository's url and hash. Repository's url are remapped to point either at AOSP or at experimental-qemu-build-internal when the AOSP repository is not ready. Some submodules are not used by the build and are not mapped. Nested repositories are relocated into ./third_party and a symlink is created to recreate the structure. """ import argparse import dataclasses import os import subprocess import sys import textwrap import xml.etree.ElementTree as ET # Host name of AOSP Git on borg repositories. # See rpc://android.googlesource.com/{repo name} AOSP_HOST = "android.googlesource.com" # Submodules that are not used by the build. REPO_TO_SKIP = { "https://boringssl.googlesource.com/boringssl", "https://chromium.googlesource.com/chromium/tools/depot_tools.git", "https://chromium.googlesource.com/chromiumos/platform/minigbm", "https://chromium.googlesource.com/chromiumos/platform/minijail", "https://chromium.googlesource.com/chromiumos/third_party/tpm2", "https://chromium.googlesource.com/crosvm/perfetto/", "https://github.com/akheron/jansson", "https://github.com/google/brotli", "https://github.com/google/wycheproof", "https://github.com/kkos/oniguruma", "https://github.com/krb5/krb5", "https://github.com/openssl/openssl", "https://github.com/pyca/cryptography.git", "https://github.com/tianocore/edk2-cmocka.git", "https://github.com/ucb-bar/berkeley-softfloat-3.git", "https://gitlab.com/libvirt/libvirt-ci.git", "https://github.com/gost-engine/engine", "https://github.com/provider-corner/libprov.git", "https://github.com/MIPI-Alliance/public-mipi-sys-t.git", "https://github.com/Zeex/subhook.git", "https://github.com/zeux/pugixml.git", "https://github.com/devicetree-org/pylibfdt.git", "https://gitlab.com/qemu-project/edk2.git", "https://gitlab.com/qemu-project/ipxe.git", "https://gitlab.com/qemu-project/libvfio-user.git", "https://gitlab.com/qemu-project/openbios.git", "https://gitlab.com/qemu-project/opensbi.git", "https://gitlab.com/qemu-project/qboot.git", "https://gitlab.com/qemu-project/qemu-palcode.git", "https://gitlab.com/qemu-project/QemuMacDrivers.git", "https://gitlab.com/qemu-project/seabios-hppa.git", "https://gitlab.com/qemu-project/seabios.git/", "https://gitlab.com/qemu-project/skiboot.git", "https://gitlab.com/qemu-project/SLOF.git", "https://gitlab.com/qemu-project/u-boot-sam460ex.git", "https://gitlab.com/qemu-project/vbootrom.git", } # Replaces repositories URLs. REPO_MAPPING = { # Submodules remapped to android AOSP. "git://anongit.freedesktop.org/git/pixman.git": ( "https://android.googlesource.com/platform/external/pixman" ), "https://boringssl.googlesource.com/boringssl": ( "https://android.googlesource.com/platform/external/boringssl" ), "https://chromium.googlesource.com/chromiumos/third_party/virglrenderer": ( "https://android.googlesource.com/platform/external/virglrenderer" ), "https://chromium.googlesource.com/crosvm/crosvm": ( "https://android.googlesource.com/platform/external/crosvm" ), "https://github.com/google/googletest.git": ( "https://android.googlesource.com/platform/external/googletest" ), "https://github.com/mesonbuild/meson.git": ( "https://android.googlesource.com/trusty/external/qemu-meson" ), "https://gitlab.com/qemu-project/dtc.git": ( "https://android.googlesource.com/platform/external/dtc" ), "https://gitlab.com/qemu-project/meson.git": ( "https://android.googlesource.com/trusty/external/qemu-meson" ), "https://gitlab.com/qemu-project/u-boot.git": ( # Can probably be removed. "https://android.googlesource.com/platform/external/u-boot" ), "https://gitlab.freedesktop.org/slirp/libslirp.git": ( "https://android.googlesource.com/trusty/external/qemu-libslirp" ), "https://gitlab.gnome.org/GNOME/glib.git": ( "https://android.googlesource.com/platform/external/bluetooth/glib" ), "https://github.com/KhronosGroup/EGL-Registry.git": ( "https://android.googlesource.com/platform/external/egl-registry" ), "https://gitlab.com/qemu-project/berkeley-softfloat-3.git": "https://android.googlesource.com/platform/external/berkeley-softfloat-3", "https://gitlab.com/qemu-project/berkeley-testfloat-3.git": "https://android.googlesource.com/platform/external/berkeley-testfloat-3", "https://salsa.debian.org/xorg-team/lib/libpciaccess.git": ( "https://android.googlesource.com/platform/external/libpciaccess" ), "https://gitlab.com/qemu-project/keycodemapdb.git": ( "https://android.googlesource.com/trusty/external/qemu-keycodemapdb" ), "https://gitlab.gnome.org/GNOME/gvdb.git": ( "https://android.googlesource.com/platform/external/gvdb" ), "https://github.com/anholt/libepoxy.git": ( "https://android.googlesource.com/platform/external/libepoxy" ), "https://gitlab.freedesktop.org/virgl/virglrenderer.git": ( "https://android.googlesource.com/platform/external/virglrenderer" ), } @dataclasses.dataclass class Submodule: """Occurrence of a module in the tree.""" path: str origin_url: str hash: str @dataclasses.dataclass class Project: """Project in a repo manifest.""" origin_url: str gob_host: str gob_path: str revision: str shallow: bool linkat: list def GetAllSubmodules(path): """Yields a Submodule for each module recursively found in the specified repo.""" yield Submodule( ".", "https://android.googlesource.com/device/google/cuttlefish_vmm", "main", ) output = subprocess.check_output([ "git", "submodule", "foreach", "--recursive", "-q", "echo ${displaypath} $(git remote get-url origin) ${sha1}", ]) for line in output.decode().strip().split("\n"): yield Submodule(*line.split(" ")) def MatchProject(url): prefix_to_remote = { "https://android.googlesource.com/": AOSP_HOST, "sso://android/": AOSP_HOST, "persistent-https://android.git.corp.google.com/": AOSP_HOST, } for prefix, remote in prefix_to_remote.items(): if url.startswith(prefix): return remote, url.removeprefix(prefix) raise ValueError( f"Url {url} could neither be mapped to AOSP to" " experimental-qemu-build-internal. If this is a new submodule, verify" " if it is used by the build. If it is, we should import it to AOSP. If" " not, add it to the REPO_TO_SKIP." ) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("input", help="git root directory") parser.add_argument( "--repo_manifest", type=argparse.FileType("w"), help="output repo tool manifest", ) args = parser.parse_args() submodules = list(GetAllSubmodules(args.input)) # Build a list of rep project. project_by_name = {} for module in submodules: repo_url = module.origin_url # Skip unused code repositories. if repo_url in REPO_TO_SKIP: continue # Remap repository to their new location or to None. if repo_url in REPO_MAPPING: repo_url = REPO_MAPPING[repo_url] gob_host, gob_path = MatchProject(repo_url) # Add the repository instance to the list merging by gob_path if gob_path not in project_by_name: project_by_name[gob_path] = Project( origin_url=module.origin_url, gob_host=gob_host, gob_path=gob_path, revision=module.hash, shallow=module.path.startswith("qemu/prebuilts/"), linkat=[], ) assert project_by_name[gob_path].gob_host == gob_host project_by_name[gob_path].linkat.append(module.path) GenerateRepoManifest(project_by_name.values(), args.repo_manifest) def SimpleName(project): return project.gob_path.replace("/", "_") def GenerateRepoManifest(projects, out): # Generate repo manifest xml. manifest = ET.Element("manifest") ET.SubElement( manifest, "remote", name="aosp", fetch="sso://android.googlesource.com", review="sso://android/", ) ET.SubElement( manifest, "remote", name="experimental-qemu-build-internal", fetch="sso://experimental-qemu-build-internal", ) ET.SubElement( manifest, "default", attrib={"sync-j": "16"}, revision="main", remote="aosp", ) for project in projects: # The project is instantiated only once. elem = ET.SubElement( manifest, "project", path=project.linkat[0], name=project.gob_path, revision=project.revision, ) if project.gob_host != AOSP_HOST: elem.attrib["remote"] = project.gob_host else: pass # Use the default value which is AOSP. if project.shallow: elem.attrib["clone-depth"] = "1" for linkat in project.linkat[1:]: ET.SubElement(elem, "linkfile", src=".", dest=linkat) tree = ET.ElementTree(manifest) tree.getroot().insert( 0, ET.Comment( "DO NOT EDIT. This file is generated by " + os.path.basename(__file__) ), ) ET.indent(tree) tree.write(out, encoding="unicode") if __name__ == "__main__": sys.exit(main())