package main import ( "flag" "fmt" "io/ioutil" "log" "os" "os/exec" "os/user" "strconv" "strings" "time" ) type OnFail int const ( IgnoreOnFail OnFail = iota WarnOnFail ExitOnFail ) type arrayFlags []string // Implemented for flag#Value interface func (s *arrayFlags) String() string { if s == nil { return "" } return fmt.Sprintf("%v", *s) } // Implemented for flag#Value interface func (s *arrayFlags) Set(value string) error { *s = append(*s, value) return nil } // Returns `"foo" "bar"` func (s *arrayFlags) AsArgs() string { var result []string for _, value := range *s { result = append(result, fmt.Sprintf("%q", value)) } return strings.Join(result, " ") } // Returns `--flag="foo" --flag="bar"` func (s *arrayFlags) AsRepeatedFlag(name string) string { var result []string for _, value := range *s { result = append(result, fmt.Sprintf(`--%s="%s"`, name, value)) } return strings.Join(result, " ") } var build_instance string var build_project string var build_zone string var dest_image string var dest_family string var dest_project string var launch_instance string var arch string var source_image_family string var source_image_project string var repository_url string var repository_branch string var version string var internal_ip_flag string var INTERNAL_extra_source string var verbose bool var username string var image_disk_size_gb int // NOTE: For `gcloud compute ssh` command, `ssh_flags` will be used as SSH_ARGS rather than // as `--ssh_flag` repeated flag. Why? because --ssh_flag is not parsed as expected when // containing quotes and spaces. var ssh_flags arrayFlags var host_orchestration_flag bool func init() { user, err := user.Current() if err != nil { panic(err) } username = user.Username flag.StringVar(&build_instance, "build_instance", username+"-build", "Instance name to create for the build") flag.StringVar(&build_project, "build_project", mustShell("gcloud config get-value project"), "Project to use for scratch") // The new get-value output format is different. The result is in 2nd line. str_list := strings.Split(build_project, "\n") if len(str_list) == 2 { build_project = str_list[1] } flag.StringVar(&build_zone, "build_zone", mustShell("gcloud config get-value compute/zone"), "Zone to use for scratch resources") // The new get-value output format is different. The result is in 2nd line. str_list = strings.Split(build_zone, "\n") if len(str_list) == 2 { build_zone = str_list[1] } flag.StringVar(&dest_image, "dest_image", "vsoc-host-scratch-"+username, "Image to create") flag.StringVar(&dest_family, "dest_family", "", "Image family to add the image to") flag.StringVar(&dest_project, "dest_project", mustShell("gcloud config get-value project"), "Project to use for the new image") // The new get-value output format is different. The result is in 2nd line. str_list = strings.Split(dest_project, "\n") if len(str_list) == 2 { dest_project = str_list[1] } flag.StringVar(&launch_instance, "launch_instance", "", "Name of the instance to launch with the new image") flag.StringVar(&arch, "arch", "gce_x86_64", "Which CPU arch, arm/x86_64/gce_x86_64") flag.StringVar(&source_image_family, "source_image_family", "debian-12", "Image familty to use as the base") flag.StringVar(&source_image_project, "source_image_project", "debian-cloud", "Project holding the base image") flag.StringVar(&repository_url, "repository_url", "https://github.com/google/android-cuttlefish.git", "URL to the repository with host changes") flag.StringVar(&repository_branch, "repository_branch", "v0.9.30", "Branch to check out") flag.StringVar(&version, "version", "", "cuttlefish-common version") flag.StringVar(&internal_ip_flag, "INTERNAL_IP", "", "INTERNAL_IP can be set to --internal-ip run on a GCE instance."+ "The instance will need --scope compute-rw.") flag.StringVar(&INTERNAL_extra_source, "INTERNAL_extra_source", "", "INTERNAL_extra_source may be set to a directory containing the source for extra packages to build.") flag.BoolVar(&verbose, "verbose", true, "print commands and output (default: true)") flag.IntVar(&image_disk_size_gb, "image_disk_size_gb", 10, "Image disk size in GB") flag.Var(&ssh_flags, "ssh_flag", "Values for --ssh-flag and --scp_flag for gcloud compute ssh/scp respectively. This flag may be repeated") flag.BoolVar(&host_orchestration_flag, "host_orchestration", false, "assembles image with host orchestration capabilities") flag.Parse() } func shell(cmd string) (string, error) { if verbose { fmt.Println(cmd) } b, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput() if verbose { fmt.Println(string(b)) } if err != nil { return "", err } return strings.TrimSpace(string(b)), nil } func mustShell(cmd string) string { if verbose { fmt.Println(cmd) } out, err := shell(cmd) if err != nil { panic(err) } if verbose { fmt.Println(out) } return strings.TrimSpace(out) } func gce(action OnFail, gceArg string, errorStr ...string) (string, error) { cmd := "gcloud " + gceArg out, err := shell(cmd) if out != "" { fmt.Println(out) } if err != nil && action != IgnoreOnFail { var buf string fmt.Sprintf(buf, "gcloud error occurred: %s", err) if len(errorStr) > 0 { buf += " [" + errorStr[0] + "]" } if action == ExitOnFail { panic(buf) } if action == WarnOnFail { fmt.Println(buf) } } return out, err } func waitForInstance(PZ string) { for { time.Sleep(5 * time.Second) _, err := gce(WarnOnFail, `compute ssh `+internal_ip_flag+` `+PZ+` `+ build_instance+` -- `+ssh_flags.AsArgs()+` uptime `) if err == nil { break } } } func packageSource(url string, branch string, subdir string) { repository_dir := url[strings.LastIndex(url, "/")+1:] repository_dir = mustShell(`basename "` + repository_dir + `" .git`) debian_dir := repository_dir if subdir != "" { debian_dir = repository_dir + "/" + subdir } mustShell("git clone " + url + " -b " + branch) mustShell("dpkg-source -b " + debian_dir) mustShell("rm -rf " + repository_dir) mustShell("ls -l") mustShell("pwd") } func createInstance(instance string, arg string) { _, err := gce(WarnOnFail, `compute instances describe "`+instance+`"`) if err != nil { gce(ExitOnFail, `compute instances create `+arg+` "`+instance+`"`) } } func main() { gpu_type := "nvidia-tesla-p100-vws" PZ := "--project=" + build_project + " --zone=" + build_zone if arch != "gce_x86_64" { // new path that generate image locally without creating GCE instance abt := os.Getenv("ANDROID_BUILD_TOP") cmd := `"` + abt + `/device/google/cuttlefish/tools/create_base_image_combined.sh"` cmd += " " + arch out, err := shell(cmd) if out != "" { fmt.Println(out) } if err != nil { fmt.Println("create_base_image arch %s error occurred: %s", arch, err) } // gce operations delete_instances := build_instance + " " + dest_image if launch_instance != "" { delete_instances += " " + launch_instance } zip_file := "disk_" + username + ".raw.tar.gz" gs_file := "gs://cloud-android-testing-esp/" + zip_file cloud_storage_file := "https://storage.googleapis.com/cloud-android-testing-esp/" + zip_file location := "us" // delete all previous instances, images and disks gce(WarnOnFail, `compute instances delete -q `+PZ+` `+delete_instances, `Not running`) gce(WarnOnFail, `compute disks delete -q `+PZ+` "`+dest_image+`"`, `No scratch disk`) gce(WarnOnFail, `compute images delete -q --project="`+build_project+`" "`+dest_image+`"`, `Not respinning`) gce(WarnOnFail, `alpha storage rm `+gs_file) // upload new local host image into GCE storage gce(WarnOnFail, `alpha storage cp `+abt+`/`+zip_file+` gs://cloud-android-testing-esp`) // create GCE image based on new uploaded host image gce(WarnOnFail, `compute images create "`+dest_image+`" --project="`+build_project+ `" --family="`+source_image_family+`" --source-uri="`+cloud_storage_file+ `" --storage-location="`+location+`" --guest-os-features=UEFI_COMPATIBLE`) // find Nvidia GPU and then create GCE instance gce(ExitOnFail, `compute accelerator-types describe "`+gpu_type+`" `+PZ, `Please use a zone with `+gpu_type+` GPUs available.`) createInstance(build_instance, PZ+ ` --machine-type=n1-standard-16 --network-interface=network-tier=PREMIUM,subnet=default`+ ` --accelerator="type=`+gpu_type+ `,count=1" --maintenance-policy=TERMINATE --provisioning-model=STANDARD`+ ` --service-account=204446994883-compute@developer.gserviceaccount.com`+ ` --scopes=https://www.googleapis.com/auth/devstorage.read_only,`+ `https://www.googleapis.com/auth/logging.write,`+ `https://www.googleapis.com/auth/monitoring.write,`+ `https://www.googleapis.com/auth/servicecontrol,`+ `https://www.googleapis.com/auth/service.management.readonly,`+ `https://www.googleapis.com/auth/trace.append`+ ` --tags=http-server --create-disk=auto-delete=yes,boot=yes,device-name=`+build_instance+ `,image=projects/cloud-android-testing/global/images/`+dest_image+ `,mode=rw,size=200,type=projects/cloud-android-testing/zones/`+build_zone+ `/diskTypes/pd-balanced --no-shielded-secure-boot --shielded-vtpm`+ ` --shielded-integrity-monitoring --reservation-affinity=any`) // enable serial-port (console) gce(WarnOnFail, `compute instances add-metadata `+build_instance+ ` --metadata serial-port-enable=TRUE`) return } dest_family_flag := "" if dest_family != "" { dest_family_flag = "--family=" + dest_family } scratch_dir, err := ioutil.TempDir("", "") if err != nil { log.Fatal(err) } oldDir, err := os.Getwd() if err != nil { log.Fatal(err) } os.Chdir(scratch_dir) packageSource(repository_url, repository_branch, "base") packageSource(repository_url, repository_branch, "frontend") os.Chdir(oldDir) abt := os.Getenv("ANDROID_BUILD_TOP") source_files := `"` + abt + `/device/google/cuttlefish/tools/create_base_image_gce.sh"` source_files += " " + `"` + abt + `/device/google/cuttlefish/tools/install_nvidia.sh"` source_files += " " + `"` + abt + `/device/google/cuttlefish/tools/update_gce_kernel.sh"` source_files += " " + `"` + abt + `/device/google/cuttlefish/tools/remove_old_gce_kernel.sh"` source_files += " " + scratch_dir + "/*" if INTERNAL_extra_source != "" { source_files += " " + INTERNAL_extra_source + "/*" } delete_instances := build_instance + " " + dest_image if launch_instance != "" { delete_instances += " " + launch_instance } gce(WarnOnFail, `compute instances delete -q `+PZ+` `+delete_instances, `Not running`) gce(WarnOnFail, `compute disks delete -q `+PZ+` "`+dest_image+ `"`, `No scratch disk`) gce(WarnOnFail, `compute images delete -q --project="`+build_project+ `" "`+dest_image+`"`, `Not respinning`) gce(WarnOnFail, `compute disks create `+PZ+` --size=`+strconv.Itoa(image_disk_size_gb)+`G `+ `--image-family="`+source_image_family+`" --image-project="`+source_image_project+`" "`+dest_image+`"`) gce(ExitOnFail, `compute accelerator-types describe "`+gpu_type+`" `+PZ, `Please use a zone with `+gpu_type+` GPUs available.`) createInstance(build_instance, PZ+ ` --machine-type=n1-standard-16 --image-family="`+source_image_family+ `" --image-project="`+source_image_project+ `" --boot-disk-size=200GiB --accelerator="type=`+gpu_type+ `,count=1" --maintenance-policy=TERMINATE --boot-disk-size=200GiB`) waitForInstance(PZ) // Ubuntu tends to mount the wrong disk as root, so help it by waiting until // it has booted before giving it access to the clean image disk gce(WarnOnFail, `compute instances attach-disk `+PZ+` "`+build_instance+ `" --disk="`+dest_image+`"`) // beta for the --internal-ip flag that may be passed via internal_ip_flag gce(ExitOnFail, `beta compute scp `+internal_ip_flag+` `+PZ+` `+source_files+ ` "`+build_instance+`:" `+ssh_flags.AsRepeatedFlag("scp-flag")) // Update the host kernel before installing any kernel modules // Needed to guarantee that the modules in the chroot aren't built for the // wrong kernel gce(WarnOnFail, `compute ssh `+internal_ip_flag+` `+PZ+` "`+build_instance+ `"`+` -- `+ssh_flags.AsArgs()+` ./update_gce_kernel.sh`) // TODO rammuthiah if the instance is clobbered with ssh commands within // 5 seconds of reboot, it becomes inaccessible. Workaround that by sleeping // 50 seconds. time.Sleep(70 * time.Second) gce(ExitOnFail, `compute ssh `+internal_ip_flag+` `+PZ+` "`+build_instance+ `"`+` -- `+ssh_flags.AsArgs()+` ./remove_old_gce_kernel.sh`) ho_arg := "" if host_orchestration_flag { ho_arg = "-o" } gce(ExitOnFail, `compute ssh `+internal_ip_flag+` `+PZ+` "`+build_instance+ `"`+` -- `+ssh_flags.AsArgs()+` ./create_base_image_gce.sh `+ho_arg) // Reboot the instance to force a clean umount of the disk's file system. gce(WarnOnFail, `compute ssh `+internal_ip_flag+` `+PZ+` "`+build_instance+ `" -- `+ssh_flags.AsArgs()+` sudo reboot`) waitForInstance(PZ) gce(ExitOnFail, `compute instances delete -q `+PZ+` "`+build_instance+`"`) gce(ExitOnFail, `compute images create --project="`+build_project+ `" --source-disk="`+dest_image+`" --source-disk-zone="`+build_zone+ `" --licenses=https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx `+ dest_family_flag+` "`+dest_image+`"`) gce(ExitOnFail, `compute disks delete -q `+PZ+` "`+dest_image+`"`) if launch_instance != "" { createInstance(launch_instance, PZ+ ` --image-project="`+build_project+`" --image="`+dest_image+ `" --machine-type=n1-standard-4 --scopes storage-ro --accelerator="type=`+ gpu_type+`,count=1" --maintenance-policy=TERMINATE --boot-disk-size=200GiB`) } fmt.Printf("Test and if this looks good, consider releasing it via:\n"+ "\n"+ "gcloud compute images create \\\n"+ " --project=\"%s\" \\\n"+ " --source-image=\"%s\" \\\n"+ " --source-image-project=\"%s\" \\\n"+ " \"%s\" \\\n"+ " \"%s\"\n", dest_project, dest_image, build_project, dest_family_flag, dest_image) }