/* * Copyright (C) 2024 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Binary ide_query generates and analyzes build artifacts. // The produced result can be consumed by IDEs to provide language features. package main import ( "bytes" "container/list" "context" "encoding/json" "flag" "fmt" "log" "os" "os/exec" "path" "slices" "strings" "google.golang.org/protobuf/proto" apb "ide_query/cc_analyzer_proto" pb "ide_query/ide_query_proto" ) // Env contains information about the current environment. type Env struct { LunchTarget LunchTarget RepoDir string OutDir string ClangToolsRoot string } // LunchTarget is a parsed Android lunch target. // Input format: -- type LunchTarget struct { Product string Release string Variant string } var _ flag.Value = (*LunchTarget)(nil) // // Get implements flag.Value. // func (l *LunchTarget) Get() any { // return l // } // Set implements flag.Value. func (l *LunchTarget) Set(s string) error { parts := strings.Split(s, "-") if len(parts) != 3 { return fmt.Errorf("invalid lunch target: %q, must have form --", s) } *l = LunchTarget{ Product: parts[0], Release: parts[1], Variant: parts[2], } return nil } // String implements flag.Value. func (l *LunchTarget) String() string { return fmt.Sprintf("%s-%s-%s", l.Product, l.Release, l.Variant) } func main() { var env Env env.OutDir = strings.TrimSuffix(os.Getenv("OUT_DIR"), "/") env.RepoDir = os.Getenv("ANDROID_BUILD_TOP") env.ClangToolsRoot = os.Getenv("PREBUILTS_CLANG_TOOLS_ROOT") flag.Var(&env.LunchTarget, "lunch_target", "The lunch target to query") flag.Parse() files := flag.Args() if len(files) == 0 { fmt.Println("No files provided.") os.Exit(1) return } var ccFiles, javaFiles []string for _, f := range files { switch { case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"): javaFiles = append(javaFiles, f) case strings.HasSuffix(f, ".cc") || strings.HasSuffix(f, ".cpp") || strings.HasSuffix(f, ".h"): ccFiles = append(ccFiles, f) default: log.Printf("File %q is supported - will be skipped.", f) } } ctx := context.Background() // TODO(michaelmerg): Figure out if module_bp_java_deps.json and compile_commands.json is outdated. runMake(ctx, env, "nothing") javaModules, err := loadJavaModules(env) if err != nil { log.Printf("Failed to load java modules: %v", err) } var targets []string javaTargetsByFile := findJavaModules(javaFiles, javaModules) for _, t := range javaTargetsByFile { targets = append(targets, t) } ccTargets, err := getCCTargets(ctx, env, ccFiles) if err != nil { log.Fatalf("Failed to query cc targets: %v", err) } targets = append(targets, ccTargets...) if len(targets) == 0 { fmt.Println("No targets found.") os.Exit(1) return } fmt.Fprintf(os.Stderr, "Running make for modules: %v\n", strings.Join(targets, ", ")) if err := runMake(ctx, env, targets...); err != nil { log.Printf("Building modules failed: %v", err) } var analysis pb.IdeAnalysis results, units := getJavaInputs(env, javaTargetsByFile, javaModules) analysis.Results = results analysis.Units = units if err != nil && analysis.Error == nil { analysis.Error = &pb.AnalysisError{ ErrorMessage: err.Error(), } } results, units, err = getCCInputs(ctx, env, ccFiles) analysis.Results = append(analysis.Results, results...) analysis.Units = append(analysis.Units, units...) if err != nil && analysis.Error == nil { analysis.Error = &pb.AnalysisError{ ErrorMessage: err.Error(), } } analysis.BuildOutDir = env.OutDir data, err := proto.Marshal(&analysis) if err != nil { log.Fatalf("Failed to marshal result proto: %v", err) } _, err = os.Stdout.Write(data) if err != nil { log.Fatalf("Failed to write result proto: %v", err) } for _, r := range analysis.Results { fmt.Fprintf(os.Stderr, "%s: %+v\n", r.GetSourceFilePath(), r.GetStatus()) } } func repoState(env Env, filePaths []string) *apb.RepoState { const compDbPath = "soong/development/ide/compdb/compile_commands.json" return &apb.RepoState{ RepoDir: env.RepoDir, ActiveFilePath: filePaths, OutDir: env.OutDir, CompDbPath: path.Join(env.OutDir, compDbPath), } } func runCCanalyzer(ctx context.Context, env Env, mode string, in []byte) ([]byte, error) { ccAnalyzerPath := path.Join(env.ClangToolsRoot, "bin/ide_query_cc_analyzer") outBuffer := new(bytes.Buffer) inBuffer := new(bytes.Buffer) inBuffer.Write(in) cmd := exec.CommandContext(ctx, ccAnalyzerPath, "--mode="+mode) cmd.Dir = env.RepoDir cmd.Stdin = inBuffer cmd.Stdout = outBuffer cmd.Stderr = os.Stderr err := cmd.Run() return outBuffer.Bytes(), err } // Execute cc_analyzer and get all the targets that needs to be build for analyzing files. func getCCTargets(ctx context.Context, env Env, filePaths []string) ([]string, error) { state, err := proto.Marshal(repoState(env, filePaths)) if err != nil { log.Fatalln("Failed to serialize state:", err) } resp := new(apb.DepsResponse) result, err := runCCanalyzer(ctx, env, "deps", state) if err != nil { return nil, err } if err := proto.Unmarshal(result, resp); err != nil { return nil, fmt.Errorf("malformed response from cc_analyzer: %v", err) } var targets []string if resp.Status != nil && resp.Status.Code != apb.Status_OK { return targets, fmt.Errorf("cc_analyzer failed: %v", resp.Status.Message) } for _, deps := range resp.Deps { targets = append(targets, deps.BuildTarget...) } return targets, nil } func getCCInputs(ctx context.Context, env Env, filePaths []string) ([]*pb.AnalysisResult, []*pb.BuildableUnit, error) { state, err := proto.Marshal(repoState(env, filePaths)) if err != nil { log.Fatalln("Failed to serialize state:", err) } resp := new(apb.IdeAnalysis) result, err := runCCanalyzer(ctx, env, "inputs", state) if err != nil { return nil, nil, fmt.Errorf("cc_analyzer failed:", err) } if err := proto.Unmarshal(result, resp); err != nil { return nil, nil, fmt.Errorf("malformed response from cc_analyzer: %v", err) } if resp.Status != nil && resp.Status.Code != apb.Status_OK { return nil, nil, fmt.Errorf("cc_analyzer failed: %v", resp.Status.Message) } var results []*pb.AnalysisResult var units []*pb.BuildableUnit for _, s := range resp.Sources { status := &pb.AnalysisResult_Status{ Code: pb.AnalysisResult_Status_CODE_OK, } if s.GetStatus().GetCode() != apb.Status_OK { status.Code = pb.AnalysisResult_Status_CODE_BUILD_FAILED status.StatusMessage = proto.String(s.GetStatus().GetMessage()) } result := &pb.AnalysisResult{ SourceFilePath: s.GetPath(), UnitId: s.GetPath(), Status: status, } results = append(results, result) var generated []*pb.GeneratedFile for _, f := range s.Generated { generated = append(generated, &pb.GeneratedFile{ Path: f.GetPath(), Contents: f.GetContents(), }) } genUnit := &pb.BuildableUnit{ Id: "genfiles_for_" + s.GetPath(), SourceFilePaths: s.GetDeps(), GeneratedFiles: generated, } unit := &pb.BuildableUnit{ Id: s.GetPath(), Language: pb.Language_LANGUAGE_CPP, SourceFilePaths: []string{s.GetPath()}, CompilerArguments: s.GetCompilerArguments(), DependencyIds: []string{genUnit.GetId()}, } units = append(units, unit, genUnit) } return results, units, nil } // findJavaModules tries to find the modules that cover the given file paths. // If a file is covered by multiple modules, the first module is returned. func findJavaModules(paths []string, modules map[string]*javaModule) map[string]string { ret := make(map[string]string) // A file may be part of multiple modules. To make the result deterministic, // check the modules in sorted order. keys := make([]string, 0, len(modules)) for name := range modules { keys = append(keys, name) } slices.Sort(keys) for _, name := range keys { if strings.HasSuffix(name, ".impl") { continue } module := modules[name] for i, p := range paths { if slices.Contains(module.Srcs, p) { ret[p] = name paths = append(paths[:i], paths[i+1:]...) break } } if len(paths) == 0 { break } } return ret } func getJavaInputs(env Env, modulesByPath map[string]string, modules map[string]*javaModule) ([]*pb.AnalysisResult, []*pb.BuildableUnit) { var results []*pb.AnalysisResult unitsById := make(map[string]*pb.BuildableUnit) for p, moduleName := range modulesByPath { r := &pb.AnalysisResult{ SourceFilePath: p, } results = append(results, r) m := modules[moduleName] if m == nil { r.Status = &pb.AnalysisResult_Status{ Code: pb.AnalysisResult_Status_CODE_NOT_FOUND, StatusMessage: proto.String("File not found in any module."), } continue } r.UnitId = moduleName r.Status = &pb.AnalysisResult_Status{Code: pb.AnalysisResult_Status_CODE_OK} if unitsById[r.UnitId] != nil { // File is covered by an already created unit. continue } u := &pb.BuildableUnit{ Id: moduleName, Language: pb.Language_LANGUAGE_JAVA, SourceFilePaths: m.Srcs, GeneratedFiles: genFiles(env, m), DependencyIds: m.Deps, } unitsById[u.Id] = u q := list.New() for _, d := range m.Deps { q.PushBack(d) } for q.Len() > 0 { name := q.Remove(q.Front()).(string) mod := modules[name] if mod == nil || unitsById[name] != nil { continue } unitsById[name] = &pb.BuildableUnit{ Id: name, SourceFilePaths: mod.Srcs, GeneratedFiles: genFiles(env, mod), DependencyIds: mod.Deps, } for _, d := range mod.Deps { q.PushBack(d) } } } units := make([]*pb.BuildableUnit, 0, len(unitsById)) for _, u := range unitsById { units = append(units, u) } return results, units } // genFiles returns the generated files (paths that start with outDir/) for the // given module. Generated files that do not exist are ignored. func genFiles(env Env, mod *javaModule) []*pb.GeneratedFile { var paths []string paths = append(paths, mod.Srcs...) paths = append(paths, mod.SrcJars...) paths = append(paths, mod.Jars...) prefix := env.OutDir + "/" var ret []*pb.GeneratedFile for _, p := range paths { relPath, ok := strings.CutPrefix(p, prefix) if !ok { continue } contents, err := os.ReadFile(path.Join(env.RepoDir, p)) if err != nil { continue } ret = append(ret, &pb.GeneratedFile{ Path: relPath, Contents: contents, }) } return ret } // runMake runs Soong build for the given modules. func runMake(ctx context.Context, env Env, modules ...string) error { args := []string{ "--make-mode", "ANDROID_BUILD_ENVIRONMENT_CONFIG=googler-cog", "SOONG_GEN_COMPDB=1", "TARGET_PRODUCT=" + env.LunchTarget.Product, "TARGET_RELEASE=" + env.LunchTarget.Release, "TARGET_BUILD_VARIANT=" + env.LunchTarget.Variant, "TARGET_BUILD_TYPE=release", "-k", } args = append(args, modules...) cmd := exec.CommandContext(ctx, "build/soong/soong_ui.bash", args...) cmd.Dir = env.RepoDir cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr return cmd.Run() } type javaModule struct { Path []string `json:"path,omitempty"` Deps []string `json:"dependencies,omitempty"` Srcs []string `json:"srcs,omitempty"` Jars []string `json:"jars,omitempty"` SrcJars []string `json:"srcjars,omitempty"` } func loadJavaModules(env Env) (map[string]*javaModule, error) { javaDepsPath := path.Join(env.RepoDir, env.OutDir, "soong/module_bp_java_deps.json") data, err := os.ReadFile(javaDepsPath) if err != nil { return nil, err } var ret map[string]*javaModule // module name -> module if err = json.Unmarshal(data, &ret); err != nil { return nil, err } // Add top level java_sdk_library for .impl modules. for name, module := range ret { if striped := strings.TrimSuffix(name, ".impl"); striped != name { ret[striped] = module } } return ret, nil }