# Copyright 2023 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. "" load("@bazel_skylib//lib:sets.bzl", "sets") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR") load("//python/private:text_util.bzl", "render") load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pip_repository_attrs.bzl", "ATTRS") load(":render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") def _get_python_interpreter_attr(rctx): """A helper function for getting the `python_interpreter` attribute or it's default Args: rctx (repository_ctx): Handle to the rule repository context. Returns: str: The attribute value or it's default """ if rctx.attr.python_interpreter: return rctx.attr.python_interpreter if "win" in rctx.os.name: return "python.exe" else: return "python3" def use_isolated(ctx, attr): """Determine whether or not to pass the pip `--isolated` flag to the pip invocation. Args: ctx: repository or module context attr: attributes for the repo rule or tag extension Returns: True if --isolated should be passed """ use_isolated = attr.isolated # The environment variable will take precedence over the attribute isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None) if isolated_env != None: if isolated_env.lower() in ("0", "false"): use_isolated = False else: use_isolated = True return use_isolated _BUILD_FILE_CONTENTS = """\ package(default_visibility = ["//visibility:public"]) # Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it exports_files(["requirements.bzl"]) """ def _pip_repository_impl(rctx): requirements_by_platform = parse_requirements( rctx, requirements_by_platform = requirements_files_by_platform( requirements_by_platform = rctx.attr.requirements_by_platform, requirements_linux = rctx.attr.requirements_linux, requirements_lock = rctx.attr.requirements_lock, requirements_osx = rctx.attr.requirements_darwin, requirements_windows = rctx.attr.requirements_windows, extra_pip_args = rctx.attr.extra_pip_args, ), extra_pip_args = rctx.attr.extra_pip_args, evaluate_markers = lambda rctx, requirements: evaluate_markers( rctx, requirements = requirements, python_interpreter = rctx.attr.python_interpreter, python_interpreter_target = rctx.attr.python_interpreter_target, srcs = rctx.attr._evaluate_markers_srcs, ), ) selected_requirements = {} options = None repository_platform = host_platform(rctx) for name, requirements in requirements_by_platform.items(): r = select_requirement( requirements, platform = None if rctx.attr.download_only else repository_platform, ) if not r: continue options = options or r.extra_pip_args selected_requirements[name] = r.requirement_line bzl_packages = sorted(selected_requirements.keys()) # Normalize cycles first requirement_cycles = { name: sorted(sets.to_list(sets.make(deps))) for name, deps in rctx.attr.experimental_requirement_cycles.items() } # Check for conflicts between cycles _before_ we normalize package names so # that reported errors use the names the user specified for i in range(len(requirement_cycles)): left_group = requirement_cycles.keys()[i] left_deps = requirement_cycles.values()[i] for j in range(len(requirement_cycles) - (i + 1)): right_deps = requirement_cycles.values()[1 + i + j] right_group = requirement_cycles.keys()[1 + i + j] for d in left_deps: if d in right_deps: fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) # And normalize the names as used in the cycle specs # # NOTE: We must check that a listed dependency is actually in the actual # requirements set for the current platform so that we can support cycles in # platform-conditional requirements. Otherwise we'll blindly generate a # label referencing a package which may not be installed on the current # platform. requirement_cycles = { normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) for name, group in requirement_cycles.items() } imports = [ # NOTE: Maintain the order consistent with `buildifier` 'load("@rules_python//python:pip.bzl", "pip_utils")', 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', ] annotations = {} for pkg, annotation in rctx.attr.annotations.items(): filename = "{}.annotation.json".format(normalize_name(pkg)) rctx.file(filename, json.encode_indent(json.decode(annotation))) annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) config = { "download_only": rctx.attr.download_only, "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, "environment": rctx.attr.environment, "envsubst": rctx.attr.envsubst, "extra_pip_args": options, "isolated": use_isolated(rctx, rctx.attr), "pip_data_exclude": rctx.attr.pip_data_exclude, "python_interpreter": _get_python_interpreter_attr(rctx), "quiet": rctx.attr.quiet, "repo": rctx.attr.name, "timeout": rctx.attr.timeout, } if rctx.attr.use_hub_alias_dependencies: config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name) else: config["repo_prefix"] = "{}_".format(rctx.attr.name) if rctx.attr.python_interpreter_target: config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) if rctx.attr.experimental_target_platforms: config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms macro_tmpl = "@%s//{}:{}" % rctx.attr.name aliases = render_pkg_aliases( aliases = { pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)] for pkg in bzl_packages or [] }, ) for path, contents in aliases.items(): rctx.file(path, contents) rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { " # %%GROUP_LIBRARY%%": """\ group_repo = "{name}__groups" group_library( name = group_repo, repo_prefix = "{name}_", groups = all_requirement_groups, )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "", "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") for p in bzl_packages ]), "%%ALL_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "pkg") for p in bzl_packages ]), "%%ALL_REQUIREMENT_GROUPS%%": render.dict(requirement_cycles), "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({ p: macro_tmpl.format(p, "whl") for p in bzl_packages }), "%%ANNOTATIONS%%": render.dict(dict(sorted(annotations.items()))), "%%CONFIG%%": render.dict(dict(sorted(config.items()))), "%%EXTRA_PIP_ARGS%%": json.encode(options), "%%IMPORTS%%": "\n".join(imports), "%%MACRO_TMPL%%": macro_tmpl, "%%NAME%%": rctx.attr.name, "%%PACKAGES%%": render.list( [ ("{}_{}".format(rctx.attr.name, p), r) for p, r in sorted(selected_requirements.items()) ], ), }) return pip_repository = repository_rule( attrs = dict( annotations = attr.string_dict( doc = """\ Optional annotations to apply to packages. Keys should be package names, with capitalization matching the input requirements file, and values should be generated using the `package_name` macro. For example usage, see [this WORKSPACE file](https://github.com/bazelbuild/rules_python/blob/main/examples/pip_repository_annotations/WORKSPACE). """, ), _template = attr.label( default = ":requirements.bzl.tmpl.workspace", ), _evaluate_markers_srcs = attr.label_list( default = EVALUATE_MARKERS_SRCS, doc = """\ The list of labels to use as SRCS for the marker evaluation code. This ensures that the code will be re-evaluated when any of files in the default changes. """, ), **ATTRS ), doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within. Those dependencies become available in a generated `requirements.bzl` file. You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. In your WORKSPACE file: ```starlark load("@rules_python//python:pip.bzl", "pip_parse") pip_parse( name = "pypi", requirements_lock = ":requirements.txt", ) load("@pypi//:requirements.bzl", "install_deps") install_deps() ``` You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following: - `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library` created after extracting the `PyYAML` package. - `@pypi//pyyaml:data` points to the extra data included in the package. - `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package. - `@pypi//pyyaml:whl` points to the wheel file that was extracted. ```starlark py_library( name = "bar", ... deps = [ "//my/other:dep", "@pypi//numpy", "@pypi//requests", ], ) ``` or ```starlark load("@pypi//:requirements.bzl", "requirement") py_library( name = "bar", ... deps = [ "//my/other:dep", requirement("numpy"), requirement("requests"), ], ) ``` In addition to the `requirement` macro, which is used to access the generated `py_library` target generated from a package's wheel, The generated `requirements.bzl` file contains functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. [whl_ep]: https://packaging.python.org/specifications/entry-points/ ```starlark load("@pypi//:requirements.bzl", "entry_point") alias( name = "pip-compile", actual = entry_point( pkg = "pip-tools", script = "pip-compile", ), ) ``` Note that for packages whose name and script are the same, only the name of the package is needed when calling the `entry_point` macro. ```starlark load("@pip//:requirements.bzl", "entry_point") alias( name = "flake8", actual = entry_point("flake8"), ) ``` :::{rubric} Vendoring the requirements.bzl file :heading-level: 3 ::: In some cases you may not want to generate the requirements.bzl file as a repository rule while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module such as a ruleset, you may want to include the requirements.bzl file rather than make your users install the WORKSPACE setup to generate it. See https://github.com/bazelbuild/rules_python/issues/608 This is the same workflow as Gazelle, which creates `go_repository` rules with [`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) To do this, use the "write to source file" pattern documented in https://blog.aspect.dev/bazel-can-write-to-the-source-folder to put a copy of the generated requirements.bzl into your project. Then load the requirements.bzl file directly rather than from the generated repository. See the example in rules_python/examples/pip_parse_vendored. """, implementation = _pip_repository_impl, environ = [ "RULES_PYTHON_PIP_ISOLATED", REPO_DEBUG_ENV_VAR, ], )