diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 2d774e511d..d5335821c9 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -500,17 +500,8 @@ bzl_library( name = "whl_library_targets", srcs = ["whl_library_targets.bzl"], deps = [ - ":env_marker_setting", - ":gen_wheel_record", - ":labels", - ":namespace_pkgs", - ":pep508_deps", - ":venv_entry_point", - ":venv_rewrite_shebang", - "//python:py_binary", - "//python:py_library", - "//python/private:normalize_name", - "@bazel_skylib//rules:copy_file", + ":whl_library_deps_targets", + ":whl_library_srcs", ], ) @@ -590,6 +581,32 @@ bzl_library( ], ) +bzl_library( + name = "whl_library_deps_targets", + srcs = ["whl_library_deps_targets.bzl"], + deps = [ + ":env_marker_setting", + ":labels", + ":pep508_deps", + "//python:py_library", + "//python/private:normalize_name", + ], +) + +bzl_library( + name = "whl_library_srcs", + srcs = ["whl_library_srcs.bzl"], + deps = [ + ":gen_wheel_record", + ":labels", + ":namespace_pkgs", + ":venv_entry_point", + ":venv_rewrite_shebang", + "//python:py_library", + "@bazel_skylib//rules:copy_file", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index ceee319cd7..bec45bb08f 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -101,7 +101,7 @@ def generate_whl_library_build_bazel( kwargs["requires_dist"] = requires_dist loads.extend([ - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "{}")""".format(fn), + """load("@rules_python//python/private/pypi:{}.bzl", "{}")""".format(fn, fn), ]) additional_content = [] diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl new file mode 100644 index 0000000000..89e9b74dc7 --- /dev/null +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -0,0 +1,215 @@ +"""Macro to generate all of the targets present in a {obj}`whl_library`.""" + +load("//python:py_library.bzl", "py_library") +load("//python/private:normalize_name.bzl", "normalize_name") +load(":env_marker_setting.bzl", "env_marker_setting") +load( + ":labels.bzl", + "DATA_LABEL", + "DIST_INFO_LABEL", + "EXTRACTED_WHEEL_FILES", + "PY_LIBRARY_IMPL_LABEL", + "PY_LIBRARY_PUBLIC_LABEL", + "PY_SRCS_LABEL", + "WHEEL_FILE", + "WHEEL_FILE_IMPL_LABEL", + "WHEEL_FILE_PUBLIC_LABEL", +) +load(":pep508_deps.bzl", "deps") + +def whl_library_deps_targets( + *, + name = None, + repo, + aliases = None, + metadata_name, + requires_dist, + extras, + include = [], + group_deps = [], + group_name = None, + dep_template, + tags = [], + visibility = ["//visibility:public"], + native = native, + rules = struct( + py_library = py_library, + env_marker_setting = env_marker_setting, + )): + """Create all of the whl_library targets. + + Args: + name: {type}`str` The wheel filename + metadata_name: {type}`str` The package name as written in wheel `METADATA`. + group_deps: {type}`list[str]` names of fellow members of the group (if + any). These will be excluded from generated deps lists so as to avoid + direct cycles. These dependencies will be provided at runtime by the + group rules which wrap this library and its fellows together. + requires_dist: {type}`list[str]` The list of `Requires-Dist` values from + the whl `METADATA`. + extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. + include: {type}`list[str]` The list of packages to include. + group_name: {type}`str | None` name of the dependency group (if any). + dep_template: {type}`str | None` The dep_template to use. + tags: {type}`list[str]` The tags set on the targets. + repo: {type}`str | Label | None` The BUILD.bazel label to the parent repo that has the + sources. If none, then will take the targets from the current dir. + aliases: {type}`dict[str, str] | None` The list of aliases to create in the parent repo. If None, will create + the default values. Empty list means no aliases. + visibility: {type}`list[str]` The visibility of the targets. + native: {type}`native` The native struct for overriding in tests. + rules: {type}`struct` A struct with references to rules for creating targets. + """ + repo_label = Label(repo).same_package_label if repo else (lambda x: x) + if aliases == None: + aliases = { + EXTRACTED_WHEEL_FILES: repo_label(EXTRACTED_WHEEL_FILES), + DIST_INFO_LABEL: repo_label(DIST_INFO_LABEL), + DATA_LABEL: repo_label(DATA_LABEL), + } + + # If this library is a member of a group, its public label aliases need to + # point to the group implementation rule not the implementation rules. We + # also need to mark the implementation rules as visible to the group + # implementation. + if group_name and "//:" in dep_template: + # This is the legacy behaviour where the group library is outside the hub repo + # + # It is expected to disappear when we drop WORKSPACE or drop the vendoring of + # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add + # another argument to the macro, but it is already full of arguments. + label_tmpl = dep_template.format( + name = "_config", + target = normalize_name(group_name) + "_{}", + ).replace( + "//:", + "//_groups:", + ) + aliases = aliases | { + PY_LIBRARY_PUBLIC_LABEL: label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), + WHEEL_FILE_PUBLIC_LABEL: label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), + } + impl_vis = [dep_template.format( + name = "_config", + target = "__pkg__", + ).replace( + "//:", + "//_groups:", + )] + + py_library_label = PY_LIBRARY_IMPL_LABEL + whl_file_label = WHEEL_FILE_IMPL_LABEL + else: + py_library_label = PY_LIBRARY_PUBLIC_LABEL + whl_file_label = WHEEL_FILE_PUBLIC_LABEL + if group_name: + impl_vis = [dep_template.format(name = "", target = "__subpackages__")] + else: + impl_vis = visibility + + if not requires_dist: + # If the package is in a group but has no deps, we still need the public labels to + # point at the srcs targets so that the group implementation can use them. We don't + # need any of the extra targets, so just create the aliases. + aliases = aliases | { + py_library_label: repo_label(PY_SRCS_LABEL), + whl_file_label: repo_label(WHEEL_FILE), + } + + for alias, actual in aliases.items(): + native.alias( + name = alias, + actual = actual, + visibility = visibility, + ) + + if not requires_dist: + # If there are extras, then they will be visible in requires_dist. + return + + package_deps = _parse_requires_dist( + name = metadata_name, + requires_dist = requires_dist, + excludes = group_deps, + extras = extras, + include = include, + ) + + _config_settings( + dependencies_with_markers = package_deps.deps_select, + rules = rules, + visibility = ["//visibility:private"], + ) + + if hasattr(native, "filegroup"): + # We include the whl file as srcs so that `$(location :whl)` expands to the whl file. + # The transitive dependencies are available via the `data` attribute. + native.filegroup( + name = whl_file_label, + srcs = [repo_label(WHEEL_FILE)], + data = _deps( + deps = [], + package_deps = package_deps, + tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + ), + visibility = impl_vis, + ) + + if hasattr(rules, "py_library"): + rules.py_library( + name = py_library_label, + # We include as srcs to ensure that the (locations :pkg) works as expected. + srcs = [repo_label(PY_SRCS_LABEL)], + deps = _deps( + # We include as deps, so that `PyInfo` and friends (e.g. `pyi_srcs`) get + # propagated. Just passing the target as `srcs` is not enough to propagate + # `pyi_srcs`, see `tests/base_rules/py_library`. + deps = [repo_label(PY_SRCS_LABEL)], + package_deps = package_deps, + tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + ), + tags = tags, + visibility = impl_vis, + ) + +def _parse_requires_dist( + *, + name, + requires_dist, + excludes, + include, + extras): + return deps( + name = normalize_name(name), + requires_dist = requires_dist, + excludes = excludes, + include = include, + extras = extras, + ) + +def _config_settings(dependencies_with_markers, rules, **kwargs): + """Generate config settings for the targets. + + Args: + dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by + each dep. + rules: used for testing + **kwargs: Extra kwargs to pass to the rule. + """ + for dep, expression in dependencies_with_markers.items(): + rules.env_marker_setting( + name = "include_{}".format(dep), + expression = expression, + **kwargs + ) + +def _deps(deps, package_deps, tmpl): + deps = [] + deps + [tmpl.format(d) for d in sorted(package_deps.deps)] + + for dep in package_deps.deps_select: + deps = deps + select({ + ":is_include_{}_true".format(dep): [tmpl.format(dep)], + "//conditions:default": [], + }) + + return deps diff --git a/python/private/pypi/whl_library_srcs.bzl b/python/private/pypi/whl_library_srcs.bzl new file mode 100644 index 0000000000..80a0d8a6dc --- /dev/null +++ b/python/private/pypi/whl_library_srcs.bzl @@ -0,0 +1,244 @@ +"""Macro to generate all of the targets present in a {obj}`whl_library`.""" + +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("//python:py_library.bzl", "py_library") +load(":gen_wheel_record.bzl", "gen_wheel_record") +load( + ":labels.bzl", + "DATA_LABEL", + "DIST_INFO_LABEL", + "EXTRACTED_WHEEL_FILES", + "PY_SRCS_LABEL", + "WHEEL_FILE", +) +load(":namespace_pkgs.bzl", _create_inits = "create_inits") +load(":venv_entry_point.bzl", "venv_entry_point") +load(":venv_rewrite_shebang.bzl", "venv_rewrite_shebang") + +# Files that are special to the Bazel processing of things. +_BAZEL_REPO_FILE_GLOBS = [ + "BUILD", + "BUILD.bazel", + "REPO.bazel", + "WORKSPACE", + "WORKSPACE.bzlmod", + "WORKSPACE.bazel", +] + +_IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes") +_VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") + +def whl_library_srcs( + *, + name, + sdist_filename = None, + data_exclude = [], + srcs_exclude = [], + tags = [], + filegroups = None, + entry_points = {}, + data = [], + copy_files = {}, + copy_executables = {}, + native = native, + enable_implicit_namespace_pkgs = False, + namespace_package_files = [], + visibility = ["//visibility:public"], + rules = struct( + copy_file = copy_file, + py_library = py_library, + venv_entry_point = venv_entry_point, + venv_rewrite_shebang = venv_rewrite_shebang, + gen_wheel_record = gen_wheel_record, + create_inits = _create_inits, + )): + """Create all of the whl_library targets. + + Args: + name: {type}`str` The file to match for including it into the `whl` + filegroup. This may be also parsed to generate extra metadata. + sdist_filename: {type}`str | None` If the wheel was built from an sdist, + the filename of the sdist. + visibility: {type}`list[str]` The visibility of the source targets. + tags: {type}`list[str]` The tags set on the `py_library`. + entry_points: {type}`list[dict]` A list of parsed entry point definitions. + filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target + names and the glob matches. If `None`, defaults will be used. + copy_executables: {type}`dict[str, str]` The mapping between src and + dest locations for the targets. + copy_files: {type}`dict[str, str]` The mapping between src and + dest locations for the targets. + data_exclude: {type}`list[str]` The globs for data attribute exclusion + in `py_library`. + srcs_exclude: {type}`list[str]` The globs for srcs attribute exclusion + in `py_library`. + data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`. + enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py + files for namespace pkgs. + namespace_package_files: {type}`list[str]` A list of labels of files whose + directories are namespace packages. + native: {type}`native` The native struct for overriding in tests. + rules: {type}`struct` A struct with references to rules for creating targets. + """ + tags = sorted(tags) + data = [] + data + + bins_for_data_label = [] + + for ep_dict in entry_points.values(): + kwargs = dict(ep_dict) + ep_name = kwargs.pop("name") + ep_target_name = "bin/{}".format(ep_name) + rules.venv_entry_point( + name = ep_target_name, + **kwargs + ) + bins_for_data_label.append(ep_target_name) + data.append(ep_target_name) + + existing_bin_names = {ep["name"].lower(): None for ep in entry_points.values()} + for p in native.glob(["bin/*"], allow_empty = True): + existing_bin_names[p[len("bin/"):].lower()] = None + + for src_path in native.glob(["rewrite-bin/*"], allow_empty = True): + script_name = src_path[len("rewrite-bin/"):] + if script_name.lower() in existing_bin_names: + continue + rewrite_target_name = "bin/{}".format(script_name) + rules.venv_rewrite_shebang( + name = rewrite_target_name, + src = src_path, + package = name, + ) + bins_for_data_label.append(rewrite_target_name) + data.append(rewrite_target_name) + + record_srcs = native.glob(["rewrite-record/*/RECORD"], allow_empty = True) + record_target_name = "record" + if record_srcs: + rules.gen_wheel_record( + name = record_target_name, + srcs = record_srcs, + tags = ["manual"], + ) + data.append(record_target_name) + + if filegroups == None: + filegroups = { + EXTRACTED_WHEEL_FILES: dict( + include = ["**"], + # The Bazel repo files are always excluded; only the sdist + # filename is conditional on `sdist_filename`. + exclude = _BAZEL_REPO_FILE_GLOBS + ( + [sdist_filename] if sdist_filename else [] + ), + ), + DIST_INFO_LABEL: dict( + include = ["site-packages/*.dist-info/**"], + ), + DATA_LABEL: dict( + include = ["data/**", "bin/**", "include/**"], + ), + } + + for filegroup_name, glob_kwargs in filegroups.items(): + glob_kwargs = {"allow_empty": True} | glob_kwargs + srcs = native.glob(**glob_kwargs) + if filegroup_name == DATA_LABEL: + srcs = srcs + bins_for_data_label + if filegroup_name == DIST_INFO_LABEL and record_srcs: + srcs = srcs + [record_target_name] + native.filegroup( + name = filegroup_name, + srcs = srcs, + visibility = visibility, + ) + + for src, dest in copy_files.items(): + rules.copy_file( + name = dest + ".copy", + src = src, + out = dest, + visibility = visibility, + ) + data.append(dest) + for src, dest in copy_executables.items(): + rules.copy_file( + name = dest + ".copy", + src = src, + out = dest, + is_executable = True, + visibility = visibility, + ) + data.append(dest) + + if hasattr(native, "filegroup"): + native.filegroup( + name = WHEEL_FILE, + srcs = [name], + visibility = visibility, + ) + + if hasattr(rules, "py_library"): + srcs = native.glob( + ["site-packages/**/*.py"], + exclude = srcs_exclude, + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ) + + # NOTE: pyi files should probably be excluded because they're carried + # by the pyi_srcs attribute. However, historical behavior included + # them in data and some tools currently rely on that. + _data_exclude = [ + "**/*.py", + "**/*.pyc", + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created + ] + if sdist_filename: + _data_exclude.append("**/*.dist-info/RECORD") + for item in data_exclude: + if item not in _data_exclude: + _data_exclude.append(item) + + data = data + native.glob( + ["site-packages/**/*"], + exclude = _data_exclude, + allow_empty = True, + ) + + pyi_srcs = native.glob( + ["site-packages/**/*.pyi"], + allow_empty = True, + ) + + if not enable_implicit_namespace_pkgs: + generated_namespace_package_files = select({ + _IS_VENV_SITE_PACKAGES_YES: [], + "//conditions:default": rules.create_inits( + srcs = srcs + data + pyi_srcs, + ignored_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. + root = "site-packages", + ), + }) + namespace_package_files += generated_namespace_package_files + srcs = srcs + generated_namespace_package_files + + # This is done after create_inits() is called so that the data scheme + # files don't have such files created in their directories. + data = data + [DATA_LABEL] + + rules.py_library( + name = PY_SRCS_LABEL, + srcs = srcs, + pyi_srcs = pyi_srcs, + data = data, + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + tags = tags, + visibility = visibility, + experimental_venvs_site_packages = _VENV_SITE_PACKAGES_FLAG, + namespace_package_files = namespace_package_files, + ) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 18a5e8b26a..b8d51d0bd7 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -14,40 +14,11 @@ """Macro to generate all of the targets present in a {obj}`whl_library`.""" -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") -load("//python:py_library.bzl", "py_library") -load("//python/private:normalize_name.bzl", "normalize_name") -load(":env_marker_setting.bzl", "env_marker_setting") -load(":gen_wheel_record.bzl", "gen_wheel_record") -load( - ":labels.bzl", - "DATA_LABEL", - "DIST_INFO_LABEL", - "EXTRACTED_WHEEL_FILES", - "PY_LIBRARY_IMPL_LABEL", - "PY_LIBRARY_PUBLIC_LABEL", - "PY_SRCS_LABEL", - "WHEEL_FILE", - "WHEEL_FILE_IMPL_LABEL", - "WHEEL_FILE_PUBLIC_LABEL", -) -load(":namespace_pkgs.bzl", _create_inits = "create_inits") -load(":pep508_deps.bzl", "deps") -load(":venv_entry_point.bzl", "venv_entry_point") -load(":venv_rewrite_shebang.bzl", "venv_rewrite_shebang") +load(":whl_library_deps_targets.bzl", _whl_library_deps_targets = "whl_library_deps_targets") +load(":whl_library_srcs.bzl", _whl_library_srcs = "whl_library_srcs") -# Files that are special to the Bazel processing of things. -_BAZEL_REPO_FILE_GLOBS = [ - "BUILD", - "BUILD.bazel", - "REPO.bazel", - "WORKSPACE", - "WORKSPACE.bzlmod", - "WORKSPACE.bazel", -] - -_IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes") -_VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") +whl_library_deps_targets = _whl_library_deps_targets +whl_library_srcs = _whl_library_srcs def whl_library_targets( *, @@ -99,7 +70,6 @@ def whl_library_targets( visibility: {type}`list[str]` The visibility of the targets. **kwargs: Extra args passed to the {obj}`whl_library_deps_targets` and {obj}`whl_library_srcs`. """ - create_extra_targets = bool(requires_dist or group_name) and dep_template whl_library_srcs( name = name, sdist_filename = sdist_filename, @@ -113,440 +83,19 @@ def whl_library_targets( copy_executables = copy_executables, enable_implicit_namespace_pkgs = enable_implicit_namespace_pkgs, namespace_package_files = namespace_package_files, - # If there are no dependencies, then let's create the targets with public labels. - # Note, we are not supporting grouping the packages in this case, but that is fine. - whl_name = WHEEL_FILE if create_extra_targets else WHEEL_FILE_PUBLIC_LABEL, - pkg_name = PY_SRCS_LABEL if create_extra_targets else PY_LIBRARY_PUBLIC_LABEL, **kwargs ) - if create_extra_targets: - whl_library_deps_targets( - name = name, - metadata_name = metadata_name, - requires_dist = requires_dist, - group_deps = group_deps, # only needed if requires_dist is present - extras = extras, # only needed if requires_dist is present - include = include, # only needed if requires_dist is present - group_name = group_name, # only needed if requires_dist is present - dep_template = dep_template, # only needed if requires_dist is present - repo = None, # set aliases in the same repo - aliases = {}, - **kwargs - ) - -def whl_library_srcs( - *, - name, - sdist_filename = None, - data_exclude = [], - srcs_exclude = [], - tags = [], - filegroups = None, - entry_points = {}, - data = [], - copy_files = {}, - copy_executables = {}, - native = native, - enable_implicit_namespace_pkgs = False, - namespace_package_files = [], - whl_name = WHEEL_FILE, - pkg_name = PY_SRCS_LABEL, - visibility = ["//visibility:public"], - rules = struct( - copy_file = copy_file, - py_library = py_library, - venv_entry_point = venv_entry_point, - venv_rewrite_shebang = venv_rewrite_shebang, - gen_wheel_record = gen_wheel_record, - create_inits = _create_inits, - )): - """Create all of the whl_library targets. - - Args: - name: {type}`str` The file to match for including it into the `whl` - filegroup. This may be also parsed to generate extra metadata. - sdist_filename: {type}`str | None` If the wheel was built from an sdist, - the filename of the sdist. - visibility: {type}`list[str]` The visibility of the source targets. - tags: {type}`list[str]` The tags set on the `py_library`. - entry_points: {type}`list[dict]` A list of parsed entry point definitions. - filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target - names and the glob matches. If `None`, defaults will be used. - copy_executables: {type}`dict[str, str]` The mapping between src and - dest locations for the targets. - copy_files: {type}`dict[str, str]` The mapping between src and - dest locations for the targets. - data_exclude: {type}`list[str]` The globs for data attribute exclusion - in `py_library`. - srcs_exclude: {type}`list[str]` The globs for srcs attribute exclusion - in `py_library`. - data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`. - enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py - files for namespace pkgs. - namespace_package_files: {type}`list[str]` A list of labels of files whose - directories are namespace packages. - whl_name: {type}`str` The label name to use for the wheel filegroup target. - pkg_name: {type}`str` The label name to use for the py_library target. - native: {type}`native` The native struct for overriding in tests. - rules: {type}`struct` A struct with references to rules for creating targets. - """ - tags = sorted(tags) - data = [] + data - - bins_for_data_label = [] - - for ep_dict in entry_points.values(): - kwargs = dict(ep_dict) - ep_name = kwargs.pop("name") - ep_target_name = "bin/{}".format(ep_name) - rules.venv_entry_point( - name = ep_target_name, - **kwargs - ) - bins_for_data_label.append(ep_target_name) - data.append(ep_target_name) - - existing_bin_names = {ep["name"].lower(): None for ep in entry_points.values()} - for p in native.glob(["bin/*"], allow_empty = True): - existing_bin_names[p[len("bin/"):].lower()] = None - - for src_path in native.glob(["rewrite-bin/*"], allow_empty = True): - script_name = src_path[len("rewrite-bin/"):] - if script_name.lower() in existing_bin_names: - continue - rewrite_target_name = "bin/{}".format(script_name) - rules.venv_rewrite_shebang( - name = rewrite_target_name, - src = src_path, - package = name, - ) - bins_for_data_label.append(rewrite_target_name) - data.append(rewrite_target_name) - - record_srcs = native.glob(["rewrite-record/*/RECORD"], allow_empty = True) - record_target_name = "record" - if record_srcs: - rules.gen_wheel_record( - name = record_target_name, - srcs = record_srcs, - tags = ["manual"], - ) - data.append(record_target_name) - - if filegroups == None: - filegroups = { - EXTRACTED_WHEEL_FILES: dict( - include = ["**"], - # The Bazel repo files are always excluded; only the sdist - # filename is conditional on `sdist_filename`. - exclude = _BAZEL_REPO_FILE_GLOBS + ( - [sdist_filename] if sdist_filename else [] - ), - ), - DIST_INFO_LABEL: dict( - include = ["site-packages/*.dist-info/**"], - ), - DATA_LABEL: dict( - include = ["data/**", "bin/**", "include/**"], - ), - } - - for filegroup_name, glob_kwargs in filegroups.items(): - glob_kwargs = {"allow_empty": True} | glob_kwargs - srcs = native.glob(**glob_kwargs) - if filegroup_name == DATA_LABEL: - srcs = srcs + bins_for_data_label - if filegroup_name == DIST_INFO_LABEL and record_srcs: - srcs = srcs + [record_target_name] - native.filegroup( - name = filegroup_name, - srcs = srcs, - visibility = visibility, - ) - - for src, dest in copy_files.items(): - rules.copy_file( - name = dest + ".copy", - src = src, - out = dest, - visibility = visibility, - ) - data.append(dest) - for src, dest in copy_executables.items(): - rules.copy_file( - name = dest + ".copy", - src = src, - out = dest, - is_executable = True, - visibility = visibility, - ) - data.append(dest) - - if hasattr(native, "filegroup"): - native.filegroup( - name = whl_name, - srcs = [name], - visibility = visibility, - ) - - if hasattr(rules, "py_library"): - srcs = native.glob( - ["site-packages/**/*.py"], - exclude = srcs_exclude, - # Empty sources are allowed to support wheels that don't have any - # pure-Python code, e.g. pymssql, which is written in Cython. - allow_empty = True, - ) - - # NOTE: pyi files should probably be excluded because they're carried - # by the pyi_srcs attribute. However, historical behavior included - # them in data and some tools currently rely on that. - _data_exclude = [ - "**/*.py", - "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created - ] - if sdist_filename: - _data_exclude.append("**/*.dist-info/RECORD") - for item in data_exclude: - if item not in _data_exclude: - _data_exclude.append(item) - - data = data + native.glob( - ["site-packages/**/*"], - exclude = _data_exclude, - allow_empty = True, - ) - - pyi_srcs = native.glob( - ["site-packages/**/*.pyi"], - allow_empty = True, - ) - - if not enable_implicit_namespace_pkgs: - generated_namespace_package_files = select({ - _IS_VENV_SITE_PACKAGES_YES: [], - "//conditions:default": rules.create_inits( - srcs = srcs + data + pyi_srcs, - ignored_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. - root = "site-packages", - ), - }) - namespace_package_files += generated_namespace_package_files - srcs = srcs + generated_namespace_package_files - - # This is done after create_inits() is called so that the data scheme - # files don't have such files created in their directories. - data = data + [DATA_LABEL] - - rules.py_library( - name = pkg_name, - srcs = srcs, - pyi_srcs = pyi_srcs, - data = data, - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["site-packages"], - tags = tags, - visibility = visibility, - experimental_venvs_site_packages = _VENV_SITE_PACKAGES_FLAG, - namespace_package_files = namespace_package_files, - ) - -def whl_library_deps_targets( - *, - name = None, - repo, - aliases = None, - metadata_name, - requires_dist, - extras, - include = [], - group_deps = [], - group_name = None, - dep_template, - tags = [], - visibility = ["//visibility:public"], - native = native, - rules = struct( - py_library = py_library, - env_marker_setting = env_marker_setting, - )): - """Create all of the whl_library targets. - - Args: - name: {type}`str` The wheel filename - metadata_name: {type}`str` The package name as written in wheel `METADATA`. - group_deps: {type}`list[str]` names of fellow members of the group (if - any). These will be excluded from generated deps lists so as to avoid - direct cycles. These dependencies will be provided at runtime by the - group rules which wrap this library and its fellows together. - requires_dist: {type}`list[str]` The list of `Requires-Dist` values from - the whl `METADATA`. - extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. - include: {type}`list[str]` The list of packages to include. - group_name: {type}`str | None` name of the dependency group (if any). - dep_template: {type}`str | None` The dep_template to use. - tags: {type}`list[str]` The tags set on the targets. - repo: {type}`str | Label | None` The BUILD.bazel label to the parent repo that has the - sources. If none, then will take the targets from the current dir. - aliases: {type}`dict[str, str] | None` The list of aliases to create in the parent repo. If None, will create - the default values. Empty list means no aliases. - visibility: {type}`list[str]` The visibility of the targets. - native: {type}`native` The native struct for overriding in tests. - rules: {type}`struct` A struct with references to rules for creating targets. - """ - repo_label = Label(repo).same_package_label if repo else (lambda x: x) - if aliases == None: - aliases = { - EXTRACTED_WHEEL_FILES: repo_label(EXTRACTED_WHEEL_FILES), - DIST_INFO_LABEL: repo_label(DIST_INFO_LABEL), - DATA_LABEL: repo_label(DATA_LABEL), - } - - # If this library is a member of a group, its public label aliases need to - # point to the group implementation rule not the implementation rules. We - # also need to mark the implementation rules as visible to the group - # implementation. - if group_name and "//:" in dep_template: - # This is the legacy behaviour where the group library is outside the hub repo - # - # It is expected to disappear when we drop WORKSPACE or drop the vendoring of - # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add - # another argument to the macro, but it is already full of arguments. - label_tmpl = dep_template.format( - name = "_config", - target = normalize_name(group_name) + "_{}", - ).replace( - "//:", - "//_groups:", - ) - aliases = aliases | { - PY_LIBRARY_PUBLIC_LABEL: label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), - WHEEL_FILE_PUBLIC_LABEL: label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), - } - impl_vis = [dep_template.format( - name = "_config", - target = "__pkg__", - ).replace( - "//:", - "//_groups:", - )] - - py_library_label = PY_LIBRARY_IMPL_LABEL - whl_file_label = WHEEL_FILE_IMPL_LABEL - else: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - if group_name: - impl_vis = [dep_template.format(name = "", target = "__subpackages__")] - else: - impl_vis = visibility - - if not requires_dist: - # If the package is in a group but has no deps, we still need the public labels to - # point at the srcs targets so that the group implementation can use them. We don't - # need any of the extra targets, so just create the aliases. - aliases = aliases | { - py_library_label: repo_label(PY_SRCS_LABEL), - whl_file_label: repo_label(WHEEL_FILE), - } - - for alias, actual in aliases.items(): - native.alias( - name = alias, - actual = actual, - visibility = visibility, - ) - - if not requires_dist: - # If there are extras, then they will be visible in requires_dist. - return - - package_deps = _parse_requires_dist( - name = metadata_name, - requires_dist = requires_dist, - excludes = group_deps, - extras = extras, - include = include, - ) - - _config_settings( - dependencies_with_markers = package_deps.deps_select, - rules = rules, - visibility = ["//visibility:private"], - ) - - if hasattr(native, "filegroup"): - # We include the whl file as srcs so that `$(location :whl)` expands to the whl file. - # The transitive dependencies are available via the `data` attribute. - native.filegroup( - name = whl_file_label, - srcs = [repo_label(WHEEL_FILE)], - data = _deps( - deps = [], - package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), - ), - visibility = impl_vis, - ) - - if hasattr(rules, "py_library"): - rules.py_library( - name = py_library_label, - # We include as srcs to ensure that the (locations :pkg) works as expected. - srcs = [repo_label(PY_SRCS_LABEL)], - deps = _deps( - # We include as deps, so that `PyInfo` and friends (e.g. `pyi_srcs`) get - # propagated. Just passing the target as `srcs` is not enough to propagate - # `pyi_srcs`, see `tests/base_rules/py_library`. - deps = [repo_label(PY_SRCS_LABEL)], - package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), - ), - tags = tags, - visibility = impl_vis, - ) - -def _parse_requires_dist( - *, - name, - requires_dist, - excludes, - include, - extras): - return deps( - name = normalize_name(name), + whl_library_deps_targets( + name = name, + metadata_name = metadata_name, requires_dist = requires_dist, - excludes = excludes, - include = include, - extras = extras, + dep_template = dep_template, # only needed if requires_dist or group_name is present + group_deps = group_deps, # only needed if group_name is present + group_name = group_name, # must specify group_deps together + extras = extras, # only needed if requires_dist is present + include = include, # only needed if requires_dist is present + repo = None, # set aliases in the same repo + aliases = {}, + **kwargs ) - -def _config_settings(dependencies_with_markers, rules, **kwargs): - """Generate config settings for the targets. - - Args: - dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by - each dep. - rules: used for testing - **kwargs: Extra kwargs to pass to the rule. - """ - for dep, expression in dependencies_with_markers.items(): - rules.env_marker_setting( - name = "include_{}".format(dep), - expression = expression, - **kwargs - ) - -def _deps(deps, package_deps, tmpl): - deps = [] + deps + [tmpl.format(d) for d in sorted(package_deps.deps)] - - for dep in package_deps.deps_select: - deps = deps + select({ - ":is_include_{}_true".format(dep): [tmpl.format(dep)], - "//conditions:default": [], - }) - - return deps diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index bb03d9a589..d3a390d9bb 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -241,6 +241,34 @@ package_metadata( _tests.append(_test_all_with_loads) +def _test_deps_targets(env): + want = """\ +load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_deps_targets.bzl", "whl_library_deps_targets") + +package(default_visibility = ["//visibility:public"]) + +whl_library_deps_targets( + metadata_name = "foo", + name = "foo.whl", + repo = "@some_repo", + tags = [ + "pypi_name=foo", + "pypi_version=0", + ], +) +""" + actual = generate_whl_library_build_bazel( + metadata_version = "0", + metadata_name = "foo", + name = "foo.whl", + repo = "@some_repo", + config_load = "", + ) + env.expect.that_str(actual.replace("@@", "@")).equals(want) + +_tests.append(_test_deps_targets) + def generate_whl_library_build_bazel_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_library_deps_targets/BUILD.bazel b/tests/pypi/whl_library_deps_targets/BUILD.bazel new file mode 100644 index 0000000000..789a86d802 --- /dev/null +++ b/tests/pypi/whl_library_deps_targets/BUILD.bazel @@ -0,0 +1,5 @@ +load(":whl_library_deps_targets_tests.bzl", "whl_library_deps_targets_test_suite") + +whl_library_deps_targets_test_suite( + name = "whl_library_deps_targets_tests", +) diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl new file mode 100644 index 0000000000..ad6ab6f5fb --- /dev/null +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -0,0 +1,147 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:whl_library_deps_targets.bzl", "whl_library_deps_targets") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +def _test_whl_library_deps_targets(env): + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + + m_glob = mocks.glob() + + m_glob.results.append([]) # bin + m_glob.results.append([]) # rewrite-bin + m_glob.results.append(["site-packages/foo/SRCS.py"]) # srcs + m_glob.results.append(["site-packages/foo/DATA.txt"]) # data + m_glob.results.append(["site-packages/foo/PYI.pyi"]) # pyi + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "Foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "foo", # this self-edge will be ignored + "bar", + "bar-baz; python_version < \"8.2\"", + "booo", # this is effectively excluded due to the list below + ], + include = ["foo", "bar", "bar_baz"], + # Overrides for testing + repo = None, + aliases = None, + extras = [], + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + config_setting = lambda **_: None, + glob = m_glob.glob, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + create_inits = lambda *args, **kwargs: ["_create_inits_target"], + venv_rewrite_shebang = lambda **kwargs: None, + ), + ) + + env.expect.that_collection(filegroup_calls).contains_exactly([ + { + "name": "whl", + # NOTE @aignas 2026-07-25: depending on the brackets position one may get different + # results in the expectation. + "srcs": ["whl_file"], + "data": ["@pypi//bar:whl"] + select({ + ":is_include_bar_baz_true": ["@pypi//bar_baz:whl"], + "//conditions:default": [], + }), + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + + env.expect.that_collection(py_library_calls).has_size(1) + if len(py_library_calls) != 1: + return + py_library_call = py_library_calls[0] + + env.expect.that_dict(py_library_call).contains_exactly({ + "name": "pkg", + "srcs": ["srcs"], + "deps": ["srcs", "@pypi//bar:pkg"] + select({ + ":is_include_bar_baz_true": ["@pypi//bar_baz:pkg"], + "//conditions:default": [], + }), + "tags": [], + "visibility": ["//visibility:public"], + }) # buildifier: @unsorted-dict-items + + env.expect.that_collection(m_glob.calls).contains_exactly([]) + + env.expect.that_collection(env_marker_setting_calls).contains_exactly([ + { + "name": "include_bar_baz", + "expression": "python_version < \"8.2\"", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_whl_library_deps_targets) + +def _test_whl_library_deps_targets_no_deps(env): + alias_calls = [] + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "Foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [], + group_name = "qux", + repo = None, + aliases = {}, + extras = [], + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: alias_calls.append(kwargs), + config_setting = lambda **_: None, + glob = lambda **_: [], + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + ), + ) + + # If the package is in a group but has no deps, then the public labels should be aliases + # to the srcs targets and no other targets should be created. + env.expect.that_collection(alias_calls).contains_exactly([ + { + "name": "pkg", + "actual": "srcs", + "visibility": ["//visibility:public"], + }, + { + "name": "whl", + "actual": "whl_file", + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + + env.expect.that_collection(filegroup_calls).contains_exactly([]) + env.expect.that_collection(py_library_calls).contains_exactly([]) + env.expect.that_collection(env_marker_setting_calls).contains_exactly([]) + +_tests.append(_test_whl_library_deps_targets_no_deps) + +def whl_library_deps_targets_test_suite(name): + """create the test suite. + + args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/whl_library_srcs/BUILD.bazel b/tests/pypi/whl_library_srcs/BUILD.bazel new file mode 100644 index 0000000000..f52514836b --- /dev/null +++ b/tests/pypi/whl_library_srcs/BUILD.bazel @@ -0,0 +1,5 @@ +load(":whl_library_srcs_tests.bzl", "whl_library_srcs_test_suite") + +whl_library_srcs_test_suite( + name = "whl_library_srcs_tests", +) diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl similarity index 56% rename from tests/pypi/whl_library_targets/whl_library_targets_tests.bzl rename to tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl index 3fe1b99768..1499170ed4 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl @@ -15,11 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load( - "//python/private/pypi:whl_library_targets.bzl", - "whl_library_deps_targets", - "whl_library_srcs", -) # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_library_srcs.bzl", "whl_library_srcs") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -107,138 +103,6 @@ def _test_copy(env): _tests.append(_test_copy) -def _test_whl_library_deps_targets(env): - filegroup_calls = [] - py_library_calls = [] - env_marker_setting_calls = [] - - m_glob = mocks.glob() - - m_glob.results.append([]) # bin - m_glob.results.append([]) # rewrite-bin - m_glob.results.append(["site-packages/foo/SRCS.py"]) # srcs - m_glob.results.append(["site-packages/foo/DATA.txt"]) # data - m_glob.results.append(["site-packages/foo/PYI.pyi"]) # pyi - - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "Foo", - dep_template = "@pypi//{name}:{target}", - requires_dist = [ - "foo", # this self-edge will be ignored - "bar", - "bar-baz; python_version < \"8.2\"", - "booo", # this is effectively excluded due to the list below - ], - include = ["foo", "bar", "bar_baz"], - # Overrides for testing - repo = None, - aliases = None, - extras = [], - native = struct( - filegroup = lambda **kwargs: filegroup_calls.append(kwargs), - alias = lambda **kwargs: None, - config_setting = lambda **_: None, - glob = m_glob.glob, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - create_inits = lambda *args, **kwargs: ["_create_inits_target"], - venv_rewrite_shebang = lambda **kwargs: None, - ), - ) - - env.expect.that_collection(filegroup_calls).contains_exactly([ - { - "name": "whl", - # NOTE @aignas 2026-07-25: depending on the brackets position one may get different - # results in the expectation. - "srcs": ["whl_file"], - "data": ["@pypi//bar:whl"] + select({ - ":is_include_bar_baz_true": ["@pypi//bar_baz:whl"], - "//conditions:default": [], - }), - "visibility": ["//visibility:public"], - }, - ]) # buildifier: @unsorted-dict-items - - env.expect.that_collection(py_library_calls).has_size(1) - if len(py_library_calls) != 1: - return - py_library_call = py_library_calls[0] - - env.expect.that_dict(py_library_call).contains_exactly({ - "name": "pkg", - "srcs": ["srcs"], - "deps": ["srcs", "@pypi//bar:pkg"] + select({ - ":is_include_bar_baz_true": ["@pypi//bar_baz:pkg"], - "//conditions:default": [], - }), - "tags": [], - "visibility": ["//visibility:public"], - }) # buildifier: @unsorted-dict-items - - env.expect.that_collection(m_glob.calls).contains_exactly([]) - - env.expect.that_collection(env_marker_setting_calls).contains_exactly([ - { - "name": "include_bar_baz", - "expression": "python_version < \"8.2\"", - "visibility": ["//visibility:private"], - }, - ]) # buildifier: @unsorted-dict-items - -_tests.append(_test_whl_library_deps_targets) - -def _test_whl_library_deps_targets_no_deps(env): - alias_calls = [] - filegroup_calls = [] - py_library_calls = [] - env_marker_setting_calls = [] - - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "Foo", - dep_template = "@pypi//{name}:{target}", - requires_dist = [], - group_name = "qux", - repo = None, - aliases = {}, - extras = [], - native = struct( - filegroup = lambda **kwargs: filegroup_calls.append(kwargs), - alias = lambda **kwargs: alias_calls.append(kwargs), - config_setting = lambda **_: None, - glob = lambda **_: [], - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - ), - ) - - # If the package is in a group but has no deps, then the public labels should be aliases - # to the srcs targets and no other targets should be created. - env.expect.that_collection(alias_calls).contains_exactly([ - { - "name": "pkg", - "actual": "srcs", - "visibility": ["//visibility:public"], - }, - { - "name": "whl", - "actual": "whl_file", - "visibility": ["//visibility:public"], - }, - ]) # buildifier: @unsorted-dict-items - - env.expect.that_collection(filegroup_calls).contains_exactly([]) - env.expect.that_collection(py_library_calls).contains_exactly([]) - env.expect.that_collection(env_marker_setting_calls).contains_exactly([]) - -_tests.append(_test_whl_library_deps_targets_no_deps) - def _test_sdist_excludes_record(env): py_library_calls = [] m_glob = mocks.glob() @@ -338,7 +202,7 @@ def _test_exclude_bazel_files(env): _tests.append(_test_exclude_bazel_files) -def whl_library_targets_test_suite(name): +def whl_library_srcs_test_suite(name): """create the test suite. args: diff --git a/tests/pypi/whl_library_targets/BUILD.bazel b/tests/pypi/whl_library_targets/BUILD.bazel deleted file mode 100644 index f3d25c2a52..0000000000 --- a/tests/pypi/whl_library_targets/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -load(":whl_library_targets_tests.bzl", "whl_library_targets_test_suite") - -whl_library_targets_test_suite( - name = "whl_library_targets_tests", -)