blob: d22f7a56757ddabde34fa03a1b3d7208be3a55dd [file] [log] [blame]
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Originally derived from:
# https://github.com/apache/incubator-heron/blob/master/tools/rules/pex_rules.bzl
"""Python pex rules for Bazel
### Setup
Add something like this to your WORKSPACE file:
git_repository(
name = "io_bazel_rules_pex",
remote = "https://github.com/benley/bazel_rules_pex.git",
tag = "0.3.0",
)
load("@io_bazel_rules_pex//pex:pex_rules.bzl", "pex_repositories")
pex_repositories()
In a BUILD file where you want to use these rules, or in your
`tools/build_rules/prelude_bazel` file if you want them present repo-wide, add:
load(
"@io_bazel_rules_pex//pex:pex_rules.bzl",
"pex_binary",
"pex_library",
"pex_test",
"pex_pytest",
)
Lastly, make sure that `tools/build_rules/BUILD` exists, even if it is empty,
so that Bazel can find your `prelude_bazel` file.
"""
pex_file_types = [".py"]
egg_file_types = [".egg", ".whl"]
PexProvider = provider(fields=["transitive_sources", "transitive_eggs", "transitive_reqs"])
def _collect_transitive_sources(ctx):
return depset(ctx.files.srcs,
transitive=[dep[PexProvider].transitive_sources for dep in ctx.attr.deps])
def _collect_transitive_eggs(ctx):
return depset(ctx.files.eggs,
transitive=[dep[PexProvider].transitive_eggs for dep in ctx.attr.deps])
def _collect_transitive_reqs(ctx):
return depset(ctx.attr.reqs,
transitive=[dep[PexProvider].transitive_reqs for dep in ctx.attr.deps])
def _collect_transitive(ctx):
return PexProvider(
# These rules don't use transitive_sources internally; it's just here for
# parity with the native py_library rule type.
transitive_sources = _collect_transitive_sources(ctx),
transitive_eggs = _collect_transitive_eggs(ctx),
transitive_reqs = _collect_transitive_reqs(ctx),
# uses_shared_libraries = ... # native py_library has this. What is it?
)
def _pex_library_impl(ctx):
transitive_files = depset(ctx.files.srcs,
transitive = [dep.default_runfiles.files for dep in ctx.attr.deps])
return struct(
providers = [_collect_transitive(ctx)],
runfiles = ctx.runfiles(
collect_default = True,
transitive_files = transitive_files,
)
)
def _gen_manifest(py, runfiles, resources):
"""Generate a manifest for pex_wrapper.
Returns:
struct(
modules = [struct(src = "path_on_disk", dest = "path_in_pex"), ...],
requirements = ["pypi_package", ...],
prebuiltLibraries = ["path_on_disk", ...],
resources = ["path_on_disk", ...],
)
"""
pex_files = []
for f in runfiles.files.to_list():
dpath = f.short_path
if dpath.startswith("../"):
dpath = dpath[3:]
pex_files.append(
struct(
src = f.path,
dest = dpath,
),
)
res_files = []
for f in resources:
dpath = f.short_path
if dpath.startswith("../"):
dpath = dpath[3:]
res_files.append(
struct(
src = f.path,
dest = dpath,
),
)
return struct(
modules = pex_files,
requirements = py.transitive_reqs.to_list(),
prebuiltLibraries = [f.path for f in py.transitive_eggs.to_list()],
resources = res_files,
)
def _pex_binary_impl(ctx):
if ctx.attr.entrypoint and ctx.file.main:
fail("Please specify either entrypoint or main, not both.")
if ctx.attr.entrypoint:
main_file = None
main_pkg = ctx.attr.entrypoint
elif ctx.file.main:
main_file = ctx.file.main
else:
main_file = ctx.files.srcs[0]
transitive_files = list(ctx.files.srcs)
if main_file:
# Translate main_file's short path into a python module name
main_pkg = main_file.short_path.replace('/', '.')[:-3]
transitive_files += [main_file]
deploy_pex = ctx.actions.declare_file('%s.pex' % ctx.attr.name)
py = _collect_transitive(ctx)
transitive_files = depset(transitive_files,
transitive = [dep.default_runfiles.files for dep in ctx.attr.deps])
runfiles = ctx.runfiles(
collect_default = True,
transitive_files = depset(transitive_files),
)
resources = ctx.files.resources
manifest_file = ctx.actions.declare_file('%s.pex_manifest' % ctx.attr.name)
manifest = _gen_manifest(py, runfiles, resources)
ctx.actions.write(
output = manifest_file,
content = manifest.to_json(),
)
pexbuilder = ctx.executable._pexbuilder
# form the arguments to pex builder
arguments = [] if ctx.attr.zip_safe else ["--not-zip-safe"]
arguments += [] if ctx.attr.pex_use_wheels else ["--no-use-wheel"]
if ctx.attr.interpreter:
arguments += ["--python", ctx.attr.interpreter]
for platform in ctx.attr.platforms:
arguments += ["--platform", platform]
for egg in py.transitive_eggs.to_list():
arguments += ["--find-links", egg.dirname]
arguments += [
"--pex-root", ".pex", # May be redundant since we also set PEX_ROOT
"--entry-point", main_pkg,
"--output-file", deploy_pex.path,
"--disable-cache",
manifest_file.path,
]
#EXTRA_PEX_ARGS#
# form the inputs to pex builder
_inputs = (
[manifest_file] +
runfiles.files.to_list() +
py.transitive_eggs.to_list() +
list(resources)
)
ctx.actions.run(
mnemonic = "PexPython",
inputs = _inputs,
outputs = [deploy_pex],
executable = pexbuilder,
execution_requirements = {
"requires-network": "1",
},
env = {
# TODO(benley): Write a repository rule to pick up certain
# PEX-related environment variables (like PEX_VERBOSE) from the
# system.
# Also, what if python is actually in /opt or something?
'PATH': '/bin:/usr/bin:/usr/local/bin',
'PEX_VERBOSE': str(ctx.attr.pex_verbosity),
'PEX_ROOT': '.pex', # So pex doesn't try to unpack into $HOME/.pex
},
arguments = arguments,
)
executable = ctx.outputs.executable
# There isn't much point in having both foo.pex and foo as identical pex
# files, but someone is probably relying on that behaviour by now so we might
# as well keep doing it.
ctx.actions.run_shell(
mnemonic = "LinkPex",
inputs = [deploy_pex],
outputs = [executable],
command = "ln -f {pex} {exe} 2>/dev/null || cp -f {pex} {exe}".format(
pex = deploy_pex.path,
exe = executable.path,
),
)
return struct(
files = depset([executable]), # Which files show up in cmdline output
runfiles = runfiles,
)
def _get_runfile_path(ctx, f):
"""Return the path to f, relative to runfiles."""
if ctx.workspace_name:
return ctx.workspace_name + "/" + f.short_path
else:
return f.short_path
def _pex_pytest_impl(ctx):
test_runner = ctx.executable.runner
output_file = ctx.outputs.executable
test_file_paths = ["${RUNFILES}/" + _get_runfile_path(ctx, f) for f in ctx.files.srcs]
ctx.actions.expand_template(
template = ctx.file.launcher_template,
output = output_file,
substitutions = {
"%test_runner%": _get_runfile_path(ctx, test_runner),
"%test_files%": " \\\n ".join(test_file_paths),
},
is_executable = True,
)
transitive_files = depset(ctx.files.srcs + [test_runner])
for dep in ctx.attr.deps:
transitive_files += dep.default_runfiles
return struct(
runfiles = ctx.runfiles(
files = [output_file],
transitive_files = transitive_files,
collect_default = True
)
)
pex_attrs = {
"srcs": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"],
allow_files = pex_file_types),
"deps": attr.label_list(allow_files = False,
providers = [PexProvider]),
"eggs": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"],
allow_files = egg_file_types),
"reqs": attr.string_list(),
"data": attr.label_list(allow_files = True),
# Used by pex_binary and pex_*test, not pex_library:
"_pexbuilder": attr.label(
default = Label("//tools/rules/pex:pex_wrapper"),
executable = True,
cfg = "host",
),
}
def _dmerge(a, b):
"""Merge two dictionaries, a+b
Workaround for https://github.com/bazelbuild/skydoc/issues/10
"""
return dict(a.items() + b.items())
pex_bin_attrs = _dmerge(pex_attrs, {
"main": attr.label(allow_single_file = True),
"entrypoint": attr.string(),
"interpreter": attr.string(),
"platforms": attr.string_list(),
"pex_use_wheels": attr.bool(default=True),
"pex_verbosity": attr.int(default=0),
"resources": attr.label_list(allow_files = True),
"zip_safe": attr.bool(
default = True,
mandatory = False,
),
})
pex_library = rule(
_pex_library_impl,
attrs = pex_attrs
)
pex_binary_outputs = {
"deploy_pex": "%{name}.pex"
}
pex_binary = rule(
_pex_binary_impl,
executable = True,
attrs = pex_bin_attrs,
outputs = pex_binary_outputs,
)
"""Build a deployable pex executable.
Args:
deps: Python module dependencies.
`pex_library` and `py_library` rules should work here.
eggs: `.egg` and `.whl` files to include as python packages.
reqs: External requirements to retrieve from pypi, in `requirements.txt` format.
This feature will reduce build determinism! It tells pex to resolve all
the transitive python dependencies and fetch them from pypi.
It is recommended that you use `eggs` instead where possible.
data: Files to include as resources in the final pex binary.
Putting other rules here will cause the *outputs* of those rules to be
embedded in this one. Files will be included as-is. Paths in the archive
will be relative to the workspace root.
resources: Similar to data, typically used for web resources.
Putting other rules here will cause the *outputs* of those rules to be
embedded in this one. Files will be included as-is. Paths in the archive
will be relative to the workspace root.
main: File to use as the entrypoint.
If unspecified, the first file from the `srcs` attribute will be used.
entrypoint: Name of a python module to use as the entrypoint.
e.g. `your.project.main`
If unspecified, the `main` attribute will be used.
It is an error to specify both main and entrypoint.
interpreter: Path to the python interpreter the pex should to use in its shebang line.
"""
pex_test = rule(
_pex_binary_impl,
executable = True,
attrs = pex_bin_attrs,
outputs = pex_binary_outputs,
test = True,
)
_pytest_pex_test = rule(
_pex_pytest_impl,
executable = True,
test = True,
attrs = _dmerge(pex_attrs, {
"runner": attr.label(
executable = True,
mandatory = True,
cfg = "target",
),
"launcher_template": attr.label(
allow_single_file = True,
default = Label("//tools/rules/pex:testlauncher.sh.template"),
),
}),
)
def pex_pytest(name, srcs, deps=[], eggs=[], data=[],
args=[],
flaky=False,
local=None,
size=None,
timeout=None,
tags=[],
**kwargs):
"""A variant of pex_test that uses py.test to run one or more sets of tests.
This produces two things:
1. A pex_binary (`<name>_runner`) containing all your code and its
dependencies, plus py.test, and the entrypoint set to the py.test
runner.
2. A small shell script to launch the `<name>_runner` executable with each
of the `srcs` enumerated as commandline arguments. This is the actual
test entrypoint for bazel.
Almost all of the attributes that can be used with pex_test work identically
here, including those not specifically mentioned in this docstring.
Exceptions are `main` and `entrypoint`, which cannot be used with this macro.
Args:
srcs: List of files containing tests that should be run.
"""
if "main" in kwargs:
fail("Specifying a `main` file makes no sense for pex_pytest.")
if "entrypoint" in kwargs:
fail("Do not specify `entrypoint` for pex_pytest.")
pex_binary(
name = "%s_runner" % name,
srcs = srcs,
deps = deps,
data = data,
eggs = eggs + [
"@pytest_whl//file",
"@py_whl//file",
],
entrypoint = "pytest",
testonly = True,
**kwargs
)
_pytest_pex_test(
name = name,
runner = ":%s_runner" % name,
args = args,
data = data,
flaky = flaky,
local = local,
size = size,
srcs = srcs,
timeout = timeout,
tags = tags,
)