third_party/pex/_pex.py - incubator-heron - Git at Google

 #!/usr/bin/env python2.7

 from __future__ import print_function
 import os
 import sys
 import functools
 import shutil
 import tempfile
 import optparse
 import zipfile


 # Try to detect if we're running from source via the repo.  Add appropriate
 # deps to our python path, so we can find the twitter libs and
 # setuptools at runtime.  Also, locate the `pkg_resources` modules
 # via our local setuptools import.
 if not zipfile.is_zipfile(sys.argv[0]):
     sys.modules.pop('twitter', None)
     sys.modules.pop('twitter.common', None)
     sys.modules.pop('twitter.common.python', None)

     root = os.path.join(
         os.sep.join(__file__.split(os.sep)[:-6]), 'pex/_pex.runfiles/__main__/third_party')
     sys.path.insert(0, os.path.join(root, 'pex'))
     sys.path.insert(0, os.path.join(root, 'setuptools'))
     setuptools_py =  os.path.join(root, 'setuptools')
     pkg_resources_py = os.path.join(root, 'setuptools/pkg_resources.py')

 # Otherwise, we're running from a PEX, so import the `pkg_resources`
 # module via a resource.
 else:
     with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "abc.txt"), "w") as foo:
         foo.write(sys.argv[0])
         foo.write(zipfile.is_zipfile(sys.argv[0]))
     import pkg_resources
     pkg_resources_py_tmp = tempfile.NamedTemporaryFile(
         prefix='pkg_resources.py')
     pkg_resources_py_tmp.write(
         pkg_resources.resource_string(__name__, 'pkg_resources.py'))
     pkg_resources_py_tmp.flush()
     pkg_resources_py = pkg_resources_py_tmp.name

 from pex.bin.pex import build_pex, configure_clp, resolve_interpreter, CANNOT_SETUP_INTERPRETER
 from pex.common import die
 from pex.interpreter import PythonInterpreter
 from pex.version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT


 def dereference_symlinks(src):
     """
     Resolve all symbolic references that `src` points to.  Note that this
     is different than `os.path.realpath` as path components leading up to
     the final location may still be symbolic links.
     """
     while os.path.islink(src):
         src = os.path.join(os.path.dirname(src), os.readlink(src))

     return src

 # The format is first line will say if it is modules/resources/nativeLibraries/prebuiltLibraries
 # Then it will follow with a tab indentation for key:value of corresponding item.
 def parse_manifest(manifest_text):
     lines = manifest_text.split('\n')
     manifest = {}
     curr_key = ''
     for line in lines:
        tokens = line.split(':')
        if len(tokens) != 2:
            continue
        elif not line.startswith('\t'):
            manifest[tokens[0]] = {}
            curr_key = tokens[0]
        else:
            # line is of form <tab>key:value
            manifest[curr_key][tokens[0][1:]] = tokens[1]
     return manifest

 def resolve_or_die(interpreter, requirement, options):
     resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)

     interpreter = resolve(interpreter, requirement)
     if interpreter is None:
       die('Could not find compatible interpreter that meets requirement %s' % requirement, CANNOT_SETUP_INTERPRETER)
     return interpreter

 def main():
     # These are the options that this class will accept from the rule
     parser = optparse.OptionParser(usage="usage: %prog [options] output")
     parser.add_option('--entry-point', default='__main__')
     parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True)
     parser.add_option('--disable-cache', action='store_true', dest='disable_cache', default=False)
     parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True)
     parser.add_option('--python', default="/usr/bin/python2.7")
     parser.add_option('--find-links', dest='find_links', default='')
     options, args = parser.parse_args()

     if len(args) == 2:
         output = args[0]
         manifest_text = open(args[1], 'r').read()
     elif len(args) == 1:
         output = args[0]
         manifest_text = sys.stdin.read()
     else:
         parser.error("'output' positional argument is required")
         return 1

     if manifest_text.startswith('"') and  manifest_text.endswith('"'):
         manifest_text = manifest_text[1:len(manifest_text) - 1]

     # The manifest is passed via stdin, as it can sometimes get too large
     # to be passed as a CLA.
     with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x:
         x.write(manifest_text)
     manifest = parse_manifest(manifest_text)

     # Setup a temp dir that the PEX builder will use as its scratch dir.
     tmp_dir = tempfile.mkdtemp()
     try:
         # These are the options that pex will use
         pparser, resolver_options_builder = configure_clp()

         # Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host.
         # Enabling Trace logging in pex/tracer.py shows this upon failure:
         #  pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl')
         #  is not compatible with CPython-2.7.3 / linux-x86_64
         poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv)
         poptions.entry_point = options.entry_point
         poptions.find_links = options.find_links
         poptions.pypi = options.pypi
         poptions.python = options.python
         poptions.zip_safe = options.zip_safe
         poptions.disable_cache = options.disable_cache


         #print("pex options: %s" % poptions)
         os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python

         # The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So
         # we keep a recent version in and force it into the process by constructing a custom
         # PythonInterpreter instance using it.
         interpreter = PythonInterpreter(
             poptions.python,
             PythonInterpreter.from_binary(options.python).identity,
             extras={
                 # TODO: Fix this to resolve automatically
                 ('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl',
                 ('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg'
             })

         # resolve setuptools
         interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions)

         # possibly resolve wheel
         if interpreter and poptions.use_wheel:
           interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions)

         # Add prebuilt libraries listed in the manifest.
         reqs = manifest.get('requirements', {}).keys()
         #if len(reqs) > 0:
         #  print("pex requirements: %s" % reqs)
         pex_builder = build_pex(reqs, poptions,
                                 resolver_options_builder, interpreter=interpreter)

         # Set whether this PEX as zip-safe, meaning everything will stayed zipped up
         # and we'll rely on python's zip-import mechanism to load modules from
         # the PEX.  This may not work in some situations (e.g. native
         # libraries, libraries that want to find resources via the FS).
         pex_builder.info.zip_safe = options.zip_safe

         # Set the starting point for this PEX.
         pex_builder.info.entry_point = options.entry_point

         pex_builder.add_source(
             dereference_symlinks(pkg_resources_py),
             os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))

         # Add the sources listed in the manifest.
         for dst, src in manifest['modules'].iteritems():
             # NOTE(agallagher): calls the `add_source` and `add_resource` below
             # hard-link the given source into the PEX temp dir.  Since OS X and
             # Linux behave different when hard-linking a source that is a
             # symbolic link (Linux does *not* follow symlinks), resolve any
             # layers of symlinks here to get consistent behavior.
             try:
                 pex_builder.add_source(dereference_symlinks(src), dst)
             except OSError as e:
                 raise Exception("Failed to add {}: {}".format(src, e))

         # Add resources listed in the manifest.
         for dst, src in manifest['resources'].iteritems():
             # NOTE(agallagher): see rationale above.
             pex_builder.add_resource(dereference_symlinks(src), dst)

         # Add prebuilt libraries listed in the manifest.
         for req in manifest.get('prebuiltLibraries', []):
             try:
                 pex_builder.add_dist_location(req)
             except Exception as e:
                 raise Exception("Failed to add {}: {}".format(req, e))

         # TODO(mikekap): Do something about manifest['nativeLibraries'].

         # Generate the PEX file.
         pex_builder.build(output)

     # Always try cleaning up the scratch dir, ignoring failures.
     finally:
         shutil.rmtree(tmp_dir, True)


 sys.exit(main())
	#!/usr/bin/env python2.7

	from __future__ import print_function
	import os
	import sys
	import functools
	import shutil
	import tempfile
	import optparse
	import zipfile


	# Try to detect if we're running from source via the repo. Add appropriate
	# deps to our python path, so we can find the twitter libs and
	# setuptools at runtime. Also, locate the `pkg_resources` modules
	# via our local setuptools import.
	if not zipfile.is_zipfile(sys.argv[0]):
	sys.modules.pop('twitter', None)
	sys.modules.pop('twitter.common', None)
	sys.modules.pop('twitter.common.python', None)

	root = os.path.join(
	os.sep.join(__file__.split(os.sep)[:-6]), 'pex/_pex.runfiles/__main__/third_party')
	sys.path.insert(0, os.path.join(root, 'pex'))
	sys.path.insert(0, os.path.join(root, 'setuptools'))
	setuptools_py = os.path.join(root, 'setuptools')
	pkg_resources_py = os.path.join(root, 'setuptools/pkg_resources.py')

	# Otherwise, we're running from a PEX, so import the `pkg_resources`
	# module via a resource.
	else:
	with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "abc.txt"), "w") as foo:
	foo.write(sys.argv[0])
	foo.write(zipfile.is_zipfile(sys.argv[0]))
	import pkg_resources
	pkg_resources_py_tmp = tempfile.NamedTemporaryFile(
	prefix='pkg_resources.py')
	pkg_resources_py_tmp.write(
	pkg_resources.resource_string(__name__, 'pkg_resources.py'))
	pkg_resources_py_tmp.flush()
	pkg_resources_py = pkg_resources_py_tmp.name

	from pex.bin.pex import build_pex, configure_clp, resolve_interpreter, CANNOT_SETUP_INTERPRETER
	from pex.common import die
	from pex.interpreter import PythonInterpreter
	from pex.version import SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT


	def dereference_symlinks(src):
	"""
	Resolve all symbolic references that `src` points to. Note that this
	is different than `os.path.realpath` as path components leading up to
	the final location may still be symbolic links.
	"""
	while os.path.islink(src):
	src = os.path.join(os.path.dirname(src), os.readlink(src))

	return src

	# The format is first line will say if it is modules/resources/nativeLibraries/prebuiltLibraries
	# Then it will follow with a tab indentation for key:value of corresponding item.
	def parse_manifest(manifest_text):
	lines = manifest_text.split('\n')
	manifest = {}
	curr_key = ''
	for line in lines:
	tokens = line.split(':')
	if len(tokens) != 2:
	continue
	elif not line.startswith('\t'):
	manifest[tokens[0]] = {}
	curr_key = tokens[0]
	else:
	# line is of form <tab>key:value
	manifest[curr_key][tokens[0][1:]] = tokens[1]
	return manifest

	def resolve_or_die(interpreter, requirement, options):
	resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)

	interpreter = resolve(interpreter, requirement)
	if interpreter is None:
	die('Could not find compatible interpreter that meets requirement %s' % requirement, CANNOT_SETUP_INTERPRETER)
	return interpreter

	def main():
	# These are the options that this class will accept from the rule
	parser = optparse.OptionParser(usage="usage: %prog [options] output")
	parser.add_option('--entry-point', default='__main__')
	parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True)
	parser.add_option('--disable-cache', action='store_true', dest='disable_cache', default=False)
	parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True)
	parser.add_option('--python', default="/usr/bin/python2.7")
	parser.add_option('--find-links', dest='find_links', default='')
	options, args = parser.parse_args()

	if len(args) == 2:
	output = args[0]
	manifest_text = open(args[1], 'r').read()
	elif len(args) == 1:
	output = args[0]
	manifest_text = sys.stdin.read()
	else:
	parser.error("'output' positional argument is required")
	return 1

	if manifest_text.startswith('"') and manifest_text.endswith('"'):
	manifest_text = manifest_text[1:len(manifest_text) - 1]

	# The manifest is passed via stdin, as it can sometimes get too large
	# to be passed as a CLA.
	with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x:
	x.write(manifest_text)
	manifest = parse_manifest(manifest_text)

	# Setup a temp dir that the PEX builder will use as its scratch dir.
	tmp_dir = tempfile.mkdtemp()
	try:
	# These are the options that pex will use
	pparser, resolver_options_builder = configure_clp()

	# Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host.
	# Enabling Trace logging in pex/tracer.py shows this upon failure:
	# pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl')
	# is not compatible with CPython-2.7.3 / linux-x86_64
	poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv)
	poptions.entry_point = options.entry_point
	poptions.find_links = options.find_links
	poptions.pypi = options.pypi
	poptions.python = options.python
	poptions.zip_safe = options.zip_safe
	poptions.disable_cache = options.disable_cache


	#print("pex options: %s" % poptions)
	os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python

	# The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So
	# we keep a recent version in and force it into the process by constructing a custom
	# PythonInterpreter instance using it.
	interpreter = PythonInterpreter(
	poptions.python,
	PythonInterpreter.from_binary(options.python).identity,
	extras={
	# TODO: Fix this to resolve automatically
	('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl',
	('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg'
	})

	# resolve setuptools
	interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions)

	# possibly resolve wheel
	if interpreter and poptions.use_wheel:
	interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions)

	# Add prebuilt libraries listed in the manifest.
	reqs = manifest.get('requirements', {}).keys()
	#if len(reqs) > 0:
	# print("pex requirements: %s" % reqs)
	pex_builder = build_pex(reqs, poptions,
	resolver_options_builder, interpreter=interpreter)

	# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
	# and we'll rely on python's zip-import mechanism to load modules from
	# the PEX. This may not work in some situations (e.g. native
	# libraries, libraries that want to find resources via the FS).
	pex_builder.info.zip_safe = options.zip_safe

	# Set the starting point for this PEX.
	pex_builder.info.entry_point = options.entry_point

	pex_builder.add_source(
	dereference_symlinks(pkg_resources_py),
	os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))

	# Add the sources listed in the manifest.
	for dst, src in manifest['modules'].iteritems():
	# NOTE(agallagher): calls the `add_source` and `add_resource` below
	# hard-link the given source into the PEX temp dir. Since OS X and
	# Linux behave different when hard-linking a source that is a
	# symbolic link (Linux does not follow symlinks), resolve any
	# layers of symlinks here to get consistent behavior.
	try:
	pex_builder.add_source(dereference_symlinks(src), dst)
	except OSError as e:
	raise Exception("Failed to add {}: {}".format(src, e))

	# Add resources listed in the manifest.
	for dst, src in manifest['resources'].iteritems():
	# NOTE(agallagher): see rationale above.
	pex_builder.add_resource(dereference_symlinks(src), dst)

	# Add prebuilt libraries listed in the manifest.
	for req in manifest.get('prebuiltLibraries', []):
	try:
	pex_builder.add_dist_location(req)
	except Exception as e:
	raise Exception("Failed to add {}: {}".format(req, e))

	# TODO(mikekap): Do something about manifest['nativeLibraries'].

	# Generate the PEX file.
	pex_builder.build(output)

	# Always try cleaning up the scratch dir, ignoring failures.
	finally:
	shutil.rmtree(tmp_dir, True)


	sys.exit(main())