flink-python/pyflink/gen_protos.py

################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

from __future__ import absolute_import
from __future__ import print_function

import glob
import logging
import multiprocessing
import os
import platform
import shutil
import subprocess
import sys
import time
import warnings

import pkg_resources

# latest grpcio-tools incompatible with latest protobuf 3.6.1.
GRPC_TOOLS = 'grpcio-tools>=1.3.5,<=1.14.2'
PROTO_PATHS = ['proto']
PYFLINK_ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
DEFAULT_PYTHON_OUTPUT_PATH = os.path.join(PYFLINK_ROOT_PATH, 'fn_execution')


def generate_proto_files(force=True, output_dir=DEFAULT_PYTHON_OUTPUT_PATH):
    try:
        import grpc_tools  # noqa  # pylint: disable=unused-import
    except ImportError:
        warnings.warn('Installing grpcio-tools is recommended for development.')

    proto_dirs = [os.path.join(PYFLINK_ROOT_PATH, path) for path in PROTO_PATHS]
    proto_files = sum(
        [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
    out_dir = os.path.join(PYFLINK_ROOT_PATH, output_dir)
    out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]

    if out_files and not proto_files and not force:
        # We have out_files but no protos; assume they're up to date.
        # This is actually the common case (e.g. installation from an sdist).
        logging.info('No proto files; using existing generated files.')
        return

    elif not out_files and not proto_files:
        raise RuntimeError(
            'No proto files found in %s.' % proto_dirs)

    # Regenerate iff the proto files or this file are newer.
    elif force or not out_files or len(out_files) < len(proto_files) or (
            min(os.path.getmtime(path) for path in out_files)
            <= max(os.path.getmtime(path)
                   for path in proto_files + [os.path.realpath(__file__)])):
        try:
            from grpc_tools import protoc
        except ImportError:
            if platform.system() == 'Windows':
                # For Windows, grpcio-tools has to be installed manually.
                raise RuntimeError(
                    'Cannot generate protos for Windows since grpcio-tools package is '
                    'not installed. Please install this package manually '
                    'using \'pip install "grpcio-tools>=1.3.5,<=1.14.2"\'.')

            # Use a subprocess to avoid messing with this process' path and imports.
            # Note that this requires a separate module from setup.py for Windows:
            # https://docs.python.org/2/library/multiprocessing.html#windows
            p = multiprocessing.Process(
                target=_install_grpcio_tools_and_generate_proto_files(force, output_dir))
            p.start()
            p.join()
            if p.exitcode:
                raise ValueError("Proto generation failed (see log for details).")
        else:
            _check_grpcio_tools_version()
            logging.info('Regenerating out-of-date Python proto definitions.')
            builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto')
            args = (
                [sys.executable] +  # expecting to be called from command line
                ['--proto_path=%s' % builtin_protos] +
                ['--proto_path=%s' % d for d in proto_dirs] +
                ['--python_out=%s' % out_dir] +
                proto_files)
            ret_code = protoc.main(args)
            if ret_code:
                raise RuntimeError(
                    'Protoc returned non-zero status (see logs for details): '
                    '%s' % ret_code)

            for output_file in os.listdir(output_dir):
                if output_file.endswith('_pb2.py'):
                    _add_license_header(output_dir, output_file)


# Though wheels are available for grpcio-tools, setup_requires uses
# easy_install which doesn't understand them. This means that it is
# compiled from scratch (which is expensive as it compiles the full
# protoc compiler). Instead, we attempt to install a wheel in a temporary
# directory and add it to the path as needed.
# See https://github.com/pypa/setuptools/issues/377
def _install_grpcio_tools_and_generate_proto_files(force, output_dir):
    install_path = os.path.join(PYFLINK_ROOT_PATH, '..', '.eggs', 'grpcio-wheels')
    build_path = install_path + '-build'
    if os.path.exists(build_path):
        shutil.rmtree(build_path)
    logging.warning('Installing grpcio-tools into %s', install_path)
    try:
        start = time.time()
        # since '--prefix' option is only supported for pip 8.0+, so here we fallback to
        # use '--install-option' when the pip version is lower than 8.0.0.
        pip_version = pkg_resources.get_distribution("pip").version
        from pkg_resources import parse_version
        if parse_version(pip_version) >= parse_version('8.0.0'):
            subprocess.check_call(
                [sys.executable, '-m', 'pip', 'install',
                 '--prefix', install_path, '--build', build_path,
                 '--upgrade', GRPC_TOOLS, "-I"])
        else:
            subprocess.check_call(
                [sys.executable, '-m', 'pip', 'install',
                 '--install-option', '--prefix=' + install_path, '--build', build_path,
                 '--upgrade', GRPC_TOOLS, "-I"])
        from distutils.dist import Distribution
        install_obj = Distribution().get_command_obj('install', create=True)
        install_obj.prefix = install_path
        install_obj.finalize_options()
        logging.warning(
            'Installing grpcio-tools took %0.2f seconds.', time.time() - start)
    finally:
        sys.stderr.flush()
        shutil.rmtree(build_path, ignore_errors=True)
    sys.path.append(install_obj.install_purelib)
    pkg_resources.working_set.add_entry(install_obj.install_purelib)
    if install_obj.install_purelib != install_obj.install_platlib:
        sys.path.append(install_obj.install_platlib)
        pkg_resources.working_set.add_entry(install_obj.install_platlib)
    try:
        generate_proto_files(force, output_dir)
    finally:
        sys.stderr.flush()


def _add_license_header(dir, file_name):
    with open(os.path.join(dir, file_name), 'r') as original_file:
        original_data = original_file.read()
        tmp_file_name = file_name + '.tmp'
        with open(os.path.join(dir, tmp_file_name), 'w') as tmp_file:
            tmp_file.write(
                '################################################################################\n'
                '#  Licensed to the Apache Software Foundation (ASF) under one\n'
                '#  or more contributor license agreements.  See the NOTICE file\n'
                '#  distributed with this work for additional information\n'
                '#  regarding copyright ownership.  The ASF licenses this file\n'
                '#  to you under the Apache License, Version 2.0 (the\n'
                '#  "License"); you may not use this file except in compliance\n'
                '#  with the License.  You may obtain a copy of the License at\n'
                '#\n'
                '#      http://www.apache.org/licenses/LICENSE-2.0\n'
                '#\n'
                '#  Unless required by applicable law or agreed to in writing, software\n'
                '#  distributed under the License is distributed on an "AS IS" BASIS,\n'
                '#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n'
                '#  See the License for the specific language governing permissions and\n'
                '# limitations under the License.\n'
                '################################################################################\n'
            )
            tmp_file.write(original_data)
    if os.path.exists(os.path.join(dir, file_name)):
        os.remove(os.path.join(dir, file_name))
    os.rename(os.path.join(dir, tmp_file_name), os.path.join(dir, file_name))


def _check_grpcio_tools_version():
    version = pkg_resources.get_distribution("grpcio-tools").parsed_version
    from pkg_resources import parse_version
    if version < parse_version('1.3.5') or version > parse_version('1.14.2'):
        raise RuntimeError(
            "Version of grpcio-tools must be between 1.3.5 and 1.14.2, got %s" % version)


if __name__ == '__main__':
    generate_proto_files()