ARROW-6963: [Packaging][Wheel][OSX] Use crossbow's command to deploy artifacts from travis builds
Closes #5726 from kszucs/osx-wheel-upload and squashes the following commits:
320b76e90 <Krisztián Szűcs> install requests security for system python
4b42380ea <Krisztián Szűcs> pass default paths as strings
80e2e5b09 <Krisztián Szűcs> better error if pygit2 is missing
1bc93a869 <Krisztián Szűcs> make pygit2 and github3 optional dependencies for crossbow
Authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py
index 50f965f..5fe5063 100755
--- a/dev/tasks/crossbow.py
+++ b/dev/tasks/crossbow.py
@@ -32,13 +32,21 @@
import click
import toolz
-import pygit2
-import github3
-import jira.client
-from jinja2 import Template, StrictUndefined
from setuptools_scm.git import parse as parse_git_version
from ruamel.yaml import YAML
+try:
+ import github3
+except ImportError:
+ github3 = object
+
+try:
+ import pygit2
+except ImportError:
+ PygitRemoteCallbacks = object
+else:
+ PygitRemoteCallbacks = pygit2.RemoteCallbacks
+
CWD = Path(__file__).parent.absolute()
@@ -117,6 +125,7 @@
def __init__(self, version, username, password,
server='https://issues.apache.org/jira'):
+ import jira.client
self.server = server
# clean version to the first numbers
self.version = '.'.join(version.split('.')[:3])
@@ -201,7 +210,7 @@
return out.getvalue().strip()
-class GitRemoteCallbacks(pygit2.RemoteCallbacks):
+class GitRemoteCallbacks(PygitRemoteCallbacks):
def __init__(self, token):
self.token = token
@@ -249,10 +258,10 @@
def __init__(self, path, github_token=None, remote_url=None,
require_https=False):
self.path = Path(path)
- self.repo = pygit2.Repository(str(self.path))
self.github_token = github_token
self.require_https = require_https
self._remote_url = remote_url
+ self._pygit_repo = None
self._github_repo = None # set by as_github_repo()
self._updated_refs = []
@@ -268,6 +277,12 @@
)
@property
+ def repo(self):
+ if self._pygit_repo is None:
+ self._pygit_repo = pygit2.Repository(str(self.path))
+ return self._pygit_repo
+
+ @property
def origin(self):
remote = self.repo.remotes['origin']
if self.require_https and remote.url.startswith('git@github.com'):
@@ -398,6 +413,10 @@
def _parse_github_user_repo(self):
m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url)
+ if m is None:
+ raise ValueError("Unable to parse the github owner and repository "
+ "from the repository's remote url '{}'"
+ .format(self.remote_url))
user, repo = m.group(1), m.group(2)
return user, repo
@@ -671,6 +690,7 @@
self._status = None # status cache
def render_files(self, **extra_params):
+ from jinja2 import Template, StrictUndefined
path = CWD / self.template
params = toolz.merge(self.params, extra_params)
template = Template(path.read_text(), undefined=StrictUndefined)
@@ -1028,11 +1048,11 @@
@click.option('--github-token', '-t', default=None,
help='OAuth token for GitHub authentication')
@click.option('--arrow-path', '-a',
- type=click.Path(exists=True), default=DEFAULT_ARROW_PATH,
+ type=click.Path(exists=True), default=str(DEFAULT_ARROW_PATH),
help='Arrow\'s repository path. Defaults to the repository of '
'this script')
@click.option('--queue-path', '-q',
- type=click.Path(exists=True), default=DEFAULT_QUEUE_PATH,
+ type=click.Path(exists=True), default=str(DEFAULT_QUEUE_PATH),
help='The repository path used for scheduling the tasks. '
'Defaults to crossbow directory placed next to arrow')
@click.option('--queue-remote', '-qr', default=None,
@@ -1059,7 +1079,7 @@
@crossbow.command()
@click.option('--changelog-path', '-c', type=click.Path(exists=True),
- default=DEFAULT_ARROW_PATH / 'CHANGELOG.md',
+ default=str(DEFAULT_ARROW_PATH / 'CHANGELOG.md'),
help='Path of changelog to update')
@click.option('--arrow-version', '-v', default=None,
help='Set target version explicitly')
@@ -1232,7 +1252,8 @@
@crossbow.command()
@click.argument('job-name', required=True)
-@click.option('-t', '--target-dir', default=DEFAULT_ARROW_PATH / 'packages',
+@click.option('-t', '--target-dir',
+ default=str(DEFAULT_ARROW_PATH / 'packages'),
type=click.Path(file_okay=False, dir_okay=True),
help='Directory to download the build artifacts')
@click.pass_obj
diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml
index 7d4177f..cdf39bc 100644
--- a/dev/tasks/python-wheels/travis.osx.yml
+++ b/dev/tasks/python-wheels/travis.osx.yml
@@ -25,10 +25,10 @@
global:
- PLAT=x86_64
- TRAVIS_TAG={{ task.tag }}
- - MACOSX_DEPLOYMENT_TARGET="10.9"
- PYARROW_VERSION={{ arrow.no_rc_version }}
- PYARROW_BUILD_VERBOSE=1
- MB_PYTHON_VERSION={{ python_version }}
+ - MACOSX_DEPLOYMENT_TARGET="10.9"
before_install:
- git clone https://github.com/matthew-brett/multibuild # TODO pin it
@@ -40,29 +40,26 @@
# ARROW-3976 Old versions of git can cause failures when Homebrew prints a
# donation solicitation. Attempt to update git
- - git --version
- - brew upgrade git
+ - brew upgrade git python
# Also remove artifacts that depend on Boost
- brew uninstall boost cgal postgis sfcgal
- brew update
- brew upgrade cmake
- travis_wait 30 brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools
- # Remove shared grpc libraries installed by brew to make sure
+ # remove shared grpc libraries installed by brew to make sure
# we are linked against the static ones.
- rm -f /usr/local/opt/grpc/lib/*.dylib
+ # source utilities required for wheel builds
- export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh
- source multibuild/common_utils.sh
- source multibuild/travis_osx_steps.sh
-
- before_install
# Fix SSL TLS issue for Python 3.5 on macOS
- pip install requests[security]
install:
- - mkdir -p dist
-
# the following functions are defined in osx-build.sh
- build_wheel arrow
@@ -78,19 +75,13 @@
# run the import tests
- run_import_tests
- # move built wheels to a top level directory
- - mv -v arrow/python/dist/* dist/
- # reinstall openssl because travis' deployment script depends on it
- brew install openssl@1.1
-
-deploy:
- provider: releases
- api_key: $CROSSBOW_GITHUB_TOKEN
- file_glob: true
- file: dist/*.whl
- skip_cleanup: true
- on:
- tags: true
+ # before_install activates a virtualenv but we need the system python3
+ - deactivate
+ # crossbow dependencies for deployment
+ - unset MACOSX_DEPLOYMENT_TARGET
+ - pip3 install click ruamel.yaml setuptools_scm github3.py toolz requests[security]
+ - python3 arrow/dev/tasks/crossbow.py --queue-path $(pwd) --queue-remote {{ queue.remote_url }} upload-artifacts --sha {{ task.branch }} --tag {{ task.tag }} --pattern "arrow/python/dist/*.whl"
notifications:
email:
