| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * License); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an AS IS BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| // Define common lifecycle tasks and artifact types |
| apply plugin: "base" |
| |
| // For some reason base doesn't define a test task so we define it below and make |
| // check depend on it. This makes the Python project similar to the task layout like |
| // Java projects, see https://docs.gradle.org/4.2.1/userguide/img/javaPluginTasks.png |
| task test {} |
| check.dependsOn test |
| |
| def envdir = "${project.buildDir}/gradleenv" |
| def tox_opts = "-c tox.ini --recreate" |
| |
| task setupVirtualenv { |
| doLast { |
| exec { |
| commandLine 'virtualenv', "${envdir}" |
| } |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && pip install --upgrade tox==3.0.0 grpcio-tools==1.3.5" |
| } |
| } |
| // Gradle will delete outputs whenever it thinks they are stale. Putting a |
| // specific binary here could make gradle delete it while pip will believe |
| // the package is fully installed. |
| outputs.dirs(envdir) |
| } |
| |
| configurations { distConfig } |
| |
| task sdist(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && python setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}" |
| } |
| def collection = fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'} |
| println "sdist archive name: ${collection.singleFile}" |
| // we need a fixed name for the artifact |
| copy { from collection.singleFile; into "${project.buildDir}"; rename { 'apache-beam.tar.gz' } } |
| } |
| } |
| |
| artifacts { |
| distConfig file: file("${project.buildDir}/apache-beam.tar.gz"), builtBy: sdist |
| } |
| |
| task cleanPython(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && python setup.py clean" |
| } |
| } |
| } |
| clean.dependsOn cleanPython |
| |
| task buildPython(dependsOn: 'setupVirtualenv') { |
| doLast { |
| println 'Building Python Dependencies' |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && python setup.py build --build-base ${project.buildDir}" |
| } |
| } |
| } |
| build.dependsOn buildPython |
| |
| task lint {} |
| check.dependsOn lint |
| |
| task lintPy27(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-lint" |
| } |
| } |
| } |
| lint.dependsOn lintPy27 |
| |
| task lintPy27_3(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-lint3" |
| } |
| } |
| } |
| lint.dependsOn lintPy27_3 |
| |
| task lintPy3(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py3-lint" |
| } |
| } |
| } |
| lint.dependsOn lintPy3 |
| |
| task testGcp(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-gcp" |
| } |
| } |
| } |
| test.dependsOn testGcp |
| |
| task testPython2(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27" |
| } |
| } |
| } |
| test.dependsOn testPython2 |
| |
| task testPython3(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py3" |
| } |
| } |
| } |
| test.dependsOn testPython3 |
| |
| task testCython(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-cython" |
| } |
| } |
| } |
| test.dependsOn testCython |
| // Ensure that testCython runs exclusively to other tests. This line is not |
| // actually required, since gradle doesn't do parallel execution within a |
| // project. |
| testCython.mustRunAfter testPython2, testGcp |
| |
| task docs(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e docs" |
| } |
| } |
| } |
| assemble.dependsOn docs |
| |
| task cover(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e cover" |
| } |
| } |
| } |
| |
| task preCommit() { |
| dependsOn "docs" |
| dependsOn "testCython" |
| dependsOn "testPython2" |
| dependsOn "testPython3" |
| dependsOn "testGcp" |
| dependsOn "lint" |
| } |
| |
| task installGcpTest(dependsOn: 'setupVirtualenv') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && pip install -e .[gcp,test]" |
| } |
| } |
| } |
| |
| task directRunnerIT(dependsOn: 'installGcpTest') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh IT batch TestDirectRunner" |
| } |
| } |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh IT streaming TestDirectRunner" |
| } |
| } |
| } |
| |
| // Before running this, you need to: |
| // |
| // 1. Build the SDK container: |
| // |
| // ./gradlew -p sdks/python/container docker |
| // |
| // 2. Either a) or b) |
| // a) If you want the Job Server to run in a Docker container: |
| // |
| // ./gradlew :beam-runners-flink_2.11-job-server-container:docker |
| // |
| // b) Otherwise, start a local JobService, for example, the Portable Flink runner |
| // (in a separate shell since it continues to run): |
| // |
| // ./gradlew :beam-runners-flink_2.11-job-server:runShadow |
| // |
| // Then you can run this example: |
| // |
| // Docker (2a): |
| // |
| // ./gradlew :beam-sdks-python:portableWordCount |
| // |
| // Local JobService (2b): |
| // |
| // ./gradlew :beam-sdks-python:portableWordCount -PjobEndpoint=localhost:8099 |
| // |
| task portableWordCount(dependsOn: 'installGcpTest') { |
| doLast { |
| // TODO: Figure out GCS credentials and use real GCS input and output. |
| def options = [ |
| "--input=/etc/profile", |
| "--output=/tmp/py-wordcount-direct", |
| "--runner=PortableRunner", |
| ] |
| if (project.hasProperty("streaming")) |
| options += ["--streaming"] |
| if (project.hasProperty("jobEndpoint")) |
| options += ["--job_endpoint=${project.property('jobEndpoint')}"] |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && python -m apache_beam.examples.wordcount ${options.join(' ')}" |
| // TODO: Check that the output file is generated and runs. |
| } |
| } |
| } |
| |
| task integrationTest(dependsOn: ['installGcpTest', 'sdist']) { |
| doLast { |
| def cmd_args = [] |
| |
| // Build test options that configures test environment and framework |
| def test_options = [ |
| "--nocapture", |
| "--nologcapture", |
| "--processes=8", |
| "--process-timeout=3000", |
| ] |
| if (project.hasProperty('testOptions')) |
| // Customize test options by adding more nose flags to -Ptest_opt in commandline |
| test_options = project.property('test_options') |
| if (project.hasProperty('attr')) |
| test_options += ["--attr=${project.property('attr')}"] |
| if (project.hasProperty('tests')) |
| test_options += ["--tests=${project.property('tests')}"] |
| cmd_args += ["--test_opts \"${test_options.join(' ')}\""] |
| |
| // Build pipeline options that configures pipeline job |
| if (project.hasProperty('pipelineOptions')) |
| cmd_args += ["--pipeline_opts \"${project.property('pipelineOptions')}\""] |
| |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && ./scripts/run_integration_test.sh ${cmd_args.join(' ')}" |
| } |
| } |
| } |
| |
| task postCommitITTests(dependsOn: 'installGcpTest') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh IT batch" |
| } |
| } |
| } |
| |
| task validatesRunnerBatchTests(dependsOn: 'installGcpTest') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh ValidatesRunner batch" |
| } |
| } |
| } |
| |
| task validatesRunnerStreamingTests(dependsOn: 'installGcpTest') { |
| doLast { |
| exec { |
| executable 'sh' |
| // TODO(BEAM-3544,BEAM-5025): Disable tests with sickbay-streaming tag. |
| args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh ValidatesRunner,'!sickbay-streaming' streaming" |
| } |
| } |
| } |
| |
| task hdfsIntegrationTest(dependsOn: 'installGcpTest') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && ./apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh" |
| } |
| } |
| } |
| |
| def flinkCompatibilityMatrix = { |
| def type = it |
| def name = 'flinkCompatibilityMatrix' + type |
| tasks.create(name: name) { |
| dependsOn 'setupVirtualenv' |
| dependsOn ':beam-sdks-python-container:docker' |
| dependsOn ':beam-runners-flink_2.11-job-server:shadowJar' |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', ". ${envdir}/bin/activate && pip install -e . && python -m apache_beam.runners.portability.flink_runner_test ${project(":beam-runners-flink_2.11-job-server:").shadowJar.archivePath} ${type}" |
| } |
| } |
| } |
| } |
| |
| flinkCompatibilityMatrix('Batch') |
| flinkCompatibilityMatrix('Streaming') |
| |
| task postCommit() { |
| dependsOn "preCommit" |
| dependsOn "directRunnerIT" |
| dependsOn "hdfsIntegrationTest" |
| dependsOn "postCommitITTests" |
| } |
| |
| task dependencyUpdates(dependsOn: ':dependencyUpdates') { |
| doLast { |
| exec { |
| executable 'sh' |
| args '-c', "./scripts/run_dependency_check.sh" |
| } |
| } |
| } |