sdks/python/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Define common lifecycle tasks and artifact types
apply plugin: "base"

// For some reason base doesn't define a test task  so we define it below and make
// check depend on it. This makes the Python project similar to the task layout like
// Java projects, see https://docs.gradle.org/4.2.1/userguide/img/javaPluginTasks.png
task test {}
check.dependsOn test

def envdir = "${project.buildDir}/gradleenv"
def tox_opts = "-c tox.ini --recreate"

task setupVirtualenv {
  doLast {
    exec {
      commandLine 'virtualenv', "${envdir}"
    }
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && pip install --upgrade tox==3.0.0 grpcio-tools==1.3.5"
    }
  }
  // Gradle will delete outputs whenever it thinks they are stale. Putting a
  // specific binary here could make gradle delete it while pip will believe
  // the package is fully installed.
  outputs.dirs(envdir)
}

configurations { distConfig }

task sdist(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && python setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}"
    }
    def collection = fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'}
    println "sdist archive name: ${collection.singleFile}"
    // we need a fixed name for the artifact
    copy { from collection.singleFile; into "${project.buildDir}"; rename { 'apache-beam.tar.gz' } }
  }
}

artifacts {
  distConfig file: file("${project.buildDir}/apache-beam.tar.gz"), builtBy: sdist
}

task cleanPython(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && python setup.py clean"
    }
  }
}
clean.dependsOn cleanPython

task buildPython(dependsOn: 'setupVirtualenv') {
  doLast {
    println 'Building Python Dependencies'
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && python setup.py build --build-base ${project.buildDir}"
    }
  }
}
build.dependsOn buildPython

task lint {}
check.dependsOn lint

task lintPy27(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-lint"
    }
  }
}
lint.dependsOn lintPy27

task lintPy27_3(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-lint3"
    }
  }
}
lint.dependsOn lintPy27_3

task lintPy3(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py3-lint"
    }
  }
}
lint.dependsOn lintPy3

task testGcp(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-gcp"
    }
  }
}
test.dependsOn testGcp

task testPython2(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27"
    }
  }
}
test.dependsOn testPython2

task testPython3(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py3"
    }
  }
}
test.dependsOn testPython3

task testCython(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e py27-cython"
    }
  }
}
test.dependsOn testCython
// Ensure that testCython runs exclusively to other tests. This line is not
// actually required, since gradle doesn't do parallel execution within a
// project.
testCython.mustRunAfter testPython2, testGcp

task docs(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e docs"
    }
  }
}
assemble.dependsOn docs

task cover(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && tox ${tox_opts} -e cover"
    }
  }
}

task preCommit() {
  dependsOn "docs"
  dependsOn "testCython"
  dependsOn "testPython2"
  dependsOn "testPython3"
  dependsOn "testGcp"
  dependsOn "lint"
}

task installGcpTest(dependsOn: 'setupVirtualenv') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && pip install -e .[gcp,test]"
    }
  }
}

task directRunnerIT(dependsOn: 'installGcpTest') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh IT batch TestDirectRunner"
    }
  }
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh IT streaming TestDirectRunner"
    }
  }
}

// Before running this, you need to:
//
// 1. Build the SDK container:
//
//    ./gradlew -p sdks/python/container docker
//
// 2. Either a) or b)
//  a) If you want the Job Server to run in a Docker container:
//
//    ./gradlew :beam-runners-flink_2.11-job-server-container:docker
//
//  b) Otherwise, start a local JobService, for example, the Portable Flink runner
//    (in a separate shell since it continues to run):
//
//    ./gradlew :beam-runners-flink_2.11-job-server:runShadow
//
// Then you can run this example:
//
//  Docker (2a):
//
//    ./gradlew :beam-sdks-python:portableWordCount
//
//  Local JobService (2b):
//
//    ./gradlew :beam-sdks-python:portableWordCount -PjobEndpoint=localhost:8099
//
task portableWordCount(dependsOn: 'installGcpTest') {
  doLast {
    // TODO: Figure out GCS credentials and use real GCS input and output.
    def options = [
            "--input=/etc/profile",
            "--output=/tmp/py-wordcount-direct",
            "--runner=PortableRunner",
    ]
    if (project.hasProperty("streaming"))
      options += ["--streaming"]
    if (project.hasProperty("jobEndpoint"))
      options += ["--job_endpoint=${project.property('jobEndpoint')}"]
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && python -m apache_beam.examples.wordcount ${options.join(' ')}"
      // TODO: Check that the output file is generated and runs.
    }
  }
}

task integrationTest(dependsOn: ['installGcpTest', 'sdist']) {
  doLast {
    def cmd_args = []

    // Build test options that configures test environment and framework
    def test_options = [
            "--nocapture",
            "--nologcapture",
            "--processes=8",
            "--process-timeout=3000",
    ]
    if (project.hasProperty('testOptions'))
      // Customize test options by adding more nose flags to -Ptest_opt in commandline
      test_options = project.property('test_options')
    if (project.hasProperty('attr'))
      test_options += ["--attr=${project.property('attr')}"]
    if (project.hasProperty('tests'))
      test_options += ["--tests=${project.property('tests')}"]
    cmd_args += ["--test_opts \"${test_options.join(' ')}\""]

    // Build pipeline options that configures pipeline job
    if (project.hasProperty('pipelineOptions'))
      cmd_args += ["--pipeline_opts \"${project.property('pipelineOptions')}\""]

    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ./scripts/run_integration_test.sh ${cmd_args.join(' ')}"
    }
  }
}

task postCommitITTests(dependsOn: 'installGcpTest') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh IT batch"
    }
  }
}

task validatesRunnerBatchTests(dependsOn: 'installGcpTest') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh ValidatesRunner batch"
    }
  }
}

task validatesRunnerStreamingTests(dependsOn: 'installGcpTest') {
  doLast {
    exec {
      executable 'sh'
      // TODO(BEAM-3544,BEAM-5025): Disable tests with sickbay-streaming tag.
      args '-c', ". ${envdir}/bin/activate && ./scripts/run_postcommit.sh ValidatesRunner,'!sickbay-streaming' streaming"
    }
  }
}

task hdfsIntegrationTest(dependsOn: 'installGcpTest') {
  doLast {
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ./apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh"
    }
  }
}

def flinkCompatibilityMatrix = {
  def type = it
  def name = 'flinkCompatibilityMatrix' + type
  tasks.create(name: name) {
    dependsOn 'setupVirtualenv'
    dependsOn ':beam-sdks-python-container:docker'
    dependsOn ':beam-runners-flink_2.11-job-server:shadowJar'
    doLast {
      exec {
        executable 'sh'
        args '-c', ". ${envdir}/bin/activate && pip install -e . && python -m apache_beam.runners.portability.flink_runner_test ${project(":beam-runners-flink_2.11-job-server:").shadowJar.archivePath} ${type}"
      }
    }
  }
}

flinkCompatibilityMatrix('Batch')
flinkCompatibilityMatrix('Streaming')

task postCommit() {
  dependsOn "preCommit"
  dependsOn "directRunnerIT"
  dependsOn "hdfsIntegrationTest"
  dependsOn "postCommitITTests"
}

task dependencyUpdates(dependsOn: ':dependencyUpdates') {
  doLast {
    exec {
      executable 'sh'
      args '-c', "./scripts/run_dependency_check.sh"
    }
  }
}