sdks/python/test-suites/dataflow/py2/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

plugins { id 'org.apache.beam.module' }
applyPythonNature()
enablePythonPerformanceTest()

dependencies {
  distTarBall project(path: ":sdks:python", configuration: "distTarBall")
}

def runScriptsDir = "${rootDir}/sdks/python/scripts"

// Basic test options for ITs running on Jenkins.
def basicTestOpts = [
    "--nocapture",  // print stdout instantly
    "--processes=8",  // run tests in parallel
    "--process-timeout=4500", // timeout of whole command execution
]

task preCommitIT {
  dependsOn 'installGcpTest'
  dependsOn ':sdks:python:sdist'
  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"

  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath

  doLast {
    // Basic integration tests to run in PreCommit
    def precommitTests = [
        "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
        "apache_beam.examples.streaming_wordcount_it_test:StreamingWordCountIT.test_streaming_wordcount_it",
    ]
    def testOpts = [
        "--tests=${precommitTests.join(',')}",
        "--nocapture",    // Print stdout instantly
        "--processes=2",    // Number of tests running in parallel
        "--process-timeout=1800",   // Timeout of whole command execution
    ]
    def cmdArgs = mapToArgString([
        "test_opts": testOpts,
        "sdk_location": files(configurations.distTarBall.files).singleFile,
        "worker_jar": dataflowWorkerJar,
        "suite": "preCommitIT-df"
    ])

    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
    }
  }
}

// Run PostCommit integration tests on default runner (TestDataflowRunner)
task postCommitIT {
  dependsOn 'installGcpTest'
  dependsOn ':sdks:python:sdist'
  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"

  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath

  doLast {
    def testOpts = basicTestOpts + ["--attr=IT"]
    def cmdArgs = mapToArgString([
        "test_opts": testOpts,
        "worker_jar": dataflowWorkerJar,
        "sdk_location": files(configurations.distTarBall.files).singleFile,
        "suite": "postCommitIT-df"
    ])
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
    }
  }
}

task validatesRunnerBatchTests {
  dependsOn 'installGcpTest'
  dependsOn ':sdks:python:sdist'
  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"

  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath

  doLast {
    def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
    def cmdArgs = mapToArgString([
        "test_opts": testOpts,
        "worker_jar": dataflowWorkerJar,
        "sdk_location": files(configurations.distTarBall.files).singleFile,
        "suite": "validatesRunnerBatchTests-df"
    ])
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
    }
  }
}

task validatesRunnerStreamingTests {
  dependsOn 'installGcpTest'
  dependsOn ':sdks:python:sdist'
  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"

  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath

  doLast {
    // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
    def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"]
    def argMap = ["test_opts": testOpts,
                  "streaming": "true",
                  "worker_jar": dataflowWorkerJar,
                  "sdk_location": files(configurations.distTarBall.files).singleFile,
                  "suite": "validatesRunnerStreamingTests-df"]
    def cmdArgs = mapToArgString(argMap)
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
    }
  }
}

task dataflowChicagoTaxiExample {
  dependsOn 'installGcpTest'
  dependsOn ':sdks:python:sdist'

  def gcsRoot = findProperty('gcsRoot')
  def runner = findProperty('runner')
  def cliArgs = "${gcsRoot} ${runner} ${files(configurations.distTarBall.files).singleFile}"

  doLast {
    exec {
      workingDir "$rootProject.projectDir/sdks/python/apache_beam/testing/benchmarks/chicago_taxi/"
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && pip install -r requirements.txt"
    }
    exec {
      workingDir "$rootProject.projectDir/sdks/python/apache_beam/testing/benchmarks/chicago_taxi/"
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && ./run_chicago.sh ${cliArgs}"
    }
  }
}