[BEAM-7730] Migrate docs and integration tests to Flink 1.9
diff --git a/.test-infra/jenkins/CommonTestProperties.groovy b/.test-infra/jenkins/CommonTestProperties.groovy
index 203e398..0d750ee 100644
--- a/.test-infra/jenkins/CommonTestProperties.groovy
+++ b/.test-infra/jenkins/CommonTestProperties.groovy
@@ -35,7 +35,7 @@
JAVA: [
DATAFLOW: ":runners:google-cloud-dataflow-java",
SPARK: ":runners:spark",
- FLINK: ":runners:flink:1.8",
+ FLINK: ":runners:flink:1.9",
DIRECT: ":runners:direct-java"
],
PYTHON: [
diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy
index e4bf45e..5a9a238 100644
--- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy
@@ -36,7 +36,7 @@
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.8:job-server:validatesCrossLanguageRunner')
+ tasks(':runners:flink:1.9:job-server:validatesCrossLanguageRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
index 038d6b3..cbcd0ba 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
@@ -40,7 +40,7 @@
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.8"' +
+ switches('-Pnexmark.runner=":runners:flink:1.9"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--streaming=false',
@@ -55,7 +55,7 @@
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.8"' +
+ switches('-Pnexmark.runner=":runners:flink:1.9"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--streaming=true',
@@ -70,7 +70,7 @@
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.8"' +
+ switches('-Pnexmark.runner=":runners:flink:1.9"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--queryLanguage=sql',
@@ -85,7 +85,7 @@
rootBuildScriptDir(commonJobProperties.checkoutDir)
tasks(':sdks:java:testing:nexmark:run')
commonJobProperties.setGradleSwitches(delegate)
- switches('-Pnexmark.runner=":runners:flink:1.8"' +
+ switches('-Pnexmark.runner=":runners:flink:1.9"' +
' -Pnexmark.args="' +
[NexmarkBigqueryProperties.nexmarkBigQueryArgs,
'--queryLanguage=sql',
diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
index 4da75f9..d618688 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Batch.groovy
@@ -36,7 +36,7 @@
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.8:job-server:validatesPortableRunnerBatch')
+ tasks(':runners:flink:1.9:job-server:validatesPortableRunnerBatch')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy
index 612c154..bf4708a 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy
@@ -36,7 +36,7 @@
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.8:job-server:validatesPortableRunnerStreaming')
+ tasks(':runners:flink:1.9:job-server:validatesPortableRunnerStreaming')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
index d5e6da9..499947e 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Flink.groovy
@@ -37,7 +37,7 @@
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.8:validatesRunner')
+ tasks(':runners:flink:1.9:validatesRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
index a2bc53e..80b2aa3 100644
--- a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
@@ -31,7 +31,7 @@
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.8:job-server:testPipelineJar')
+ tasks(':runners:flink:1.9:job-server:testPipelineJar')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index b7e9d2a..b824535 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1303,7 +1303,7 @@
}
if (runner?.equalsIgnoreCase('flink')) {
- testRuntime it.project(path: ":runners:flink:1.8", configuration: 'testRuntime')
+ testRuntime it.project(path: ":runners:flink:1.9", configuration: 'testRuntime')
}
if (runner?.equalsIgnoreCase('spark')) {
@@ -1724,7 +1724,7 @@
dependsOn setupTask
// We need flink-job-server-container dependency since Python PortableRunner automatically
// brings the flink-job-server-container up when --job_endpoint is not specified.
- dependsOn ':runners:flink:1.8:job-server-container:docker'
+ dependsOn ':runners:flink:1.9:job-server-container:docker'
}
mainTask.dependsOn pythonTask
cleanupTask.mustRunAfter pythonTask
@@ -1906,7 +1906,7 @@
project.task('portableWordCount' + (isStreaming ? 'Streaming' : 'Batch')) {
dependsOn = ['installGcpTest']
mustRunAfter = [
- ':runners:flink:1.8:job-server-container:docker',
+ ':runners:flink:1.9:job-server-container:docker',
':sdks:python:container:py2:docker',
':sdks:python:container:py35:docker',
':sdks:python:container:py36:docker',
diff --git a/release/build.gradle b/release/build.gradle
index 44e9f98..d3a13cc 100644
--- a/release/build.gradle
+++ b/release/build.gradle
@@ -34,7 +34,7 @@
dependsOn ":runners:google-cloud-dataflow-java:runQuickstartJavaDataflow"
dependsOn ":runners:apex:runQuickstartJavaApex"
dependsOn ":runners:spark:runQuickstartJavaSpark"
- dependsOn ":runners:flink:1.8:runQuickstartJavaFlinkLocal"
+ dependsOn ":runners:flink:1.9:runQuickstartJavaFlinkLocal"
dependsOn ":runners:direct-java:runMobileGamingJavaDirect"
dependsOn ":runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow"
}
diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh
index 887e821..3f025d8 100755
--- a/release/src/main/scripts/run_rc_validation.sh
+++ b/release/src/main/scripts/run_rc_validation.sh
@@ -209,7 +209,7 @@
echo "*************************************************************"
echo "* Running Java Quickstart with Flink local runner"
echo "*************************************************************"
- ./gradlew :runners:flink:1.8:runQuickstartJavaFlinkLocal \
+ ./gradlew :runners:flink:1.9:runQuickstartJavaFlinkLocal \
-Prepourl=${REPO_URL} \
-Pver=${RELEASE_VER}
else
diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle
index 08b03a0..368f442 100644
--- a/runners/flink/flink_runner.gradle
+++ b/runners/flink/flink_runner.gradle
@@ -208,5 +208,5 @@
dependsOn validatesRunnerStreaming
}
-// Generates :runners:flink:1.8:runQuickstartJavaFlinkLocal
+// Generates :runners:flink:1.9:runQuickstartJavaFlinkLocal
createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'FlinkLocal')
diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle
index 77fd3be..c453ccc 100644
--- a/sdks/go/test/build.gradle
+++ b/sdks/go/test/build.gradle
@@ -49,12 +49,12 @@
task flinkValidatesRunner {
dependsOn ":sdks:go:test:goBuild"
- dependsOn ":runners:flink:1.8:job-server:shadowJar"
+ dependsOn ":runners:flink:1.9:job-server:shadowJar"
doLast {
def options = [
"--runner flink",
"--parallel 1", // prevent memory overuse
- "--flink_job_server_jar ${project(":runners:flink:1.8:job-server").shadowJar.archivePath}",
+ "--flink_job_server_jar ${project(":runners:flink:1.9:job-server").shadowJar.archivePath}",
]
exec {
executable "sh"
diff --git a/sdks/java/testing/nexmark/build.gradle b/sdks/java/testing/nexmark/build.gradle
index 1fdfbed..9a62c3a 100644
--- a/sdks/java/testing/nexmark/build.gradle
+++ b/sdks/java/testing/nexmark/build.gradle
@@ -102,7 +102,7 @@
//
// Parameters:
// -Pnexmark.runner
-// Specify a runner subproject, such as ":runners:spark" or ":runners:flink:1.8"
+// Specify a runner subproject, such as ":runners:spark" or ":runners:flink:1.9"
// Defaults to ":runners:direct-java"
//
// -Pnexmark.args
diff --git a/sdks/python/apache_beam/runners/interactive/README.md b/sdks/python/apache_beam/runners/interactive/README.md
index 6f187de..bdcb85d 100644
--- a/sdks/python/apache_beam/runners/interactive/README.md
+++ b/sdks/python/apache_beam/runners/interactive/README.md
@@ -225,7 +225,7 @@
```bash
$ ./gradlew -p sdks/python/container/py35 docker # Optionally replace py35 with the Python version of your choice
- $ ./gradlew :runners:flink:1.8:job-server:runShadow # Blocking
+ $ ./gradlew :runners:flink:1.9:job-server:runShadow # Blocking
```
* Run `$ jupyter notebook` in another terminal.
diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle
index 690c09d..1ea51ca 100644
--- a/sdks/python/test-suites/portable/common.gradle
+++ b/sdks/python/test-suites/portable/common.gradle
@@ -45,7 +45,7 @@
extra_experiments.add('pre_optimize=all')
tasks.create(name: name) {
dependsOn 'setupVirtualenv'
- dependsOn ':runners:flink:1.8:job-server:shadowJar'
+ dependsOn ':runners:flink:1.9:job-server:shadowJar'
if (workerType.toLowerCase() == 'docker')
dependsOn pythonContainerTask
else if (workerType.toLowerCase() == 'process')
@@ -53,7 +53,7 @@
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && cd ${pythonRootDir} && pip install -e .[test] && python -m apache_beam.runners.portability.flink_runner_test --flink_job_server_jar=${project(":runners:flink:1.8:job-server:").shadowJar.archivePath} --environment_type=${workerType} ${environment_config} ${streaming ? '--streaming' : ''} ${extra_experiments ? '--extra_experiments=' + extra_experiments.join(',') : ''}"
+ args '-c', ". ${envdir}/bin/activate && cd ${pythonRootDir} && pip install -e .[test] && python -m apache_beam.runners.portability.flink_runner_test --flink_job_server_jar=${project(":runners:flink:1.9:job-server:").shadowJar.archivePath} --environment_type=${workerType} ${environment_config} ${streaming ? '--streaming' : ''} ${extra_experiments ? '--extra_experiments=' + extra_experiments.join(',') : ''}"
}
}
}
diff --git a/sdks/python/test-suites/portable/py2/build.gradle b/sdks/python/test-suites/portable/py2/build.gradle
index 20a3de4..2b95296 100644
--- a/sdks/python/test-suites/portable/py2/build.gradle
+++ b/sdks/python/test-suites/portable/py2/build.gradle
@@ -28,7 +28,7 @@
addPortableWordCountTasks()
task preCommitPy2() {
- dependsOn ':runners:flink:1.8:job-server-container:docker'
+ dependsOn ':runners:flink:1.9:job-server-container:docker'
dependsOn ':sdks:python:container:py2:docker'
dependsOn portableWordCountBatch
dependsOn portableWordCountStreaming
@@ -45,12 +45,12 @@
// 2. Either a) or b)
// a) If you want the Job Server to run in a Docker container:
//
-// ./gradlew :runners:flink:1.8:job-server-container:docker
+// ./gradlew :runners:flink:1.9:job-server-container:docker
//
// b) Otherwise, start a local JobService, for example, the Portable Flink runner
// (in a separate shell since it continues to run):
//
-// ./gradlew :runners:flink:1.8:job-server:runShadow
+// ./gradlew :runners:flink:1.9:job-server:runShadow
//
// Then you can run this example:
//
@@ -87,7 +87,7 @@
task crossLanguagePythonJavaFlink {
dependsOn 'setupVirtualenv'
- dependsOn ':runners:flink:1.8:job-server-container:docker'
+ dependsOn ':runners:flink:1.9:job-server-container:docker'
dependsOn ':sdks:python:container:py2:docker'
dependsOn ':sdks:java:container:docker'
dependsOn ':sdks:java:testing:expansion-service:buildTestExpansionServiceJar'
@@ -112,7 +112,7 @@
task crossLanguagePortableWordCount {
dependsOn 'setupVirtualenv'
- dependsOn ':runners:flink:1.8:job-server-container:docker'
+ dependsOn ':runners:flink:1.9:job-server-container:docker'
dependsOn ':sdks:python:container:py2:docker'
dependsOn ':sdks:java:container:docker'
dependsOn ':sdks:java:testing:expansion-service:buildTestExpansionServiceJar'
diff --git a/sdks/python/test-suites/portable/py35/build.gradle b/sdks/python/test-suites/portable/py35/build.gradle
index b0d670c..42667c7 100644
--- a/sdks/python/test-suites/portable/py35/build.gradle
+++ b/sdks/python/test-suites/portable/py35/build.gradle
@@ -25,7 +25,7 @@
addPortableWordCountTasks()
task preCommitPy35() {
- dependsOn ':runners:flink:1.8:job-server-container:docker'
+ dependsOn ':runners:flink:1.9:job-server-container:docker'
dependsOn ':sdks:python:container:py35:docker'
dependsOn portableWordCountBatch
dependsOn portableWordCountStreaming
diff --git a/sdks/python/test-suites/portable/py36/build.gradle b/sdks/python/test-suites/portable/py36/build.gradle
index 70fbdce..d536d14 100644
--- a/sdks/python/test-suites/portable/py36/build.gradle
+++ b/sdks/python/test-suites/portable/py36/build.gradle
@@ -25,7 +25,7 @@
addPortableWordCountTasks()
task preCommitPy36() {
- dependsOn ':runners:flink:1.8:job-server-container:docker'
+ dependsOn ':runners:flink:1.9:job-server-container:docker'
dependsOn ':sdks:python:container:py36:docker'
dependsOn portableWordCountBatch
dependsOn portableWordCountStreaming
diff --git a/sdks/python/test-suites/portable/py37/build.gradle b/sdks/python/test-suites/portable/py37/build.gradle
index fa2ead2..da57c93 100644
--- a/sdks/python/test-suites/portable/py37/build.gradle
+++ b/sdks/python/test-suites/portable/py37/build.gradle
@@ -25,7 +25,7 @@
addPortableWordCountTasks()
task preCommitPy37() {
- dependsOn ':runners:flink:1.8:job-server-container:docker'
+ dependsOn ':runners:flink:1.9:job-server-container:docker'
dependsOn ':sdks:python:container:py37:docker'
dependsOn portableWordCountBatch
dependsOn portableWordCountStreaming
diff --git a/website/src/contribute/release-guide.md b/website/src/contribute/release-guide.md
index 03fb6cc..1987986 100644
--- a/website/src/contribute/release-guide.md
+++ b/website/src/contribute/release-guide.md
@@ -993,7 +993,7 @@
```
Flink Local Runner
```
- ./gradlew :runners:flink:1.8:runQuickstartJavaFlinkLocal \
+ ./gradlew :runners:flink:1.9:runQuickstartJavaFlinkLocal \
-Prepourl=https://repository.apache.org/content/repositories/orgapachebeam-${KEY} \
-Pver=${RELEASE_VERSION}
```
diff --git a/website/src/documentation/dsls/sql/shell.md b/website/src/documentation/dsls/sql/shell.md
index 1317575..025b031 100644
--- a/website/src/documentation/dsls/sql/shell.md
+++ b/website/src/documentation/dsls/sql/shell.md
@@ -31,7 +31,7 @@
To use Beam SQL shell, you must first clone the [Beam SDK repository](https://github.com/apache/beam). Then, from the root of the repository clone, execute the following commands to run the shell:
```
-./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.8,:sdks:java:io:kafka' installDist
+./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.9,:sdks:java:io:kafka' installDist
./sdks/java/extensions/sql/shell/build/install/shell/bin/shell
```
@@ -119,7 +119,7 @@
1. Make sure the SQL shell includes the desired runner. Add the corresponding project id to the `-Pbeam.sql.shell.bundled` parameter of the Gradle invocation ([source code](https://github.com/apache/beam/blob/master/sdks/java/extensions/sql/shell/build.gradle), [project ids](https://github.com/apache/beam/blob/master/settings.gradle)). For example, use the following command to include Flink runner and KafkaIO:
```
- ./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.8,:sdks:java:io:kafka' installDist
+ ./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.9,:sdks:java:io:kafka' installDist
```
_Note: You can bundle multiple runners (using a comma-separated list) or other additional components in the same manner. For example, you can add support for more I/Os._
@@ -145,7 +145,7 @@
You can also build your own standalone package for SQL shell using `distZip` or `distTar` tasks. For example:
```
-./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.8,:sdks:java:io:kafka' distZip
+./gradlew -p sdks/java/extensions/sql/shell -Pbeam.sql.shell.bundled=':runners:flink:1.9,:sdks:java:io:kafka' distZip
ls ./sdks/java/extensions/sql/shell/build/distributions/
beam-sdks-java-extensions-sql-shell-2.6.0-SNAPSHOT.tar beam-sdks-java-extensions-sql-shell-2.6.0-SNAPSHOT.zip
diff --git a/website/src/documentation/runners/flink.md b/website/src/documentation/runners/flink.md
index 515d8e2..2667929 100644
--- a/website/src/documentation/runners/flink.md
+++ b/website/src/documentation/runners/flink.md
@@ -103,7 +103,11 @@
<th>Artifact Id</th>
</tr>
<tr>
- <td rowspan="2">2.17.0</td>
+ <td rowspan="3">2.17.0</td>
+ <td>1.9.x</td>
+ <td>beam-runners-flink-1.9</td>
+</tr>
+<tr>
<td>1.8.x</td>
<td>beam-runners-flink-1.8</td>
</tr>
@@ -271,7 +275,7 @@
available. To run a pipeline on an embedded Flink cluster:
</span>
-<span class="language-py">1. Start the JobService endpoint: `./gradlew :runners:flink:1.8:job-server:runShadow`
+<span class="language-py">1. Start the JobService endpoint: `./gradlew :runners:flink:1.9:job-server:runShadow`
</span>
<span class="language-py">
@@ -304,7 +308,7 @@
<span class="language-py">1. Start a Flink cluster which exposes the Rest interface on `localhost:8081` by default.
</span>
-<span class="language-py">2. Start JobService with Flink Rest endpoint: `./gradlew :runners:flink:1.8:job-server:runShadow -PflinkMasterUrl=localhost:8081`.
+<span class="language-py">2. Start JobService with Flink Rest endpoint: `./gradlew :runners:flink:1.9:job-server:runShadow -PflinkMasterUrl=localhost:8081`.
</span>
<span class="language-py">3. Submit the pipeline as above.
diff --git a/website/src/documentation/sdks/nexmark.md b/website/src/documentation/sdks/nexmark.md
index d5230da..b73023b 100644
--- a/website/src/documentation/sdks/nexmark.md
+++ b/website/src/documentation/sdks/nexmark.md
@@ -149,7 +149,7 @@
-P nexmark.runner
The Gradle project name of the runner, such as ":runners:direct-java" or
- ":runners:flink:1.8. The project names can be found in the root
+ ":runners:flink:1.9. The project names can be found in the root
`settings.gradle`.
Test data is deterministically synthesized on demand. The test
@@ -557,7 +557,7 @@
Batch Mode:
./gradlew :sdks:java:testing:nexmark:run \
- -Pnexmark.runner=":runners:flink:1.8" \
+ -Pnexmark.runner=":runners:flink:1.9" \
-Pnexmark.args="
--runner=FlinkRunner
--suite=SMOKE
@@ -570,7 +570,7 @@
Streaming Mode:
./gradlew :sdks:java:testing:nexmark:run \
- -Pnexmark.runner=":runners:flink:1.8" \
+ -Pnexmark.runner=":runners:flink:1.9" \
-Pnexmark.args="
--runner=FlinkRunner
--suite=SMOKE
