[release-2.26.0][BEAM-11409] Revert java-versioning of SDK container #12505
diff --git a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy
index 05a6b53..ed6ecff 100644
--- a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy
@@ -46,7 +46,7 @@
gradle {
rootBuildScriptDir(properties.checkoutDir)
tasks 'javaExamplesDataflowPreCommit'
- switches '-PimageJavaVersion=11'
+ switches '-Pdockerfile=Dockerfile-java11'
switches '-PdisableSpotlessCheck=true'
switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11
switches '-PcompileAndRunTestsWithJava11'
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 39c10a8..6c611b2 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1613,7 +1613,7 @@
// development image at docker.io (see sdks/CONTAINERS.md):
//
// format: apache/beam_$NAME_sdk:latest
- // ie: apache/beam_python3.7_sdk:latest apache/beam_java8_sdk:latest apache/beam_go_sdk:latest
+ // ie: apache/beam_python3.7_sdk:latest apache/beam_java_sdk:latest apache/beam_go_sdk:latest
//
// Both the root and tag can be defined using properties or explicitly provided.
project.ext.containerImageName = {
diff --git a/release/src/main/scripts/publish_docker_images.sh b/release/src/main/scripts/publish_docker_images.sh
index be8ba4a..4767928 100755
--- a/release/src/main/scripts/publish_docker_images.sh
+++ b/release/src/main/scripts/publish_docker_images.sh
@@ -27,7 +27,6 @@
DOCKER_IMAGE_DEFAULT_REPO_ROOT=apache
DOCKER_IMAGE_DEFAULT_REPO_PREFIX=beam_
-JAVA_VER=("java8")
PYTHON_VER=("python3.6" "python3.7" "python3.8")
FLINK_VER=("1.8" "1.9" "1.10")
@@ -49,7 +48,7 @@
echo '-------------------Tagging and Pushing Python images-----------------'
for ver in "${PYTHON_VER[@]}"; do
- # Pull verified RC from dockerhub.
+ # Pull varified RC from dockerhub.
docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
# Tag with ${RELEASE} and push to dockerhub.
@@ -67,23 +66,21 @@
done
echo '-------------------Tagging and Pushing Java images-----------------'
- for ver in "${JAVA_VER[@]}"; do
- # Pull verified RC from dockerhub.
- docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
+ # Pull varified RC from dockerhub.
+ docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION}
- # Tag with ${RELEASE} and push to dockerhub.
- docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
- docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
+ # Tag with ${RELEASE} and push to dockerhub.
+ docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}
+ docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}
- # Tag with latest and push to dockerhub.
- docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
- docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
+ # Tag with latest and push to dockerhub.
+ docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:latest
+ docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:latest
- # Cleanup images from local
- docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
- docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
- docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
- end
+ # Cleanup images from local
+ docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION}
+ docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}
+ docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:latest
echo '-------------Tagging and Pushing Flink job server images-------------'
echo "Publishing images for the following Flink versions:" "${FLINK_VER[@]}"
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
index b3b821f..6944684 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
@@ -75,38 +75,6 @@
public static final String ENVIRONMENT_EMBEDDED = "EMBEDDED"; // Non Public urn for testing
public static final String ENVIRONMENT_LOOPBACK = "LOOPBACK"; // Non Public urn for testing
- public enum JavaVersion {
- v8("java8", "1.8"),
- v11("java11", "11");
-
- private final String name;
- private final String specification;
-
- JavaVersion(final String name, final String specification) {
- this.name = name;
- this.specification = specification;
- }
-
- @Override
- public String toString() {
- return this.name;
- }
-
- public String specification() {
- return this.specification;
- }
-
- public static JavaVersion forSpecification(String specification) {
- for (JavaVersion ver : JavaVersion.values()) {
- if (ver.specification.equals(specification)) {
- return ver;
- }
- }
- throw new UnsupportedOperationException(
- String.format("unsupported Java version: %s", specification));
- }
- }
-
/* For development, use the container build by the current user to ensure that the SDK harness and
* the SDK agree on how they should interact. This should be changed to a version-specific
* container during a release.
@@ -114,9 +82,12 @@
* See https://beam.apache.org/contribute/docker-images/ for more information on how to build a
* container.
*/
-
private static final String JAVA_SDK_HARNESS_CONTAINER_URL =
- getDefaultJavaSdkHarnessContainerUrl();
+ ReleaseInfo.getReleaseInfo().getDefaultDockerRepoRoot()
+ + "/"
+ + ReleaseInfo.getReleaseInfo().getDefaultDockerRepoPrefix()
+ + "java_sdk:"
+ + ReleaseInfo.getReleaseInfo().getSdkVersion();
public static final Environment JAVA_SDK_HARNESS_ENVIRONMENT =
createDockerEnvironment(JAVA_SDK_HARNESS_CONTAINER_URL);
@@ -351,10 +322,6 @@
return capabilities.build();
}
- public static JavaVersion getJavaVersion() {
- return JavaVersion.forSpecification(System.getProperty("java.specification.version"));
- }
-
public static String createStagingFileName(File path, HashCode hash) {
String encodedHash = Base64Variants.MODIFIED_FOR_URL.encode(hash.asBytes());
String fileName = Files.getNameWithoutExtension(path.getAbsolutePath());
@@ -393,13 +360,4 @@
return env;
}
}
-
- private static String getDefaultJavaSdkHarnessContainerUrl() {
- return String.format(
- "%s/%s%s_sdk:%s",
- ReleaseInfo.getReleaseInfo().getDefaultDockerRepoRoot(),
- ReleaseInfo.getReleaseInfo().getDefaultDockerRepoPrefix(),
- getJavaVersion().toString(),
- ReleaseInfo.getReleaseInfo().getSdkVersion());
- }
}
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java
index 3e6960e..7d8476d 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java
@@ -20,7 +20,6 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import java.io.IOException;
@@ -34,7 +33,6 @@
import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
import org.apache.beam.model.pipeline.v1.RunnerApi.ProcessPayload;
import org.apache.beam.model.pipeline.v1.RunnerApi.StandardEnvironments;
-import org.apache.beam.runners.core.construction.Environments.JavaVersion;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -166,17 +164,4 @@
env1,
equalTo(components.toComponents().getEnvironmentsOrThrow(ptransform.getEnvironmentId())));
}
-
- @Test
- public void testJavaVersion() {
- assertEquals(JavaVersion.v8, JavaVersion.forSpecification("1.8"));
- assertEquals("java8", JavaVersion.v8.toString());
- assertEquals(JavaVersion.v11, JavaVersion.forSpecification("11"));
- assertEquals("java11", JavaVersion.v11.toString());
- }
-
- @Test(expected = UnsupportedOperationException.class)
- public void testJavaVersionInvalid() {
- assertEquals(JavaVersion.v8, JavaVersion.forSpecification("invalid"));
- }
}
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index 05b3e6c..62efe09 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -243,7 +243,7 @@
dependsOn ":sdks:java:container:docker"
finalizedBy 'cleanUpDockerImages'
def defaultDockerImageName = containerImageName(
- name: project.docker_image_default_repo_prefix + "java8_sdk",
+ name: project.docker_image_default_repo_prefix + "java_sdk",
root: "apache",
tag: project.sdk_version)
doLast {
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index fcbbe3b..0ba890e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -382,16 +382,16 @@
}
// Adding the Java version to the SDK name for user's and support convenience.
- String agentJavaVer =
- (Environments.getJavaVersion() == Environments.JavaVersion.v8)
- ? "(JRE 8 environment)"
- : "(JDK 11 environment)";
+ String javaVersion =
+ Float.parseFloat(System.getProperty("java.specification.version")) >= 9
+ ? "(JDK 11 environment)"
+ : "(JRE 8 environment)";
DataflowRunnerInfo dataflowRunnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo();
String userAgent =
String.format(
"%s/%s%s",
- dataflowRunnerInfo.getName(), dataflowRunnerInfo.getVersion(), agentJavaVer)
+ dataflowRunnerInfo.getName(), dataflowRunnerInfo.getVersion(), javaVersion)
.replace(" ", "_");
dataflowOptions.setUserAgent(userAgent);
@@ -2145,9 +2145,8 @@
static String getContainerImageForJob(DataflowPipelineOptions options) {
String workerHarnessContainerImage = options.getWorkerHarnessContainerImage();
- Environments.JavaVersion javaVersion = Environments.getJavaVersion();
String javaVersionId =
- (javaVersion == Environments.JavaVersion.v8) ? "java" : javaVersion.toString();
+ Float.parseFloat(System.getProperty("java.specification.version")) >= 9 ? "java11" : "java";
if (!workerHarnessContainerImage.contains("IMAGE")) {
return workerHarnessContainerImage;
} else if (hasExperiment(options, "beam_fn_api")) {
diff --git a/sdks/java/container/Dockerfile b/sdks/java/container/Dockerfile
index 95f20cd..b637384 100644
--- a/sdks/java/container/Dockerfile
+++ b/sdks/java/container/Dockerfile
@@ -15,8 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
-ARG java_version
-FROM openjdk:${java_version}
+
+FROM openjdk:8
MAINTAINER "Apache Beam <dev@beam.apache.org>"
ARG pull_licenses
@@ -43,7 +43,7 @@
# copy an extra LICENSE file then remove it.
COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
-RUN if [ "${pull_licenses}" = "false" ] ; then \
+RUN if [ "$pull_licenses" = "false" ] ; then \
# Remove above license dir if pull licenses false
rm -rf /opt/apache/beam/third_party_licenses ; \
fi
diff --git a/sdks/java/container/Dockerfile-java11 b/sdks/java/container/Dockerfile-java11
new file mode 100644
index 0000000..1d85e85
--- /dev/null
+++ b/sdks/java/container/Dockerfile-java11
@@ -0,0 +1,45 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM openjdk:11
+MAINTAINER "Apache Beam <dev@beam.apache.org>"
+
+ARG pull_licenses
+
+ADD target/slf4j-api.jar /opt/apache/beam/jars/
+ADD target/slf4j-jdk14.jar /opt/apache/beam/jars/
+ADD target/beam-sdks-java-harness.jar /opt/apache/beam/jars/
+
+# Required to run cross-language pipelines with KafkaIO
+# TODO May be removed once custom environments are supported
+ADD target/beam-sdks-java-io-kafka.jar /opt/apache/beam/jars/
+ADD target/kafka-clients.jar /opt/apache/beam/jars/
+
+ADD target/linux_amd64/boot /opt/apache/beam/
+
+# Add golang licenses. Because the go-license directory may be empty if
+# pull_licenses is false, and COPY fails if there are no files,
+# copy an extra LICENSE file then remove it.
+COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
+RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
+RUN if [ "$pull_licenses" = "false" ] ; then \
+ # Remove above license dir if pull licenses false
+ rm -rf /opt/apache/beam/third_party_licenses ; \
+ fi
+
+ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle
index 042dcf6..bdd073f 100644
--- a/sdks/java/container/build.gradle
+++ b/sdks/java/container/build.gradle
@@ -54,6 +54,8 @@
dockerDependency library.java.kafka_clients
}
+def dockerfileName = project.findProperty('dockerfile') ?: 'Dockerfile'
+
task copyDockerfileDependencies(type: Copy) {
from configurations.dockerDependency
rename "slf4j-api.*", "slf4j-api.jar"
@@ -86,22 +88,18 @@
renderers = [new JsonReportRenderer()]
}
-def imageJavaVersion = project.hasProperty('imageJavaVersion') ? project.findProperty('imageJavaVersion') : '8'
docker {
name containerImageName(
- name: "${project.docker_image_default_repo_prefix}java${imageJavaVersion}_sdk",
+ name: project.docker_image_default_repo_prefix + "java_sdk",
root: project.rootProject.hasProperty(["docker-repository-root"]) ?
project.rootProject["docker-repository-root"] :
project.docker_image_default_repo_root,
tag: project.rootProject.hasProperty(["docker-tag"]) ?
project.rootProject["docker-tag"] : project.sdk_version)
- dockerfile project.file("./Dockerfile")
+ dockerfile project.file("./${dockerfileName}")
files "./build/"
- buildArgs([
- 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) ||
- project.rootProject.hasProperty(["isRelease"]),
- 'java_version': imageJavaVersion,
- ])
+ buildArgs(['pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) ||
+ project.rootProject.hasProperty(["isRelease"])])
}
task pullLicenses(type: Exec) {
diff --git a/sdks/python/apache_beam/examples/kafkataxi/README.md b/sdks/python/apache_beam/examples/kafkataxi/README.md
index aba8fde..11b15f0 100644
--- a/sdks/python/apache_beam/examples/kafkataxi/README.md
+++ b/sdks/python/apache_beam/examples/kafkataxi/README.md
@@ -136,7 +136,7 @@
```sh
export DOCKER_ROOT="Your Docker Repository Root"
./gradlew :sdks:java:container:docker -Pdocker-repository-root=$DOCKER_ROOT -Pdocker-tag=latest
-docker push $DOCKER_ROOT/beam_java8_sdk:latest
+docker push $DOCKER_ROOT/beam_java_sdk:latest
```
For portable Flink/Spark in local mode, instead of above command just build the
@@ -184,6 +184,6 @@
--num_workers $NUM_WORKERS \
--job_name $JOB_NAME \
--bootstrap_servers $BOOTSTRAP_SERVER \
- --sdk_harness_container_image_overrides ".*java.*,${DOCKER_ROOT}/beam_java8_sdk:latest" \
+ --sdk_harness_container_image_overrides ".*java.*,${DOCKER_ROOT}/beam_java_sdk:latest" \
--experiments=use_runner_v2
```
diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md
index d10fbf8..13c800a 100644
--- a/website/www/site/content/en/contribute/release-guide.md
+++ b/website/www/site/content/en/contribute/release-guide.md
@@ -575,7 +575,7 @@
```
- For Java SDK images, there should be around 200 dependencies.
```
- docker run -it --entrypoint=/bin/bash apache/beam_java${ver}_sdk:${RELEASE}_rc{RC_NUM}
+ docker run -it --entrypoint=/bin/bash apache/beam_java_sdk:${RELEASE}_rc{RC_NUM}
ls -al /opt/apache/beam/third_party_licenses/ | wc -l
```
1. Publish staging artifacts
diff --git a/website/www/site/content/en/documentation/runtime/environments.md b/website/www/site/content/en/documentation/runtime/environments.md
index 93ecbae..0089722 100644
--- a/website/www/site/content/en/documentation/runtime/environments.md
+++ b/website/www/site/content/en/documentation/runtime/environments.md
@@ -116,8 +116,8 @@
To examine the containers that you built, run `docker images` from anywhere in the command line. If you successfully built all of the container images, the command prints a table like the following:
```
-REPOSITORY TAG IMAGE ID CREATED SIZE
-apache/beam_java8_sdk latest 16ca619d489e 2 weeks ago 550MB
+REPOSITORY TAG IMAGE ID CREATED SIZE
+apache/beam_java_sdk latest 16ca619d489e 2 weeks ago 550MB
apache/beam_python2.7_sdk latest b6fb40539c29 2 weeks ago 1.78GB
apache/beam_python3.5_sdk latest bae309000d09 2 weeks ago 1.85GB
apache/beam_python3.6_sdk latest 42faad307d1a 2 weeks ago 1.86GB
