[FLINK-20598] [training] Update playgrounds for Flink 1.12 (#21)
* Update ops-playground to v1.12.1
* Create local dirs for the checkpoints&savepoints before spinning up the docker compose application.
* Upgrade table-walkthrough with flink-1.12.1 version.
* Upgrade pyflink-walkthrough with flink-1.12.1 version.
* [FLINK-20599] Change flink-playground-clickcountjob version from 1.11.0 to 1.12.1.
* Update operations-playground/README.md
Co-authored-by: David Anderson <david@alpinegizmo.com>
diff --git a/docker/ops-playground-image/Dockerfile b/docker/ops-playground-image/Dockerfile
index 83d7b20..d897a7e 100644
--- a/docker/ops-playground-image/Dockerfile
+++ b/docker/ops-playground-image/Dockerfile
@@ -32,7 +32,7 @@
# Build Operations Playground Image
###############################################################################
-FROM flink:1.11.0-scala_2.11
+FROM apache/flink:1.12.1-scala_2.11-java8
WORKDIR /opt/flink/bin
diff --git a/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml b/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml
index f759ede..838f08a 100644
--- a/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml
+++ b/docker/ops-playground-image/java/flink-playground-clickcountjob/pom.xml
@@ -22,7 +22,7 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-playground-clickcountjob</artifactId>
- <version>1-FLINK-1.11_2.11</version>
+ <version>1-FLINK-1.12_2.11</version>
<name>flink-playground-clickcountjob</name>
<packaging>jar</packaging>
@@ -44,7 +44,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <flink.version>1.11.0</flink.version>
+ <flink.version>1.12.1</flink.version>
<java.version>1.8</java.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
diff --git a/operations-playground/README.md b/operations-playground/README.md
index 605d62a..fca9a01 100644
--- a/operations-playground/README.md
+++ b/operations-playground/README.md
@@ -26,6 +26,15 @@
docker-compose build
```
+### Preparing the Checkpoint and Savepoint Directories
+
+Create the checkpoint and savepoint directories on the Docker host machine (these volumes are mounted by the jobmanager and taskmanager, as specified in docker-compose.yaml):
+
+```bash
+mkdir -p /tmp/flink-checkpoints-directory
+mkdir -p /tmp/flink-savepoints-directory
+```
+
### Starting the Playground
Once you built the Docker image, run the following command to start the playground
@@ -52,4 +61,4 @@
## Further instructions
The playground setup and more detailed instructions are presented in the
-["Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-release-1.11/try-flink/flink-operations-playground.html) of Flink's documentation.
+["Getting Started" guide](https://ci.apache.org/projects/flink/flink-docs-release-1.12/try-flink/flink-operations-playground.html) of Flink's documentation.
diff --git a/operations-playground/docker-compose.yaml b/operations-playground/docker-compose.yaml
index 0d795e7..d9b8695 100644
--- a/operations-playground/docker-compose.yaml
+++ b/operations-playground/docker-compose.yaml
@@ -20,7 +20,7 @@
services:
client:
build: ../docker/ops-playground-image
- image: apache/flink-ops-playground:1-FLINK-1.11-scala_2.11
+ image: apache/flink-ops-playground:1-FLINK-1.12-scala_2.11
command: "flink run -d /opt/ClickCountJob.jar --bootstrap.servers kafka:9092 --checkpointing --event-time"
depends_on:
- jobmanager
@@ -30,29 +30,29 @@
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
clickevent-generator:
- image: apache/flink-ops-playground:1-FLINK-1.11-scala_2.11
+ image: apache/flink-ops-playground:1-FLINK-1.12-scala_2.11
command: "java -classpath /opt/ClickCountJob.jar:/opt/flink/lib/* org.apache.flink.playgrounds.ops.clickcount.ClickEventGenerator --bootstrap.servers kafka:9092 --topic input"
depends_on:
- kafka
jobmanager:
- image: flink:1.11.0-scala_2.11
+ image: apache/flink:1.12.1-scala_2.11-java8
command: "jobmanager.sh start-foreground"
ports:
- 8081:8081
volumes:
- ./conf:/opt/flink/conf
- - flink-checkpoints-directory:/tmp/flink-checkpoints-directory
+ - /tmp/flink-checkpoints-directory:/tmp/flink-checkpoints-directory
- /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
taskmanager:
- image: flink:1.11.0-scala_2.11
+ image: apache/flink:1.12.1-scala_2.11-java8
depends_on:
- jobmanager
command: "taskmanager.sh start-foreground"
volumes:
- ./conf:/opt/flink/conf
- - flink-checkpoints-directory:/tmp/flink-checkpoints-directory
+ - /tmp/flink-checkpoints-directory:/tmp/flink-checkpoints-directory
- /tmp/flink-savepoints-directory:/tmp/flink-savepoints-directory
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
@@ -69,5 +69,3 @@
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
ports:
- 9094:9094
-volumes:
- flink-checkpoints-directory:
diff --git a/pyflink-walkthrough/Dockerfile b/pyflink-walkthrough/Dockerfile
index 9224a58..3612785 100644
--- a/pyflink-walkthrough/Dockerfile
+++ b/pyflink-walkthrough/Dockerfile
@@ -20,8 +20,8 @@
# Build PyFlink Playground Image
###############################################################################
-FROM flink:1.11.0-scala_2.11
-ARG FLINK_VERSION=1.11.1
+FROM apache/flink:1.12.1-scala_2.11-java8
+ARG FLINK_VERSION=1.12.1
# Install pyflink
RUN set -ex; \
@@ -33,7 +33,7 @@
ln -s /usr/bin/pip3 /usr/bin/pip; \
apt-get update; \
python -m pip install --upgrade pip; \
- pip install apache-flink==1.11.1; \
+ pip install apache-flink==1.12.1; \
pip install kafka-python;
diff --git a/pyflink-walkthrough/README.md b/pyflink-walkthrough/README.md
index d721790..fa2f3e5 100644
--- a/pyflink-walkthrough/README.md
+++ b/pyflink-walkthrough/README.md
@@ -5,7 +5,7 @@
In this playground, you will learn how to build and run an end-to-end PyFlink pipeline for data analytics, covering the following steps:
* Reading data from a Kafka source;
-* Creating data using a [UDF](https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/python/table-api-users-guide/udfs/python_udfs.html);
+* Creating data using a [UDF](https://ci.apache.org/projects/flink/flink-docs-release-1.12/dev/python/table-api-users-guide/udfs/python_udfs.html);
* Performing a simple aggregation over the source data;
* Writing the results to Elasticsearch and visualizing them in Kibana.
@@ -153,4 +153,4 @@
```
Furthermore, you can also [create new Kibana dashboards](https://www.elastic.co/guide/en/kibana/7.8/dashboard-create-new-dashboard.html)
-to visualize other aspects of the data in Elasticsearch.
\ No newline at end of file
+to visualize other aspects of the data in Elasticsearch.
diff --git a/pyflink-walkthrough/docker-compose.yml b/pyflink-walkthrough/docker-compose.yml
index 96e4c6f..ef07632 100644
--- a/pyflink-walkthrough/docker-compose.yml
+++ b/pyflink-walkthrough/docker-compose.yml
@@ -20,7 +20,7 @@
services:
jobmanager:
build: .
- image: pyflink/pyflink:1.11.0-scala_2.11
+ image: pyflink/pyflink:1.12.1-scala_2.11
volumes:
- .:/opt/pyflink-walkthrough
hostname: "jobmanager"
@@ -32,7 +32,7 @@
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
taskmanager:
- image: pyflink/pyflink:1.11.0-scala_2.11
+ image: pyflink/pyflink:1.12.1-scala_2.11
volumes:
- .:/opt/pyflink-walkthrough
expose:
diff --git a/table-walkthrough/Dockerfile b/table-walkthrough/Dockerfile
index 9210b4d..ee223d6 100644
--- a/table-walkthrough/Dockerfile
+++ b/table-walkthrough/Dockerfile
@@ -22,12 +22,12 @@
COPY ./src /opt/src
RUN cd /opt; mvn clean install -Dmaven.test.skip
-FROM flink:1.11.0-scala_2.11
+FROM apache/flink:1.12.1-scala_2.11-java8
# Download connector libraries
-RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka_2.11/1.11.0/flink-sql-connector-kafka_2.11-1.11.0.jar; \
- wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc_2.11/1.11.0/flink-connector-jdbc_2.11-1.11.0.jar; \
- wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/1.11.0/flink-csv-1.11.0.jar; \
+RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka_2.11/1.12.1/flink-sql-connector-kafka_2.11-1.12.1.jar; \
+ wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc_2.11/1.12.1/flink-connector-jdbc_2.11-1.12.1.jar; \
+ wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/1.12.1/flink-csv-1.12.1.jar; \
wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.19/mysql-connector-java-8.0.19.jar;
COPY --from=builder /opt/target/spend-report-*.jar /opt/flink/usrlib/spend-report.jar
diff --git a/table-walkthrough/docker-compose.yml b/table-walkthrough/docker-compose.yml
index a3e232a..65be5ea 100644
--- a/table-walkthrough/docker-compose.yml
+++ b/table-walkthrough/docker-compose.yml
@@ -19,7 +19,7 @@
version: '2.1'
services:
jobmanager:
- image: apache/flink-table-walkthrough:1-FLINK-1.11-scala_2.11
+ image: apache/flink-table-walkthrough:1-FLINK-1.12-scala_2.11
build: .
hostname: "jobmanager"
expose:
@@ -33,7 +33,7 @@
- kafka
- mysql
taskmanager:
- image: apache/flink-table-walkthrough:1-FLINK-1.11-scala_2.11
+ image: apache/flink-table-walkthrough:1-FLINK-1.12-scala_2.11
build: .
expose:
- "6121"
diff --git a/table-walkthrough/pom.xml b/table-walkthrough/pom.xml
index 207fc51..529d5e9 100644
--- a/table-walkthrough/pom.xml
+++ b/table-walkthrough/pom.xml
@@ -30,7 +30,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <flink.version>1.11.0</flink.version>
+ <flink.version>1.12.1</flink.version>
<java.version>1.8</java.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
