[improvement] update build.sh and pom for support 1.15-1.17 (#126)
* update build.sh
* update readme
* update pom
---------
Co-authored-by: wudi <>
diff --git a/README.md b/README.md
index 1ba2211..a1b02c2 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@
## Flink Doris Connector
-Flink Doris Connector now support flink version from 1.11 to 1.15.
+Flink Doris Connector now support flink version from 1.11 to 1.17.
If you wish to contribute or use a connector from flink 1.13 (and earlier), please use the [branch-for-flink-before-1.13](https://github.com/apache/doris-flink-connector/tree/branch-for-flink-before-1.13)
diff --git a/flink-doris-connector/build.sh b/flink-doris-connector/build.sh
index ac812ab..6263436 100755
--- a/flink-doris-connector/build.sh
+++ b/flink-doris-connector/build.sh
@@ -116,7 +116,7 @@
selectFlink() {
echo 'Flink-Doris-Connector supports multiple versions of flink. Which version do you need ?'
- select flink in "1.15.x" "1.16.x"
+ select flink in "1.15.x" "1.16.x" "1.17.x"
do
case $flink in
"1.15.x")
@@ -125,6 +125,9 @@
"1.16.x")
return 2
;;
+ "1.17.x")
+ return 3
+ ;;
*)
echo "invalid selected, exit.."
exit 1
@@ -140,22 +143,17 @@
FLINK_VERSION="1.15.0"
elif [ ${flinkVer} -eq 2 ]; then
FLINK_VERSION="1.16.0"
+elif [ ${flinkVer} -eq 3 ]; then
+ FLINK_VERSION="1.17.0"
fi
-# extract minor version:
-# eg: 3.1.2 -> 3
-FLINK_MINOR_VERSION=0
-[ ${FLINK_VERSION} != 0 ] && FLINK_MINOR_VERSION=${FLINK_VERSION%.*}
-
-echo_g " flink version: ${FLINK_VERSION}, minor version: ${FLINK_MINOR_VERSION}"
+echo_g " flink version: ${FLINK_VERSION}"
echo_g " build starting..."
${MVN_BIN} clean package \
-Dflink.version=${FLINK_VERSION} \
- -Dflink.minor.version=${FLINK_MINOR_VERSION} \
-Dthrift.binary=${THRIFT_BIN} "$@"
-
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
DIST_DIR=${DORIS_HOME}/dist
diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml
index e0e5d83..089f530 100644
--- a/flink-doris-connector/pom.xml
+++ b/flink-doris-connector/pom.xml
@@ -26,10 +26,11 @@
<version>23</version>
</parent>
<groupId>org.apache.doris</groupId>
- <artifactId>flink-doris-connector-${flink.minor.version}</artifactId>
+ <artifactId>flink-doris-connector</artifactId>
<version>1.4.0-SNAPSHOT</version>
<name>Flink Doris Connector</name>
<url>https://doris.apache.org/</url>
+
<licenses>
<license>
<name>Apache 2.0 License</name>
@@ -37,16 +38,19 @@
<distribution>repo</distribution>
</license>
</licenses>
+
<scm>
<connection>scm:git:https://git@github.com/apache/doris-flink-connector.git</connection>
<developerConnection>scm:git:https://git@github.com/apache/doris-flink-connector.git</developerConnection>
<url>scm:git:https://git@github.com/apache/doris-flink-connector.git</url>
<tag>HEAD</tag>
</scm>
+
<issueManagement>
<system>GitHub</system>
<url>https://github.com/apache/doris/issues</url>
</issueManagement>
+
<mailingLists>
<mailingList>
<name>Dev Mailing List</name>
@@ -61,68 +65,18 @@
<unsubscribe>commits-unsubscribe@doris.apache.org</unsubscribe>
</mailingList>
</mailingLists>
+
<properties>
<flink.version>1.15.0</flink.version>
- <flink.minor.version>1.15</flink.minor.version>
<libthrift.version>0.13.0</libthrift.version>
<arrow.version>5.0.0</arrow.version>
- <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
+ <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version>
<maven-javadoc-plugin.version>3.3.0</maven-javadoc-plugin.version>
<maven-source-plugin.version>3.2.1</maven-source-plugin.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<log4j2.version>2.18.0</log4j2.version>
<project.scm.id>github</project.scm.id>
</properties>
- <profiles>
- <profile>
- <id>thirdparty</id>
- <activation>
- <property>
- <name>env.DORIS_THIRDPARTY</name>
- </property>
- </activation>
- <properties>
- <doris.thirdparty>${env.DORIS_THIRDPARTY}</doris.thirdparty>
- </properties>
- </profile>
- <!-- for custom internal repository -->
- <profile>
- <id>custom-env</id>
- <activation>
- <property>
- <name>env.CUSTOM_MAVEN_REPO</name>
- </property>
- </activation>
- <repositories>
- <repository>
- <id>custom-nexus</id>
- <url>${env.CUSTOM_MAVEN_REPO}</url>
- </repository>
- </repositories>
- <pluginRepositories>
- <pluginRepository>
- <id>custom-nexus</id>
- <url>${env.CUSTOM_MAVEN_REPO}</url>
- </pluginRepository>
- </pluginRepositories>
- </profile>
- <!-- for general repository -->
- <profile>
- <id>general-env</id>
- <activation>
- <property>
- <name>!env.CUSTOM_MAVEN_REPO</name>
- </property>
- </activation>
- <repositories>
- <repository>
- <id>central</id>
- <name>central maven repo https</name>
- <url>https://repo.maven.apache.org/maven2</url>
- </repository>
- </repositories>
- </profile>
- </profiles>
<dependencies>
<dependency>
@@ -131,30 +85,35 @@
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner-loader</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-base</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-runtime</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
+
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
@@ -170,6 +129,7 @@
</exclusion>
</exclusions>
</dependency>
+
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@@ -181,16 +141,19 @@
</exclusion>
</exclusions>
</dependency>
+
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.15</version>
</dependency>
+
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${arrow.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
@@ -215,27 +178,32 @@
</exclusion>
</exclusions>
</dependency>
+
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.77.Final</version>
</dependency>
+
<!-- jackson -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.13.3</version>
</dependency>
+
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.13.3</version>
</dependency>
+
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.13.3</version>
</dependency>
+
<!--Test-->
<dependency>
<groupId>org.hamcrest</groupId>
@@ -243,24 +211,28 @@
<version>1.3</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>2.27.0</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>com.ververica</groupId>
<artifactId>flink-connector-mysql-cdc</artifactId>
@@ -268,151 +240,259 @@
<scope>test</scope>
</dependency>
</dependencies>
+
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>sign-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.thrift.tools</groupId>
+ <artifactId>maven-thrift-plugin</artifactId>
+ <version>0.1.11</version>
+ <configuration>
+ <thriftExecutable>${thrift.binary}</thriftExecutable>
+ <generator>java:fullcamel</generator>
+ </configuration>
+ <executions>
+ <execution>
+ <id>thrift-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.4</version>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.arrow</pattern>
+ <shadedPattern>org.apache.doris.shaded.org.apache.arrow</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>org.apache.doris.shaded.io.netty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.doris.shaded.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.codec</pattern>
+ <shadedPattern>org.apache.doris.shaded.org.apache.commons.codec</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.flatbuffers</pattern>
+ <shadedPattern>org.apache.doris.shaded.com.google.flatbuffers</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.thrift</pattern>
+ <shadedPattern>org.apache.doris.shaded.org.apache.thrift</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <source>8</source>
+ <target>8</target>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>${maven-javadoc-plugin.version}</version>
+ <configuration>
+ <skip>true</skip>
+ <source>8</source>
+ <failOnError>false</failOnError>
+ </configuration>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>${maven-source-plugin.version}</version>
+ <configuration>
+ <attach>true</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>compile</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>2.0.0</version>
+ <executions>
+ <execution>
+ <id>add-third-party</id>
+ <goals>
+ <goal>add-third-party</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-release-plugin</artifactId>
+ <version>3.0.0-M5</version>
+ <configuration>
+ <arguments>${releaseArgs}</arguments>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-gpg-plugin</artifactId>
- <executions>
- <execution>
- <id>sign-artifacts</id>
- <phase>verify</phase>
- <goals>
- <goal>sign</goal>
- </goals>
- </execution>
- </executions>
+ <artifactId>maven-shade-plugin</artifactId>
</plugin>
+
<plugin>
<groupId>org.apache.thrift.tools</groupId>
<artifactId>maven-thrift-plugin</artifactId>
- <version>0.1.11</version>
- <configuration>
- <thriftExecutable>${thrift.binary}</thriftExecutable>
- <generator>java:fullcamel</generator>
- </configuration>
- <executions>
- <execution>
- <id>thrift-sources</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.2.1</version>
- <configuration>
- <artifactSet>
- <excludes>
- <exclude>com.google.code.findbugs:*</exclude>
- <exclude>org.slf4j:*</exclude>
- </excludes>
- </artifactSet>
- <relocations>
- <relocation>
- <pattern>org.apache.arrow</pattern>
- <shadedPattern>org.apache.doris.shaded.org.apache.arrow</shadedPattern>
- </relocation>
- <relocation>
- <pattern>io.netty</pattern>
- <shadedPattern>org.apache.doris.shaded.io.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.doris.shaded.com.fasterxml.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons.codec</pattern>
- <shadedPattern>org.apache.doris.shaded.org.apache.commons.codec</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.flatbuffers</pattern>
- <shadedPattern>org.apache.doris.shaded.com.google.flatbuffers</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.thrift</pattern>
- <shadedPattern>org.apache.doris.shaded.org.apache.thrift</shadedPattern>
- </relocation>
- </relocations>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>${maven-compiler-plugin.version}</version>
- <configuration>
- <source>8</source>
- <target>8</target>
- </configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>${maven-javadoc-plugin.version}</version>
- <configuration>
- <skip>true</skip>
- <source>8</source>
- <failOnError>false</failOnError>
- </configuration>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <version>${maven-source-plugin.version}</version>
- <configuration>
- <attach>true</attach>
- </configuration>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>license-maven-plugin</artifactId>
- <version>2.0.0</version>
- <executions>
- <execution>
- <id>add-third-party</id>
- <goals>
- <goal>add-third-party</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-release-plugin</artifactId>
- <version>3.0.0-M5</version>
- <configuration>
- <arguments>${releaseArgs}</arguments>
- </configuration>
- </plugin>
-
</plugins>
+
</build>
-</project>
+
+ <profiles>
+
+ <profile>
+ <id>thirdparty</id>
+ <activation>
+ <property>
+ <name>env.DORIS_THIRDPARTY</name>
+ </property>
+ </activation>
+ <properties>
+ <doris.thirdparty>${env.DORIS_THIRDPARTY}</doris.thirdparty>
+ </properties>
+ </profile>
+
+ <!-- for custom internal repository -->
+ <profile>
+ <id>custom-env</id>
+ <activation>
+ <property>
+ <name>env.CUSTOM_MAVEN_REPO</name>
+ </property>
+ </activation>
+ <repositories>
+ <repository>
+ <id>custom-nexus</id>
+ <url>${env.CUSTOM_MAVEN_REPO}</url>
+ </repository>
+ </repositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <id>custom-nexus</id>
+ <url>${env.CUSTOM_MAVEN_REPO}</url>
+ </pluginRepository>
+ </pluginRepositories>
+ </profile>
+
+ <!-- for general repository -->
+ <profile>
+ <id>general-env</id>
+ <activation>
+ <property>
+ <name>!env.CUSTOM_MAVEN_REPO</name>
+ </property>
+ </activation>
+ <repositories>
+ <repository>
+ <id>central</id>
+ <name>central maven repo https</name>
+ <url>https://repo.maven.apache.org/maven2</url>
+ </repository>
+ </repositories>
+ </profile>
+
+ <profile>
+ <id>apache-release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-release-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
\ No newline at end of file
