paimon-e2e-tests/pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <parent>
        <artifactId>paimon-parent</artifactId>
        <groupId>org.apache.paimon</groupId>
        <version>1.4-SNAPSHOT</version>
    </parent>

    <artifactId>paimon-e2e-tests</artifactId>
    <name>Paimon : End to End Tests</name>

    <properties>
        <test.java.version>java8</test.java.version>
        <flink.shaded.hadoop.version>2.8.3-10.0</flink.shaded.hadoop.version>
        <flink.cdc.version>3.1.1</flink.cdc.version>
        <flink.sql.connector.hive>flink-sql-connector-hive-2.3.10_${flink.scala.binary.version}</flink.sql.connector.hive>
    </properties>

    <dependencies>
        <!-- Add dependencies to affect the order of mvn build -->

        <dependency>
            <groupId>org.apache.paimon</groupId>
            <artifactId>paimon-flink-${test.flink.main.version}</artifactId>
            <version>${project.version}</version>
            <scope>runtime</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.paimon</groupId>
            <artifactId>paimon-flink-action</artifactId>
            <version>${project.version}</version>
            <scope>runtime</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.paimon</groupId>
            <artifactId>paimon-hive-connector-common</artifactId>
            <version>${project.version}</version>
            <scope>runtime</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.paimon</groupId>
            <artifactId>paimon-spark-${test.spark.main.version}_${scala.binary.version}</artifactId>
            <version>${project.version}</version>
            <scope>runtime</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.paimon</groupId>
            <artifactId>paimon-flink-common</artifactId>
            <version>${project.version}</version>
            <scope>test</scope>
            <type>test-jar</type>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-mysql-cdc</artifactId>
            <version>${flink.cdc.version}</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.paimon</groupId>
            <artifactId>paimon-test-utils</artifactId>
            <version>${project.version}</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.paimon</groupId>
            <artifactId>paimon-flink-cdc</artifactId>
            <version>${project.version}</version>
            <scope>test</scope>
            <type>test-jar</type>
        </dependency>

        <!-- testcontainers -->

        <dependency>
            <groupId>org.testcontainers</groupId>
            <artifactId>testcontainers</artifactId>
            <version>${testcontainers.version}</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.testcontainers</groupId>
            <artifactId>kafka</artifactId>
            <version>${testcontainers.version}</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.testcontainers</groupId>
            <artifactId>mysql</artifactId>
            <version>${testcontainers.version}</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>${test.mysql.connector.java.version}</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-deploy-plugin</artifactId>
                <configuration>
                    <skip>true</skip>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-dependency-plugin</artifactId>
                <executions>
                    <execution>
                        <id>copy-jars</id>
                        <phase>process-resources</phase>
                        <goals>
                            <goal>copy</goal>
                        </goals>
                    </execution>
                </executions>
                <configuration>
                    <artifactItems>
                        <artifactItem>
                            <groupId>org.apache.paimon</groupId>
                            <artifactId>paimon-flink-${test.flink.main.version}</artifactId>
                            <version>${project.version}</version>
                            <destFileName>paimon-flink.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <artifactItem>
                            <groupId>org.apache.paimon</groupId>
                            <artifactId>paimon-flink-action</artifactId>
                            <version>${project.version}</version>
                            <destFileName>paimon-flink-action.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <artifactItem>
                            <groupId>org.apache.paimon</groupId>
                            <artifactId>paimon-hive-connector-common</artifactId>
                            <version>${project.version}</version>
                            <destFileName>paimon-hive-connector.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <artifactItem>
                            <groupId>org.apache.paimon</groupId>
                            <artifactId>paimon-spark-${test.spark.main.version}_${scala.binary.version}</artifactId>
                            <version>${project.version}</version>
                            <destFileName>paimon-spark.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <artifactItem>
                            <groupId>org.apache.flink</groupId>
                            <artifactId>flink-shaded-hadoop-2-uber</artifactId>
                            <version>${flink.shaded.hadoop.version}</version>
                            <destFileName>bundled-hadoop.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <artifactItem>
                            <groupId>org.apache.flink</groupId>
                            <artifactId>flink-sql-connector-mysql-cdc</artifactId>
                            <version>${flink.cdc.version}</version>
                            <destFileName>mysql-cdc.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <artifactItem>
                            <groupId>mysql</groupId>
                            <artifactId>mysql-connector-java</artifactId>
                            <version>${test.mysql.connector.java.version}</version>
                            <destFileName>mysql-connector-java.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <!-- test paimon with kafka sql jar -->
                        <artifactItem>
                            <groupId>org.apache.flink</groupId>
                            <artifactId>flink-sql-connector-kafka</artifactId>
                            <version>${test.flink.connector.kafka.version}</version>
                            <destFileName>flink-sql-connector-kafka.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                        <!-- flink hive catalog needs flink hive sql jar -->
                        <artifactItem>
                            <groupId>org.apache.flink</groupId>
                            <artifactId>${flink.sql.connector.hive}</artifactId>
                            <version>${test.flink.connector.hive.version}</version>
                            <destFileName>flink-sql-connector-hive.jar</destFileName>
                            <type>jar</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>/tmp/paimon-e2e-tests-jars
                            </outputDirectory>
                        </artifactItem>
                    </artifactItems>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-antrun-plugin</artifactId>
                <executions>
                    <execution>
                        <id>FLINK-31695</id>
                        <phase>process-resources</phase>
                        <goals>
                            <goal>run</goal>
                        </goals>
                        <configuration>
                            <!--
                            Due to FLINK-31695, we need to delete conflicting classes from jar file.
                            Remove this work-around after we make our own e2e docker image,
                            with both Flink and Hadoop in the same docker.
                            -->
                            <target>
                                <!-- Unzip the jar file to a temporary directory -->
                                <unzip src="/tmp/paimon-e2e-tests-jars/bundled-hadoop.jar" dest="target/temp"/>
                                <!-- Delete the conflicting file -->
                                <delete file="target/temp/org/apache/commons/cli/CommandLine.class"/>
                                <!-- Repackage the modified content -->
                                <zip destfile="/tmp/paimon-e2e-tests-jars/bundled-hadoop.jar" basedir="target/temp"/>
                                <!-- Delete the extracted temporary folder -->
                                <delete dir="target/temp"/>
                            </target>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>

        <resources>
            <resource>
                <directory>src/test/resources</directory>
                <filtering>false</filtering>
            </resource>
            <resource>
                <directory>src/test/resources-filtered</directory>
                <filtering>true</filtering>
            </resource>
        </resources>
    </build>

    <profiles>
        <!-- Activate these profiles with -Pflink-x.xx to build and test against different Flink versions -->

        <profile>
            <id>flink-2.1</id>
            <properties>
                <test.flink.main.version>2.1</test.flink.main.version>
                <test.flink.version>2.1.0</test.flink.version>
                <test.flink.connector.kafka.version>4.0.0-2.0</test.flink.connector.kafka.version>
            </properties>
        </profile>

        <profile>
            <id>flink-2.0</id>
            <properties>
                <test.flink.main.version>2.0</test.flink.main.version>
                <test.flink.version>2.0.0</test.flink.version>
                <test.flink.connector.kafka.version>4.0.0-${test.flink.main.version}</test.flink.connector.kafka.version>
            </properties>
        </profile>

        <profile>
            <id>flink-1.19</id>
            <properties>
                <test.flink.main.version>1.19</test.flink.main.version>
                <test.flink.version>1.19.1</test.flink.version>
                <test.flink.connector.kafka.version>3.2.0-${test.flink.main.version}</test.flink.connector.kafka.version>
                <flink.sql.connector.hive>flink-sql-connector-hive-2.3.9_${scala.binary.version}</flink.sql.connector.hive>
                <test.flink.connector.hive.version>${test.flink.version}</test.flink.connector.hive.version>
            </properties>
        </profile>

        <profile>
            <id>flink-1.18</id>
            <properties>
                <test.flink.main.version>1.18</test.flink.main.version>
                <test.flink.version>1.18.1</test.flink.version>
                <test.flink.connector.kafka.version>3.0.1-${test.flink.main.version}</test.flink.connector.kafka.version>
                <flink.sql.connector.hive>flink-sql-connector-hive-2.3.9_${scala.binary.version}</flink.sql.connector.hive>
                <test.flink.connector.hive.version>${test.flink.version}</test.flink.connector.hive.version>
            </properties>
        </profile>

        <profile>
            <id>flink-1.17</id>
            <properties>
                <test.flink.main.version>1.17</test.flink.main.version>
                <test.flink.version>1.17.2</test.flink.version>
                <test.flink.connector.kafka.version>${test.flink.version}</test.flink.connector.kafka.version>
                <flink.sql.connector.hive>flink-sql-connector-hive-2.3.9_${scala.binary.version}</flink.sql.connector.hive>
                <test.flink.connector.hive.version>${test.flink.version}</test.flink.connector.hive.version>
            </properties>
        </profile>

        <profile>
            <id>flink-1.16</id>
            <properties>
                <test.flink.main.version>1.16</test.flink.main.version>
                <test.flink.version>1.16.3</test.flink.version>
                <test.flink.connector.kafka.version>${test.flink.version}</test.flink.connector.kafka.version>
                <flink.sql.connector.hive>flink-sql-connector-hive-2.3.9_${scala.binary.version}</flink.sql.connector.hive>
                <test.flink.connector.hive.version>${test.flink.version}</test.flink.connector.hive.version>
            </properties>
        </profile>

        <profile>
            <id>java11</id>
            <activation>
                <jdk>[11,)</jdk>
            </activation>
            <properties>
                <test.java.version>java11</test.java.version>
            </properties>
        </profile>
    </profiles>
</project>