| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # NOTE: |
| # This config file defines how Azure CI runs tests with Spark 2.4 and Flink 1.17 profiles. |
| # PRs will need to keep in sync with master's version to trigger the CI runs. |
| |
| trigger: |
| branches: |
| include: |
| - '*' # must quote since "*" is a YAML reserved character; we want a string |
| |
| pool: |
| vmImage: 'ubuntu-22.04' |
| |
| parameters: |
| - name: job1Modules |
| type: object |
| default: |
| - 'hudi-common' |
| - 'hudi-flink-datasource' |
| - 'hudi-flink-datasource/hudi-flink' |
| - 'hudi-flink-datasource/hudi-flink1.13.x' |
| - 'hudi-flink-datasource/hudi-flink1.14.x' |
| - 'hudi-flink-datasource/hudi-flink1.15.x' |
| - 'hudi-flink-datasource/hudi-flink1.16.x' |
| - 'hudi-flink-datasource/hudi-flink1.17.x' |
| - name: job2Modules |
| type: object |
| default: |
| - 'hudi-client/hudi-spark-client' |
| - name: job3UTModules |
| type: object |
| default: |
| - 'hudi-spark-datasource' |
| - 'hudi-spark-datasource/hudi-spark' |
| - 'hudi-spark-datasource/hudi-spark3.2.x' |
| - 'hudi-spark-datasource/hudi-spark3.2plus-common' |
| - 'hudi-spark-datasource/hudi-spark3-common' |
| - 'hudi-spark-datasource/hudi-spark-common' |
| - name: job4UTModules |
| type: object |
| default: |
| - '!hudi-client/hudi-spark-client' |
| - '!hudi-common' |
| - '!hudi-examples' |
| - '!hudi-examples/hudi-examples-common' |
| - '!hudi-examples/hudi-examples-flink' |
| - '!hudi-examples/hudi-examples-java' |
| - '!hudi-examples/hudi-examples-spark' |
| - '!hudi-flink-datasource' |
| - '!hudi-flink-datasource/hudi-flink' |
| - '!hudi-flink-datasource/hudi-flink1.13.x' |
| - '!hudi-flink-datasource/hudi-flink1.14.x' |
| - '!hudi-flink-datasource/hudi-flink1.15.x' |
| - '!hudi-flink-datasource/hudi-flink1.16.x' |
| - '!hudi-flink-datasource/hudi-flink1.17.x' |
| - '!hudi-spark-datasource' |
| - '!hudi-spark-datasource/hudi-spark' |
| - '!hudi-spark-datasource/hudi-spark3.2.x' |
| - '!hudi-spark-datasource/hudi-spark3.2plus-common' |
| - '!hudi-spark-datasource/hudi-spark3-common' |
| - '!hudi-spark-datasource/hudi-spark-common' |
| - name: job4FTModules |
| type: object |
| default: |
| - '!hudi-client/hudi-spark-client' |
| - '!hudi-common' |
| - '!hudi-examples' |
| - '!hudi-examples/hudi-examples-common' |
| - '!hudi-examples/hudi-examples-flink' |
| - '!hudi-examples/hudi-examples-java' |
| - '!hudi-examples/hudi-examples-spark' |
| - '!hudi-flink-datasource' |
| - '!hudi-flink-datasource/hudi-flink' |
| - '!hudi-flink-datasource/hudi-flink1.13.x' |
| - '!hudi-flink-datasource/hudi-flink1.14.x' |
| - '!hudi-flink-datasource/hudi-flink1.15.x' |
| - '!hudi-flink-datasource/hudi-flink1.16.x' |
| - '!hudi-flink-datasource/hudi-flink1.17.x' |
| |
| variables: |
| BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.17' |
| PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn' |
| MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS)' |
| MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)' |
| JOB1_MODULES: ${{ join(',',parameters.job1Modules) }} |
| JOB2_MODULES: ${{ join(',',parameters.job2Modules) }} |
| JOB3_MODULES: ${{ join(',',parameters.job3UTModules) }} |
| JOB4_UT_MODULES: ${{ join(',',parameters.job4UTModules) }} |
| JOB4_FT_MODULES: ${{ join(',',parameters.job4FTModules) }} |
| |
| stages: |
| - stage: test |
| jobs: |
| - job: UT_FT_1 |
| displayName: UT FT common & flink & UT client/spark-client |
| timeoutInMinutes: '150' |
| steps: |
| - task: Maven@4 |
| displayName: maven install |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'clean install' |
| options: $(MVN_OPTS_INSTALL) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| - task: Maven@4 |
| displayName: UT common flink client/spark-client |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'test' |
| options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| mavenOptions: '-Xmx4g' |
| - task: Maven@4 |
| displayName: FT common flink |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'test' |
| options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| mavenOptions: '-Xmx4g' |
| - script: | |
| grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 |
| displayName: Top 100 long-running testcases |
| - job: UT_FT_2 |
| displayName: FT client/spark-client |
| timeoutInMinutes: '150' |
| steps: |
| - task: Maven@4 |
| displayName: maven install |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'clean install' |
| options: $(MVN_OPTS_INSTALL) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| - task: Maven@4 |
| displayName: FT client/spark-client |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'test' |
| options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| mavenOptions: '-Xmx4g' |
| - script: | |
| grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 |
| displayName: Top 100 long-running testcases |
| - job: UT_FT_3 |
| displayName: UT spark-datasource |
| timeoutInMinutes: '180' |
| steps: |
| - task: Maven@4 |
| displayName: maven install |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'clean install' |
| options: $(MVN_OPTS_INSTALL) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| - task: Maven@4 |
| displayName: UT spark-datasource |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'test' |
| options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| mavenOptions: '-Xmx4g' |
| - script: | |
| grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 |
| displayName: Top 100 long-running testcases |
| - job: UT_FT_4 |
| displayName: UT FT other modules |
| timeoutInMinutes: '180' |
| steps: |
| - task: Maven@4 |
| displayName: maven install |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'clean install' |
| options: $(MVN_OPTS_INSTALL) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| - task: Maven@4 |
| displayName: UT other modules |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'test' |
| options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| mavenOptions: '-Xmx4g' |
| - task: Maven@4 |
| displayName: FT other modules |
| inputs: |
| mavenPomFile: 'pom.xml' |
| goals: 'test' |
| options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_FT_MODULES) |
| publishJUnitResults: false |
| jdkVersionOption: '1.8' |
| mavenOptions: '-Xmx4g' |
| - script: | |
| grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 |
| displayName: Top 100 long-running testcases |
