| name: core |
| |
| on: |
| push: |
| branches-ignore: |
| - 'dependabot/**' |
| pull_request: |
| branches: |
| - master |
| - 'branch-*' |
| |
| env: |
| # Disable keepAlive and pool |
| # https://github.com/actions/virtual-environments/issues/1499#issuecomment-689467080 |
| MAVEN_OPTS: >- |
| -Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit |
| -Dhttp.keepAlive=false |
| -Dmaven.wagon.http.pool=false |
| -Dmaven.wagon.http.retryHandler.count=3 |
| MAVEN_ARGS: >- |
| -B --no-transfer-progress |
| ZEPPELIN_HELIUM_REGISTRY: helium |
| SPARK_PRINT_LAUNCH_COMMAND: "true" |
| SPARK_LOCAL_IP: 127.0.0.1 |
| ZEPPELIN_LOCAL_IP: 127.0.0.1 |
| |
| # Use the bash login, because we are using miniconda |
| defaults: |
| run: |
| shell: bash -l {0} |
| |
| permissions: |
| contents: read # to fetch code (actions/checkout) |
| |
| jobs: |
| # test on core-modules (zeppelin-interpreter,zeppelin-zengine,zeppelin-server), |
| # some interpreters are included, because zeppelin-server test depends on them: spark, shell & markdown |
| core-modules: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| java: [ 11 ] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| ~/conda_pkgs_dir |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: install application with some interpreter |
| run: ./mvnw install -Pbuild-distr -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Pweb-classic -Phelium-dev -Pexamples ${MAVEN_ARGS} |
| - name: install and test plugins |
| run: ./mvnw package -pl zeppelin-plugins -amd ${MAVEN_ARGS} |
| - name: Setup conda environment with python 3.9 and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_3_with_R |
| environment-file: testing/env_python_3.9_with_R.yml |
| python-version: 3.9 |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: Make IRkernel available to Jupyter |
| run: | |
| R -e "IRkernel::installspec()" |
| conda list |
| conda info |
| - name: run tests # skip spark test because we would run them in other CI |
| run: ./mvnw verify -Pusing-packaged-distr -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Pweb-classic -Phelium-dev -Pexamples -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false |
| |
| # test interpreter modules except spark, flink, python, rlang, jupyter |
| interpreter-test-non-core: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| java: [ 11 ] |
| env: |
| INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,alluxio,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell' |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: install environment |
| run: ./mvnw install -DskipTests -am -pl ${INTERPRETERS} ${MAVEN_ARGS} |
| - name: Setup conda environment with python 3.9 and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_3_with_R_and_tensorflow |
| environment-file: testing/env_python_3_with_R_and_tensorflow.yml |
| python-version: 3.9 |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: verify interpreter |
| run: ./mvnw verify -am -pl ${INTERPRETERS} ${MAVEN_ARGS} |
| |
| # test interpreter modules for jupyter, python, rlang |
| interpreter-test-jupyter-python-rlang: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| python: [ 3.9 ] |
| java: [ 11 ] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: Setup conda environment with python ${{ matrix.python }} and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_3_with_R |
| environment-file: testing/env_python_${{ matrix.python }}_with_R.yml |
| python-version: ${{ matrix.python }} |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: Make IRkernel available to Jupyter |
| run: | |
| R -e "IRkernel::installspec()" |
| - name: install environment |
| run: | |
| ./mvnw install -DskipTests -pl python,rlang,zeppelin-jupyter-interpreter -am ${MAVEN_ARGS} |
| - name: run tests with ${{ matrix.python }} |
| run: | |
| ./mvnw test -pl python,rlang,zeppelin-jupyter-interpreter -DfailIfNoTests=false ${MAVEN_ARGS} |
| |
| # zeppelin integration test except Spark & Flink |
| zeppelin-integration-test: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| java: [ 11 ] |
| steps: |
| # user/password => root/root |
| - name: Start mysql |
| run: sudo systemctl start mysql.service |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: install environment |
| run: | |
| ./mvnw install -DskipTests -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell -am -Pweb-classic -Pflink-117 ${MAVEN_ARGS} |
| ./mvnw package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} |
| - name: Setup conda environment with python 3.9 and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_3_with_R |
| environment-file: testing/env_python_3_with_R.yml |
| python-version: 3.9 |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: Make IRkernel available to Jupyter |
| run: | |
| R -e "IRkernel::installspec()" |
| - name: run tests |
| run: ./mvnw test -pl zeppelin-interpreter-integration -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest |
| - name: Print zeppelin logs |
| if: always() |
| run: if [ -d "logs" ]; then cat logs/*; fi |
| |
| flink-test-and-flink-integration-test: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| python: [ 3.9 ] |
| flink: [116, 117] |
| include: |
| # Flink 1.15 supports Python 3.6, 3.7, and 3.8 |
| # https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/dev/python/installation/ |
| - python: 3.8 |
| flink: 115 |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK 8 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 11 |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: install environment for flink |
| run: | |
| ./mvnw install -DskipTests -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration ${MAVEN_ARGS} |
| ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} |
| - name: Setup conda environment with python ${{ matrix.python }} and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_3_with_flink |
| environment-file: testing/env_python_3_with_flink_${{ matrix.flink }}.yml |
| python-version: ${{ matrix.python }} |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: run tests for flink |
| run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -am -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} |
| - name: Print zeppelin logs |
| if: always() |
| run: if [ -d "logs" ]; then cat logs/*; fi |
| |
| |
| spark-integration-test: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| java: [ 11 ] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: install environment |
| run: | |
| ./mvnw install -DskipTests -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown -am -Pweb-classic -Pintegration ${MAVEN_ARGS} |
| ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} |
| - name: Setup conda environment with python 3.9 and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_3_with_R |
| environment-file: testing/env_python_3_with_R.yml |
| python-version: 3.9 |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: Make IRkernel available to Jupyter |
| run: | |
| R -e "IRkernel::installspec()" |
| - name: run tests |
| run: ./mvnw test -pl zeppelin-interpreter-integration -Pintegration -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest32,SparkIntegrationTest32,ZeppelinSparkClusterTest33,SparkIntegrationTest33 -DfailIfNoTests=false ${MAVEN_ARGS} |
| |
| # test on spark for each spark version & scala version |
| spark-test: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| python: [ 3.9 ] |
| java: [ 11, 17 ] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: install environment |
| run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-2.12,spark/scala-2.13 -am ${MAVEN_ARGS} |
| - name: Setup conda environment with python ${{ matrix.python }} and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_3_with_R |
| environment-file: testing/env_python_${{ matrix.python }}_with_R.yml |
| python-version: ${{ matrix.python }} |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: Make IRkernel available to Jupyter |
| run: | |
| R -e "IRkernel::installspec()" |
| - name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }} |
| if: ${{ matrix.java == 11 }} |
| run: | |
| rm -rf spark/interpreter/metastore_db |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} |
| - name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }} |
| if: ${{ matrix.java == 11 }} |
| run: | |
| rm -rf spark/interpreter/metastore_db |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} |
| - name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python }} |
| if: ${{ matrix.java == 11 }} |
| run: | |
| rm -rf spark/interpreter/metastore_db |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} |
| - name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python }} |
| if: ${{ matrix.java == 11 }} |
| run: | |
| rm -rf spark/interpreter/metastore_db |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} |
| - name: run spark-4.0 tests with scala-2.13 and python-${{ matrix.python }} |
| if: ${{ matrix.java == 17 }} |
| run: | |
| rm -rf spark/interpreter/metastore_db |
| ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-4.0 -Pspark-scala-2.13 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} |
| |
| # Livy 0.8.0 tested with Spark 3.2 |
| # https://github.com/apache/incubator-livy/blob/v0.8.0-incubating/dev/docker/livy-dev-spark/Dockerfile#L20 |
| livy-0-8-with-spark-3-2-under-python39: |
| runs-on: ubuntu-24.04 |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK 11 |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: 11 |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: install environment |
| run: | |
| ./mvnw install -DskipTests -pl livy -am ${MAVEN_ARGS} |
| ./testing/downloadSpark.sh "3.2.4" "3.2" |
| ./testing/downloadLivy.sh "0.8.0-incubating" "2.12" |
| - name: Setup conda environment with python 3.9 and R |
| uses: conda-incubator/setup-miniconda@v3 |
| with: |
| activate-environment: python_39_with_R |
| environment-file: testing/env_python_3.9_with_R.yml |
| python-version: 3.9 |
| channels: conda-forge,defaults |
| channel-priority: strict |
| auto-activate-base: false |
| use-mamba: true |
| - name: Make IRkernel available to Jupyter |
| run: | |
| R -e "IRkernel::installspec()" |
| - name: run tests |
| run: | |
| export SPARK_HOME=$PWD/spark-3.2.4-bin-hadoop3.2 |
| export LIVY_HOME=$PWD/apache-livy-0.8.0-incubating_2.12-bin |
| ./mvnw verify -pl livy ${MAVEN_ARGS} |
| |
| default-build: |
| runs-on: ubuntu-24.04 |
| strategy: |
| fail-fast: false |
| matrix: |
| java: [ 11 ] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Tune Runner VM |
| uses: ./.github/actions/tune-runner-vm |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~/.m2/repository |
| !~/.m2/repository/org/apache/zeppelin/ |
| ~/.spark-dist |
| ~/.cache |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: build without any profiles |
| run: ./mvnw clean verify -DskipTests ${MAVEN_ARGS} |
| |
| windows-build: |
| runs-on: windows-latest |
| strategy: |
| fail-fast: false |
| matrix: |
| java: [ 11 ] |
| steps: |
| - name: Checkout |
| uses: actions/checkout@v4 |
| - name: Set up JDK ${{ matrix.java }} |
| uses: actions/setup-java@v4 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ matrix.java }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v4 |
| with: |
| path: | |
| ~\.m2\repository |
| !~\.m2\repository\org\apache\zeppelin\ |
| key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| ${{ runner.os }}-zeppelin- |
| - name: Build on Windows |
| run: | |
| .\mvnw.cmd clean package -DskipTests ^ |
| -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade.mojo.ShadeMojo=warn ^ |
| -Dorg.slf4j.simpleLogger.log.com.googlecode.download.maven.plugin.internal.WGet=warn ^ |
| -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency.fromDependencies.CopyDependenciesMojo=warn ^ |
| --no-transfer-progress |
| shell: cmd |
