| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| sudo: required |
| |
| before_cache: |
| - sudo chown -R travis:travis $HOME/.m2 |
| |
| cache: |
| apt: true |
| directories: |
| - ${HOME}/.m2 |
| - submarine-workbench/workbench-web/node |
| - submarine-workbench/workbench-web/node_modules |
| |
| addons: |
| apt: |
| sources: |
| - mysql-5.7-trusty |
| packages: |
| - mysql-server |
| - mysql-client |
| |
| services: |
| - mysql |
| # https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI |
| - xvfb |
| |
| env: |
| global: |
| # If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE |
| - SUBMARINE="org.apache.submarine" |
| - EXCLUDE_SUBMARINE="!:submarine-all,!:submarine-client,!:submarine-dist" |
| - EXCLUDE_SERVER="!:submarine-server-api,!:submarine-server-core,!:submarine-server-rpc" |
| - EXCLUDE_CLIENT="!:submarine-client" |
| - EXCLUDE_ALL="!:submarine-all" |
| - EXCLUDE_WORKBENCH="!:submarine-workbench,!:submarine-workbench-web" |
| - EXCLUDE_INTERPRETER="!:submarine-interpreter,!:submarine-interpreter-core,!:submarine-python-interpreter,!:submarine-spark-interpreter" |
| - EXCLUDE_SUBMITTER_K8S="!:submarine-submitter-k8s" |
| - EXCLUDE_SUBMITTER_YARN="!:submarine-submitter-yarn" |
| - EXCLUDE_SUBMITTER="!:submarine-server-submitter,${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMITTER_YARN}" |
| - EXCLUDE_COMMONS="!:submarine-commons-cluster,!:submarine-commons-metastore,!:submarine-commons-rpc,!:submarine-commons-runtime,!:submarine-commons-utils" |
| - EXCLUDE_CLOUD="!:submarine-cloud" |
| - EXCLUDE_DIST="!:submarine-dist" |
| - EXCLUDE_TEST="!:submarine-test,!:submarine-test-e2e,!:submarine-test-k8s" |
| - EXCLUDE_SPARK_SECURTITY="!:submarine-spark-security" |
| - MOZ_HEADLESS=1 |
| |
| before_install: |
| # mysql |
| - sudo service mysql restart |
| - ./docs/database/init-database.sh |
| - ./dev-support/travis/install_external_dependencies.sh |
| # protobuf 3.10.1 |
| - PROTOBUF_VERSION=3.10.1 |
| - PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip |
| - pushd /home/travis |
| - wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME} |
| - unzip ${PROTOC_FILENAME} |
| - bin/protoc --version |
| - popd |
| |
| matrix: |
| include: |
| - name: Test submarine test-k8s |
| dist: xenial |
| services: docker |
| language: java |
| jdk: openjdk8 |
| addons: |
| chrome: stable |
| firefox: latest |
| # Because submarine on k8s uses port 80, it needs to be set `-Durl=http://127.0.0.1` |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am -Durl=http://127.0.0.1" |
| - MODULES="" |
| - TEST_MODULES="-pl ${EXCLUDE_SERVER},${EXCLUDE_COMMONS},${EXCLUDE_CLIENT},:submarine-test-k8s" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine test-e2e |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| addons: |
| chrome: stable |
| firefox: latest |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" |
| - TEST_FLAG="verify -DskipRat -am" |
| - TEST_MODULES="-pl org.apache.submarine:submarine-test-e2e" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine commons-cluster |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" |
| - TEST_MODULES="-pl org.apache.submarine:submarine-commons-cluster" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine commons-metastore |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" |
| - TEST_MODULES="-pl org.apache.submarine:submarine-commons-metastore" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine commons-rpc |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" |
| - TEST_MODULES="-pl org.apache.submarine:submarine-commons-rpc" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine commons-runtime |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" |
| - TEST_MODULES="-pl org.apache.submarine:submarine-commons-runtime" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine commons-unixusersync |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" |
| - TEST_MODULES="-pl org.apache.submarine:submarine-commons-unixusersync" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine server |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" |
| - TEST_MODULES="-pl ${EXCLUDE_COMMONS},org.apache.submarine:submarine-server-core" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine submitter on hadoop-2.9 (default) |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.9" |
| - BUILD_FLAG="clean package install -DskipTests -DskipRat" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" |
| - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine submitter on hadoop-2.10 |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-2.10" |
| - BUILD_FLAG="clean package install -DskipTests -DskipRat" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" |
| - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine submitter on hadoop-3.1 |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-3.1" |
| - BUILD_FLAG="clean package install -DskipTests -DskipRat" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" |
| - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine submitter on hadoop-3.2 |
| language: java |
| jdk: openjdk8 |
| dist: xenial |
| env: |
| - PROFILE="-Phadoop-3.2" |
| - BUILD_FLAG="clean package install -DskipTests -DskipRat" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" |
| - TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine interpreter |
| language: java |
| jdk: "openjdk8" |
| dist: xenial |
| env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS="" |
| |
| - name: Test submarine workbench-web Angular |
| language: node_js |
| node_js: |
| - "10" |
| before_install: |
| - cd submarine-workbench/workbench-web |
| before_script: |
| npm install |
| addons: |
| chrome: stable |
| firefox: latest |
| script: |
| - npm run test -- --no-watch --no-progress --browsers=ChromeHeadlessCI |
| - npm run test -- --no-watch --no-progress --browsers=FirefoxHeadless |
| - npm run webdriver |
| - npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js |
| env: |
| - BUILD_FLAG="clean package -DskipTests" |
| - TEST_FLAG="test -DskipRat -am" |
| - MODULES="-pl org.apache.submarine:submarine-workbench-web" |
| - TEST_MODULES="-pl org.apache.submarine:submarine-workbench-web" |
| - TEST_PROJECTS="" |
| |
| - name: Test submarine spark security with spark 2.3 and ranger 1.0 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.3 -Pranger-1.0" |
| - MODULES="-pl :submarine-spark-security" |
| |
| - name: Test submarine spark security with spark 2.3 and ranger 1.1 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.3 -Pranger-1.1" |
| - MODULES="-pl :submarine-spark-security" |
| |
| - name: Test submarine spark security with spark 2.3 and ranger 1.2 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.3 -Pranger-1.2" |
| - MODULES="-pl :submarine-spark-security" |
| |
| - name: Test submarine spark security with spark 2.3 and ranger 2.0 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.3 -Pranger-2.0" |
| - MODULES="-pl :submarine-spark-security" |
| |
| - name: Test submarine spark security with spark 2.4 and ranger 1.0 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.4 -Pranger-1.0" |
| - MODULES="-pl :submarine-spark-security" |
| |
| - name: Test submarine spark security with spark 2.4 and ranger 1.1 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.4 -Pranger-1.1" |
| - MODULES="-pl :submarine-spark-security" |
| |
| - name: Test submarine spark security with spark 2.4 and ranger 1.2 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.4 -Pranger-1.2" |
| - MODULES="-pl :submarine-spark-security" |
| |
| - name: Test submarine spark security with spark 2.4 and ranger 2.0 |
| language: scala |
| jdk: openjdk8 |
| env: |
| - MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" |
| - BUILD_FLAG="--batch-mode clean install -Dmaven.javadoc.skip=true" |
| - TEST_FLAG=$BUILD_FLAG |
| - PROFILE="-Pspark-2.4 -Pranger-2.0" |
| - MODULES="-pl :submarine-spark-security" |
| install: |
| - mvn --version |
| - echo ">>> mvn $BUILD_FLAG $MODULES $PROFILE -B" |
| - mvn $BUILD_FLAG $MODULES $PROFILE -B |
| |
| before_script: |
| # display info log for debugging |
| - if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi |
| |
| script: |
| - if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi |
| - echo ">>> mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS" |
| - mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS |
| |
| after_success: |
| - echo "Travis exited with ${TRAVIS_TEST_RESULT}" |
| |
| after_failure: |
| - echo "Travis exited with ${TRAVIS_TEST_RESULT}" |
| - find . -name rat.txt | xargs cat |
| - cat logs/* |
