blob: e514fa4a2f6669e42acdf37fd6f7fb0f4ddddb07 [file] [log] [blame]
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
sudo: required
before_cache:
- sudo chown -R travis:travis $HOME/.m2
cache:
apt: true
directories:
- ${HOME}/.m2
- submarine-workbench/workbench-web/node
- submarine-workbench/workbench-web/node_modules
- submarine-workbench/workbench-web-ng/node
- submarine-workbench/workbench-web-ng/node_modules
addons:
apt:
sources:
- mysql-5.7-trusty
packages:
- mysql-server
- mysql-client
services:
- mysql
# https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
- xvfb
env:
global:
# If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
- SUBMARINE="org.apache.submarine"
- EXCLUDE_SUBMARINE="!${SUBMARINE}:submarine-all,!${SUBMARINE}:submarine-client,!${SUBMARINE}:submarine-dist"
- EXCLUDE_SERVER="!${SUBMARINE}:submarine-server-api,!${SUBMARINE}:submarine-server-core,!${SUBMARINE}:submarine-server-rpc"
- EXCLUDE_CLIENT="!${SUBMARINE}:submarine-client"
- EXCLUDE_ALL="!${SUBMARINE}:submarine-all"
- EXCLUDE_WORKBENCH="!${SUBMARINE}:submarine-workbench,!${SUBMARINE}:submarine-workbench-web,!${SUBMARINE}:submarine-workbench-web-ng"
- EXCLUDE_INTERPRETER="" # Template disable by SUBMARINE-381 "!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter
- EXCLUDE_SUBMITTER_K8S="!${SUBMARINE}:submarine-submitter-k8s"
- EXCLUDE_SUBMITTER_YARN="!${SUBMARINE}:submarine-submitter-yarn"
- EXCLUDE_SUBMITTER="!${SUBMARINE}:submarine-server-submitter,${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMITTER_YARN}"
- EXCLUDE_COMMONS="!${SUBMARINE}:submarine-commons-cluster,!${SUBMARINE}:submarine-commons-metastore,!${SUBMARINE}:submarine-commons-rpc,!${SUBMARINE}:submarine-commons-runtime,!${SUBMARINE}:submarine-commons-utils"
- EXCLUDE_CLOUD="!${SUBMARINE}:submarine-cloud"
- EXCLUDE_DIST="!${SUBMARINE}:submarine-dist"
- EXCLUDE_TEST="!${SUBMARINE}:submarine-test,!${SUBMARINE}:submarine-test-e2e,!${SUBMARINE}:submarine-test-k8s"
- EXCLUDE_SPARK_SECURTITY="!${SUBMARINE}:submarine-spark-security"
- MOZ_HEADLESS=1
before_install:
# maven 3.6.1 (3.6.2 build tony failed!!!)
- echo "Download Maven 3.6.1"
- wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz
- tar zxvf apache-maven-3.6.1-bin.tar.gz || travis_terminate 1
- export M2_HOME=$PWD/apache-maven-3.6.1
- export PATH=$M2_HOME/bin:$PATH
# mysql
- sudo service mysql restart
- mysql -e "create database submarine_test;"
- mysql -e "CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';"
- mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine_test'@'%';"
- mysql -e "use submarine_test; source ./docs/database/submarine.sql; show tables;"
- mysql -e "create database metastore_test;"
- mysql -e "CREATE USER 'metastore_test'@'%' IDENTIFIED BY 'password_test';"
- mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore_test'@'%';"
- mysql -e "use metastore_test; source ./docs/database/metastore.sql; show tables;"
# For e2e tests
- mysql -e "create database submarine;"
- mysql -e "CREATE USER 'submarine'@'%' IDENTIFIED BY 'password';"
- mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'submarine'@'%';"
- mysql -e "use submarine; source ./docs/database/submarine.sql; source ./docs/database/submarine-data.sql; show tables;"
- mysql -e "create database metastore;"
- mysql -e "CREATE USER 'metastore'@'%' IDENTIFIED BY 'password';"
- mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'metastore'@'%';"
- mysql -e "use metastore; source ./docs/database/metastore.sql; show tables;"
- ./dev-support/travis/install_external_dependencies.sh
# protobuf 3.10.1
- PROTOBUF_VERSION=3.10.1
- PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip
- pushd /home/travis
- wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME}
- unzip ${PROTOC_FILENAME}
- bin/protoc --version
- popd
matrix:
include:
- name: Test submarine test-k8s
dist: xenial
services: docker
language: java
jdk: openjdk8
addons:
chrome: stable
firefox: latest
# Because submarine on k8s uses port 80, it needs to be set `-Durl=http://127.0.0.1`
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am -Durl=http://127.0.0.1"
- MODULES=""
- TEST_MODULES="-pl ${EXCLUDE_SERVER},${EXCLUDE_COMMONS},${EXCLUDE_CLIENT},org.apache.submarine:submarine-test-k8s"
- TEST_PROJECTS=""
- name: Test submarine test-e2e
language: java
jdk: openjdk8
dist: xenial
addons:
chrome: stable
firefox: latest
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests"
- TEST_FLAG="verify -DskipRat -am"
- TEST_MODULES="-pl org.apache.submarine:submarine-test-e2e"
- TEST_PROJECTS=""
- name: Test submarine commons-cluster
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
- TEST_MODULES="-pl org.apache.submarine:submarine-commons-cluster"
- TEST_PROJECTS=""
- name: Test submarine commons-metastore
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
- TEST_MODULES="-pl org.apache.submarine:submarine-commons-metastore"
- TEST_PROJECTS=""
- name: Test submarine commons-rpc
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
- TEST_MODULES="-pl org.apache.submarine:submarine-commons-rpc"
- TEST_PROJECTS=""
- name: Test submarine commons-runtime
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
- TEST_MODULES="-pl org.apache.submarine:submarine-commons-runtime"
- TEST_PROJECTS=""
- name: Test submarine server
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}"
- TEST_MODULES="-pl ${EXCLUDE_COMMONS},org.apache.submarine:submarine-server-core"
- TEST_PROJECTS=""
- name: Test submarine submitter on hadoop-2.9 (default)
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests -DskipRat"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
- TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
- TEST_PROJECTS=""
- name: Test submarine submitter on hadoop-2.10
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-2.10"
- BUILD_FLAG="clean package install -DskipTests -DskipRat"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
- TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
- TEST_PROJECTS=""
- name: Test submarine submitter on hadoop-3.1
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-3.1"
- BUILD_FLAG="clean package install -DskipTests -DskipRat"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
- TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
- TEST_PROJECTS=""
- name: Test submarine submitter on hadoop-3.2
language: java
jdk: openjdk8
dist: xenial
env:
- PROFILE="-Phadoop-3.2"
- BUILD_FLAG="clean package install -DskipTests -DskipRat"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}"
- TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
- TEST_PROJECTS=""
- name: Test submarine submitter on Kubernetes
dist: xenial
services: docker
language: java
jdk: openjdk8
env: BUILD_FLAG="clean package install -DskipTests -am" TEST_FLAG="test -am" MODULES="-pl org.apache.submarine:submarine-submitter-k8s" TEST_MODULES="-pl org.apache.submarine:submarine-submitter-k8s" TEST_PROJECTS=""
before_install:
# deploy Kubernetes cluster
- curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
- chmod +x kubectl
- sudo mv kubectl /usr/bin
- docker run -d --name kube --privileged -p 8443:8443 -p 10080:10080 bsycorp/kind:latest-1.15
- until curl -s --fail http://127.0.0.1:10080/kubernetes-ready; do
sleep 1;
done
- echo "Kubernetes ready - run tests!"
- mkdir $HOME/.kube
- curl http://127.0.0.1:10080/config > $HOME/.kube/config
- export KUBECONFIG=$HOME/.kube/config
- echo $KUBECONFIG
- kubectl config set clusters.kubernetes.server https://127.0.0.1:8443
- kubectl get nodes
- kubectl create namespace submarine
- kubectl apply -f ./dev-support/travis/tf-operator/crd_v1.yaml
- kubectl apply -f ./dev-support/travis/tf-operator/tfevent-volume/.
# prepare pytroch job CRD
- kubectl apply -f ./dev-support/k8s/pytorchjob/crd.yaml
# Template disable by SUBMARINE-381
#- name: Test submarine interpreter
# language: java
# jdk: "openjdk8"
# dist: xenial
# env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH}" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
- name: Test submarine workbench-web VUE
language: node_js
node_js:
- 10.15.0
before_install:
- cd submarine-workbench/workbench-web
install:
- yarn install
script:
- yarn
- yarn run lint --no-fix && yarn run build
env: NAME="Build workbench-web"
- name: Test submarine workbench-web Angular
language: node_js
node_js:
- "10"
before_install:
- cd submarine-workbench/workbench-web-ng
before_script:
npm install
addons:
chrome: stable
firefox: latest
script:
- npm run test -- --no-watch --no-progress --browsers=ChromeHeadlessCI
- npm run test -- --no-watch --no-progress --browsers=FirefoxHeadless
- npm run webdriver
- npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js
env:
- BUILD_FLAG="clean package -DskipTests"
- TEST_FLAG="test -DskipRat -am"
- MODULES="-pl org.apache.submarine:submarine-workbench-web-ng"
- TEST_MODULES="-pl org.apache.submarine:submarine-workbench-web-ng"
- TEST_PROJECTS=""
- name: Test submarine spark security with spark 2.3 and ranger 1.0
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.3 -Pranger-1.0"
- MODULES="-pl :submarine-spark-security"
- name: Test submarine spark security with spark 2.3 and ranger 1.1
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.3 -Pranger-1.1"
- MODULES="-pl :submarine-spark-security"
- name: Test submarine spark security with spark 2.3 and ranger 1.2
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.3 -Pranger-1.2"
- MODULES="-pl :submarine-spark-security"
- name: Test submarine spark security with spark 2.3 and ranger 2.0
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.3 -Pranger-2.0"
- MODULES="-pl :submarine-spark-security"
- name: Test submarine spark security with spark 2.4 and ranger 1.0
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.4 -Pranger-1.0"
- MODULES="-pl :submarine-spark-security"
- name: Test submarine spark security with spark 2.4 and ranger 1.1
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.4 -Pranger-1.1"
- MODULES="-pl :submarine-spark-security"
- name: Test submarine spark security with spark 2.4 and ranger 1.2
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.4 -Pranger-1.2"
- MODULES="-pl :submarine-spark-security"
- name: Test submarine spark security with spark 2.4 and ranger 2.0
language: scala
jdk: openjdk8
env:
- MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
- BUILD_FLAG="--no-transfer-progress clean install -Dmaven.javadoc.skip=true"
- TEST_FLAG=$BUILD_FLAG
- PROFILE="-Pspark-2.4 -Pranger-2.0"
- MODULES="-pl :submarine-spark-security"
install:
- mvn --version
- echo ">>> mvn $BUILD_FLAG $MODULES $PROFILE -B"
- mvn $BUILD_FLAG $MODULES $PROFILE -B
before_script:
# display info log for debugging
- if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi
script:
- if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi
- echo ">>> mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS"
- mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS
after_success:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
after_failure:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
- find . -name rat.txt | xargs cat
- cat logs/*