| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| language: java |
| |
| dist: xenial |
| |
| jdk: |
| - openjdk8 |
| |
| cache: |
| directories: |
| - $HOME/.m2 |
| |
| env: |
| global: |
| - DOCKER_IP=127.0.0.1 # for integration tests |
| - MVN="mvn -B" |
| - > # Various options to make execution of maven goals faster (e.g., mvn install) |
| MAVEN_SKIP="-Pskip-static-checks -Ddruid.console.skip=true -Dmaven.javadoc.skip=true" |
| - MAVEN_SKIP_TESTS="-Pskip-tests" |
| |
| addons: |
| apt: |
| packages: |
| - maven |
| - python3 |
| |
| # Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not |
| # needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any |
| # output. To compensate, use travis_wait to extend the timeout. |
| install: ./check_test_suite.py && travis_terminate 0 || echo 'Running maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} |
| |
| # There are 3 stages of tests |
| # 1. Tests - phase 1 |
| # 2. Tests - phase 2 |
| # 3. cron |
| # |
| # The cron type only runs jobs that are marked with stage cron. |
| # The test type is split into 2 stages. This is done so that more PRs can run their validation |
| # in parallel. The first phase is meant to include sanity test jobs. The jobs in this phase are |
| # meant to be fast. The second phase is meant to run all other tests. |
| # Jobs with known flaky tests should be put in the second phase since the second phase will not |
| # start if there are any failures in the second stage. |
| stages: |
| - name: Tests - phase 1 |
| if: type != cron |
| - name: Tests - phase 2 |
| if: type != cron |
| - name: cron |
| if: type = cron |
| |
| jobs: |
| include: |
| - name: "animal sniffer checks" |
| stage: Tests - phase 1 |
| script: ${MVN} animal-sniffer:check --fail-at-end |
| |
| - name: "checkstyle" |
| script: ${MVN} checkstyle:checkstyle --fail-at-end |
| |
| - name: "enforcer checks" |
| script: ${MVN} enforcer:enforce --fail-at-end |
| |
| - name: "forbidden api checks" |
| script: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end |
| |
| - name: "pmd checks" |
| script: ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check |
| |
| - name: "spotbugs checks" |
| script: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' |
| |
| - name: "license checks" |
| install: skip |
| before_script: &setup_generate_license |
| - sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y |
| - ./check_test_suite.py && travis_terminate 0 || echo 'Continuing setup' |
| - pip3 install wheel # install wheel first explicitly |
| - pip3 install pyyaml |
| script: |
| - > |
| ${MVN} apache-rat:check -Prat --fail-at-end |
| -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn |
| -Drat.consoleOutput=true |
| # Generate dependency reports and checks they are valid. When running on Travis CI, 2 cores are available |
| # (https://docs.travis-ci.com/user/reference/overview/#virtualisation-environment-vs-operating-system). |
| - mkdir -p target |
| - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 |
| - distribution/bin/check-licenses.py licenses.yaml target/license-reports |
| - name: "script checks" |
| install: skip |
| # who watches the watchers? |
| script: ./check_test_suite_test.py |
| |
| - name: "(openjdk8) strict compilation" |
| install: skip |
| # Strict compilation requires more than 2 GB |
| script: > |
| ./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end |
| -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} |
| |
| - name: "analyze dependencies" |
| script: |- |
| MAVEN_OPTS='-Xmx3000m' ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || { echo " |
| |
| The dependency analysis has found a dependency that is either: |
| |
| 1) Used and undeclared: These are available as a transitive dependency but should be explicitly |
| added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is |
| shown above. |
| |
| 2) Unused and declared: These are not needed and removing them from the POM will speed up the build |
| and reduce the artifact size. The dependencies to remove are shown above. |
| |
| If there are false positive dependency analysis warnings, they can be suppressed: |
| https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies |
| https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html |
| |
| For more information, refer to: |
| https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html |
| |
| " && false; } |
| |
| - name: "intellij inspections" |
| script: > |
| ./check_test_suite.py && travis_terminate 0 || docker run --rm |
| -v $(pwd):/project |
| -v ~/.m2:/home/inspect/.m2 |
| ccaominh/intellij-inspect:1.0.0 |
| /project/pom.xml |
| /project/.idea/inspectionProfiles/Druid.xml |
| --levels ERROR |
| --scope JavaInspectionsScope |
| |
| - &package |
| name: "(openjdk8) packaging check" |
| install: skip |
| before_script: *setup_generate_license |
| script: > |
| MAVEN_OPTS='-Xmx3000m' ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end |
| -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Ddruid.console.skip=false -T1C |
| |
| - <<: *package |
| name: "(openjdk11) packaging check" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *package |
| name: "(openjdk15) packaging check" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_processing_module |
| name: "(openjdk8) processing module test" |
| stage: Tests - phase 1 |
| env: |
| - MAVEN_PROJECTS='processing' |
| before_script: |
| - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=true |
| script: |
| - unset _JAVA_OPTIONS |
| # Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when |
| # resolving the SIGAR dependency. |
| - > |
| MAVEN_OPTS='-Xmx1100m' ${MVN} test -pl ${MAVEN_PROJECTS} |
| ${MAVEN_SKIP} -Dremoteresources.skip=true -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} |
| - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0" |
| - free -m |
| - ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report |
| # Add merge target branch to determine diff (see https://github.com/travis-ci/travis-ci/issues/6069). |
| # This is not needed for build triggered by tags, since there will be no code diff. |
| - echo "TRAVIS_BRANCH=${TRAVIS_BRANCH}" # for debugging |
| - if [[ -z "${TRAVIS_TAG}" ]]; then git remote set-branches --add origin ${TRAVIS_BRANCH} && git fetch; fi |
| # Determine the modified files that match the maven projects being tested. We use maven project lists that |
| # either exclude (starts with "!") or include (does not start with "!"), so both cases need to be handled. |
| # If the build is triggered by a tag, an error will be printed, but `all_files` will be correctly set to empty |
| # so that the coverage check is skipped. |
| - all_files="$(git diff --name-only origin/${TRAVIS_BRANCH}...HEAD | grep "\.java$" || [[ $? == 1 ]])" |
| - for f in ${all_files}; do echo $f; done # for debugging |
| - > |
| if [[ "${MAVEN_PROJECTS}" = \!* ]]; then |
| regex="${MAVEN_PROJECTS:1}"; |
| regex="^${regex//,\!/\\|^}"; |
| project_files="$(echo "${all_files}" | grep -v "${regex}" || [[ $? == 1 ]])"; |
| else |
| regex="^${MAVEN_PROJECTS//,/\\|^}"; |
| project_files="$(echo "${all_files}" | grep "${regex}" || [[ $? == 1 ]])"; |
| fi |
| - for f in ${project_files}; do echo $f; done # for debugging |
| # Check diff code coverage for the maven projects being tested (retry install in case of network error). |
| # Currently, the function coverage check is not reliable, so it is disabled. |
| - > |
| if [ -n "${project_files}" ]; then |
| travis_retry npm install @connectis/diff-test-coverage@1.5.3 |
| && git diff origin/${TRAVIS_BRANCH}...HEAD -- ${project_files} |
| | node_modules/.bin/diff-test-coverage |
| --coverage "**/target/site/jacoco/jacoco.xml" |
| --type jacoco |
| --line-coverage 50 |
| --branch-coverage 50 |
| --function-coverage 0 |
| --log-template "coverage-lines-complete" |
| --log-template "coverage-files-complete" |
| --log-template "totals-complete" |
| --log-template "errors" |
| -- |
| || { printf "\n\n****FAILED****\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n" && false; } |
| fi |
| after_success: |
| # retry in case of network error |
| - travis_retry curl -o codecov.sh -s https://codecov.io/bash |
| - travis_retry bash codecov.sh -X gcov |
| |
| - <<: *test_processing_module |
| name: "(openjdk11) processing module test" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_processing_module |
| name: "(openjdk15) processing module test" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_processing_module_sqlcompat |
| <<: *test_processing_module |
| name: "(openjdk8) processing module test (SQL Compatibility)" |
| stage: Tests - phase 1 |
| before_script: &setup_sqlcompat |
| - export DRUID_USE_DEFAULT_VALUE_FOR_NULL=false |
| |
| - <<: *test_processing_module_sqlcompat |
| name: "(openjdk11) processing module test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_processing_module_sqlcompat |
| name: "(openjdk15) processing module test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_indexing_module |
| <<: *test_processing_module |
| name: "(openjdk8) indexing modules test" |
| env: |
| - MAVEN_PROJECTS='indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service' |
| |
| - <<: *test_indexing_module |
| name: "(openjdk11) indexing modules test" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_indexing_module |
| name: "(openjdk15) indexing modules test" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_indexing_module_sqlcompat |
| <<: *test_indexing_module |
| name: "(openjdk8) indexing modules test (SQL Compatibility)" |
| stage: Tests - phase 1 |
| before_script: *setup_sqlcompat |
| |
| - <<: *test_indexing_module_sqlcompat |
| name: "(openjdk11) indexing modules test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_indexing_module_sqlcompat |
| name: "(openjdk15) indexing modules test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_server_module |
| <<: *test_processing_module |
| name: "(openjdk8) server module test" |
| env: |
| - MAVEN_PROJECTS='server' |
| |
| - <<: *test_server_module |
| name: "(openjdk11) server module test" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_server_module |
| name: "(openjdk15) server module test" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_server_module_sqlcompat |
| <<: *test_server_module |
| name: "(openjdk8) server module test (SQL Compatibility)" |
| before_script: *setup_sqlcompat |
| |
| - <<: *test_server_module_sqlcompat |
| name: "(openjdk11) server module test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_server_module_sqlcompat |
| name: "(openjdk15) server module test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_other_modules |
| <<: *test_processing_module |
| name: "(openjdk8) other modules test" |
| env: |
| - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests' |
| |
| - <<: *test_other_modules |
| name: "(openjdk11) other modules test" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_other_modules |
| name: "(openjdk15) other modules test" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - &test_other_modules_sqlcompat |
| <<: *test_other_modules |
| name: "(openjdk8) other modules test (SQL Compatibility)" |
| before_script: *setup_sqlcompat |
| |
| - <<: *test_other_modules_sqlcompat |
| name: "(openjdk11) other modules test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk11 |
| |
| - <<: *test_other_modules_sqlcompat |
| name: "(openjdk15) other modules test (SQL Compatibility)" |
| stage: Tests - phase 2 |
| jdk: openjdk15 |
| |
| - name: "web console" |
| install: skip |
| stage: Tests - phase 1 |
| script: |
| - ./check_test_suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console' |
| after_success: |
| - (cd web-console && travis_retry npm run codecov) # retry in case of network error |
| |
| - name: "web console end-to-end test" |
| before_install: *setup_generate_license |
| install: web-console/script/druid build |
| before_script: |
| - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...' |
| - nvm install 10.24.0 |
| - web-console/script/druid start |
| script: (cd web-console && npm run test-e2e) |
| after_script: web-console/script/druid stop |
| |
| - name: "docs" |
| install: ./check_test_suite.py && travis_terminate 0 || (cd website && npm install) |
| script: |- |
| (cd website && npm run lint && npm run spellcheck) || { echo " |
| |
| If there are spell check errors: |
| |
| 1) Suppressing False Positives: Edit website/.spelling to add suppressions. Instructions |
| are at the top of the file and explain how to suppress false positives either globally or |
| within a particular file. |
| |
| 2) Running Spell Check Locally: cd website && npm install && npm run spellcheck |
| |
| For more information, refer to: https://www.npmjs.com/package/markdown-spellcheck |
| |
| " && false; } |
| |
| # Integration tests Java Compile version is set by the machine environment jdk (set by the jdk key) |
| # Integration tests Java Runtime version is set by the JVM_RUNTIME env property (set env key to -Djvm.runtime=<JVM_RUNTIME_VERSION>) |
| # Integration tests will either use MiddleManagers or Indexers |
| # (Currently integration tests only support running with jvm runtime 8 and 11) |
| # START - Integration tests for Compile with Java 8 and Run with Java 8 |
| - &integration_batch_index |
| name: "(Compile=openjdk8, Run=openjdk8) batch index integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: &integration_test_services |
| - docker |
| env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: &run_integration_test |
| - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP} |
| after_failure: &integration_test_diags |
| - for v in ~/shared/logs/*.log ; do |
| echo $v logtail ======================== ; tail -100 $v ; |
| done |
| - for v in broker middlemanager overlord router coordinator historical ; do |
| echo $v dmesg ======================== ; |
| docker exec -it druid-$v sh -c 'dmesg | tail -3' ; |
| done |
| |
| - <<: *integration_batch_index |
| name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_input_format |
| name: "(Compile=openjdk8, Run=openjdk8) input format integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_input_format |
| name: "(Compile=openjdk8, Run=openjdk8) input format integration test with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_input_source |
| name: "(Compile=openjdk8, Run=openjdk8) input source integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_input_source |
| name: "(Compile=openjdk8, Run=openjdk8) input source integration test with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_perfect_rollup_parallel_batch_index |
| name: "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test" |
| jdk: openjdk8 |
| stage: Tests - phase 2 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_perfect_rollup_parallel_batch_index |
| name: "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_kafka_index |
| name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=kafka-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_kafka_index |
| name: "(Compile=openjdk8, Run=openjdk8) kafka index, transactional kafka index integration test with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=kafka-index,kafka-transactional-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_kafka_index_slow |
| name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test slow" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=kafka-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_kafka_index_slow |
| name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test slow with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=kafka-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_kafka_transactional_index |
| name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - &integration_kafka_transactional_index_slow |
| name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test slow" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_kafka_transactional_index_slow |
| name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test slow with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_kafka_format_tests |
| name: "(Compile=openjdk8, Run=openjdk8) Kafka index integration test with various formats" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_kafka_format_tests |
| name: "(Compile=openjdk8, Run=openjdk8) Kafka index integration test with various formats with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_query |
| name: "(Compile=openjdk8, Run=openjdk8) query integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - &integration_query_retry |
| name: "(Compile=openjdk8, Run=openjdk8) query retry integration test for missing segments" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=query-retry' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - &integration_query_error |
| name: "(Compile=openjdk8, Run=openjdk8) query error integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=query-error' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - &integration_security |
| name: "(Compile=openjdk8, Run=openjdk8) security integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=security' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - &integration_ldap_security |
| name: "(Compile=openjdk8, Run=openjdk8) ldap security integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=ldap-security' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - &integration_realtime_index |
| name: "(Compile=openjdk8, Run=openjdk8) realtime index integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=realtime-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - &integration_append_ingestion |
| name: "(Compile=openjdk8, Run=openjdk8) append ingestion integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=append-ingestion' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_append_ingestion |
| name: "(Compile=openjdk8, Run=openjdk8) append ingestion integration test with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=append-ingestion' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_compaction_tests |
| name: "(Compile=openjdk8, Run=openjdk8) compaction integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-Dgroups=compaction' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_compaction_tests |
| name: "(Compile=openjdk8, Run=openjdk8) compaction integration test with Indexer" |
| env: TESTNG_GROUPS='-Dgroups=compaction' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - &integration_tests |
| name: "(Compile=openjdk8, Run=openjdk8) other integration tests" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: *integration_test_services |
| env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| script: *run_integration_test |
| after_failure: *integration_test_diags |
| |
| - <<: *integration_tests |
| name: "(Compile=openjdk8, Run=openjdk8) other integration tests with Indexer" |
| env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer' |
| |
| - <<: *integration_tests |
| name: "(Compile=openjdk8, Run=openjdk8) leadership and high availability integration tests" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' |
| |
| # END - Integration tests for Compile with Java 8 and Run with Java 8 |
| |
| # START - Integration tests for Compile with Java 8 and Run with Java 11 |
| - <<: *integration_batch_index |
| name: "(Compile=openjdk8, Run=openjdk11) batch index integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_input_format |
| name: "(Compile=openjdk8, Run=openjdk11) input format integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_input_source |
| name: "(Compile=openjdk8, Run=openjdk11) input source integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_perfect_rollup_parallel_batch_index |
| name: "(Compile=openjdk8, Run=openjdk11) perfect rollup parallel batch index integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_query |
| name: "(Compile=openjdk8, Run=openjdk11) query integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_query_retry |
| name: "(Compile=openjdk8, Run=openjdk11) query retry integration test for missing segments" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=query-retry' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_query_error |
| name: "(Compile=openjdk8, Run=openjdk11) query error integration test for missing segments" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=query-error' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_security |
| name: "(Compile=openjdk8, Run=openjdk11) security integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=security' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_ldap_security |
| name: "(Compile=openjdk8, Run=openjdk11) ldap security integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=ldap-security' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_realtime_index |
| name: "(Compile=openjdk8, Run=openjdk11) realtime index integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=realtime-index' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_append_ingestion |
| name: "(Compile=openjdk8, Run=openjdk11) append ingestion integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=append-ingestion' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_compaction_tests |
| name: "(Compile=openjdk8, Run=openjdk11) compaction integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=compaction' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_tests |
| name: "(Compile=openjdk8, Run=openjdk11) other integration test" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| - <<: *integration_tests |
| name: "(Compile=openjdk8, Run=openjdk11) leadership and high availability integration tests" |
| jdk: openjdk8 |
| env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' |
| |
| # END - Integration tests for Compile with Java 8 and Run with Java 11 |
| |
| - &integration_batch_index_k8s |
| name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test" |
| stage: Tests - phase 2 |
| jdk: openjdk8 |
| services: &integration_test_services_k8s |
| - docker |
| env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default BUILD_DRUID_CLSUTER=true |
| script: &run_integration_test_k8s |
| - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLSUTER} |
| after_failure: &integration_test_diags_k8s |
| - for v in broker middlemanager router coordinator historical ; do |
| echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------"; |
| sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0; |
| done |
| - name: "security vulnerabilities" |
| stage: cron |
| install: skip |
| script: |- |
| ${MVN} dependency-check:aggregate -pl '!integration-tests' || { echo " |
| |
| The OWASP dependency check has found security vulnerabilities. Please use a newer version |
| of the dependency that does not have vulnerabilities. To see a report run |
| `mvn dependency-check:check` |
| If the analysis has false positives, |
| they can be suppressed by adding entries to owasp-dependency-check-suppressions.xml (for more |
| information, see https://jeremylong.github.io/DependencyCheck/general/suppression.html). |
| |
| " && false; } |
| |
| # Travis CI only supports per build (and not per-job notifications): https://github.com/travis-ci/travis-ci/issues/9888 |
| notifications: |
| email: |
| if: type = cron |
| recipients: |
| # This is the string "dev@druid.apache.org" encrypted against the apache/druid repo so that forks are unable to |
| # use this notification: |
| # https://github.com/travis-ci/travis-ci/issues/1094#issuecomment-215019909 |
| # https://github.com/travis-ci/travis-ci/issues/2711 |
| - secure: "MupjX/0jLwh3XzHPl74BTk2/Kp5r+8TrEewfRhpQdWKFMBXLKNqu0k2VXf5C/NIg3uvPianq3REk+qeTHI8dL2ShjiWS/eIRkJOHLfObdNNBuos5fo4TxAuBQcXyT4VjAq5jnAkH84Pxf2Nl0rkisWoIhvwSX7+kNrjW1qdu7K0=" |
| on_success: change |
| on_failure: change |