HDDS-4933. Make CI checks fail faster (#2037)
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 96c237d..a8e5ed3 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -18,13 +18,15 @@
push:
schedule:
- cron: 30 0,12 * * *
+env:
+ FAIL_FAST: ${{ github.event_name == 'pull_request' }}
jobs:
compile:
runs-on: ubuntu-18.04
strategy:
matrix:
java: [ 8, 11 ]
- fail-fast: false
+ fail-fast: ${{ github.event_name == 'pull_request' }}
steps:
- name: Checkout project
uses: actions/checkout@v2
@@ -187,7 +189,7 @@
- secure
- unsecure
- misc
- fail-fast: false
+ fail-fast: ${{ github.event_name == 'pull_request' }}
steps:
- name: Checkout project
uses: actions/checkout@v2
@@ -232,7 +234,7 @@
- client
- filesystem-hdds
- ozone
- fail-fast: false
+ fail-fast: ${{ github.event_name == 'pull_request' }}
steps:
- name: Setup link to SSD
run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
diff --git a/hadoop-ozone/dev-support/checks/junit.sh b/hadoop-ozone/dev-support/checks/junit.sh
index fbb9a61..8452362 100644
--- a/hadoop-ozone/dev-support/checks/junit.sh
+++ b/hadoop-ozone/dev-support/checks/junit.sh
@@ -22,9 +22,23 @@
: ${CHECK:="unit"}
: ${ITERATIONS:="1"}
+declare -i ITERATIONS
+if [[ ${ITERATIONS} -le 0 ]]; then
+ ITERATIONS=1
+fi
+
export MAVEN_OPTS="-Xmx4096m"
MAVEN_OPTIONS='-B -Dskip.npx -Dskip.installnpx'
-mvn ${MAVEN_OPTIONS} -DskipTests clean install
+
+if [[ "${FAIL_FAST:-}" == "true" ]]; then
+ MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-fast -Dsurefire.skipAfterFailureCount=1"
+else
+ MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-at-end"
+fi
+
+if [[ "${CHECK}" == "integration" ]] || [[ ${ITERATIONS} -gt 1 ]]; then
+ mvn ${MAVEN_OPTIONS} -DskipTests clean install
+fi
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/${CHECK}"}
mkdir -p "$REPORT_DIR"
@@ -37,7 +51,7 @@
mkdir -p "${REPORT_DIR}"
fi
- mvn ${MAVEN_OPTIONS} -fae "$@" test \
+ mvn ${MAVEN_OPTIONS} "$@" test \
| tee "${REPORT_DIR}/output.log"
irc=$?
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
index 4e0151b..df77a59 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/test.sh
@@ -23,16 +23,7 @@
tests=$(find_tests)
cd "$SCRIPT_DIR"
-RESULT=0
-# shellcheck disable=SC2044
-for t in ${tests}; do
- d="$(dirname "${t}")"
-
- if ! run_test_script "${d}"; then
- RESULT=1
- fi
-
- copy_results "${d}" "${ALL_RESULT_DIR}"
-done
+run_test_scripts ${tests}
+RESULT=$?
exit ${RESULT}
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh
index 3ef2dcf..dfb89c0 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -36,17 +36,8 @@
tests=$(find_tests)
cd "$SCRIPT_DIR"
-RESULT=0
-# shellcheck disable=SC2044
-for t in ${tests}; do
- d="$(dirname "${t}")"
-
- if ! run_test_script "${d}"; then
- RESULT=1
- fi
-
- copy_results "${d}" "${ALL_RESULT_DIR}"
-done
+run_test_scripts ${tests}
+RESULT=$?
rebot --nostatusrc -N acceptance -d "$ALL_RESULT_DIR" "$ALL_RESULT_DIR"/*.xml
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 3ba5bcd..c2a75b9 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -40,11 +40,11 @@
## @description find all the test.sh scripts in the immediate child dirs
find_tests(){
if [[ -n "${OZONE_ACCEPTANCE_SUITE}" ]]; then
- tests=$(find . -mindepth 2 -maxdepth 2 -name test.sh | xargs grep -l "^#suite:${OZONE_ACCEPTANCE_SUITE}$" | sort)
+ tests=$(find . -mindepth 2 -maxdepth 2 -name test.sh | cut -c3- | xargs grep -l "^#suite:${OZONE_ACCEPTANCE_SUITE}$" | sort)
# 'misc' is default suite, add untagged tests, too
if [[ "misc" == "${OZONE_ACCEPTANCE_SUITE}" ]]; then
- untagged="$(find . -mindepth 2 -maxdepth 2 -name test.sh | xargs grep -L "^#suite:")"
+ untagged="$(find . -mindepth 2 -maxdepth 2 -name test.sh | cut -c3- | xargs grep -L "^#suite:")"
if [[ -n "${untagged}" ]]; then
tests=$(echo ${tests} ${untagged} | xargs -n1 | sort)
fi
@@ -55,7 +55,7 @@
exit 1
fi
else
- tests=$(find . -mindepth 2 -maxdepth 2 -name test.sh | grep "${OZONE_TEST_SELECTOR:-""}" | sort)
+ tests=$(find . -mindepth 2 -maxdepth 2 -name test.sh | cut -c3- | grep "${OZONE_TEST_SELECTOR:-""}" | sort)
fi
echo $tests
}
@@ -331,6 +331,26 @@
return ${ret}
}
+run_test_scripts() {
+ ret=0
+
+ for t in "$@"; do
+ d="$(dirname "${t}")"
+
+ if ! run_test_script "${d}"; then
+ ret=1
+ fi
+
+ copy_results "${d}" "${ALL_RESULT_DIR}"
+
+ if [[ "${ret}" == "1" ]] && [[ "${FAIL_FAST:-}" == "true" ]]; then
+ break
+ fi
+ done
+
+ return ${ret}
+}
+
## @description Make `OZONE_VOLUME_OWNER` the owner of the `OZONE_VOLUME`
## directory tree (required in Github Actions runner environment)
fix_data_dir_permissions() {
diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
index 929be05..0c63a67 100755
--- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh
@@ -24,17 +24,8 @@
tests=$(find_tests)
cd "$SCRIPT_DIR"
-RESULT=0
-# shellcheck disable=SC2044
-for t in ${tests}; do
- d="$(dirname "${t}")"
-
- if ! run_test_script "${d}"; then
- RESULT=1
- fi
-
- copy_results "${d}" "${ALL_RESULT_DIR}"
-done
+run_test_scripts ${tests}
+RESULT=$?
generate_report "upgrade" "${ALL_RESULT_DIR}"
diff --git a/hadoop-ozone/dist/src/test/shell/compose_testlib.bats b/hadoop-ozone/dist/src/test/shell/compose_testlib.bats
index 058da64..3fcf363 100644
--- a/hadoop-ozone/dist/src/test/shell/compose_testlib.bats
+++ b/hadoop-ozone/dist/src/test/shell/compose_testlib.bats
@@ -19,19 +19,19 @@
@test "Find test recursive, only on one level" {
cd $BATS_TEST_DIRNAME
run find_tests
- [[ "$output" == "./test1/test.sh ./test2/test.sh ./test4/test.sh" ]]
+ [[ "$output" == "test1/test.sh test2/test.sh test4/test.sh" ]]
}
@test "Find test by suite" {
OZONE_ACCEPTANCE_SUITE=one
cd $BATS_TEST_DIRNAME
run find_tests
- [[ "$output" == "./test4/test.sh" ]]
+ [[ "$output" == "test4/test.sh" ]]
}
@test "Find test default suite" {
OZONE_ACCEPTANCE_SUITE=misc
cd $BATS_TEST_DIRNAME
run find_tests
- [[ "$output" == "./test1/test.sh ./test2/test.sh" ]]
+ [[ "$output" == "test1/test.sh test2/test.sh" ]]
}
