GEODE-8763: run benchmark tests up to 5 times (#5864)
* run benchmark tests up to 5 times, this time correctly reporting failure if still didn't pass on the 5th try
* increase benchmark timeouts
* run benchmarks on every commit even when multiple commits come in close together
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 9ada84d..3e5b03d 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -34,6 +34,7 @@
trigger: true
passed:
- Build
+ version: every
{% endmacro %}
{%- macro deep_merge(a, b): %}
@@ -476,6 +477,7 @@
- get: geode-build-version
trigger: true
passed: *benchmark-inputs
+ version: every
- put: concourse-metadata-resource
- do:
- task: run_benchmarks{{ run_var.title }}
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index eebb29b..79f2f99 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -29,12 +29,12 @@
flag: '-PwithSsl -PtestJVM=/usr/lib/jvm/bellsoft-java11-amd64'
options: '--tests=*GetBenchmark --tests=*PutBenchmark'
max_in_flight: 1
- timeout: 3h
+ timeout: 8h
- title: '_with_security_manager'
flag: '-PwithSecurityManager'
options: '--tests=Partitioned*'
max_in_flight: 2
- timeout: 5h
+ timeout: 12h
build_test:
ARTIFACT_SLUG: build
diff --git a/ci/scripts/run_benchmarks.sh b/ci/scripts/run_benchmarks.sh
index 05156f5..3314bf3 100755
--- a/ci/scripts/run_benchmarks.sh
+++ b/ci/scripts/run_benchmarks.sh
@@ -85,10 +85,14 @@
./run_on_cluster.sh -t ${CLUSTER_TAG} -- rm /home/geode/locator10334view.dat;
if ./run_against_baseline.sh -t ${CLUSTER_TAG} -b ${GEODE_SHA} -r ${GEODE_REPO} -p ${BENCHMARKS_REPO} ${BASELINE_OPTION} -e ${BENCHMARKS_BRANCH} -o ${RESULTS_DIR} -m "'source':'geode-ci',${METADATA_BASELINE},'baseline_branch':'${BASELINE_BRANCH}','geode_branch':'${GEODE_SHA}'" --ci -- ${FLAGS} ${TEST_OPTIONS} ; then
+ STATUS=0
break
+ else
+ STATUS=1
fi
set -e
done
popd
+exit $STATUS
\ No newline at end of file
