autest shard support through AUTEST_SHARD environment
Adds support for autest sharding, defaults to 4 shards.
Also should fix branch branch build detection folder.
diff --git a/jenkins/branch/autest.pipeline b/jenkins/branch/autest.pipeline
index 00d73a4..cbc12d3 100644
--- a/jenkins/branch/autest.pipeline
+++ b/jenkins/branch/autest.pipeline
@@ -7,6 +7,7 @@
label 'branch'
}
}
+
environment {
CCACHE_DIR = "/tmp/ccache"
}
@@ -15,10 +16,27 @@
stage('Initialization') {
steps {
script {
- currentBuild.displayName = "#${BUILD_NUMBER} ${GITHUB_BRANCH}"
if (env.SHA1) {
currentBuild.description = env.SHA1
}
+ if (env.AUTEST_SHARD) {
+ String[] arr = env.AUTEST_SHARD.split('of')
+ if (2 == arr.length) {
+ shard = arr[0] as int
+ shardcnt = arr[1] as int
+ if (shard < shardcnt) {
+ env.SHARD = shard
+ env.SHARDCNT = shardcnt
+ currentBuild.displayName = "#${BUILD_NUMBER} ${GITHUB_BRANCH} ${AUTEST_SHARD}"
+ }
+ }
+ }
+
+ if (!env.SHARD || ! env.SHARDCNT) {
+ currentBuild.displayName = "#${BUILD_NUMBER} ${GITHUB_BRANCH}"
+ env.SHARD = 0
+ env.SHARDCNT = 0
+ }
}
}
}
@@ -79,14 +97,35 @@
steps {
echo 'Starting AuTest'
dir('src/tests') {
- sh '''
- set +e
+
+ sh '''#!/bin/bash -x
+ #set +e
# We want to pick up the OpenSSL-QUIC version of curl in /opt/bin.
# The HTTP/3 AuTests depend upon this, so update the PATH accordingly.
export PATH=/opt/bin:${PATH}
mkdir -p ${WORKSPACE}/output/${GITHUB_BRANCH}
- ./autest.sh --ats-bin /tmp/ats/bin/ --sandbox /tmp/sandbox || true
+
+ if [ ${SHARDCNT} -le 0 ]; then
+ ./autest.sh --ats-bin /tmp/ats/bin/ --sandbox /tmp/sandbox || true
+ else
+ testsall=( $( find . -iname "*.test.py" | awk -F'/' '{print $NF}' | awk -F'.' '{print $1}' ) )
+
+ testsall=( $(
+ for el in "${testsall[@]}" ; do
+ echo $el
+ done | sort) )
+ ntests=${#testsall[@]}
+
+ shardsize=$((${ntests} / ${SHARDCNT}))
+ [ 0 -ne $((${ntests} % ${shardsize})) ] && shardsize=$((${shardsize} + 1))
+ shardbeg=$((${shardsize} * ${SHARD}))
+ sliced=${testsall[@]:${shardbeg}:${shardsize}}
+ ./autest.sh --ats-bin /tmp/ats/bin/ --sandbox /tmp/sandbox -f ${sliced[@]} || true
+
+ exit
+ fi
+
if [ -n "$(ls -A /tmp/sandbox/)" ]; then
cp -rf /tmp/sandbox/ ${WORKSPACE}/output/${GITHUB_BRANCH}/
sudo chmod -R 777 ${WORKSPACE}
diff --git a/jenkins/branch/branch_build.pipeline b/jenkins/branch/branch_build.pipeline
index 1f816f1..62cd3b2 100644
--- a/jenkins/branch/branch_build.pipeline
+++ b/jenkins/branch/branch_build.pipeline
@@ -8,7 +8,7 @@
Jenkins.instance.getAllItems(Job.class).each {
if (it.class == org.jenkinsci.plugins.workflow.job.WorkflowJob && it.isBuildable()) {
def jobname = it.fullName
- if (0 == jobname.indexOf(env.GITHUB_BRANCH + '/os-')) {
+ if (0 == jobname.indexOf(env.JOB_DIR + '/os-')) {
jobnames.add(jobname)
}
}
@@ -41,6 +41,7 @@
parameters: [
string(name: 'GITHUB_URL', value: GITHUB_URL),
string(name: 'GITHUB_BRANCH', value: GITHUB_BRANCH),
+ string(name: 'JOB_DIR', value: JOB_DIR),
string(name: 'SHA1', value: SHA1),
]
)
@@ -50,6 +51,24 @@
return result
}
+String autestJob(String ghcontext, String jobName, String shard) {
+ echo "Autest Build of: " + jobName + " " + shard
+ if (DUMMY) { jobName = "Branch_Builds/dummy" }
+ def jobBuild = build(job: jobName, propagate: false,
+ parameters: [
+ string(name: 'GITHUB_URL', value: GITHUB_URL),
+ string(name: 'GITHUB_BRANCH', value: GITHUB_BRANCH),
+ string(name: 'JOB_DIR', value: JOB_DIR),
+ string(name: 'SHA1', value: SHA1),
+ string(name: 'AUTEST_SHARD', value: shard),
+ ]
+ )
+ def result = jobBuild.getResult()
+ echo "Build of " + jobName + " returned result: " + result
+ if ('FAILURE' == result) { error("${jobName} failed") }
+ return result
+}
+
def shaForBranch(url, branch) {
sha1 = sh (
script: "set -- `git ls-remote -h $url refs/heads/$branch`; echo \${1}",
@@ -66,6 +85,15 @@
agent { label 'master' }
steps {
script {
+ if (! env.AUTEST_SHARDS) {
+ env.AUTEST_SHARDS = 4
+ }
+ if (! env.JOB_DIR) {
+ def bparts = env.JOB_NAME.split('/')
+ bparts = bparts - bparts.last()
+ env.JOB_DIR = bparts.join('/')
+ }
+
if (! env.GITHUB_BRANCH) {
def bparts = env.JOB_NAME.split('/')
if (2 != bparts.length) {
@@ -81,6 +109,7 @@
currentBuild.description = env.SHA1
sh 'printenv'
+
}
}
}
@@ -90,28 +119,28 @@
stage('In Tree') {
steps {
script {
- buildJob('in_tree', env.GITHUB_BRANCH + '/in_tree')
+ buildJob('in_tree', env.JOB_DIR + '/in_tree')
}
}
}
stage('Out Of Tree') {
steps {
script {
- buildJob('out_of_tree', env.GITHUB_BRANCH + '/out_of_tree')
+ buildJob('out_of_tree', env.JOB_DIR + '/out_of_tree')
}
}
}
stage('RAT') {
steps {
script {
- buildJob('rat', env.GITHUB_BRANCH + '/rat')
+ buildJob('rat', env.JOB_DIR + '/rat')
}
}
}
stage('clang format') {
steps {
script {
- buildJob('clang_format', env.GITHUB_BRANCH + '/clang_format')
+ buildJob('clang_format', env.JOB_DIR + '/clang_format')
}
}
}
@@ -121,7 +150,7 @@
stage('Snapshot') {
steps {
script {
- //buildJob('snapshot', env.GITHUB_BRANCH + '/snapshot')
+ //buildJob('snapshot', env.JOB_DIR + '/snapshot')
echo "Skipping snapshot"
}
}
@@ -142,35 +171,46 @@
stage('clang analyzer') {
steps {
script {
- buildJob('clang-analyzer', env.GITHUB_BRANCH + '/clang_analyzer')
+ buildJob('clang-analyzer', env.JOB_DIR + '/clang_analyzer')
}
}
}
stage('autests') {
steps {
script {
- buildJob('autest', env.GITHUB_BRANCH + '/autest')
+ if (env.AUTEST_SHARDS) {
+ def nshards = env.AUTEST_SHARDS as int
+ def jobs = [:]
+ for (ind = 0 ; ind < nshards ; ind++) {
+ index = ind
+ String shard = index + "of" + env.AUTEST_SHARDS
+ jobs[shard] = { autestJob('autest', env.JOB_DIR + '/autest', shard) }
+ }
+ parallel jobs
+ } else {
+ buildJob('autest', env.JOB_DIR + '/autest')
+ }
}
}
}
stage('docs') {
steps {
script {
- buildJob('docs', env.GITHUB_BRANCH + '/docs')
+ buildJob('docs', env.JOB_DIR + '/docs')
}
}
}
stage('cache_tests') {
steps {
script {
- buildJob('cache-tests', env.GITHUB_BRANCH + '/cache-tests')
+ buildJob('cache-tests', env.JOB_DIR + '/cache-tests')
}
}
}
stage('coverage') {
steps {
script {
- buildJob('coverage', env.GITHUB_BRANCH + '/coverage')
+ buildJob('coverage', env.JOB_DIR + '/coverage')
}
}
}
diff --git a/jenkins/branch/distro_builds.pipeline b/jenkins/branch/distro_builds.pipeline
index 350b9f6..21067df 100644
--- a/jenkins/branch/distro_builds.pipeline
+++ b/jenkins/branch/distro_builds.pipeline
@@ -12,7 +12,7 @@
}
}
- String jjob = env.GITHUB_BRANCH + '/os_build'
+ String jjob = env.JOB_DIR + '/os_build'
def builders = [:]
@@ -68,6 +68,12 @@
agent { label 'master' }
steps {
script {
+ if (! env.JOB_DIR) {
+ def bparts = env.JOB_NAME.split('/')
+ bparts.pop()
+ env.JOB_DIR = bparts.join('/')
+ }
+
if (! env.DISTRO) {
def dparts = env.JOB_BASE_NAME.split("_|-")
if (3 != dparts.length) {
@@ -75,6 +81,7 @@
}
env.DISTRO = [ dparts[1], dparts[2] ].join(':')
}
+
if (! env.GITHUB_BRANCH) {
def bparts = env.JOB_NAME.split('/')
if (2 != bparts.length) {
