YETUS-811. Add an API for robots to do their own artifact URLs
Signed-off-by: Allen Wittenauer <aw@apache.org>
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 758ccf8..dfdff7c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -31,24 +31,19 @@
steps:
- checkout
-
- # Download and cache dependencies
- - restore_cache:
- keys:
- - v1-dependencies-{{ checksum "pom.xml" }}
- # fallback to using the latest cache if no exact match is found
- - v1-dependencies-
-
+ - run: mkdir -p /tmp/yetus-out
+ - run: echo "bootstrap" > /tmp/yetus-out/bootstrap
+ - store_artifacts:
+ path: /tmp/yetus-out
- run: >
~/repo/precommit/src/main/shell/test-patch.sh
--plugins=all
--patch-dir=/tmp/yetus-out
+ --html-report-file=/tmp/yetus-out/report.html
+ --console-report-file=/tmp/yetus-out/console.txt
+ --brief-report-file=/tmp/yetus-out/brief.txt
+ --bugcomments=briefreport,htmlout
--tests-filter=checkstyle,javadoc,rubocop,test4tests
- - save_cache:
- paths:
- - ~/.m2
- key: v1-dependencies-{{ checksum "pom.xml" }}
-
- store_artifacts:
path: /tmp/yetus-out
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 486ab52..1c8ee4c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,6 +24,10 @@
--patch-dir=/tmp/yetus-out
--plugins=all
--java-home=/usr/lib/jvm/java-8-openjdk-amd64
+ --html-report-file=/tmp/yetus-out/report.html
+ --console-report-file=/tmp/yetus-out/console.txt
+ --brief-report-file=/tmp/yetus-out/brief.txt
+ --bugcomments=briefreport,htmlout,gitlab
--tests-filter=checkstyle,javadoc,rubocop,test4tests
artifacts:
diff --git a/.travis.yml b/.travis.yml
index 3b7c36c..e96a21a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,4 +32,8 @@
--patch-dir=/tmp/yetus-out
--plugins=all
--docker-cache-from=apache/yetus:master
+ --html-report-file=/tmp/yetus-out/report.html
+ --console-report-file=/tmp/yetus-out/console.txt
+ --brief-report-file=/tmp/yetus-out/brief.txt
+ --bugcomments=briefreport,htmlout
--tests-filter=checkstyle,javadoc,rubocop,test4tests
diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.md b/asf-site-src/source/documentation/in-progress/precommit-robots.md
index 653c4df..6fb44ab 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-robots.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-robots.md
@@ -65,7 +65,7 @@
TRIGGER: ${CIRCLECI}=true
-Circle CI support in `test-patch` is limited to github.com. Artifacts (the `--patch-dir` directory) location needs to be handled set on the command line. Linking to the logs is not currently supported.
+Circle CI support in `test-patch` is limited to github.com.
To use the pre-built Apache Yetus Docker image from docker hub as the build environment, use the following snippet in the `.circleci/config.yaml` file, substituting the tag for the version of Apache Yetus that should be used and replacing the JAVA_HOME with the appropriate version as bundled mentioned in the Dockerfile:
@@ -81,6 +81,25 @@
...
```
+Artifacts need some special handling. In order to get links, the storage of artifacts must be 'primed' prior to launching test-patch and then again to actually store the content. Additionally, the location needs to be handled set on the command line. In practice, this configuration looks similar to this:
+
+```yaml
+jobs:
+ build:
+ steps:
+ ...
+ - run: mkdir -p /tmp/yetus-out
+ - run: echo "bootstrap" > /tmp/yetus-out/bootstrap
+ - store_artifacts:
+ path: /tmp/yetus-out
+ - run: >
+ test-patch.sh
+ --patch-dir=/tmp/yetus-out
+ ...
+ - store_artifacts:
+ path: /tmp/yetus-out
+```
+
See also
* See also the source tree's `.circleci/config.yaml` for some tips and tricks.
diff --git a/precommit/src/main/shell/core.d/01-common.sh b/precommit/src/main/shell/core.d/01-common.sh
index e2c8d68..a79b404 100755
--- a/precommit/src/main/shell/core.d/01-common.sh
+++ b/precommit/src/main/shell/core.d/01-common.sh
@@ -720,6 +720,22 @@
fi
}
+## @description Get the base URL of the artifacts
+## @audience public
+## @stability evolving
+## @replaceable yes
+## @return urlstring
+function get_artifact_url
+{
+ if [[ ${ROBOT} == true ]]; then
+ if declare -f "${ROBOTTYPE}"_artifact_url >/dev/null; then
+ "${ROBOTTYPE}"_artifact_url
+ fi
+ elif [[ -n "${BUILD_URL}" ]] && [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then
+ echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}"
+ fi
+}
+
## @description attempt to guess what the build tool should be
## @audience public
## @stability evolving
diff --git a/precommit/src/main/shell/core.d/builtin-bugsystem.sh b/precommit/src/main/shell/core.d/builtin-bugsystem.sh
index 9ed531d..4294227 100755
--- a/precommit/src/main/shell/core.d/builtin-bugsystem.sh
+++ b/precommit/src/main/shell/core.d/builtin-bugsystem.sh
@@ -45,6 +45,7 @@
declare seccoladj=0
declare spcfx=${PATCH_DIR}/spcl.txt
declare calctime
+ declare url
if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
exec 6>&1
@@ -153,15 +154,17 @@
echo "============================================================================"
i=0
+ if [[ "${CONSOLE_USE_BUILD_URL}" = true ]]; then
+ url=$(get_artifact_url)
+ fi
+
+ if [[ -z "${url}" ]]; then
+ url=${PATCH_DIR}
+ fi
+
until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
- if [[ "${CONSOLE_USE_BUILD_URL}" = true &&
- -n "${BUILD_URL}" ]]; then
- comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
- else
- comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
- fi
+ comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
+ "${SED}" -e "s,@@BASE@@,${url},g")
printf '%s\n' "${comment}"
((i=i+1))
done
diff --git a/precommit/src/main/shell/robots.d/circleci.sh b/precommit/src/main/shell/robots.d/circleci.sh
index 4095866..9b9c2eb 100755
--- a/precommit/src/main/shell/robots.d/circleci.sh
+++ b/precommit/src/main/shell/robots.d/circleci.sh
@@ -27,12 +27,11 @@
# needs to get rewritten first before this can be used
BUILD_URL="https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}"
- BUILD_URL_CONSOLE='/'
- CONSOLE_USE_BUILD_URL=false
+ BUILD_URL_CONSOLE=''
+ CONSOLE_USE_BUILD_URL=true
ROBOT=true
ROBOTTYPE=circleci
-
yetus_comma_to_array CPR "${CIRCLE_PULL_REQUESTS}"
if [[ "${#CIRCLE_PULL_REQUESTS[@]}" -ne 1 ]]; then
@@ -55,12 +54,35 @@
CIRCLE_PULL_REQUESTS \
CIRCLE_PROJECT_USERNAME \
CIRCLE_PROJECT_REPONAME \
- CIRCLE_REPOSITORY_URL
+ CIRCLE_REPOSITORY_URL \
+ CIRCLE_TOKEN
yetus_add_array_element EXEC_MODES Circle_CI
fi
fi
+function circleci_artifact_url
+{
+ declare apiurl
+ declare baseurl
+
+ if [[ -z "${CIRCLECI_ARTIFACTS}" ]]; then
+ apiurl="https://circleci.com/api/v1.1/project/github/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}/artifacts"
+
+ if "${CURL}" --silent --fail \
+ --output "${PATCH_DIR}/circleci.txt" \
+ --location \
+ "${apiurl}"; then
+ baseurl=$("${GREP}" url "${PATCH_DIR}/circleci.txt" | head -1 | cut -f2- -d:)
+ baseurl=${baseurl//\"/}
+ baseurl=${baseurl%/*}
+ rm "${PATCH_DIR}/circleci.txt" 2>/dev/null
+ CIRCLECI_ARTIFACTS=${baseurl}
+ fi
+ fi
+ echo "${CIRCLECI_ARTIFACTS}"
+}
+
function circleci_set_plugin_defaults
{
if [[ ${CIRCLE_REPOSITORY_URL} =~ github.com ]]; then
@@ -71,3 +93,16 @@
GITHUB_REPO=${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}
fi
}
+
+function circleci_finalreport
+{
+ add_footer_table "Console output" "${BUILD_URL}"
+}
+
+#function circleci_unittest_footer
+#{
+# declare statusjdk=$1
+# declare extra
+#
+# add_footer_table "${statusjdk} Test Results" "${BUILD_URL}#tests/containers/0"
+#}
\ No newline at end of file
diff --git a/precommit/src/main/shell/robots.d/gitlabci.sh b/precommit/src/main/shell/robots.d/gitlabci.sh
index 17f96d1..fff8d9f 100755
--- a/precommit/src/main/shell/robots.d/gitlabci.sh
+++ b/precommit/src/main/shell/robots.d/gitlabci.sh
@@ -50,4 +50,14 @@
{
# shellcheck disable=2034
GITLAB_REPO=${CI_PROJECT_PATH}
+}
+
+function gitlabci_finalreport
+{
+ add_footer_table "Console output" "${BUILD_URL}"
+}
+
+function gitlabci_artifact_url
+{
+ echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}"
}
\ No newline at end of file
diff --git a/precommit/src/main/shell/robots.d/jenkins.sh b/precommit/src/main/shell/robots.d/jenkins.sh
index 6850317..2d52945 100755
--- a/precommit/src/main/shell/robots.d/jenkins.sh
+++ b/precommit/src/main/shell/robots.d/jenkins.sh
@@ -168,4 +168,9 @@
function jenkins_finalreport
{
add_footer_table "Console output" "${BUILD_URL}${BUILD_URL_CONSOLE}"
+}
+
+function jenkins_artifact_url
+{
+ echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}"
}
\ No newline at end of file
diff --git a/precommit/src/main/shell/robots.d/travisci.sh b/precommit/src/main/shell/robots.d/travisci.sh
index 937b6ec..1334c1f 100755
--- a/precommit/src/main/shell/robots.d/travisci.sh
+++ b/precommit/src/main/shell/robots.d/travisci.sh
@@ -50,6 +50,7 @@
TRAVIS \
TRAVIS_BRANCH \
TRAVIS_BUILD_ID \
+ TRAVIS_BUILD_WEB_URL \
TRAVIS_PULL_REQUEST \
TRAVIS_REPO_SLUG
@@ -75,6 +76,11 @@
GITHUB_REPO=${TRAVIS_REPO_SLUG}
}
+function travisci_finalreport
+{
+ add_footer_table "Console output" "${TRAVIS_BUILD_WEB_URL}"
+}
+
#function travisci_verify_patchdir
#{
# declare commentfile=$1
@@ -84,8 +90,3 @@
#{
# declare statusjdk=$1
#}
-
-#function travisci_finalreport
-#{
-# declare foo
-#}
\ No newline at end of file
diff --git a/precommit/src/main/shell/test-patch.d/briefreport.sh b/precommit/src/main/shell/test-patch.d/briefreport.sh
index a764389..1780164 100755
--- a/precommit/src/main/shell/test-patch.d/briefreport.sh
+++ b/precommit/src/main/shell/test-patch.d/briefreport.sh
@@ -218,6 +218,8 @@
version=$(cat "${BINDIR}/VERSION")
fi
+ url=$(get_artifact_url)
+
i=0
until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
if [[ "${TP_FOOTER_TABLE[${i}]}" =~ \@\@BASE\@\@ ]] \
@@ -242,10 +244,10 @@
# shellcheck disable=SC2016
size=$(du -sh "${fn}" | "${AWK}" '{print $1}')
fi
- if [[ -n "${BUILD_URL}" ]]; then
+ if [[ -n "${url}" ]]; then
comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
cut -f3 -d\| |
- "${SED}" -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+ "${SED}" -e "s,@@BASE@@,${url},g")
fi
{
if [[ "${subs}" != "${vote}" ]]; then
diff --git a/precommit/src/main/shell/test-patch.d/github.sh b/precommit/src/main/shell/test-patch.d/github.sh
index 9d95085..d0446be 100755
--- a/precommit/src/main/shell/test-patch.d/github.sh
+++ b/precommit/src/main/shell/test-patch.d/github.sh
@@ -568,6 +568,7 @@
declare i
declare commentfile=${PATCH_DIR}/gitcommentfile.$$
declare comment
+ declare url
rm "${commentfile}" 2>/dev/null
@@ -630,10 +631,13 @@
echo "|----------:|:-------------|"
} >> "${commentfile}"
+
+ url=$(get_artifact_url)
+
i=0
until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+ "${SED}" -e "s,@@BASE@@,${url},g")
printf '%s\n' "${comment}" >> "${commentfile}"
((i=i+1))
done
diff --git a/precommit/src/main/shell/test-patch.d/gitlab.sh b/precommit/src/main/shell/test-patch.d/gitlab.sh
index e9f23ec..595d444 100755
--- a/precommit/src/main/shell/test-patch.d/gitlab.sh
+++ b/precommit/src/main/shell/test-patch.d/gitlab.sh
@@ -401,6 +401,7 @@
declare i
declare commentfile=${PATCH_DIR}/gitcommentfile.$$
declare comment
+ declare url
if [[ "${GITLAB_WRITE_ENABLED}" == "false" ]]; then
return 0
@@ -468,10 +469,13 @@
echo "|----------:|:-------------|"
} >> "${commentfile}"
+
+ url=$(get_artifact_url)
+
i=0
until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- "${SED}" -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+ "${SED}" -e "s,@@BASE@@,${url},g")
printf '%s\n' "${comment}" >> "${commentfile}"
((i=i+1))
done
diff --git a/precommit/src/main/shell/test-patch.d/htmlout.sh b/precommit/src/main/shell/test-patch.d/htmlout.sh
index fc896f4..9d34886 100755
--- a/precommit/src/main/shell/test-patch.d/htmlout.sh
+++ b/precommit/src/main/shell/test-patch.d/htmlout.sh
@@ -33,6 +33,7 @@
{
declare i
declare fn
+ declare url
for i in "$@"; do
case ${i} in
@@ -224,6 +225,7 @@
echo "</tr>"
} >> "${commentfile}"
+ url=$(get_artifact_url)
i=0
until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
@@ -239,7 +241,7 @@
for j in ${TP_FOOTER_TABLE[${i}]}; do
if [[ "${j}" =~ ^@@BASE@@ ]]; then
t1=${j#@@BASE@@/}
- t2=$(echo "${j}" | "${SED}" -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+ t2=$(echo "${j}" | "${SED}" -e "s,@@BASE@@,${url},g")
if [[ -n "${BUILD_URL}" ]]; then
t2="<a href=\"${t2}\">${t1}</a>"
fi
diff --git a/precommit/src/main/shell/test-patch.d/jira.sh b/precommit/src/main/shell/test-patch.d/jira.sh
index f8f4a2d..c13c5f2 100755
--- a/precommit/src/main/shell/test-patch.d/jira.sh
+++ b/precommit/src/main/shell/test-patch.d/jira.sh
@@ -412,6 +412,7 @@
declare color
declare comment
declare calctime
+ declare url
rm "${commentfile}" 2>/dev/null
@@ -512,11 +513,13 @@
{ echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
+ url=$(get_artifact_url)
+
echo "|| Subsystem || Report/Notes ||" >> "${commentfile}"
i=0
until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- "${SED}" -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+ "${SED}" -e "s,@@BASE@@,${url},g")
printf '%s\n' "${comment}" >> "${commentfile}"
((i=i+1))
done
