| #!/usr/bin/env bash |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # Make sure that bash version meets the pre-requisite |
| |
| if [[ -z "${BASH_VERSINFO[0]}" ]] \ |
| || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \ |
| || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then |
| echo "bash v3.2+ is required. Sorry." |
| exit 1 |
| fi |
| |
| this="${BASH_SOURCE-$0}" |
| BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) |
| BINNAME=${this##*/} |
| BINNAME=${BINNAME%.sh} |
| STARTINGDIR=$(pwd) |
| USER_PARAMS=("$@") |
| |
| #shellcheck disable=SC2034 |
| QATESTMODE=false |
| |
| # global arrays |
| declare -a CHANGED_FILES |
| declare -a CHANGED_MODULES |
| declare -a TP_HEADER |
| declare -a TP_VOTE_TABLE |
| declare -a TP_TEST_TABLE |
| declare -a TP_FOOTER_TABLE |
| declare -a MODULE |
| declare -a MODULE_BACKUP_STATUS |
| declare -a MODULE_BACKUP_STATUS_TIMER |
| declare -a MODULE_BACKUP_STATUS_MSG |
| declare -a MODULE_BACKUP_STATUS_LOG |
| declare -a MODULE_BACKUP_COMPILE_LOG |
| declare -a MODULE_STATUS |
| declare -a MODULE_STATUS_TIMER |
| declare -a MODULE_STATUS_MSG |
| declare -a MODULE_STATUS_LOG |
| declare -a MODULE_COMPILE_LOG |
| declare -a USER_MODULE_LIST |
| |
| TP_HEADER_COUNTER=0 |
| TP_VOTE_COUNTER=0 |
| TP_TEST_COUNTER=0 |
| TP_FOOTER_COUNTER=0 |
| |
| ## @description Setup the default global variables |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| function setup_defaults |
| { |
| declare version="in-progress" |
| |
| common_defaults |
| GLOBALTIMER=$("${AWK}" 'BEGIN {srand(); print srand()}') |
| |
| set_yetus_version |
| |
| #shellcheck disable=SC2153 |
| if [[ ${VERSION} =~ SNAPSHOT$ ]]; then |
| version="in-progress" |
| fi |
| |
| PATCH_NAMING_RULE="https://yetus.apache.org/documentation/${version}/precommit-patchnames" |
| INSTANCE=${RANDOM} |
| RELOCATE_PATCH_DIR=false |
| |
| ALLOWSUMMARIES=true |
| |
| BUILD_NATIVE=${BUILD_NATIVE:-true} |
| |
| BUILD_URL_ARTIFACTS=artifact/patchprocess |
| BUILD_URL_CONSOLE=console |
| BUILDTOOLCWD=module |
| |
| # shellcheck disable=SC2034 |
| CHANGED_UNION_MODULES="" |
| |
| CONTINUOUS_IMPROVEMENT=false |
| |
| PROC_LIMIT=1000 |
| REEXECED=false |
| RESETREPO=false |
| BUILDMODE=${BUILDMODE:-patch} |
| # shellcheck disable=SC2034 |
| BUILDMODEMSG=${BUILDMODEMSG:-"The patch"} |
| ISSUE=${ISSUE:-""} |
| TIMER=$("${AWK}" 'BEGIN {srand(); print srand()}') |
| JVM_REQUIRED=true |
| yetus_add_array_element JDK_TEST_LIST compile |
| yetus_add_array_element JDK_TEST_LIST unit |
| } |
| |
| |
| ## @description Convert time in seconds to m + s |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @param seconds |
| function clock_display |
| { |
| local -r elapsed=$1 |
| |
| if [[ ${elapsed} -lt 0 ]]; then |
| echo "N/A" |
| else |
| printf "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) |
| fi |
| } |
| |
| ## @description Activate the local timer |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| function start_clock |
| { |
| TIMER=$(yetus_get_ctime) |
| } |
| |
| ## @description Print the elapsed time in seconds since the start of the local timer |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| function stop_clock |
| { |
| local -r stoptime=$(yetus_get_ctime) |
| local -r elapsed=$((stoptime-TIMER)) |
| |
| echo ${elapsed} |
| } |
| |
| ## @description Print the elapsed time in seconds since the start of the global timer |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| function stop_global_clock |
| { |
| local -r stoptime=$(yetus_get_ctime) |
| local -r elapsed=$((stoptime-GLOBALTIMER)) |
| echo ${elapsed} |
| } |
| |
| ## @description Add time to the local timer |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @param seconds |
| function offset_clock |
| { |
| declare off=$1 |
| |
| yetus_debug "offset clock by ${off}" |
| |
| if [[ -n ${off} ]]; then |
| ((TIMER=TIMER-off)) |
| else |
| yetus_error "ASSERT: no offset passed to offset_clock: ${index}" |
| generate_stack |
| fi |
| } |
| |
| ## @description Add to the header of the display |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @param string |
| function add_header_line |
| { |
| # shellcheck disable=SC2034 |
| TP_HEADER[${TP_HEADER_COUNTER}]="$*" |
| ((TP_HEADER_COUNTER=TP_HEADER_COUNTER+1 )) |
| } |
| |
| ## @description Add to the output table. If the first parameter is a number |
| ## @description that is the vote for that column and calculates the elapsed time |
| ## @description based upon the last start_clock(). The second parameter is the reporting |
| ## @description subsystem (or test) that is providing the vote. The second parameter |
| ## @description is always required. The third parameter is any extra verbage that goes |
| ## @description with that subsystem. |
| ## @description if the vote is H, then that designates that "subsystem" should be a |
| ## @description header in the vote table comment output. The other parameters are |
| ## @description ignored |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @param +1/0/-1/H |
| ## @param subsystem |
| ## @param string |
| function add_vote_table |
| { |
| declare value=$1 |
| declare subsystem=$2 |
| shift 2 |
| |
| # apparently shellcheck doesn't know about declare -r |
| #shellcheck disable=SC2155 |
| declare -r elapsed=$(stop_clock) |
| declare filt |
| |
| yetus_debug "add_vote_table ${value} ${subsystem} ${elapsed} ${*}" |
| |
| if [[ "${value}" = H ]]; then |
| TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="|${value}| | | ${subsystem} |" |
| ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1)) |
| return |
| fi |
| |
| if [[ ${value} == "1" ]]; then |
| value="+1" |
| fi |
| |
| for filt in "${VOTE_FILTER[@]}"; do |
| if [[ "${subsystem}" == "${filt}" && "${value}" == -1 ]]; then |
| value=-0 |
| fi |
| done |
| |
| # shellcheck disable=SC2034 |
| TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${elapsed} | $* |" |
| ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1)) |
| |
| if [[ "${value}" = -1 ]]; then |
| ((RESULT = RESULT + 1)) |
| fi |
| } |
| |
| ## @description Report the JVM version of the given directory |
| ## @stability stable |
| ## @audience private |
| ## @replaceable yes |
| ## @param directory |
| ## @return version |
| function report_jvm_version |
| { |
| #shellcheck disable=SC2016 |
| "${1}/bin/java" -version 2>&1 | head -1 | "${AWK}" '{print $NF}' | tr -d \" |
| } |
| |
| ## @description Verify if a given test is multijdk |
| ## @audience public |
| ## @stability stable |
| ## @replaceable yes |
| ## @param test |
| ## @return 0 = yes |
| ## @return 1 = no |
| function verify_multijdk_test |
| { |
| local i=$1 |
| |
| if [[ "${#JDK_DIR_LIST[@]}" -lt 2 ]] ; then |
| yetus_debug "MultiJDK not configured." |
| return 1 |
| fi |
| |
| if yetus_ver_array_element JDK_TEST_LIST "${i}"; then |
| yetus_debug "${i} is in JDK_TEST_LIST and MultiJDK configured." |
| return 0 |
| fi |
| return 1 |
| } |
| |
| ## @description Put the opening environment information at the bottom |
| ## @description of the footer table |
| ## @stability stable |
| ## @audience private |
| ## @replaceable yes |
| function prepopulate_footer |
| { |
| # shellcheck disable=SC2155 |
| declare -r unamea=$(uname -a) |
| |
| add_footer_table "uname" "${unamea}" |
| add_footer_table "Build tool" "${BUILDTOOL}" |
| |
| if [[ -n ${REEXECPERSONALITY} ]]; then |
| add_footer_table "Personality" "${REEXECPERSONALITY}" |
| elif [[ -n ${PERSONALITY} ]]; then |
| add_footer_table "Personality" "${PERSONALITY}" |
| fi |
| |
| gitrev=$("${GIT}" rev-parse --verify --short HEAD) |
| |
| add_footer_table "git revision" "${PATCH_BRANCH} / ${gitrev}" |
| } |
| |
| ## @description Last minute entries on the footer table |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| function finish_footer_table |
| { |
| declare counter |
| |
| if [[ -f "${PATCH_DIR}/threadcounter.txt" ]]; then |
| counter=$(cat "${PATCH_DIR}/threadcounter.txt") |
| add_footer_table "Max. process+thread count" "${counter} (vs. ulimit of ${PROC_LIMIT})" |
| fi |
| |
| add_footer_table "modules" "C: ${CHANGED_MODULES[*]} U: ${CHANGED_UNION_MODULES}" |
| } |
| |
| ## @description Put the final elapsed time at the bottom of the table. |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| function finish_vote_table |
| { |
| |
| local -r elapsed=$(stop_global_clock) |
| local calctime |
| |
| calctime=$(clock_display "${elapsed}") |
| |
| echo "" |
| echo "Total Elapsed time: ${calctime}" |
| echo "" |
| |
| # shellcheck disable=SC2034 |
| TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${elapsed} | |" |
| ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1 )) |
| } |
| |
| ## @description Add to the footer of the display. @@BASE@@ will get replaced with the |
| ## @description correct location for the local filesystem in dev mode or the URL for |
| ## @description Jenkins mode. |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @param subsystem |
| ## @param string |
| function add_footer_table |
| { |
| local subsystem=$1 |
| shift 1 |
| |
| # shellcheck disable=SC2034 |
| TP_FOOTER_TABLE[${TP_FOOTER_COUNTER}]="| ${subsystem} | $* |" |
| ((TP_FOOTER_COUNTER=TP_FOOTER_COUNTER+1 )) |
| } |
| |
| ## @description Special table just for unit test failures |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @param failurereason |
| ## @param testlist |
| function add_test_table |
| { |
| local failure=$1 |
| shift 1 |
| |
| # shellcheck disable=SC2034 |
| TP_TEST_TABLE[${TP_TEST_COUNTER}]="| ${failure} | $* |" |
| ((TP_TEST_COUNTER=TP_TEST_COUNTER+1 )) |
| } |
| |
| ## @description Large display for the user console |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @param string |
| ## @return large chunk of text |
| function big_console_header |
| { |
| local text="$*" |
| local spacing=$(( (75+${#text}) /2 )) |
| printf '\n\n' |
| echo "============================================================================" |
| echo "============================================================================" |
| printf '%*s\n' ${spacing} "${text}" |
| echo "============================================================================" |
| echo "============================================================================" |
| printf '\n\n' |
| } |
| |
| ## @description Find the largest size of a column of an array |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @return size |
| function findlargest |
| { |
| local column=$1 |
| shift |
| local a=("$@") |
| local sizeofa=${#a[@]} |
| local i=0 |
| local string |
| local maxlen=0 |
| |
| until [[ ${i} -eq ${sizeofa} ]]; do |
| # shellcheck disable=SC2086 |
| string=$( echo ${a[$i]} | cut -f$((column + 1)) -d\| ) |
| if [[ ${#string} -gt ${maxlen} ]]; then |
| maxlen=${#string} |
| fi |
| i=$((i+1)) |
| done |
| echo "${maxlen}" |
| } |
| |
| ## @description Write the contents of a file to all of the bug systems |
| ## @description (so content should avoid special formatting) |
| ## @param filename |
| ## @stability stable |
| ## @audience public |
| function write_comment |
| { |
| local -r commentfile=${1} |
| declare bug |
| |
| for bug in ${BUGCOMMENTS}; do |
| if declare -f "${bug}_write_comment" >/dev/null; then |
| "${bug}_write_comment" "${commentfile}" |
| fi |
| done |
| } |
| |
| ## @description Verify that the patch directory is still in working order |
| ## @description since bad actors on some systems wipe it out. If not, |
| ## @description recreate it and then exit |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable yes |
| ## @return may exit on failure |
| function verify_patchdir_still_exists |
| { |
| local -r commentfile=/tmp/testpatch.$$.${RANDOM} |
| |
| if [[ ! -d ${PATCH_DIR} ]]; then |
| rm "${commentfile}" 2>/dev/null |
| |
| echo "(!) The patch artifact directory has been removed! " > "${commentfile}" |
| echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}" |
| echo |
| cat ${commentfile} |
| echo |
| if [[ ${ROBOT} == true ]]; then |
| if declare -f "${ROBOTTYPE}"_verify_patchdir >/dev/null; then |
| "${ROBOTTYPE}"_verify_patchdir "${commentfile}" |
| fi |
| |
| write_comment ${commentfile} |
| fi |
| |
| rm "${commentfile}" |
| cleanup_and_exit "${RESULT}" |
| fi |
| } |
| |
| ## @description generate a list of all files and line numbers in $GITDIFFLINES that |
| ## @description that were added/changed in the source repo. $GITDIFFCONTENT |
| ## @description is same file, but also includes the content of those lines |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| function compute_gitdiff |
| { |
| local file |
| local line |
| local startline |
| local counter |
| local numlines |
| local actual |
| local content |
| local outfile="${PATCH_DIR}/computegitdiff.${RANDOM}" |
| |
| pushd "${BASEDIR}" >/dev/null || return 1 |
| "${GIT}" add --all --intent-to-add |
| while read -r line; do |
| if [[ ${line} =~ ^\+\+\+ ]]; then |
| file=$(echo "${line}" | cut -f2- -d/) |
| continue |
| elif [[ ${line} =~ ^@@ ]]; then |
| startline=$(echo "${line}" | cut -f3 -d' ' | cut -f1 -d, | tr -d + ) |
| numlines=$(echo "${line}" | cut -f3 -d' ' | cut -s -f2 -d, ) |
| # if this is empty, then just this line |
| # if it is 0, then no lines were added and this part of the patch |
| # is strictly a delete |
| if [[ ${numlines} == 0 ]]; then |
| continue |
| elif [[ -z ${numlines} ]]; then |
| numlines=1 |
| fi |
| counter=0 |
| # it isn't obvious, but on MOST platforms under MOST use cases, |
| # this is faster than using sed, and definitely faster than using |
| # awk. |
| # http://unix.stackexchange.com/questions/47407/cat-line-x-to-line-y-on-a-huge-file |
| # has a good discussion w/benchmarks |
| # |
| # note that if tail is still sending data through the pipe, but head gets enough |
| # to do what was requested, head will exit, leaving tail with a broken pipe. |
| # we're going to send stderr to /dev/null and ignore the error since head's |
| # output is really what we're looking for |
| tail -n "+${startline}" "${file}" 2>/dev/null | head -n ${numlines} > "${outfile}" |
| oldifs=${IFS} |
| IFS='' |
| while read -r content; do |
| ((actual=counter+startline)) |
| echo "${file}:${actual}:" >> "${GITDIFFLINES}" |
| printf '%s:%s:%s\n' "${file}" "${actual}" "${content}" >> "${GITDIFFCONTENT}" |
| ((counter=counter+1)) |
| done < "${outfile}" |
| rm "${outfile}" |
| IFS=${oldifs} |
| fi |
| done < <("${GIT}" diff --unified=0 --no-color) |
| |
| if [[ ! -f "${GITDIFFLINES}" ]]; then |
| touch "${GITDIFFLINES}" |
| fi |
| |
| if [[ ! -f "${GITDIFFCONTENT}" ]]; then |
| touch "${GITDIFFCONTENT}" |
| fi |
| |
| if [[ -s "${GITDIFFLINES}" ]]; then |
| compute_unidiff |
| fi |
| |
| popd >/dev/null || return 1 |
| } |
| |
| ## @description generate an index of unified diff lines vs. modified/added lines |
| ## @description ${GITDIFFLINES} must exist. |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| function compute_unidiff |
| { |
| declare fn |
| declare filen |
| declare tmpfile="${PATCH_DIR}/tmp.$$.${RANDOM}" |
| |
| # now that we know what lines are where, we can deal |
| # with github's pain-in-the-butt API. It requires |
| # that the client provides the line number of the |
| # unified diff on a per file basis. |
| |
| # First, build a per-file unified diff, pulling |
| # out the 'extra' lines, grabbing the adds with |
| # the line number in the diff file along the way, |
| # finally rewriting the line so that it is in |
| # 'filename:diff line:content' format |
| |
| for fn in "${CHANGED_FILES[@]}"; do |
| if [[ -f "${filen}" ]]; then |
| "${GIT}" diff "${filen}" \ |
| | tail -n +6 \ |
| | "${GREP}" -n '^+' \ |
| | "${GREP}" -vE '^[0-9]*:\+\+\+' \ |
| | "${SED}" -e 's,^\([0-9]*:\)\+,\1,g' \ |
| -e "s,^,${filen}:,g" \ |
| >> "${tmpfile}" |
| fi |
| done |
| |
| # if every file that got changed was excluded, there |
| # is no data to calculate. |
| |
| if [[ -s "${tmpfile}" ]]; then |
| |
| # at this point, tmpfile should be in the same format |
| # as gitdiffcontent, just with different line numbers. |
| # let's do a merge (using gitdifflines because it's easier) |
| |
| # filename:real number:diff number |
| # shellcheck disable=SC2016 |
| paste -d: "${GITDIFFLINES}" "${tmpfile}" \ |
| | "${AWK}" -F: '{print $1":"$2":"$5":"$6}' \ |
| >> "${GITUNIDIFFLINES}" |
| |
| rm "${tmpfile}" |
| fi |
| } |
| |
| |
| ## @description Print the command to be executing to the screen. Then |
| ## @description run the command, sending stdout and stderr to the given filename |
| ## @description This will also ensure that any directories in ${BASEDIR} have |
| ## @description the exec bit set as a pre-exec step. |
| ## @audience public |
| ## @stability stable |
| ## @param filename |
| ## @param command |
| ## @param [..] |
| ## @replaceable no |
| ## @return $? |
| function echo_and_redirect |
| { |
| declare logfile=$1 |
| shift |
| |
| verify_patchdir_still_exists |
| |
| find "${BASEDIR}" -type d -exec chmod +x {} \; |
| # to the screen |
| echo "cd $(pwd)" |
| echo "${*} > ${logfile} 2>&1" |
| |
| if [[ ${BASH_VERSINFO[0]} -gt 3 ]]; then |
| |
| # use a coprocessor with the |
| # lower proc limit so that yetus can |
| # do stuff unimpacted by it |
| |
| e_a_r_helper "${logfile}" "${@}" >> "${COPROC_LOGFILE}" 2>&1 |
| |
| # now that it's off as a separate process, we need to wait |
| # for it to finish. wait will either return 0, exit code |
| # of the coproc, or 127. all of which is |
| # perfectly fine for us. |
| |
| |
| # shellcheck disable=SC2154,SC2086 |
| wait ${yrr_coproc_PID} |
| |
| else |
| |
| # if bash < 4 (e.g., OS X), just run it |
| # the ulimit was set earlier |
| |
| yetus_run_and_redirect "${logfile}" "${@}" |
| fi |
| } |
| |
| ## @description Print the usage information |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| function yetus_usage |
| { |
| declare bugsys |
| declare jdktlist |
| declare buildtools |
| |
| importplugins |
| |
| bugsys=$(yetus_array_to_comma BUGSYSTEMS ) |
| jdktlist=$(yetus_array_to_comma JDK_TEST_LIST ) |
| buildtools=$(yetus_array_to_comma BUILDTOOLS ) |
| |
| bugsys=${bugsys:-"None: no plugins enabled"} |
| jdktlist=${jdktlist:-"None: no plugins enabled"} |
| buildtools=${buildtools:-"None: no plugins enabled"} |
| |
| if [[ "${BUILDMODE}" = patch ]]; then |
| echo "${BINNAME} [OPTIONS] patch" |
| echo "" |
| echo "Where:" |
| echo " patch is a file, URL, or bugsystem-compatible location of the patch file" |
| else |
| echo "${BINNAME} [OPTIONS]" |
| fi |
| echo "" |
| echo "Options:" |
| echo "" |
| yetus_add_option "--archive-list=<list>" "Comma delimited list of pattern matching notations to copy to patch-dir" |
| yetus_add_option "--basedir=<dir>" "The directory to apply the patch to (default: current directory)" |
| yetus_add_option "--branch=<ref>" "Forcibly set the branch" |
| yetus_add_option "--branch-default=<ref>" "If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'master')" |
| yetus_add_option "--build-native=<bool>" "If true, then build native components (default 'true')" |
| # shellcheck disable=SC2153 |
| yetus_add_option "--build-tool=<tool>" "Pick which build tool to focus around (default: autodetect from '${buildtools}')" |
| yetus_add_option "--bugcomments=<bug>" "Only write comments to the screen and this comma delimited list (default: '${bugsys}')" |
| yetus_add_option "--contrib-guide=<url>" "URL to point new users towards project conventions. (default: ${PATCH_NAMING_RULE} )" |
| yetus_add_option "--continuous-improvement=<bool>" "If true, then do not exit with failure on branches (default: ${CONTINUOUS_IMPROVEMENT})" |
| yetus_add_option "--debug" "If set, then output some extra stuff to stderr" |
| yetus_add_option "--dirty-workspace" "Allow the local git workspace to have uncommitted changes" |
| yetus_add_option "--empty-patch" "Create a summary of the current source tree" |
| yetus_add_option "--excludes=<file>" "File of regexs to keep project files out of the set of changes passed to plugins." |
| yetus_add_option "--git-offline" "Do not fail if git cannot do certain remote operations" |
| yetus_add_option "--git-shallow" "Repo does not know about other branches or tags" |
| yetus_add_option "--ignore-unknown-options=<bool>" "Continue despite unknown options (default: ${IGNORE_UNKNOWN_OPTIONS})" |
| yetus_add_option "--java-home=<path>" "Set JAVA_HOME (In Docker mode, this should be local to the image)" |
| yetus_add_option "--linecomments=<bug>" "Only write line comments to this comma delimited list (default: same as --bugcomments)" |
| yetus_add_option "--list-plugins" "List all installed plug-ins and then exit" |
| yetus_add_option "--multijdkdirs=<paths>" "Comma delimited lists of JDK paths to use for multi-JDK tests" |
| yetus_add_option "--multijdktests=<list>" "Comma delimited tests to use when multijdkdirs is used. (default: '${jdktlist}')" |
| yetus_add_option "--modulelist=<list>" "Specify additional modules to test (comma delimited)" |
| yetus_add_option "--offline" "Avoid connecting to the network" |
| yetus_add_option "--patch-dir=<dir>" "The directory for working and output files (default '/tmp/test-patch-${PROJECT_NAME}/pid')" |
| yetus_add_option "--personality=<file>" "The personality file to load" |
| yetus_add_option "--proclimit=<num>" "Limit on the number of processes (default: ${PROC_LIMIT})" |
| yetus_add_option "--project=<name>" "The short name for project currently using test-patch (default 'yetus')" |
| yetus_add_option "--plugins=<list>" "Specify which plug-ins to add/delete (comma delimited; use 'all' for all found) e.g. --plugins=all,-ant,-scalac (all plugins except ant and scalac)" |
| yetus_add_option "--resetrepo" "Forcibly clean the repo" |
| yetus_add_option "--run-tests" "Run all relevant tests below the base directory" |
| yetus_add_option "--skip-dirs=<list>" "Skip following directories for module finding" |
| yetus_add_option "--skip-system-plugins" "Do not load plugins from ${BINDIR}/test-patch.d" |
| yetus_add_option "--summarize=<bool>" "Allow tests to summarize results" |
| yetus_add_option "--test-parallel=<bool>" "Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)" |
| yetus_add_option "--test-threads=<int>" "Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)" |
| yetus_add_option "--unit-test-filter-file=<file>" "The unit test filter file to load" |
| yetus_add_option "--tests-filter=<list>" "Lists of tests to turn failures into warnings" |
| yetus_add_option "--user-plugins=<dir>" "A directory of user provided plugins. see test-patch.d for examples (default empty)" |
| yetus_add_option "--version" "Print release version information and exit" |
| |
| yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" |
| yetus_reset_usage |
| |
| echo "" |
| echo "Shell binary overrides:" |
| yetus_add_option "--awk-cmd=<cmd>" "The 'awk' command to use (default 'awk')" |
| yetus_add_option "--curl-cmd=<cmd>" "The 'curl' command to use (default 'curl')" |
| yetus_add_option "--diff-cmd=<cmd>" "The GNU-compatible 'diff' command to use (default 'diff')" |
| yetus_add_option "--file-cmd=<cmd>" "The 'file' command to use (default 'file')" |
| yetus_add_option "--git-cmd=<cmd>" "The 'git' command to use (default 'git')" |
| yetus_add_option "--grep-cmd=<cmd>" "The 'grep' command to use (default 'grep')" |
| yetus_add_option "--patch-cmd=<cmd>" "The 'patch' command to use (default 'patch')" |
| yetus_add_option "--sed-cmd=<cmd>" "The 'sed' command to use (default 'sed')" |
| |
| yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" |
| yetus_reset_usage |
| |
| echo "" |
| echo "Automation options:" |
| yetus_add_option "--build-url=<url>" "Set the build location web page (Default: '${BUILD_URL}')" |
| yetus_add_option "--build-url-console=<location>" "Location relative to --build-url of the console (Default: '${BUILD_URL_CONSOLE}')" |
| yetus_add_option "--build-url-artifacts=<location>" "Location relative to --build-url of the --patch-dir (Default: '${BUILD_URL_ARTIFACTS}')" |
| yetus_add_option "--console-report-file=<file>" "Save the final console-based report to a file in addition to the screen" |
| yetus_add_option "--console-urls" "Use the build URL instead of path on the console report" |
| yetus_add_option "--instance=<string>" "Parallel execution identifier string" |
| yetus_add_option "--mv-patch-dir" "Move the patch-dir into the basedir during cleanup" |
| yetus_add_option "--robot" "Assume this is an automated run (default: auto-detect supported CI system)" |
| yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)" |
| |
| yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" |
| yetus_reset_usage |
| |
| |
| echo "" |
| echo "Docker options:" |
| docker_usage |
| yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" |
| yetus_reset_usage |
| |
| echo "" |
| echo "Reaper options:" |
| reaper_usage |
| yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" |
| yetus_reset_usage |
| |
| # shellcheck disable=SC2153 |
| for plugin in "${BUILDTOOLS[@]}" "${TESTTYPES[@]}" "${BUGSYSTEMS[@]}" "${TESTFORMATS[@]}"; do |
| if declare -f "${plugin}_usage" >/dev/null 2>&1; then |
| echo "" |
| echo "'${plugin}' plugin usage options:" |
| "${plugin}_usage" |
| yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}" |
| yetus_reset_usage |
| fi |
| done |
| } |
| |
| ## @description Interpret the command line parameters |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| ## @param $@ |
| ## @return May exit on failure |
| function parse_args |
| { |
| declare i |
| declare j |
| |
| common_args "$@" |
| |
| for i in "$@"; do |
| case ${i} in |
| --archive-list=*) |
| delete_parameter "${i}" |
| yetus_comma_to_array ARCHIVE_LIST "${i#*=}" |
| yetus_debug "Set to archive: ${ARCHIVE_LIST[*]}" |
| ;; |
| --bugcomments=*) |
| delete_parameter "${i}" |
| BUGCOMMENTS=${i#*=} |
| BUGCOMMENTS=${BUGCOMMENTS//,/ } |
| ;; |
| --build-native=*) |
| delete_parameter "${i}" |
| BUILD_NATIVE=${i#*=} |
| ;; |
| --build-tool=*) |
| delete_parameter "${i}" |
| BUILDTOOL=${i#*=} |
| ;; |
| --build-url=*) |
| delete_parameter "${i}" |
| BUILD_URL=${i#*=} |
| ;; |
| --build-url-artifacts=*) |
| delete_parameter "${i}" |
| # shellcheck disable=SC2034 |
| BUILD_URL_ARTIFACTS=${i#*=} |
| ;; |
| --build-url-console=*) |
| delete_parameter "${i}" |
| # shellcheck disable=SC2034 |
| BUILD_URL_CONSOLE=${i#*=} |
| ;; |
| --console-report-file=*) |
| delete_parameter "${i}" |
| CONSOLE_REPORT_FILE=${i#*=} |
| ;; |
| --console-urls) |
| delete_parameter "${i}" |
| # shellcheck disable=SC2034 |
| CONSOLE_USE_BUILD_URL=true |
| ;; |
| --contrib-guide=*) |
| delete_parameter "${i}" |
| PATCH_NAMING_RULE=${i#*=} |
| ;; |
| --continuous-improvement=*) |
| delete_parameter "${i}" |
| CONTINUOUS_IMPROVEMENT=${i#*=} |
| ;; |
| --dirty-workspace) |
| delete_parameter "${i}" |
| DIRTY_WORKSPACE=true |
| ;; |
| --excludes=*) |
| delete_parameter "${i}" |
| EXCLUDE_PATHS_FILE="${i#*=}" |
| ;; |
| --instance=*) |
| delete_parameter "${i}" |
| INSTANCE=${i#*=} |
| ;; |
| --empty-patch) |
| delete_parameter "${i}" |
| BUILDMODE=full |
| ;; |
| --java-home=*) |
| delete_parameter "${i}" |
| JAVA_HOME=${i#*=} |
| ;; |
| --linecomments=*) |
| delete_parameter "${i}" |
| BUGLINECOMMENTS=${i#*=} |
| BUGLINECOMMENTS=${BUGLINECOMMENTS//,/ } |
| if [[ -z "${BUGLINECOMMENTS}" ]]; then |
| BUGLINECOMMENTS=" " |
| fi |
| ;; |
| --modulelist=*) |
| delete_parameter "${i}" |
| yetus_comma_to_array USER_MODULE_LIST "${i#*=}" |
| yetus_debug "Manually forcing modules ${USER_MODULE_LIST[*]}" |
| ;; |
| --multijdkdirs=*) |
| delete_parameter "${i}" |
| yetus_comma_to_array JDK_DIR_LIST "${i#*=}" |
| yetus_debug "Multi-JDK mode activated with ${JDK_DIR_LIST[*]}" |
| yetus_add_array_element EXEC_MODES MultiJDK |
| ;; |
| --multijdktests=*) |
| delete_parameter "${i}" |
| yetus_comma_to_array JDK_TEST_LIST "${i#*=}" |
| yetus_debug "MultiJDK test list=${JDK_TEST_LIST[*]}" |
| ;; |
| --mv-patch-dir) |
| delete_parameter "${i}" |
| RELOCATE_PATCH_DIR=true; |
| ;; |
| --personality=*) |
| delete_parameter "${i}" |
| PERSONALITY=${i#*=} |
| ;; |
| --proclimit=*) |
| delete_parameter "${i}" |
| PROC_LIMIT=${i#*=} |
| ;; |
| --reexec) |
| delete_parameter "${i}" |
| REEXECED=true |
| ;; |
| --resetrepo) |
| delete_parameter "${i}" |
| RESETREPO=true |
| ;; |
| --robot) |
| delete_parameter "${i}" |
| ROBOT=true |
| ;; |
| --run-tests) |
| delete_parameter "${i}" |
| RUN_TESTS=true |
| ;; |
| --sentinel) |
| delete_parameter "${i}" |
| # shellcheck disable=SC2034 |
| SENTINEL=true |
| yetus_add_array_element EXEC_MODES Sentinel |
| ;; |
| --skip-dirs=*) |
| delete_parameter "${i}" |
| MODULE_SKIPDIRS=${i#*=} |
| MODULE_SKIPDIRS=${MODULE_SKIPDIRS//,/ } |
| yetus_debug "Setting skipdirs to ${MODULE_SKIPDIRS}" |
| ;; |
| --summarize=*) |
| delete_parameter "${i}" |
| ALLOWSUMMARIES=${i#*=} |
| ;; |
| --test-parallel=*) |
| delete_parameter "${i}" |
| # shellcheck disable=SC2034 |
| TEST_PARALLEL=${i#*=} |
| ;; |
| --test-threads=*) |
| delete_parameter "${i}" |
| # shellcheck disable=SC2034 |
| TEST_THREADS=${i#*=} |
| ;; |
| --unit-test-filter-file=*) |
| delete_parameter "${i}" |
| UNIT_TEST_FILTER_FILE=${i#*=} |
| ;; |
| --tests-filter=*) |
| delete_parameter "${i}" |
| yetus_comma_to_array VOTE_FILTER "${i#*=}" |
| ;; |
| --tpglobaltimer=*) |
| delete_parameter "${i}" |
| GLOBALTIMER=${i#*=} |
| ;; |
| --tpinstance=*) |
| delete_parameter "${i}" |
| INSTANCE=${i#*=} |
| ;; |
| --tpperson=*) |
| delete_parameter "${i}" |
| REEXECPERSONALITY=${i#*=} |
| ;; |
| --tpreexectimer=*) |
| delete_parameter "${i}" |
| REEXECLAUNCHTIMER=${i#*=} |
| ;; |
| --*) |
| ## PATCH_OR_ISSUE can't be a --. So this is probably |
| ## a plugin thing. |
| continue |
| ;; |
| *) |
| PATCH_OR_ISSUE=${i} |
| ;; |
| esac |
| done |
| |
| docker_parse_args "$@" |
| |
| reaper_parse_args "$@" |
| |
| if [[ -z "${PATCH_OR_ISSUE}" |
| && "${BUILDMODE}" = patch ]]; then |
| yetus_usage |
| exit 1 |
| fi |
| |
| set_buildmode |
| |
| if [[ ${ROBOT} = true ]]; then |
| # shellcheck disable=SC2034 |
| TEST_PARALLEL=true |
| RESETREPO=true |
| RUN_TESTS=true |
| ISSUE=${PATCH_OR_ISSUE} |
| yetus_add_array_element EXEC_MODES Robot |
| fi |
| |
| if [[ -n $UNIT_TEST_FILTER_FILE ]]; then |
| if [[ -f $UNIT_TEST_FILTER_FILE ]]; then |
| UNIT_TEST_FILTER_FILE=$(yetus_abs "${UNIT_TEST_FILTER_FILE}") |
| else |
| yetus_error "ERROR: Unit test filter file (${UNIT_TEST_FILTER_FILE}) does not exist!" |
| cleanup_and_exit 1 |
| fi |
| fi |
| |
| if [[ -n ${REEXECLAUNCHTIMER} ]]; then |
| TIMER=${REEXECLAUNCHTIMER}; |
| else |
| start_clock |
| fi |
| |
| if [[ "${DOCKERMODE}" = true || "${DOCKERSUPPORT}" = true ]]; then |
| if [[ "${DOCKER_DESTRCUTIVE}" = true ]]; then |
| yetus_add_array_element EXEC_MODES DestructiveDocker |
| else |
| yetus_add_array_element EXEC_MODES Docker |
| fi |
| add_vote_table 0 reexec "Docker mode activated." |
| start_clock |
| elif [[ "${REEXECED}" = true ]]; then |
| yetus_add_array_element EXEC_MODES Re-exec |
| add_vote_table 0 reexec "Precommit patch detected." |
| start_clock |
| fi |
| |
| # we need absolute dir for ${BASEDIR} |
| cd "${STARTINGDIR}" || cleanup_and_exit 1 |
| BASEDIR=$(yetus_abs "${BASEDIR}") |
| |
| if [[ -n ${USER_PATCH_DIR} ]]; then |
| PATCH_DIR="${USER_PATCH_DIR}" |
| fi |
| |
| # we need absolute dir for PATCH_DIR |
| cd "${STARTINGDIR}" || cleanup_and_exit 1 |
| if [[ ! -d ${PATCH_DIR} ]]; then |
| if mkdir -p "${PATCH_DIR}"; then |
| echo "${PATCH_DIR} has been created" |
| else |
| echo "Unable to create ${PATCH_DIR}" |
| cleanup_and_exit 1 |
| fi |
| fi |
| PATCH_DIR=$(yetus_abs "${PATCH_DIR}") |
| COPROC_LOGFILE="${PATCH_DIR}/coprocessors.txt" |
| |
| if [[ -n "${EXCLUDE_PATHS_FILE}" ]]; then |
| if [[ -f "${EXCLUDE_PATHS_FILE}" ]]; then |
| EXCLUDE_PATHS_FILE=$(yetus_abs "${EXCLUDE_PATHS_FILE}") |
| elif [[ -f "${BASEDIR}/${EXCLUDE_PATHS_FILE}" ]]; then |
| EXCLUDE_PATHS_FILE=$(yetus_abs "${BASEDIR}/${EXCLUDE_PATHS_FILE}") |
| elif [[ -f "${PATCH_DIR}/precommit/excluded.txt" ]]; then |
| EXCLUDE_PATHS_FILE="${PATCH_DIR}/precommit/excluded.txt" |
| else |
| yetus_error "ERROR: Excluded paths file (${EXCLUDE_PATHS_FILE}}) does not exist!" |
| cleanup_and_exit 1 |
| fi |
| fi |
| |
| # we need absolute dir for ${CONSOLE_REPORT_FILE} |
| if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then |
| if : > "${CONSOLE_REPORT_FILE}"; then |
| CONSOLE_REPORT_FILE_ORIG="${CONSOLE_REPORT_FILE}" |
| CONSOLE_REPORT_FILE=$(yetus_abs "${CONSOLE_REPORT_FILE_ORIG}") |
| else |
| yetus_error "ERROR: cannot write to ${CONSOLE_REPORT_FILE}. Disabling console report file." |
| unset CONSOLE_REPORT_FILE |
| fi |
| fi |
| |
| if [[ ${RESETREPO} == "true" ]] ; then |
| yetus_add_array_element EXEC_MODES ResetRepo |
| fi |
| |
| if [[ ${RUN_TESTS} == "true" ]] ; then |
| yetus_add_array_element EXEC_MODES UnitTests |
| fi |
| |
| if [[ -n "${USER_PLUGIN_DIR}" ]]; then |
| USER_PLUGIN_DIR=$(yetus_abs "${USER_PLUGIN_DIR}") |
| fi |
| |
| GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt" |
| GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt" |
| GITUNIDIFFLINES="${PATCH_DIR}/gitdiffunilines.txt" |
| |
| if [[ "${REEXECED}" = true |
| && -f "${PATCH_DIR}/precommit/personality/provided.sh" ]]; then |
| PERSONALITY="${PATCH_DIR}/precommit/personality/provided.sh" |
| fi |
| } |
| |
| ## @description Switch BUILDMODE. Callers are responsible for setting |
| ## @description the appropriate vars. Use with caution. |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| function set_buildmode |
| { |
| # if both a patch and --empty-patch has been set, a choice needs |
| # to be made. If our exec is called qbt, then go full. |
| # otherwise, defer to the patch |
| if [[ -n "${PATCH_OR_ISSUE}" && "${BUILDMODE}" == full ]]; then |
| if [[ "${BINNAME}" =~ qbt ]]; then |
| BUILDMODE="full" |
| else |
| BUILDMODE="patch" |
| fi |
| fi |
| |
| if [[ "${BUILDMODE}" == full ]]; then |
| # shellcheck disable=SC2034 |
| BUILDMODEMSG="The source tree" |
| else |
| BUILDMODEMSG="The patch" |
| fi |
| } |
| |
| ## @description check if repo requires creds to do remote operations |
| ## @description also sets and uses GIT_OFFLINE as appropriate |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| ## @return 0 = no |
| ## @return 1 = yes |
| function git_requires_creds |
| { |
| declare status |
| |
| if [[ "${GIT_OFFLINE}" == true ]]; then |
| return 1 |
| fi |
| |
| pushd "${BASEDIR}" >/dev/null || cleanup_and_exit 1 |
| |
| if ! "${GIT}" fetch --dry-run >/dev/null 2>&1; then |
| GIT_OFFLINE=true |
| return 1 |
| fi |
| |
| popd >/dev/null || cleanup_and_exit 1 |
| GIT_OFFLINE=false |
| return 0 |
| } |
| |
| ## @description git clean the repository |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @return 0 on success |
| function git_clean |
| { |
| declare exemptdir |
| |
| if [[ ${RESETREPO} == "true" ]]; then |
| # if PATCH_DIR is in BASEDIR, then we don't want |
| # git wiping it out. |
| exemptdir=$(yetus_relative_dir "${BASEDIR}" "${PATCH_DIR}") |
| if [[ $? == 1 ]]; then |
| "${GIT}" clean -xdf |
| else |
| # we do, however, want it emptied of all _files_. |
| # we need to leave _directories_ in case we are in |
| # re-exec mode (which places a directory full of stuff in it) |
| yetus_debug "Exempting ${exemptdir} from clean" |
| "${GIT}" clean -xdf -e "${exemptdir}" |
| fi |
| fi |
| } |
| |
| ## @description Forcibly reset the tree back to it's original state |
| ## @audience public |
| ## @stability stable |
| ## @replaceable no |
| ## @return 0 on success |
| function git_checkout_force |
| { |
| declare exemptdir |
| |
| if [[ ${RESETREPO} == "true" ]]; then |
| git_clean && "${GIT}" checkout --force "${PATCH_BRANCH}" |
| fi |
| } |
| |
| ## @description git checkout the appropriate branch to test. Additionally, this calls |
| ## @description 'determine_branch' based upon the context provided |
| ## @description in ${PATCH_DIR} and in git after checkout. |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| ## @return 0 on success. May exit on failure. |
| function git_checkout |
| { |
| declare currentbranch |
| declare exemptdir |
| declare status |
| |
| big_console_header "Confirming git environment" |
| |
| git_requires_creds |
| |
| if [[ ${ROBOT} == true ]]; then |
| if declare -f "${ROBOTTYPE}"_pre_git_checkout >/dev/null; then |
| "${ROBOTTYPE}"_pre_git_checkout |
| fi |
| fi |
| |
| cd "${BASEDIR}" || cleanup_and_exit 1 |
| if [[ ! -e .git ]]; then |
| yetus_error "ERROR: ${BASEDIR} is not a git repo." |
| cleanup_and_exit 1 |
| fi |
| |
| if [[ ${RESETREPO} == "true" ]] ; then |
| |
| if [[ -d .git/rebase-apply ]]; then |
| yetus_error "ERROR: a previous rebase failed. Aborting it." |
| "${GIT}" rebase --abort |
| fi |
| |
| if ! "${GIT}" reset --hard; then |
| yetus_error "ERROR: git reset is failing" |
| cleanup_and_exit 1 |
| fi |
| |
| if ! git_clean; then |
| yetus_error "ERROR: git clean is failing" |
| cleanup_and_exit 1 |
| fi |
| |
| # if PATCH_DIR is in BASEDIR, then we don't want |
| # git wiping it out. |
| if yetus_relative_dir "${BASEDIR}" "${PATCH_DIR}" >/dev/null; then |
| # we need to empty out PATCH_DIR, but |
| # leave _directories_ in case we are in |
| # re-exec mode (which places a directory full of stuff in it) |
| yetus_debug "Exempting ${exemptdir} from clean" |
| rm "${PATCH_DIR}/*" 2>/dev/null |
| fi |
| |
| if [[ "${GIT_SHALLOW}" == false ]]; then |
| if ! "${GIT}" checkout --force "${PATCH_BRANCH_DEFAULT}"; then |
| yetus_error "WARNING: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing; assuming shallow" |
| GIT_SHALLOW=true |
| fi |
| fi |
| |
| determine_branch |
| |
| # we need to explicitly fetch in case the |
| # git ref hasn't been brought in tree yet |
| if [[ ${GIT_OFFLINE} == false ]]; then |
| if ! "${GIT}" pull --rebase; then |
| yetus_error "ERROR: git pull is failing" |
| cleanup_and_exit 1 |
| fi |
| fi |
| |
| if [[ ${GIT_SHALLOW} == false ]]; then |
| # forcibly checkout this branch or git ref |
| if ! "${GIT}" checkout --force "${PATCH_BRANCH}"; then |
| yetus_error "ERROR: git checkout ${PATCH_BRANCH} is failing" |
| cleanup_and_exit 1 |
| fi |
| fi |
| |
| # if we've selected a feature branch that has new changes |
| # since our last build, we'll need to reset to the latest FETCH_HEAD. |
| if [[ ${OFFLINE} == false ]]; then |
| |
| # previous clause where GIT_OFFLINE would get set is also |
| # protected by OFFLINE == false |
| |
| if [[ "${GIT_OFFLINE}" == false ]]; then |
| if ! "${GIT}" fetch; then |
| yetus_error "ERROR: git fetch is failing" |
| cleanup_and_exit 1 |
| fi |
| |
| if ! "${GIT}" reset --hard FETCH_HEAD; then |
| yetus_error "ERROR: git reset is failing" |
| cleanup_and_exit 1 |
| fi |
| fi |
| |
| if ! git_clean; then |
| yetus_error "ERROR: git clean is failing" |
| cleanup_and_exit 1 |
| fi |
| fi |
| |
| else |
| |
| status=$("${GIT}" status --porcelain) |
| if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then |
| yetus_error "ERROR: --dirty-workspace option not provided." |
| yetus_error "ERROR: can't run in a workspace that contains the following modifications" |
| yetus_error "${status}" |
| cleanup_and_exit 1 |
| fi |
| |
| determine_branch |
| |
| currentbranch=$("${GIT}" rev-parse --abbrev-ref HEAD) |
| if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then |
| if [[ "${BUILDMODE}" = patch ]]; then |
| echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}." |
| echo "WARNING: Continuing anyway..." |
| fi |
| PATCH_BRANCH=${currentbranch} |
| fi |
| fi |
| |
| return 0 |
| } |
| |
| ## @description Confirm the given branch is a git reference |
| ## @descriptoin or a valid gitXYZ commit hash |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branch |
| ## @return 0 on success, if gitXYZ was passed, PATCH_BRANCH=xyz |
| ## @return 1 on failure |
| function verify_valid_branch |
| { |
| local check=$1 |
| local i |
| local hash |
| |
| # shortcut some common |
| # non-resolvable names |
| if [[ -z ${check} ]]; then |
| return 1 |
| fi |
| |
| if [[ ${check} =~ ^git ]]; then |
| hash=$(echo "${check}" | cut -f2- -dt) |
| if [[ -n ${hash} ]]; then |
| if "${GIT}" cat-file -t "${hash}" >/dev/null 2>&1; then |
| PATCH_BRANCH=${hash} |
| return 0 |
| fi |
| return 1 |
| else |
| return 1 |
| fi |
| fi |
| |
| "${GIT}" show-ref "${check}" >/dev/null 2>&1 |
| return $? |
| } |
| |
| ## @description Try to guess the branch being tested using a variety of heuristics |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @return 0 on success, with PATCH_BRANCH updated appropriately |
| ## @return 1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT |
| function determine_branch |
| { |
| declare bugs |
| declare retval=1 |
| |
| # something has already set this, so move on |
| if [[ -n ${PATCH_BRANCH} ]]; then |
| return |
| fi |
| |
| pushd "${BASEDIR}" > /dev/null || return 1 |
| |
| yetus_debug "Determine branch" |
| |
| # something has already set this, so move on |
| if [[ -n ${PATCH_BRANCH} ]]; then |
| return |
| fi |
| |
| # developer mode, existing checkout, whatever |
| if [[ "${DIRTY_WORKSPACE}" == true ]];then |
| PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD) |
| echo "dirty workspace mode; applying against existing branch" |
| return |
| fi |
| |
| for bugs in "${BUGSYSTEMS[@]}"; do |
| if declare -f "${bugs}_determine_branch" >/dev/null;then |
| "${bugs}_determine_branch" |
| retval=$? |
| if [[ ${retval} == 0 ]]; then |
| break |
| fi |
| fi |
| done |
| |
| if [[ ${retval} != 0 ]]; then |
| PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}" |
| fi |
| popd >/dev/null || return 1 |
| } |
| |
| ## @description Try to guess the issue being tested using a variety of heuristics |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @return 0 on success, with ISSUE updated appropriately |
| ## @return 1 on failure, with ISSUE updated to "Unknown" |
| function determine_issue |
| { |
| declare bugsys |
| |
| yetus_debug "Determine issue" |
| |
| for bugsys in "${BUGSYSTEMS[@]}"; do |
| if declare -f "${bugsys}_determine_issue" >/dev/null; then |
| if "${bugsys}_determine_issue" "${PATCH_OR_ISSUE}"; then |
| yetus_debug "${bugsys} says ${ISSUE}" |
| return 0 |
| fi |
| fi |
| done |
| return 1 |
| } |
| |
| ## @description Use some heuristics to determine which long running |
| ## @description tests to run |
| ## @audience private |
| ## @stability stable |
| ## @replaceable no |
| function determine_needed_tests |
| { |
| declare i |
| declare plugin |
| |
| big_console_header "Determining needed tests" |
| echo "(Depending upon input size and number of plug-ins, this may take a while)" |
| |
| exclude_paths_from_changed_files |
| |
| for i in "${CHANGED_FILES[@]}"; do |
| yetus_debug "Determining needed tests for ${i}" |
| personality_file_tests "${i}" |
| |
| for plugin in "${TESTTYPES[@]}" ${BUILDTOOL}; do |
| if declare -f "${plugin}_filefilter" >/dev/null 2>&1; then |
| "${plugin}_filefilter" "${i}" |
| fi |
| done |
| done |
| |
| add_footer_table "Optional Tests" "${NEEDED_TESTS[*]}" |
| } |
| |
| ## @description Given ${INPUT_APPLIED_FILE}, actually apply the patch |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @return 0 on success |
| ## @return exit on failure |
| function apply_patch_file |
| { |
| |
| if [[ "${INPUT_APPLIED_FILE}" == "${INPUT_DIFF_FILE}" ]]; then |
| BUGLINECOMMENTS="" |
| add_vote_table '-0' patch "Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary." |
| big_console_header "Applying diff to ${PATCH_BRANCH}" |
| else |
| big_console_header "Applying patch to ${PATCH_BRANCH}" |
| fi |
| |
| if ! patchfile_apply_driver "${INPUT_APPLIED_FILE}"; then |
| echo "PATCH APPLICATION FAILED" |
| ((RESULT = RESULT + 1)) |
| add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase required? Wrong Branch? See ${PATCH_NAMING_RULE} for help." |
| bugsystem_finalreport 1 |
| cleanup_and_exit 1 |
| fi |
| return 0 |
| } |
| |
| ## @description copy the test-patch binary bits to a new working dir, |
| ## @description setting USER_PLUGIN_DIR and PERSONALITY to the new |
| ## @description locations. |
| ## @description this is used for test-patch in docker and reexec mode |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| function copytpbits |
| { |
| # we need to copy/consolidate all the bits that might have changed |
| # that are considered part of test-patch. This *might* break |
| # things that do off-path includes, but there isn't much we can |
| # do about that, I don't think. |
| |
| # if we've already copied, then don't bother doing it again |
| if [[ ${STARTINGDIR} == ${PATCH_DIR}/precommit ]]; then |
| yetus_debug "Skipping copytpbits; already copied once" |
| return |
| fi |
| |
| pushd "${STARTINGDIR}" >/dev/null || return 1 |
| mkdir -p "${PATCH_DIR}/precommit/user-plugins" |
| mkdir -p "${PATCH_DIR}/precommit/personality" |
| mkdir -p "${PATCH_DIR}/precommit/test-patch-docker" |
| |
| # copy our entire universe, preserving links, etc. |
| yetus_debug "copying '${BINDIR}' over to '${PATCH_DIR}/precommit'" |
| # shellcheck disable=SC2164 |
| (cd "${BINDIR}"; tar cpf - . ) | (cd "${PATCH_DIR}/precommit"; tar xpf - ) |
| |
| echo "${VERSION}" > "${PATCH_DIR}/precommit/VERSION" |
| |
| if [[ -n "${USER_PLUGIN_DIR}" |
| && -d "${USER_PLUGIN_DIR}" ]]; then |
| yetus_debug "copying '${USER_PLUGIN_DIR}' over to ${PATCH_DIR}/precommit/user-plugins" |
| cp -pr "${USER_PLUGIN_DIR}"/. \ |
| "${PATCH_DIR}/precommit/user-plugins" |
| fi |
| # Set to be relative to ${PATCH_DIR}/precommit |
| USER_PLUGIN_DIR="${PATCH_DIR}/precommit/user-plugins" |
| |
| if [[ -n ${EXCLUDE_PATHS_FILE} |
| && -f ${EXCLUDE_PATHS_FILE} ]]; then |
| yetus_debug "copying '${EXCLUDE_PATHS_FILE}' over to '${PATCH_DIR}/precommit/excluded.txt'" |
| cp -pr "${EXCLUDE_PATHS_FILE}" "${PATCH_DIR}/precommit/excluded.txt" |
| fi |
| if [[ -n ${PERSONALITY} |
| && -f ${PERSONALITY} ]]; then |
| yetus_debug "copying '${PERSONALITY}' over to '${PATCH_DIR}/precommit/personality/provided.sh'" |
| cp -pr "${PERSONALITY}" "${PATCH_DIR}/precommit/personality/provided.sh" |
| fi |
| |
| if [[ -n ${UNIT_TEST_FILTER_FILE} |
| && -f ${UNIT_TEST_FILTER_FILE} ]]; then |
| yetus_debug "copying '${UNIT_TEST_FILTER_FILE}' over to '${PATCH_DIR}/precommit/unit_test_filter_file.txt'" |
| cp -pr "${UNIT_TEST_FILTER_FILE}" "${PATCH_DIR}/precommit/unit_test_filter_file.txt" |
| fi |
| |
| popd >/dev/null || return 1 |
| } |
| |
| ## @description change the working directory to execute the buildtool |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param MODULE_ index |
| function buildtool_cwd |
| { |
| declare modindex=$1 |
| |
| BUILDTOOLCWD="${BUILDTOOLCWD//@@@BASEDIR@@@/${BASEDIR}}" |
| BUILDTOOLCWD="${BUILDTOOLCWD//@@@MODULEDIR@@@/${BASEDIR}/${MODULE[${modindex}]}}" |
| |
| if [[ "${BUILDTOOLCWD}" =~ ^/ ]]; then |
| yetus_debug "buildtool_cwd: ${BUILDTOOLCWD}" |
| if [[ ! -e "${BUILDTOOLCWD}" ]]; then |
| mkdir -p "${BUILDTOOLCWD}" || return 1 |
| fi |
| pushd "${BUILDTOOLCWD}" >/dev/null || return 1 |
| return 0 |
| fi |
| |
| case "${BUILDTOOLCWD}" in |
| basedir) |
| pushd "${BASEDIR}" >/dev/null || return 1 |
| ;; |
| module) |
| if [[ ! -d "${BASEDIR}/${MODULE[${modindex}]}" ]]; then |
| return 1 |
| fi |
| pushd "${BASEDIR}/${MODULE[${modindex}]}" >/dev/null || return 1 |
| ;; |
| *) |
| pushd "$(pwd)" >/dev/null || return 1 |
| ;; |
| esac |
| } |
| |
| ## @description If this patches actually patches test-patch.sh, then |
| ## @description run with the patched version for the test. |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @return none; otherwise relaunches |
| function check_reexec |
| { |
| declare commentfile=${PATCH_DIR}/tp.${RANDOM} |
| declare tpdir |
| declare copy=false |
| declare testdir |
| declare plugin |
| |
| if [[ ${REEXECED} == true ]]; then |
| big_console_header "Re-exec mode detected. Continuing." |
| return |
| fi |
| |
| # determine if the patch hits |
| # any test-patch sensitive bits |
| # if so, we need to copy the universe |
| # after patching it (copy=true) |
| for testdir in "${BINDIR}" \ |
| "${PERSONALITY}" \ |
| "${USER_PLUGIN_DIR}" \ |
| "${DOCKERFILE}"; do |
| tpdir=$(yetus_relative_dir "${BASEDIR}" "${testdir}") |
| # shellcheck disable=SC2181 |
| if [[ $? == 0 |
| && "${CHANGED_FILES[*]}" =~ ${tpdir} ]]; then |
| copy=true |
| fi |
| done |
| |
| if [[ ${copy} == true && "${BUILDMODE}" != full ]]; then |
| big_console_header "precommit patch detected" |
| |
| if [[ ${RESETREPO} == false ]]; then |
| ((RESULT = RESULT + 1)) |
| yetus_debug "can't destructively change the working directory. run with '--resetrepo' please. :(" |
| add_vote_table -1 precommit "Couldn't test precommit changes because we aren't configured to destructively change the working directory." |
| else |
| |
| apply_patch_file |
| |
| if [[ ${ROBOT} == true ]]; then |
| rm "${commentfile}" 2>/dev/null |
| echo "(!) A patch to the testing environment has been detected. " > "${commentfile}" |
| echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}" |
| echo "The console is at ${BUILD_URL}${BUILD_URL_CONSOLE} in case of problems." >> "${commentfile}" |
| write_comment "${commentfile}" |
| rm "${commentfile}" |
| fi |
| fi |
| fi |
| |
| if [[ ${DOCKERSUPPORT} == false |
| && ${copy} == false ]]; then |
| return |
| fi |
| |
| if [[ ${DOCKERSUPPORT} == true |
| && ${copy} == false ]]; then |
| big_console_header "Re-execing under Docker" |
| fi |
| |
| # copy our universe |
| copytpbits |
| |
| if [[ ${DOCKERSUPPORT} == true ]]; then |
| |
| #if we are doing docker, then we re-exec, but underneath the |
| #container |
| |
| docker_handler |
| exit $? |
| else |
| |
| # if we aren't doing docker, then just call ourselves |
| # but from the new path with the new flags |
| #shellcheck disable=SC2164 |
| cd "${PATCH_DIR}/precommit/" |
| exec "${PATCH_DIR}/precommit/test-patch.sh" \ |
| "${USER_PARAMS[@]}" \ |
| --reexec \ |
| --basedir="${BASEDIR}" \ |
| --branch="${PATCH_BRANCH}" \ |
| --patch-dir="${PATCH_DIR}" \ |
| --tpglobaltimer="${GLOBALTIMER}" \ |
| --tpreexectimer="${TIMER}" \ |
| --personality="${PERSONALITY}" \ |
| --tpinstance="${INSTANCE}" \ |
| --user-plugins="${USER_PLUGIN_DIR}" |
| fi |
| } |
| |
| ## @description Save file names and directory to the patch dir |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| function archive |
| { |
| declare pmn |
| declare fn |
| declare line |
| declare srcdir |
| declare tmpfile="${PATCH_DIR}/tmp.$$.${RANDOM}" |
| |
| if [[ ${#ARCHIVE_LIST[@]} -eq 0 ]]; then |
| return |
| fi |
| |
| if ! verify_command "rsync" "${RSYNC}"; then |
| yetus_error "WARNING: Cannot use the archive function" |
| return |
| fi |
| |
| yetus_debug "Starting archiving process" |
| # get the list of files. these will be with |
| # the full path |
| # (this is pretty expensive) |
| |
| rm "${tmpfile}" 2>/dev/null |
| for pmn in "${ARCHIVE_LIST[@]}"; do |
| find "${BASEDIR}" -name "${pmn}" >> "${tmpfile}" |
| done |
| |
| # read the list, stripping of both |
| # the BASEDIR and any leading /. |
| # with our filename fragment, |
| # call faster_dirname with a prepended / |
| while read -r line; do |
| yetus_debug "Archiving: ${line}" |
| srcdir=$(faster_dirname "/${line}") |
| mkdir -p "${PATCH_DIR}/archiver${srcdir}" |
| "${RSYNC}" -av "${BASEDIR}/${line}" "${PATCH_DIR}/archiver${srcdir}" >/dev/null 2>&1 |
| done < <("${SED}" -e "s,${BASEDIR},,g" \ |
| -e "s,^/,,g" "${tmpfile}") |
| rm "${tmpfile}" 2>/dev/null |
| yetus_debug "Ending archiving process" |
| |
| } |
| |
| ## @description Reset the test results |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| function modules_reset |
| { |
| MODULE_STATUS=() |
| MODULE_STATUS_TIMER=() |
| MODULE_STATUS_MSG=() |
| MODULE_STATUS_LOG=() |
| MODULE_COMPILE_LOG=() |
| } |
| |
| ## @description Backup the MODULE globals prior to loop processing |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| function modules_backup |
| { |
| MODULE_BACKUP_STATUS=("${MODULE_STATUS[@]}") |
| MODULE_BACKUP_STATUS_TIMER=("${MODULE_STATUS_TIMER[@]}") |
| MODULE_BACKUP_STATUS_MSG=("${MODULE_STATUS_MSG[@]}") |
| MODULE_BACKUP_STATUS_LOG=("${MODULE_STATUS_LOG[@]}") |
| MODULE_BACKUP_COMPILE_LOG=("${MODULE_COMPILE_LOG[@]}") |
| } |
| |
| ## @description Restore the backup |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| function modules_restore |
| { |
| MODULE_STATUS=("${MODULE_BACKUP_STATUS[@]}") |
| MODULE_STATUS_TIMER=("${MODULE_BACKUP_STATUS_TIMER[@]}") |
| MODULE_STATUS_MSG=("${MODULE_BACKUP_STATUS_MSG[@]}") |
| MODULE_STATUS_LOG=("${MODULE_BACKUP_STATUS_LOG[@]}") |
| MODULE_COMPILE_LOG=("${MODULE_BACKUP_COMPILE_LOG[@]}") |
| } |
| |
| ## @description Utility to print standard module errors |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param repostatus |
| ## @param testtype |
| ## @param summarymode |
| function modules_messages |
| { |
| declare repostatus=$1 |
| declare testtype=$2 |
| declare summarymode=$3 |
| shift 3 |
| declare modindex=0 |
| declare repo |
| declare goodtime=0 |
| declare failure=false |
| declare oldtimer |
| declare statusjdk |
| declare multijdkmode=false |
| |
| if [[ "${BUILDMODE}" == full ]]; then |
| repo="the source" |
| elif [[ "${repostatus}" == branch ]]; then |
| repo=${PATCH_BRANCH} |
| else |
| repo="the patch" |
| fi |
| |
| if verify_multijdk_test "${testtype}"; then |
| multijdkmode=true |
| fi |
| |
| oldtimer=${TIMER} |
| |
| if [[ ${summarymode} == true |
| && ${ALLOWSUMMARIES} == true ]]; then |
| |
| until [[ ${modindex} -eq ${#MODULE[@]} ]]; do |
| |
| if [[ ${multijdkmode} == true ]]; then |
| statusjdk=${MODULE_STATUS_JDK[${modindex}]} |
| fi |
| |
| if [[ "${MODULE_STATUS[${modindex}]}" == '+1' ]]; then |
| ((goodtime=goodtime + ${MODULE_STATUS_TIMER[${modindex}]})) |
| else |
| failure=true |
| start_clock |
| echo "" |
| echo "${MODULE_STATUS_MSG[${modindex}]}" |
| echo "" |
| offset_clock "${MODULE_STATUS_TIMER[${modindex}]}" |
| add_vote_table "${MODULE_STATUS[${modindex}]}" "${testtype}" "${MODULE_STATUS_MSG[${modindex}]}" |
| if [[ ${MODULE_STATUS[${modindex}]} == -1 |
| && -n "${MODULE_STATUS_LOG[${modindex}]}" ]]; then |
| add_footer_table "${testtype}" "@@BASE@@/${MODULE_STATUS_LOG[${modindex}]}" |
| fi |
| fi |
| ((modindex=modindex+1)) |
| done |
| |
| if [[ ${failure} == false ]]; then |
| start_clock |
| offset_clock "${goodtime}" |
| add_vote_table +1 "${testtype}" "${repo} passed${statusjdk}" |
| fi |
| else |
| until [[ ${modindex} -eq ${#MODULE[@]} ]]; do |
| start_clock |
| echo "" |
| echo "${MODULE_STATUS_MSG[${modindex}]}" |
| echo "" |
| offset_clock "${MODULE_STATUS_TIMER[${modindex}]}" |
| add_vote_table "${MODULE_STATUS[${modindex}]}" "${testtype}" "${MODULE_STATUS_MSG[${modindex}]}" |
| if [[ ${MODULE_STATUS[${modindex}]} == -1 |
| && -n "${MODULE_STATUS_LOG[${modindex}]}" ]]; then |
| add_footer_table "${testtype}" "@@BASE@@/${MODULE_STATUS_LOG[${modindex}]}" |
| fi |
| ((modindex=modindex+1)) |
| done |
| fi |
| TIMER=${oldtimer} |
| } |
| |
| ## @description Add or update a test result. Update requires |
| ## @description at least the first two parameters. |
| ## @description WARNING: If the message is updated, |
| ## @description then the JDK version is also calculated to match |
| ## @description the current JAVA_HOME. |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param moduleindex |
| ## @param -1-0|0|+1 |
| ## @param logvalue |
| ## @param message |
| function module_status |
| { |
| declare index=$1 |
| declare value=$2 |
| shift 2 |
| declare log=$1 |
| shift |
| |
| declare jdk |
| |
| jdk=$(report_jvm_version "${JAVA_HOME}") |
| |
| if [[ -n ${index} |
| && ${index} =~ ^[0-9]+$ ]]; then |
| MODULE_STATUS[${index}]="${value}" |
| if [[ -n ${log} ]]; then |
| MODULE_STATUS_LOG[${index}]="${log}" |
| fi |
| if [[ -n $1 ]]; then |
| MODULE_STATUS_JDK[${index}]=" with JDK v${jdk}" |
| MODULE_STATUS_MSG[${index}]="${*}" |
| fi |
| else |
| yetus_error "ASSERT: module_status given bad index: ${index}" |
| yetus_error "ASSERT: module_stats $*" |
| generate_stack |
| exit 1 |
| fi |
| } |
| |
| ## @description run the tests for the queued modules |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param repostatus |
| ## @param testtype |
| ## @param mvncmdline |
| function modules_workers |
| { |
| declare repostatus=$1 |
| declare testtype=$2 |
| shift 2 |
| declare modindex=0 |
| declare fn |
| declare savestart=${TIMER} |
| declare savestop |
| declare repo |
| declare modulesuffix |
| declare jdk="" |
| declare jdkindex=0 |
| declare statusjdk |
| declare result=0 |
| declare argv |
| declare execvalue |
| |
| if [[ "${BUILDMODE}" = full ]]; then |
| repo="the source" |
| elif [[ ${repostatus} == branch ]]; then |
| repo=${PATCH_BRANCH} |
| else |
| repo="the patch" |
| fi |
| |
| modules_reset |
| |
| if verify_multijdk_test "${testtype}"; then |
| jdk=$(report_jvm_version "${JAVA_HOME}") |
| statusjdk=" with JDK v${jdk}" |
| jdk="-jdk${jdk}" |
| jdk=${jdk// /} |
| yetus_debug "Starting MultiJDK mode${statusjdk} on ${testtype}" |
| fi |
| |
| until [[ ${modindex} -eq ${#MODULE[@]} ]]; do |
| start_clock |
| |
| fn=$(module_file_fragment "${MODULE[${modindex}]}") |
| fn="${fn}${jdk}" |
| modulesuffix=$(basename "${MODULE[${modindex}]}") |
| if [[ ${modulesuffix} = \. ]]; then |
| modulesuffix="root" |
| fi |
| |
| if ! buildtool_cwd "${modindex}"; then |
| echo "${BASEDIR}/${MODULE[${modindex}]} no longer exists. Skipping." |
| ((modindex=modindex+1)) |
| savestop=$(stop_clock) |
| MODULE_STATUS_TIMER[${modindex}]=${savestop} |
| continue |
| fi |
| |
| argv=("${@//@@@MODULEFN@@@/${fn}}") |
| argv=("${argv[@]//@@@MODULEDIR@@@/${BASEDIR}/${MODULE[${modindex}]}}") |
| |
| # shellcheck disable=2086,2046 |
| echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \ |
| $("${BUILDTOOL}_executor" "${testtype}") \ |
| ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} \ |
| "${argv[@]}" |
| execvalue=$? |
| |
| reaper_post_exec "${modulesuffix}" "${repostatus}-${testtype}-${fn}" |
| ((execvalue = execvalue + $? )) |
| |
| if [[ ${execvalue} == 0 ]] ; then |
| module_status \ |
| ${modindex} \ |
| +1 \ |
| "${repostatus}-${testtype}-${fn}.txt" \ |
| "${modulesuffix} in ${repo} passed${statusjdk}." |
| else |
| module_status \ |
| ${modindex} \ |
| -1 \ |
| "${repostatus}-${testtype}-${fn}.txt" \ |
| "${modulesuffix} in ${repo} failed${statusjdk}." |
| ((result = result + 1)) |
| fi |
| |
| # compile is special |
| if [[ ${testtype} = compile ]]; then |
| MODULE_COMPILE_LOG[${modindex}]="${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" |
| yetus_debug "Compile log set to ${MODULE_COMPILE_LOG[${modindex}]}" |
| fi |
| |
| savestop=$(stop_clock) |
| MODULE_STATUS_TIMER[${modindex}]=${savestop} |
| # shellcheck disable=SC2086 |
| echo "Elapsed: $(clock_display ${savestop})" |
| popd >/dev/null || return 1 |
| ((modindex=modindex+1)) |
| done |
| |
| TIMER=${savestart} |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Reset the queue for tests |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| function clear_personality_queue |
| { |
| yetus_debug "Personality: clear queue" |
| MODCOUNT=0 |
| MODULE=() |
| } |
| |
| ## @description Build the queue for tests |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param module |
| ## @param profiles/flags/etc |
| function personality_enqueue_module |
| { |
| yetus_debug "Personality: enqueue $*" |
| local module=$1 |
| shift |
| |
| MODULE[${MODCOUNT}]=${module} |
| MODULEEXTRAPARAM[${MODCOUNT}]=${*} |
| ((MODCOUNT=MODCOUNT+1)) |
| } |
| |
| ## @description Remove a module |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param modulenames |
| function dequeue_personality_module |
| { |
| declare modname=$1 |
| declare oldmodule=("${MODULE[@]}") |
| declare oldmodparams=("${MODULEESXTRAPARAM[@]}") |
| declare modindex=0 |
| |
| yetus_debug "Personality: dequeue $*" |
| |
| clear_personality_queue |
| |
| until [[ ${modindex} -eq ${#oldmodule[@]} ]]; do |
| if [[ "${oldmodule[${modindex}]}" = "${modname}" ]]; then |
| yetus_debug "Personality: removing ${modindex}, ${oldmodule[${modindex}]} = ${modname}" |
| else |
| personality_enqueue_module "${oldmodule[${modindex}]}" "${oldmodparams[${modindex}]}" |
| fi |
| ((modindex=modindex+1)) |
| done |
| } |
| |
| ## @description Utility to push many tests into the failure list |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @param testdesc |
| ## @param testlist |
| function populate_test_table |
| { |
| local reason=$1 |
| shift |
| local first="" |
| local i |
| |
| for i in "$@"; do |
| if [[ -z "${first}" ]]; then |
| add_test_table "${reason}" "${i}" |
| first="${reason}" |
| else |
| add_test_table " " "${i}" |
| fi |
| done |
| } |
| |
| ## @description Run and verify the output of the appropriate unit tests |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @return 0 on success |
| ## @return 1 on failure |
| function check_unittests |
| { |
| declare i |
| declare testsys |
| declare test_logfile |
| declare result=0 |
| declare -r savejavahome=${JAVA_HOME} |
| declare multijdkmode |
| declare jdk="" |
| declare jdkindex=0 |
| declare -a jdklist |
| declare statusjdk |
| declare formatresult=0 |
| declare needlog |
| |
| if ! verify_needed_test unit; then |
| return 0 |
| fi |
| |
| big_console_header "Running unit tests" |
| |
| if verify_multijdk_test unit; then |
| multijdkmode=true |
| jdklist=("${JDK_DIR_LIST[@]}") |
| else |
| multijdkmode=false |
| jdklist=("${JAVA_HOME}") |
| fi |
| |
| for jdkindex in "${jdklist[@]}"; do |
| if [[ ${multijdkmode} == true ]]; then |
| JAVA_HOME=${jdkindex} |
| jdk=$(report_jvm_version "${JAVA_HOME}") |
| statusjdk="JDK v${jdk} " |
| jdk="-jdk${jdk}" |
| jdk=${jdk// /} |
| fi |
| |
| personality_modules patch unit |
| "${BUILDTOOL}_modules_worker" patch unit |
| |
| ((result=result+$?)) |
| |
| i=0 |
| until [[ $i -eq ${#MODULE[@]} ]]; do |
| module=${MODULE[${i}]} |
| fn=$(module_file_fragment "${module}") |
| fn="${fn}${jdk}" |
| test_logfile="${PATCH_DIR}/patch-unit-${fn}.txt" |
| |
| buildtool_cwd "${i}" |
| |
| needlog=0 |
| for testsys in "${TESTFORMATS[@]}"; do |
| if declare -f "${testsys}_process_tests" >/dev/null; then |
| yetus_debug "Calling ${testsys}_process_tests" |
| "${testsys}_process_tests" "${module}" "${test_logfile}" "${fn}" |
| formatresult=$? |
| ((result=result+formatresult)) |
| if [[ "${formatresult}" != 0 ]]; then |
| needlog=1 |
| fi |
| fi |
| done |
| |
| if [[ ${needlog} == 1 ]]; then |
| module_status ${i} -1 "patch-unit-${fn}.txt" |
| fi |
| |
| popd >/dev/null || return 1 |
| |
| ((i=i+1)) |
| done |
| |
| for testsys in "${TESTFORMATS[@]}"; do |
| if declare -f "${testsys}_finalize_results" >/dev/null; then |
| yetus_debug "Calling ${testsys}_finalize_results" |
| "${testsys}_finalize_results" "${statusjdk}" |
| fi |
| done |
| |
| done |
| JAVA_HOME=${savejavahome} |
| |
| modules_messages patch unit false |
| |
| if [[ "${ROBOT}" == true ]]; then |
| if declare -f "${ROBOTTYPE}"_unittest_footer >/dev/null; then |
| "${ROBOTTYPE}"_unittest_footer "${statusjdk}" |
| fi |
| fi |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Queue up comments to write into bug systems |
| ## @description that have code review support, if such support |
| ## @description enabled/available. |
| ## @description File should be in the form of "file:line:comment" |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param plugin |
| ## @param filename |
| function bugsystem_linecomments_queue |
| { |
| declare plugin=$1 |
| declare fn=$2 |
| declare line |
| declare linenum |
| declare text |
| |
| if [[ ! -f "${GITUNIDIFFLINES}" ]]; then |
| return |
| fi |
| |
| while read -r line; do |
| file=$(echo "${line}" | cut -f1 -d:) |
| if [[ "${file}" =~ ^\./ ]]; then |
| file=${file:2} |
| fi |
| linenum=$(echo "${line}" | cut -f2 -d:) |
| text=$(echo "${line}" | cut -f3- -d:) |
| |
| echo "${file}:${linenum}:${plugin}:${text}" >> "${PATCH_DIR}/linecomments-in.txt" |
| |
| done < "${fn}" |
| } |
| |
| ## @description Call relevant bug systems to write comments |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| function bugsystem_linecomments_writer |
| { |
| declare fn=$1 |
| declare linenum=$2 |
| shift 2 |
| declare -a text |
| text=("$@") |
| declare idxline |
| declare uniline |
| |
| if [[ ! -f "${GITUNIDIFFLINES}" ]]; then |
| return |
| fi |
| |
| if [[ -z "${fn}" ]]; then |
| return |
| fi |
| |
| idxline="${fn}:${linenum}:" |
| uniline=$("${GREP}" "${idxline}" "${GITUNIDIFFLINES}" | cut -f3 -d: ) |
| |
| for bugs in ${BUGLINECOMMENTS}; do |
| if declare -f "${bugs}_linecomments" >/dev/null;then |
| "${bugs}_linecomments" "${fn}" "${linenum}" "${uniline}" "${linetext[@]}" |
| fi |
| done |
| } |
| |
| ## @description Write all of the bugsystem linecomments |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| function bugsystem_linecomments_trigger |
| { |
| declare line |
| declare fn |
| declare linenum |
| declare text |
| declare -a linetext |
| declare prevnum |
| declare prevfn |
| declare plugin |
| |
| if [[ ! -f "${GITUNIDIFFLINES}" ]]; then |
| return 0 |
| fi |
| |
| if [[ ! -f "${PATCH_DIR}/linecomments-in.txt" ]]; then |
| return 0 |
| fi |
| |
| # sort the file such that all files and lines are now next to each other |
| sort "${PATCH_DIR}/linecomments-in.txt" > "${PATCH_DIR}/linecomments-sorted.txt" |
| |
| while read -r line;do |
| fn=$(echo "${line}" | cut -f1 -d:) |
| if [[ -z "${prevfn}" ]]; then |
| prevfn=${fn} |
| fi |
| |
| linenum=$(echo "${line}" | cut -f2 -d:) |
| if [[ -z "${prevnum}" ]]; then |
| prevnum=${linenum} |
| fi |
| |
| text=$(echo "${line}" | cut -f3- -d:) |
| |
| # if, for some reason either one of these |
| # isn't a number, force it to be zero. |
| if [[ ! ${prevnum} =~ ^[0-9]+$ ]]; then |
| prevnum=0 |
| fi |
| |
| if [[ ! ${linenum} =~ ^[0-9]+$ ]]; then |
| linenum=0 |
| fi |
| |
| if [[ "${prevfn}" == "${fn}" ]] && |
| [[ ${prevnum} -eq ${linenum} ]]; then |
| linetext+=("${text}") |
| continue |
| fi |
| |
| bugsystem_linecomments_writer "${prevfn}" "${prevnum}" "${linetext[@]}" |
| prevfn=${fn} |
| prevnum=${linenum} |
| linetext=("${text}") |
| done < "${PATCH_DIR}/linecomments-sorted.txt" |
| |
| bugsystem_linecomments_writer "${prevfn}" "${prevnum}" "${linetext[@]}" |
| if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then |
| yetus_debug "Keeping linecomments files for debugging" |
| else |
| rm "${PATCH_DIR}/linecomments-in.txt" "${PATCH_DIR}/linecomments-sorted.txt" |
| fi |
| } |
| |
| ## @description Write the final output to the selected bug system |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| function bugsystem_finalreport |
| { |
| declare version |
| declare bugs |
| |
| if [[ "${ROBOT}" = true ]]; then |
| if declare -f "${ROBOTTYPE}"_finalreport >/dev/null; then |
| "${ROBOTTYPE}"_finalreport |
| fi |
| fi |
| |
| if [[ "${#VERSION_DATA[@]}" -gt 0 ]]; then |
| add_footer_table "versions" "${VERSION_DATA[@]}" |
| fi |
| |
| add_footer_table "Powered by" "Apache Yetus ${VERSION} http://yetus.apache.org" |
| |
| bugsystem_linecomments_trigger |
| |
| for bugs in ${BUGCOMMENTS}; do |
| if declare -f "${bugs}_finalreport" >/dev/null;then |
| "${bugs}_finalreport" "${@}" |
| fi |
| done |
| } |
| |
| ## @description Clean the filesystem as appropriate and then exit |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| ## @param runresult |
| function cleanup_and_exit |
| { |
| local result=$1 |
| |
| if [[ ${ROBOT} == "true" ]]; then |
| if declare -f "${ROBOTTYPE}"_cleanup_and_exit >/dev/null; then |
| "${ROBOTTYPE}"_cleanup_and_exit "${result}" |
| fi |
| |
| if [[ ${RELOCATE_PATCH_DIR} == "true" && \ |
| -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then |
| # if PATCH_DIR is already inside BASEDIR, then |
| # there is no need to move it since we assume that |
| # Jenkins or whatever already knows where it is at |
| # since it told us to put it there! |
| yetus_relative_dir "${BASEDIR}" "${PATCH_DIR}" >/dev/null |
| if [[ $? == 1 ]]; then |
| yetus_debug "mv ${PATCH_DIR} ${BASEDIR}" |
| mv "${PATCH_DIR}" "${BASEDIR}" |
| fi |
| fi |
| fi |
| |
| # docker mode will print this after exit |
| if [[ "${DOCKERMODE}" == false ]]; then |
| big_console_header "Finished build." |
| fi |
| |
| if [[ "${DOCKERMODE}" != true ]]; then |
| rm "${PATCH_DIR}/pidfile.txt" |
| fi |
| |
| if [[ "${BUILDMODE}" == 'full' ]] && [[ "${CONTINUOUS_IMPROVEMENT}" == true ]]; then |
| exit 0 |
| fi |
| |
| #shellcheck disable=SC2086 |
| exit ${result} |
| } |
| |
| ## @description Driver to execute _tests routines |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| function runtests |
| { |
| local plugin |
| |
| if [[ ${RUN_TESTS} == "true" ]] ; then |
| |
| verify_patchdir_still_exists |
| check_unittests |
| fi |
| |
| for plugin in "${TESTTYPES[@]}"; do |
| verify_patchdir_still_exists |
| if declare -f "${plugin}_tests" >/dev/null 2>&1; then |
| modules_reset |
| yetus_debug "Running ${plugin}_tests" |
| "${plugin}_tests" |
| fi |
| done |
| archive |
| } |
| |
| ## @description Calculate the differences between the specified files |
| ## @description using just the column+ messages (third+ column in a |
| ## @descriptoin colon delimated flie) and output it to stdout. |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branchlog |
| ## @param patchlog |
| ## @return differences |
| function column_calcdiffs |
| { |
| declare branch=$1 |
| declare patch=$2 |
| declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} |
| declare j |
| |
| # first, strip filenames:line: |
| # this keeps column: in an attempt to increase |
| # accuracy in case of multiple, repeated errors |
| # since the column number shouldn't change |
| # if the line of code hasn't been touched |
| cut -f3- -d: "${branch}" > "${tmp}.branch" |
| cut -f3- -d: "${patch}" > "${tmp}.patch" |
| |
| # compare the errors, generating a string of line |
| # numbers. Sorry portability: GNU diff makes this too easy |
| "${DIFF}" --unchanged-line-format="" \ |
| --old-line-format="" \ |
| --new-line-format="%dn " \ |
| "${tmp}.branch" \ |
| "${tmp}.patch" > "${tmp}.lined" |
| |
| if [[ "${BUILDMODE}" == full ]]; then |
| cat "${patch}" |
| else |
| # now, pull out those lines of the raw output |
| # shellcheck disable=SC2013 |
| for j in $(cat "${tmp}.lined"); do |
| head -"${j}" "${patch}" | tail -1 |
| done |
| fi |
| |
| rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null |
| } |
| |
| ## @description Calculate the differences between the specified files |
| ## @description using just the error messages (last column in a |
| ## @descriptoin colon delimated flie) and output it to stdout. |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branchlog |
| ## @param patchlog |
| ## @return differences |
| function error_calcdiffs |
| { |
| declare branch=$1 |
| declare patch=$2 |
| declare tmp=${PATCH_DIR}/pl.$$.${RANDOM} |
| declare j |
| |
| # first, pull out just the errors |
| # shellcheck disable=SC2016 |
| "${AWK}" -F: '{print $NF}' "${branch}" > "${tmp}.branch" |
| |
| # shellcheck disable=SC2016 |
| "${AWK}" -F: '{print $NF}' "${patch}" > "${tmp}.patch" |
| |
| # compare the errors, generating a string of line |
| # numbers. Sorry portability: GNU diff makes this too easy |
| "${DIFF}" --unchanged-line-format="" \ |
| --old-line-format="" \ |
| --new-line-format="%dn " \ |
| "${tmp}.branch" \ |
| "${tmp}.patch" > "${tmp}.lined" |
| |
| if [[ "${BUILDMODE}" == full ]]; then |
| cat "${patch}" |
| else |
| |
| # now, pull out those lines of the raw output |
| # shellcheck disable=SC2013 |
| for j in $(cat "${tmp}.lined"); do |
| head -"${j}" "${patch}" | tail -1 |
| done |
| fi |
| |
| rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null |
| } |
| |
| ## @description Wrapper to call specific version of calcdiffs if available |
| ## @description otherwise calls error_calcdiffs |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branchlog |
| ## @param patchlog |
| ## @param testtype |
| ## @return differences |
| function calcdiffs |
| { |
| declare branchlog=$1 |
| declare patchlog=$2 |
| declare testtype=$3 |
| |
| # ensure that both log files exist |
| if [[ ! -f "${branchlog}" ]]; then |
| touch "${branchlog}" |
| fi |
| if [[ ! -f "${patchlog}" ]]; then |
| touch "${patchlog}" |
| fi |
| |
| if declare -f "${PROJECT_NAME}_${testtype}_calcdiffs" >/dev/null; then |
| "${PROJECT_NAME}_${testtype}_calcdiffs" "${branchlog}" "${patchlog}" |
| elif declare -f "${BUILDTOOL}_${testtype}_calcdiffs" >/dev/null; then |
| "${BUILDTOOL}_${testtype}_calcdiffs" "${branchlog}" "${patchlog}" |
| elif declare -f "${testtype}_calcdiffs" >/dev/null; then |
| "${testtype}_calcdiffs" "${branchlog}" "${patchlog}" |
| else |
| error_calcdiffs "${branchlog}" "${patchlog}" |
| fi |
| } |
| |
| ## @description generate a standarized calcdiff status message |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param totalbranchissues |
| ## @param totalpatchissues |
| ## @param newpatchissues |
| ## @return errorstring |
| function generic_calcdiff_status |
| { |
| declare -i numbranch=$1 |
| declare -i numpatch=$2 |
| declare -i addpatch=$3 |
| declare -i samepatch |
| declare -i fixedpatch |
| |
| ((samepatch=numpatch-addpatch)) |
| ((fixedpatch=numbranch-numpatch+addpatch)) |
| |
| if [[ "${BUILDMODE}" = full ]]; then |
| printf "has %i issues." "${addpatch}" |
| else |
| printf "generated %i new + %i unchanged - %i fixed = %i total (was %i)" \ |
| "${addpatch}" \ |
| "${samepatch}" \ |
| "${fixedpatch}" \ |
| "${numpatch}" \ |
| "${numbranch}" |
| fi |
| } |
| |
| ## @description Helper routine for plugins to ask projects, etc |
| ## @description to count problems in a log file |
| ## @description and output it to stdout. |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @return number of issues |
| function generic_logfilter |
| { |
| declare testtype=$1 |
| declare input=$2 |
| declare output=$3 |
| |
| if declare -f "${PROJECT_NAME}_${testtype}_logfilter" >/dev/null; then |
| "${PROJECT_NAME}_${testtype}_logfilter" "${input}" "${output}" |
| elif declare -f "${BUILDTOOL}_${testtype}_logfilter" >/dev/null; then |
| "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}" |
| elif declare -f "${testtype}_logfilter" >/dev/null; then |
| "${testtype}_logfilter" "${input}" "${output}" |
| else |
| yetus_error "ERROR: ${testtype}: No function defined to filter problems." |
| echo 0 |
| fi |
| } |
| |
| ## @description Helper routine for plugins to do a pre-patch prun |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param testype |
| ## @param multijdk |
| ## @return 1 on failure |
| ## @return 0 on success |
| function generic_pre_handler |
| { |
| declare testtype=$1 |
| declare multijdkmode=$2 |
| declare result=0 |
| declare -r savejavahome=${JAVA_HOME} |
| declare multijdkmode |
| declare jdkindex=0 |
| declare -a jdklist |
| |
| if ! verify_needed_test "${testtype}"; then |
| return 0 |
| fi |
| |
| big_console_header "Pre-patch ${testtype} verification on ${PATCH_BRANCH}" |
| |
| if verify_multijdk_test "${testtype}"; then |
| multijdkmode=true |
| jdklist=("${JDK_DIR_LIST[@]}") |
| else |
| multijdkmode=false |
| jdklist=("${JAVA_HOME}") |
| fi |
| |
| for jdkindex in "${jdklist[@]}"; do |
| if [[ ${multijdkmode} == true ]]; then |
| JAVA_HOME=${jdkindex} |
| fi |
| |
| personality_modules branch "${testtype}" |
| "${BUILDTOOL}_modules_worker" branch "${testtype}" |
| |
| ((result=result + $?)) |
| modules_messages branch "${testtype}" true |
| |
| done |
| JAVA_HOME=${savejavahome} |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Generic post-patch log handler |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @return 0 on success |
| ## @return 1 on failure |
| ## @param origlog |
| ## @param testtype |
| ## @param multijdkmode |
| function generic_postlog_compare |
| { |
| declare origlog=$1 |
| declare testtype=$2 |
| declare multijdk=$3 |
| declare result=0 |
| declare i |
| declare fn |
| declare jdk |
| declare statusjdk |
| declare -i numbranch=0 |
| declare -i numpatch=0 |
| declare -i addpatch=0 |
| declare -i samepatch=0 |
| declare -i fixedpatch=0 |
| declare summarize=true |
| |
| if [[ ${multijdk} == true ]]; then |
| jdk=$(report_jvm_version "${JAVA_HOME}") |
| statusjdk=" with JDK v${jdk}" |
| jdk="-jdk${jdk}" |
| jdk=${jdk// /} |
| fi |
| |
| i=0 |
| until [[ ${i} -eq ${#MODULE[@]} ]]; do |
| if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then |
| ((result=result+1)) |
| ((i=i+1)) |
| continue |
| fi |
| |
| fn=$(module_file_fragment "${MODULE[${i}]}") |
| fn="${fn}${jdk}" |
| module_suffix=$(basename "${MODULE[${i}]}") |
| if [[ ${module_suffix} == \. ]]; then |
| module_suffix=root |
| fi |
| |
| yetus_debug "${testtype}: branch-${origlog}-${fn}.txt vs. patch-${origlog}-${fn}.txt" |
| |
| # if it was a new module, this won't exist. |
| if [[ ! -f "${PATCH_DIR}/branch-${origlog}-${fn}.txt" ]]; then |
| touch "${PATCH_DIR}/branch-${origlog}-${fn}.txt" |
| fi |
| |
| if [[ ! -f "${PATCH_DIR}/patch-${origlog}-${fn}.txt" ]]; then |
| touch "${PATCH_DIR}/patch-${origlog}-${fn}.txt" |
| fi |
| |
| generic_logfilter "${testtype}" "${PATCH_DIR}/branch-${origlog}-${fn}.txt" "${PATCH_DIR}/branch-${origlog}-${testtype}-${fn}.txt" |
| generic_logfilter "${testtype}" "${PATCH_DIR}/patch-${origlog}-${fn}.txt" "${PATCH_DIR}/patch-${origlog}-${testtype}-${fn}.txt" |
| |
| # shellcheck disable=SC2016 |
| numbranch=$(wc -l "${PATCH_DIR}/branch-${origlog}-${testtype}-${fn}.txt" | "${AWK}" '{print $1}') |
| # shellcheck disable=SC2016 |
| numpatch=$(wc -l "${PATCH_DIR}/patch-${origlog}-${testtype}-${fn}.txt" | "${AWK}" '{print $1}') |
| |
| calcdiffs \ |
| "${PATCH_DIR}/branch-${origlog}-${testtype}-${fn}.txt" \ |
| "${PATCH_DIR}/patch-${origlog}-${testtype}-${fn}.txt" \ |
| "${testtype}" \ |
| > "${PATCH_DIR}/diff-${origlog}-${testtype}-${fn}.txt" |
| |
| # shellcheck disable=SC2016 |
| addpatch=$(wc -l "${PATCH_DIR}/diff-${origlog}-${testtype}-${fn}.txt" | "${AWK}" '{print $1}') |
| |
| ((fixedpatch=numbranch-numpatch+addpatch)) |
| |
| statstring=$(generic_calcdiff_status "${numbranch}" "${numpatch}" "${addpatch}" ) |
| |
| if [[ ${addpatch} -gt 0 ]]; then |
| ((result = result + 1)) |
| module_status "${i}" -1 "diff-${origlog}-${testtype}-${fn}.txt" "${fn}${statusjdk} ${statstring}" |
| elif [[ ${fixedpatch} -gt 0 ]]; then |
| module_status "${i}" +1 "${MODULE_STATUS_LOG[${i}]}" "${fn}${statusjdk} ${statstring}" |
| summarize=false |
| fi |
| ((i=i+1)) |
| done |
| |
| modules_messages patch "${testtype}" "${summarize}" |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Generic post-patch handler |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @return 0 on success |
| ## @return 1 on failure |
| ## @param origlog |
| ## @param testtype |
| ## @param multijdkmode |
| ## @param run commands |
| function generic_post_handler |
| { |
| declare origlog=$1 |
| declare testtype=$2 |
| declare multijdkmode=$3 |
| declare need2run=$4 |
| declare i |
| declare result=0 |
| declare fn |
| declare -r savejavahome=${JAVA_HOME} |
| declare jdk="" |
| declare jdkindex=0 |
| declare statusjdk |
| declare -i numbranch=0 |
| declare -i numpatch=0 |
| |
| if ! verify_needed_test "${testtype}"; then |
| yetus_debug "${testtype} not needed" |
| return 0 |
| fi |
| |
| big_console_header "${testtype} verification: ${BUILDMODE}" |
| |
| for jdkindex in "${JDK_DIR_LIST[@]}"; do |
| if [[ ${multijdkmode} == true ]]; then |
| JAVA_HOME=${jdkindex} |
| yetus_debug "Using ${JAVA_HOME} to run this set of tests" |
| fi |
| |
| if [[ ${need2run} = true ]]; then |
| personality_modules "${codebase}" "${testtype}" |
| "${BUILDTOOL}_modules_worker" "${codebase}" "${testtype}" |
| |
| if [[ ${UNSUPPORTED_TEST} = true ]]; then |
| return 0 |
| fi |
| fi |
| |
| generic_postlog_compare "${origlog}" "${testtype}" "${multijdkmode}" |
| ((result=result+$?)) |
| done |
| JAVA_HOME=${savejavahome} |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Execute the compile phase. This will callout |
| ## @description to _compile |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branch|patch |
| ## @return 0 on success |
| ## @return 1 on failure |
| function compile_jvm |
| { |
| declare codebase=$1 |
| declare result=0 |
| declare -r savejavahome=${JAVA_HOME} |
| declare multijdkmode |
| declare jdkindex=0 |
| declare -a jdklist |
| |
| if verify_multijdk_test compile; then |
| multijdkmode=true |
| jdklist=("${JDK_DIR_LIST[@]}") |
| else |
| multijdkmode=false |
| jdklist=("${JAVA_HOME}") |
| fi |
| |
| for jdkindex in "${jdklist[@]}"; do |
| if [[ ${multijdkmode} == true ]]; then |
| JAVA_HOME=${jdkindex} |
| fi |
| |
| compile_nonjvm "${codebase}" "${multijdkmode}" |
| |
| done |
| JAVA_HOME=${savejavahome} |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Execute the compile phase. This will callout |
| ## @description to _compile |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branch|patch |
| ## @return 0 on success |
| ## @return 1 on failure |
| function compile_nonjvm |
| { |
| declare codebase=$1 |
| declare result=0 |
| declare -r savejavahome=${JAVA_HOME} |
| declare multijdkmode=${2:-false} |
| declare jdkindex=0 |
| |
| personality_modules "${codebase}" compile |
| "${BUILDTOOL}_modules_worker" "${codebase}" compile |
| modules_messages "${codebase}" compile true |
| |
| modules_backup |
| |
| for plugin in "${TESTTYPES[@]}"; do |
| modules_restore |
| verify_patchdir_still_exists |
| if declare -f "${plugin}_compile" >/dev/null 2>&1; then |
| yetus_debug "Running ${plugin}_compile ${codebase} ${multijdkmode}" |
| "${plugin}_compile" "${codebase}" "${multijdkmode}" |
| ((result = result + $?)) |
| archive |
| fi |
| done |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Execute the compile phase. This will callout |
| ## @description to _compile |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branch|patch |
| ## @return 0 on success |
| ## @return 1 on failure |
| function compile |
| { |
| declare codebase=$1 |
| |
| if ! verify_needed_test compile; then |
| return 0 |
| fi |
| |
| if [[ ${codebase} = "branch" ]]; then |
| big_console_header "${PATCH_BRANCH} compilation: pre-patch" |
| else |
| big_console_header "${PATCH_BRANCH} compilation: ${BUILDMODE}" |
| fi |
| |
| yetus_debug "Is JVM Required? ${JVM_REQUIRED}" |
| if [[ "${JVM_REQUIRED}" = true ]]; then |
| compile_jvm "${codebase}" |
| else |
| compile_nonjvm "${codebase}" |
| fi |
| } |
| |
| ## @description Execute the static analysis test cycle. |
| ## @description This will callout to _precompile, compile, _postcompile and _rebuild |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branch|patch |
| ## @return 0 on success |
| ## @return 1 on failure |
| function compile_cycle |
| { |
| declare codebase=$1 |
| declare result=0 |
| declare plugin |
| |
| find_changed_modules "${codebase}" |
| |
| for plugin in ${PROJECT_NAME} ${BUILDTOOL} "${TESTTYPES[@]}" "${TESTFORMATS[@]}"; do |
| if declare -f "${plugin}_precompile" >/dev/null 2>&1; then |
| yetus_debug "Running ${plugin}_precompile" |
| if ! "${plugin}_precompile" "${codebase}"; then |
| ((result = result+1)) |
| fi |
| archive |
| fi |
| done |
| |
| compile "${codebase}" |
| |
| for plugin in ${PROJECT_NAME} ${BUILDTOOL} "${TESTTYPES[@]}" "${TESTFORMATS[@]}"; do |
| if declare -f "${plugin}_postcompile" >/dev/null 2>&1; then |
| yetus_debug "Running ${plugin}_postcompile" |
| if ! "${plugin}_postcompile" "${codebase}"; then |
| ((result = result+1)) |
| fi |
| archive |
| fi |
| done |
| |
| for plugin in ${PROJECT_NAME} ${BUILDTOOL} "${TESTTYPES[@]}" "${TESTFORMATS[@]}"; do |
| if declare -f "${plugin}_rebuild" >/dev/null 2>&1; then |
| yetus_debug "Running ${plugin}_rebuild" |
| if ! "${plugin}_rebuild" "${codebase}"; then |
| ((result = result+1)) |
| fi |
| archive |
| fi |
| done |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Execute the patch file test phase. Calls out to |
| ## @description to _patchfile |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param branch|patch |
| ## @return 0 on success |
| ## @return 1 on failure |
| function patchfiletests |
| { |
| declare plugin |
| declare result=0 |
| |
| for plugin in ${BUILDTOOL} "${TESTTYPES[@]}" "${TESTFORMATS[@]}"; do |
| if declare -f "${plugin}_patchfile" >/dev/null 2>&1; then |
| yetus_debug "Running ${plugin}_patchfile" |
| if ! "${plugin}_patchfile" "${INPUT_APPLIED_FILE}"; then |
| ((result = result+1)) |
| fi |
| archive |
| fi |
| done |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| |
| ## @description Wipe the repo clean to not invalidate tests |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @return 0 on success |
| ## @return 1 on failure |
| function distclean |
| { |
| declare result=0 |
| declare plugin |
| |
| big_console_header "Cleaning the source tree" |
| |
| for plugin in "${TESTTYPES[@]}" "${TESTFORMATS[@]}"; do |
| if declare -f "${plugin}_clean" >/dev/null 2>&1; then |
| yetus_debug "Running ${plugin}_clean" |
| if ! "${plugin}_clean"; then |
| ((result = result+1)) |
| fi |
| fi |
| done |
| |
| personality_modules branch distclean |
| "${BUILDTOOL}_modules_worker" branch distclean |
| (( result = result + $? )) |
| |
| if [[ ${result} -gt 0 ]]; then |
| return 1 |
| fi |
| return 0 |
| } |
| |
| ## @description Start any coprocessors |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable yes |
| function start_coprocessors |
| { |
| |
| declare filename |
| |
| # Eventually, we might open this up for plugins |
| # and other operating environments |
| # but for now, this is private and only for us |
| |
| if [[ "${BASH_VERSINFO[0]}" -gt 3 ]]; then |
| |
| for filename in "${BINDIR}/coprocs.d"/*; do |
| # shellcheck disable=SC1091 |
| # shellcheck source=coprocs.d/process_counter.sh |
| . "${filename}" |
| done |
| |
| determine_user |
| |
| process_counter_coproc_start |
| |
| reaper_coproc_start |
| |
| fi |
| } |
| |
| ## @description Stop any coprocessors |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable yes |
| function stop_coprocessors |
| { |
| if [[ "${BASH_VERSINFO[0]}" -gt 3 ]]; then |
| # shellcheck disable=SC2154 |
| if [[ -n "${process_counter_coproc_PID}" ]]; then |
| # shellcheck disable=SC2086 |
| echo exit >&${process_counter_coproc[1]} |
| fi |
| |
| #shellcheck disable=SC2154 |
| if [[ -n "${reaper_coproc_PID}" ]]; then |
| # shellcheck disable=SC2086 |
| echo exit >&${reaper_coproc[1]} |
| fi |
| fi |
| } |
| |
| ## @description Additional setup work when in patch mode, including |
| ## @description setting ${INPUT_APPLIED_FILE} so that the system knows |
| ## @description which one to use because it will have passed dryrun. |
| ## @audience private |
| ## @stability evolving |
| ## @replaceable no |
| function patch_setup_work |
| { |
| |
| # from here on out, we'll be in ${BASEDIR} for cwd |
| # plugins need to pushd/popd if they change. |
| determine_issue |
| |
| if ! dryrun_both_files; then |
| ((RESULT = RESULT + 1)) |
| yetus_error "ERROR: ${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}." |
| add_vote_table -1 patch "${PATCH_OR_ISSUE} does not apply to ${PATCH_BRANCH}. Rebase required? Wrong Branch? See ${PATCH_NAMING_RULE} for help." |
| bugsystem_finalreport 1 |
| cleanup_and_exit 1 |
| fi |
| |
| if [[ "${ISSUE}" == 'Unknown' ]]; then |
| echo "" |
| echo "Testing ${INPUT_APPLY_TYPE} on ${PATCH_BRANCH}." |
| else |
| echo "" |
| echo "Testing ${ISSUE} ${INPUT_APPLY_TYPE} on ${PATCH_BRANCH}." |
| fi |
| } |
| |
| ## @description Setup to execute |
| ## @audience public |
| ## @stability evolving |
| ## @replaceable no |
| ## @param $@ |
| ## @return 0 on success |
| ## @return 1 on failure |
| function initialize |
| { |
| setup_defaults |
| |
| parse_args "$@" |
| |
| importplugins |
| |
| if [[ -z "${BUILDTOOL}" ]]; then |
| guess_build_tool |
| fi |
| |
| parse_args_plugins "$@" |
| |
| if declare -f personality_parse_args >/dev/null; then |
| personality_parse_args "$@" |
| fi |
| |
| BUGCOMMENTS=${BUGCOMMENTS:-"${BUGSYSTEMS[@]}"} |
| if [[ ! ${BUGCOMMENTS} =~ console ]]; then |
| BUGCOMMENTS="${BUGCOMMENTS} console" |
| fi |
| |
| if [[ "${BUGLINECOMMENTS}" == " " ]]; then |
| BUGLINECOMMENTS="" |
| else |
| BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}} |
| fi |
| |
| # we need to do this BEFORE plugins initialize |
| # because they may change what they do based upon |
| # docker support |
| # note that docker support still isn't guaranteed |
| # to be working even after this is executed here! |
| if declare -f docker_initialize >/dev/null; then |
| docker_initialize |
| fi |
| |
| if [[ "${DOCKERMODE}" != true ]]; then |
| echo "$$" > "${PATCH_DIR}/pidfile.txt" |
| fi |
| |
| plugins_initialize |
| if [[ ${RESULT} != 0 ]]; then |
| cleanup_and_exit 1 |
| fi |
| |
| echo "Modes: ${EXEC_MODES[*]}" |
| |
| if [[ "${BUILDMODE}" = patch ]]; then |
| locate_patch |
| fi |
| |
| git_checkout |
| |
| if [[ "${BUILDMODE}" = patch ]]; then |
| patch_setup_work |
| fi |
| |
| find_changed_files |
| |
| # re-verify that our dockerfile is still there (branch switch, etc) |
| # note that there is still a chance that docker mode will be |
| # disabled from here. Plug-ins should plan appropriately! |
| if declare -f docker_fileverify >/dev/null; then |
| docker_fileverify |
| fi |
| |
| check_reexec |
| |
| if [[ "${#PARAMETER_TRACKER}" -gt 0 ]]; then |
| yetus_error "ERROR: Unprocessed flag(s): ${PARAMETER_TRACKER[*]}" |
| if [[ "${IGNORE_UNKNOWN_OPTIONS}" == true ]]; then |
| add_vote_table "-0" yetus "Unprocessed flag(s): ${PARAMETER_TRACKER[*]}" |
| else |
| add_vote_table -1 yetus "Unprocessed flag(s): ${PARAMETER_TRACKER[*]}" |
| bugsystem_finalreport 1 |
| cleanup_and_exit 1 |
| fi |
| fi |
| |
| determine_needed_tests |
| |
| prepopulate_footer |
| } |
| |
| ## @description perform prechecks |
| ## @audience private |
| ## @stability evolving |
| ## @return exits on failure |
| function prechecks |
| { |
| declare plugin |
| declare result=0 |
| |
| for plugin in ${BUILDTOOL} "${NEEDED_TESTS[@]}" "${TESTFORMATS[@]}"; do |
| verify_patchdir_still_exists |
| |
| if declare -f "${plugin}_precheck" >/dev/null 2>&1; then |
| |
| yetus_debug "Running ${plugin}_precheck" |
| "${plugin}_precheck" |
| |
| (( result = result + $? )) |
| if [[ ${result} != 0 ]] ; then |
| bugsystem_finalreport 1 |
| cleanup_and_exit 1 |
| fi |
| fi |
| done |
| } |
| |
| ## @description import core library routines |
| ## @audience private |
| ## @stability evolving |
| function import_core |
| { |
| declare filename |
| |
| for filename in "${BINDIR}/core.d"/*; do |
| # shellcheck disable=SC1091 |
| # shellcheck source=core.d/00-yetuslib.sh |
| # shellcheck source=core.d/01-common.sh |
| . "${filename}" |
| done |
| } |
| |
| ## @description setup the parameter tracker for param errors |
| ## @audience private |
| ## @stability evolving |
| function setup_parameter_tracker |
| { |
| declare i |
| |
| for i in "${USER_PARAMS[@]}"; do |
| if [[ "${i}" =~ ^-- ]]; then |
| i=${i%=*} |
| PARAMETER_TRACKER+=("${i}") |
| fi |
| done |
| } |
| |
| ############################################################################### |
| ############################################################################### |
| ############################################################################### |
| |
| # robots will change USER_PARAMS so must |
| # do this before importing other code |
| setup_parameter_tracker |
| |
| import_core |
| |
| if [[ "${BINNAME}" =~ qbt ]]; then |
| initialize --empty-patch "$@" |
| else |
| initialize "$@" |
| fi |
| |
| |
| if [[ ${BASH_VERSINFO[0]} -gt 3 ]]; then |
| yetus_debug "Starting coprocessors" |
| |
| # we need to catch out and err bz the coproc |
| # command is extremely noisy on both startup |
| # and shutdown |
| start_coprocessors >> "${COPROC_LOGFILE}" 2>&1 |
| else |
| |
| # If we aren't using bash4 (e.g. OS X), then set the ulimit now. |
| # bash4 gets it set in an (on demand) coprocessor |
| |
| ulimit -Su "${PROC_LIMIT}" |
| yetus_debug "Changed process/Java native thread limit to ${PROC_LIMIT}" |
| fi |
| |
| add_vote_table H "Prechecks" |
| |
| prechecks |
| |
| if [[ "${BUILDMODE}" = patch ]]; then |
| |
| patchfiletests |
| |
| add_vote_table H "${PATCH_BRANCH} Compile Tests" |
| |
| compile_cycle branch |
| |
| distclean |
| |
| apply_patch_file |
| |
| exclude_paths_from_changed_files |
| |
| compute_gitdiff |
| |
| add_vote_table H "Patch Compile Tests" |
| |
| else |
| |
| add_vote_table H "Compile Tests" |
| |
| fi |
| |
| compile_cycle patch |
| |
| add_vote_table H "Other Tests" |
| |
| runtests |
| |
| stop_coprocessors |
| |
| finish_vote_table |
| |
| finish_footer_table |
| |
| bugsystem_finalreport ${RESULT} |
| cleanup_and_exit ${RESULT} |
