| #!/usr/bin/env bash |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # This is a Yetus precommit "personality" (aka customized configuration) for Lucene/Solr. |
| # |
| # See the Yetus precommit documentation at https://yetus.apache.org/documentation/0.7.0/ |
| # and especially https://yetus.apache.org/documentation/0.7.0/precommit-advanced/. |
| # See also the Yetus source code for other projects' personality examples at |
| # https://git-wip-us.apache.org/repos/asf?p=yetus.git;f=precommit/personality;a=tree;hb=HEAD |
| # |
| # To add a new validation method (aka "plugin"): |
| # 1) Add its name to the PLUGIN_LIST below |
| # 2) Invoke "add_test_type" with it below |
| # 3) Add a "<plugin>_filefilter" function to decide whether the plugin needs to be run based on changed files |
| # 4) Add a "<plugin>_rebuild" function to call out to ant to perform the validation method. |
| # See examples of the above-described function types ^^ below. |
| |
| # Both compile+javac plugins are required, as well as unit+junit: in both cases, neither work individually |
| PLUGIN_LIST="ant,jira,compile,javac,unit,junit,test4tests" |
| PLUGIN_LIST+=",testoutput,checkluceneversion,ratsources,checkforbiddenapis,checklicenses" |
| PLUGIN_LIST+=",validatesourcepatterns,validaterefguide" |
| personality_plugins "${PLUGIN_LIST}" |
| |
| add_test_type "checkluceneversion" |
| add_test_type "ratsources" |
| add_test_type "checkforbiddenapis" |
| add_test_type "checklicenses" |
| add_test_type "validatesourcepatterns" |
| add_test_type "validaterefguide" |
| |
| add_test_format "testoutput" |
| |
| ## @description Globals specific to this personality |
| ## @audience private |
| ## @stability evolving |
| function personality_globals |
| { |
| #shellcheck disable=SC2034 |
| PATCH_BRANCH_DEFAULT=master |
| #shellcheck disable=SC2034 |
| JIRA_ISSUE_RE='^(LUCENE|SOLR)-[0-9]+$' |
| #shellcheck disable=SC2034 |
| JIRA_STATUS_RE='Patch Available' |
| #shellcheck disable=SC2034 |
| GITHUB_REPO="apache/lucene-solr" |
| #shellcheck disable=SC2034 |
| BUILDTOOL=ant |
| } |
| |
| ## @description Queue up modules for this personality |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| ## @param testtype |
| function personality_modules |
| { |
| local repostatus=$1 |
| local testtype=$2 |
| |
| local module |
| local extra |
| |
| local moduleType="submodules" |
| |
| yetus_debug "Personality (lucene-solr): ${repostatus} ${testtype}" |
| |
| clear_personality_queue |
| |
| case ${testtype} in |
| clean|distclean|validatesourcepatterns) |
| moduleType="top" |
| ;; |
| checkluceneversion) |
| moduleType="solr" |
| ;; |
| ratsources) |
| moduleType="submodules" |
| ;; |
| checkforbiddenapis) |
| moduleType="both" |
| ;; |
| checklicenses) |
| moduleType="mains" |
| ;; |
| validaterefguide) |
| moduleType="solr-ref-guide" |
| ;; |
| compile) |
| moduleType="submodules" |
| extra="compile-test" |
| ;; |
| junit|unit) |
| moduleType="submodules" |
| extra="-Dtests.badapples=false test" |
| ;; |
| *) |
| ;; |
| esac |
| |
| case ${moduleType} in |
| submodules) |
| for module in "${CHANGED_MODULES[@]}"; do |
| if [[ ! "${module}" =~ ^lucene/(licenses|site) ]]; then # blacklist lucene/ dirs that aren't modules |
| if [[ "${module}" =~ ^(lucene/(analysis/[^/]+|[^/]+)) ]]; then |
| local lucene_module=${BASH_REMATCH[0]} |
| personality_enqueue_module "${lucene_module}" "${extra}" |
| elif [[ "${module}" =~ ^solr/(core|solrj|test-framework|solr-ref-guide|contrib/[^.]+) ]]; then # whitelist solr/ modules |
| local solr_module=${BASH_REMATCH[0]} |
| # In solr-ref-guide module, do not execute "compile" or "unit" plugins |
| if [[ ! "${solr_module}" == solr/solr-ref-guide || ! ${testtype} =~ ^(compile|unit)$ ]]; then |
| personality_enqueue_module "${solr_module}" "${extra}" |
| fi |
| fi |
| fi |
| done |
| ;; |
| lucene|solr) |
| personality_enqueue_module "${moduleType}" "${extra}" |
| ;; |
| top) |
| personality_enqueue_module . "${extra}" |
| ;; |
| mains) |
| personality_enqueue_module "lucene" "${extra}" |
| personality_enqueue_module "solr" "${extra}" |
| ;; |
| both) # solr, lucene, or both |
| # personality_enqueue_module KEEPS duplicates, so de-dupe first |
| local doSolr=0,doLucene=0 |
| for module in "${CHANGED_MODULES[@]}"; do |
| if [[ "${module}" =~ ^solr/ ]]; then doSolr=1; fi |
| if [[ "${module}" =~ ^lucene/ ]]; then doLucene=1; fi |
| done |
| if [[ ${doLucene} == 1 ]]; then |
| if [[ ${doSolr} == 1 ]]; then |
| personality_enqueue_module . "${extra}" |
| else |
| personality_enqueue_module "lucene" "${extra}" |
| fi |
| elif [[ ${doSolr} == 1 ]]; then |
| personality_enqueue_module "solr" "${extra}" |
| fi |
| ;; |
| solr-ref-guide) |
| for module in "${CHANGED_MODULES[@]}"; do |
| if [[ "${module}" =~ ^solr/solr-ref-guide ]]; then |
| personality_enqueue_module "solr/solr-ref-guide" "${extra}" |
| fi |
| done |
| ;; |
| *) |
| ;; |
| esac |
| } |
| |
| ## @description Add tests based upon personality needs |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function personality_file_tests |
| { |
| declare filename=$1 |
| |
| yetus_debug "Using Lucene/Solr-specific personality_file_tests" |
| |
| if [[ ! ${filename} =~ solr-ref-guide ]]; then |
| if [[ ${filename} =~ build\.xml$ || ${filename} =~ /src/(java|resources|test|test-files|tools) ]]; then |
| yetus_debug "tests/unit: ${filename}" |
| add_test compile |
| add_test javac |
| add_test unit |
| fi |
| fi |
| } |
| |
| ## @description hook to reroute junit folder to search test results based on the module |
| ## @audience private |
| ## @stability evolving |
| ## @param module |
| ## @param buildlogfile |
| function testoutput_process_tests |
| { |
| # shellcheck disable=SC2034 |
| declare module=$1 |
| declare buildlogfile=$2 |
| if [[ "${module}" =~ ^lucene/analysis/ ]]; then |
| JUNIT_TEST_OUTPUT_DIR="../../build/${module#*/}" |
| elif [[ "${module}" =~ ^lucene/ ]]; then |
| JUNIT_TEST_OUTPUT_DIR="../build/${module#*/}" |
| elif [[ "${module}" =~ ^solr/contrib/extraction ]]; then |
| JUNIT_TEST_OUTPUT_DIR="../../build/contrib/solr-cell" |
| elif [[ "${module}" =~ ^solr/contrib/(.*) ]]; then |
| JUNIT_TEST_OUTPUT_DIR="../../build/contrib/solr-${BASH_REMATCH[1]}" |
| elif [[ "${module}" =~ ^solr/(.*) ]]; then |
| JUNIT_TEST_OUTPUT_DIR="../build/solr-${BASH_REMATCH[1]}" |
| fi |
| yetus_debug "Rerouting build dir for junit to ${JUNIT_TEST_OUTPUT_DIR}" |
| } |
| |
| ## @description checkluceneversion file filter |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function checkluceneversion_filefilter |
| { |
| local filename=$1 |
| if [[ ${filename} =~ ^solr/(example|server/solr/configsets) ]]; then |
| yetus_debug "tests/checkluceneversion: ${filename}" |
| add_test checkluceneversion |
| fi |
| } |
| |
| ## @description checkluceneversion test |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| function checkluceneversion_rebuild |
| { |
| local repostatus=$1 |
| lucene_ant_command ${repostatus} "checkluceneversion" "check-example-lucene-match-version" "Check configsets' lucene version" |
| } |
| |
| ## @description ratsources file filter |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function ratsources_filefilter |
| { |
| local filename=$1 |
| if [[ ${filename} =~ /src/|\.xml$ ]] ; then |
| yetus_debug "tests/ratsources: ${filename}" |
| add_test ratsources |
| fi |
| } |
| |
| ## @description ratsources test |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| function ratsources_rebuild |
| { |
| local repostatus=$1 |
| lucene_ant_command ${repostatus} "ratsources" "rat-sources" "Release audit (RAT)" |
| } |
| |
| ## @description checkforbiddenapis file filter |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function checkforbiddenapis_filefilter |
| { |
| local filename=$1 |
| if [[ ${filename} =~ \.java$ ]] ; then |
| yetus_debug "tests/checkforbiddenapis: ${filename}" |
| add_test checkforbiddenapis |
| fi |
| } |
| |
| ## @description checkforbiddenapis test |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| function checkforbiddenapis_rebuild |
| { |
| local repostatus=$1 |
| lucene_ant_command ${repostatus} "checkforbiddenapis" "check-forbidden-apis" "Check forbidden APIs" |
| } |
| |
| ## @description checklicenses file filter |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function checklicenses_filefilter |
| { |
| local filename=$1 |
| if [[ ${filename} =~ (lucene|solr)/licenses/|lucene/ivy-versions.properties$ ]]; then |
| yetus_debug "tests/checklicenses: ${filename}" |
| add_test checklicenses |
| fi |
| } |
| |
| ## @description checklicenses test |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| function checklicenses_rebuild |
| { |
| local repostatus=$1 |
| lucene_ant_command ${repostatus} "checklicenses" "check-licenses" "Check licenses" |
| } |
| |
| ## @description validaterefguide file filter |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function validaterefguide_filefilter |
| { |
| local filename=$1 |
| if [[ ${filename} =~ solr/solr-ref-guide ]]; then |
| yetus_debug "tests/validaterefguide: ${filename}" |
| add_test validaterefguide |
| fi |
| } |
| |
| ## @description validaterefguide test |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| function validaterefguide_rebuild |
| { |
| local repostatus=$1 |
| lucene_ant_command ${repostatus} "validaterefguide" "bare-bones-html-validation" "Validate ref guide" |
| } |
| |
| ## @description validatesourcepatterns file filter |
| ## @audience private |
| ## @stability evolving |
| ## @param filename |
| function validatesourcepatterns_filefilter |
| { |
| local filename=$1 |
| if [[ ${filename} =~ \.(java|jflex|py|pl|g4|jj|html|js|css|xml|xsl|vm|sh|cmd|bat|policy|properties|mdtext|groovy|template|adoc|json)$ ]]; then |
| yetus_debug "tests/validatesourcepatterns: ${filename}" |
| add_test validatesourcepatterns |
| fi |
| } |
| |
| ## @description validatesourcepatterns test |
| ## @audience private |
| ## @stability evolving |
| ## @param repostatus |
| function validatesourcepatterns_rebuild |
| { |
| local repostatus=$1 |
| lucene_ant_command ${repostatus} "validatesourcepatterns" "validate-source-patterns" "Validate source patterns" |
| } |
| |
| function lucene_ant_command |
| { |
| declare repostatus=$1 |
| declare testname=$2 |
| declare antcommand=$3 |
| declare title=$4 |
| |
| declare result=0 |
| declare i=0 |
| declare module |
| declare fn |
| declare result |
| |
| if [[ "${repostatus}" = branch ]]; then |
| return 0 |
| fi |
| |
| if ! verify_needed_test ${testname}; then |
| echo "${BUILDMODEMSG} does not need ${testname} testing." |
| return 0 |
| fi |
| |
| big_console_header "${title}" |
| personality_modules ${repostatus} ${testname} |
| until [[ ${i} -eq ${#MODULE[@]} ]]; do |
| if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then |
| ((result=result+1)) |
| ((i=i+1)) |
| continue |
| fi |
| ANT_ARGS=${antcommand} |
| |
| start_clock |
| module=${MODULE[${i}]} |
| fn=$(module_file_fragment "${module}") |
| logfilename="${repostatus}-${antcommand}-${fn}.txt"; |
| logfile="${PATCH_DIR}/${logfilename}" |
| buildtool_cwd "${i}" |
| echo_and_redirect "${logfile}" $(ant_executor) |
| |
| if [[ $? == 0 ]] ; then |
| module_status ${i} +1 "${logfilename}" "${title}" "${antcommand} passed" |
| else |
| module_status ${i} -1 "${logfilename}" "${title}" "${antcommand} failed" |
| ((result=result+1)) |
| fi |
| ((i=i+1)) |
| savestop=$(stop_clock) |
| MODULE_STATUS_TIMER[${i}]=${savestop} |
| done |
| ANT_ARGS="" |
| if [[ ${result} -gt 0 ]]; then |
| modules_messages ${repostatus} "${title}" false |
| return 1 |
| fi |
| modules_messages ${repostatus} "${title}" true |
| return 0 |
| } |