blob: 566b9d0e2dbf328d446ec2c4bc69dd5e3cd4e5e5 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* Start of util methods.
*/
def envVariable = { name, defaultVar ->
def res = System.getenv(name as String)
if (res == 'null' || res == null)
return defaultVar
res
}
def envVariableAsList = { name, defaultList ->
def list = System.getenv(name as String)?.split(' ') as List
if (list == 'null' || list == null)
return defaultList
list
}
/**
* Monitors given process and show errors if exist.
*/
def checkprocess = { process ->
process.waitFor()
if (process.exitValue() != 0) {
println "Return code: " + process.exitValue()
// println "Errout:\n" + process.err.text
assert process.exitValue() == 0 || process.exitValue() == 128
}
}
def exec = {command, envp, dir ->
println "Executing command '$command'..."
def ps = command.execute(envp, dir)
try {
println "Command output:"
println ps.text
}
catch (Throwable e) {
// Do nothing.
println "Error: could not get caommand output."
}
checkprocess ps
}
def execGit = {command ->
exec(command, null, new File("../"))
}
/**
* Util method to send http request.
*/
def sendHttpRequest = { requestMethod, urlString, user, pwd, postData, contentType ->
URL url = new URL(urlString as String);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
String encoded = "$user:$pwd".getBytes().encodeBase64().toString();
conn.setRequestProperty("Authorization", "Basic " + encoded);
conn.setDoOutput(true);
conn.setRequestMethod(requestMethod);
if (postData) {
conn.setRequestProperty("Content-Type", contentType);
conn.setRequestProperty("Content-Length", String.valueOf(postData.length()));
OutputStream os = conn.getOutputStream();
os.write(postData.getBytes());
os.flush();
os.close();
}
conn.connect();
// Read response.
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String response = "";
String line;
while ((line = br.readLine()) != null)
response += line
br.close();
response
}
/**
* Util method to send http POST request.
*/
def sendPostRequest = { urlString, user, pwd, postData, contentType ->
sendHttpRequest("POST", urlString, user, pwd, postData, contentType);
}
/**
* Util method to send http GET request.
*/
def sendGetRequest = { urlString, user, pwd->
sendHttpRequest("GET", urlString, user, pwd, null, null);
}
/*
* End of util methods.
*/
/**
* Parsing a special filter from Apache Ignite JIRA and picking up latest by ID
* attachments to process.
*/
final GIT_REPO = "https://git1-us-west.apache.org/repos/asf/ignite.git"
final JIRA_URL = "https://issues.apache.org"
final ATTACHMENT_URL = "$JIRA_URL/jira/secure/attachment"
final HISTORY_FILE = "${System.getProperty("user.home")}/validated-jira.txt"
final LAST_SUCCESSFUL_ARTIFACT = "guestAuth/repository/download/Ignite_PatchValidation_PatchChecker/.lastSuccessful/$HISTORY_FILE"
final NL = System.getProperty("line.separator")
final def JIRA_CMD = System.getProperty('JIRA_COMMAND', 'jira.sh')
// Envariement variables.
final def TC_PROJECT_NAME = envVariable("PROJECT_NAME", "Ignite")
final def TC_BUILD_EXCLUDE_LIST = envVariableAsList("BUILD_ID_EXCLUDES", ["Ignite_RunAllTestBuilds"])
final def JIRA_USER = System.getenv('JIRA_USER')
final def JIRA_PWD = System.getenv('JIRA_PWD')
final def CONTRIBUTORS = []
def contributors = {
if (!CONTRIBUTORS) {
def response = sendGetRequest(
"$JIRA_URL/jira/rest/api/2/project/12315922/role/10010" as String,
JIRA_USER,
JIRA_PWD)
println "Response on contributors request = $response"
def json = new groovy.json.JsonSlurper().parseText(response)
json.actors.each {
CONTRIBUTORS.add(it.name)
}
println "Contributors list: $CONTRIBUTORS"
}
CONTRIBUTORS
}
/**
* Gets jiras for which test tasks were already triggered.
*
* @return List of pairs [jira,attachemntId].
*/
def readHistory = {
final int MAX_HISTORY = 5000
List validated_list = []
def validated = new File(HISTORY_FILE)
if (validated.exists()) {
validated_list = validated.text.split('\n')
}
// Let's make sure the preserved history isn't too long
if (validated_list.size > MAX_HISTORY) {
validated_list = validated_list[validated_list.size - MAX_HISTORY..validated_list.size - 1]
validated.delete()
validated << validated_list.join(NL)
}
println "History=$validated_list"
validated_list
}
/**
* Accepting the <jira> XML element from JIRA stream
* @return <code>null</code> or <code>JIRA-###,latest_attach_id</code>
*/
def getLatestAttachment = { jira ->
def attachment = jira.attachments[0].attachment.list()
.sort { it.@id.toInteger() }
.reverse()
.find {
def fName = it.@name.toString()
contributors().contains(it.@author as String) &&
(fName.endsWith(".patch") || fName.endsWith(".txt") || fName.endsWith(".diff"))
}
String row = null
if (attachment == null) {
println "${jira.key} is in invalid state: there was not found '.{patch/txt/diff}'-file from approved user."
}
else {
row = "${jira.key},${attachment.@id}"
}
}
/**
* Checks all "Patch availiable" jiras on attached ".patch"-files from approved users.
*/
def findAttachments = {
// See https://issues.apache.org/jira/issues/?filter=12330308 (the same).
def JIRA_FILTER =
"https://issues.apache.org/jira/sr/jira.issueviews:searchrequest-xml/12330308/SearchRequest-12330308.xml?tempMax=100&field=key&field=attachments"
def rss = new XmlSlurper().parse(JIRA_FILTER)
final List history = readHistory {}
LinkedHashMap<String, String> attachments = [:]
rss.channel.item.each { jira ->
String row = getLatestAttachment(jira)
if (row != null && !history.contains(row)) {
def pair = row.split(',')
attachments.put(pair[0] as String, pair[1] as String)
}
}
attachments
}
/**
* Store jira with attachment id to hostory.
*/
def addToHistory = {jira, attachmentId ->
def validated = new File(HISTORY_FILE)
assert validated.exists(), "History file does not exist."
validated << NL + "$jira,$attachmentId"
}
def tryGitAmAbort = {
try {
checkprocess "git am --abort".execute(null, new File("../"))
println "Succsessfull: git am --abort."
}
catch (Throwable e) {
println "Error: git am --abort fails: "
}
}
/**
* Applys patch from jira to given git state.
*/
def applyPatch = { jira, attachementURL ->
// Delete all old IGNITE-*-*.patch files.
def directory = new File("./")
println "Remove IGNITE-*-*.patch files in ${directory.absolutePath} and its subdirectories..."
def classPattern = ~/.*IGNITE-.*-.*\.patch/
directory.eachFileRecurse(groovy.io.FileType.FILES)
{ file ->
if (file ==~ classPattern){
println "Deleting ${file}..."
file.delete()
}
}
// Main logic.
println "Patch apllying with jira='$jira' and attachment='$ATTACHMENT_URL/$attachementURL/'."
def userEmail = System.getenv("env.GIT_USER_EMAIL");
def userName = System.getenv("env.GIT_USER_NAME");
def patchFile = new File("${jira}-${attachementURL}.patch")
println "Getting patch content."
def attachmentUrl = new URL("$ATTACHMENT_URL/$attachementURL/")
HttpURLConnection conn = (HttpURLConnection)attachmentUrl.openConnection();
conn.setRequestProperty("Content-Type", "text/x-patch;charset=utf-8");
conn.setRequestProperty("X-Content-Type-Options", "nosniff");
conn.connect();
patchFile << conn.getInputStream()
println "Got patch content."
try {
tryGitAmAbort()
execGit "git branch"
execGit "git config user.email \"$userEmail\""
execGit "git config user.name \"$userName\""
// Create a new uniqueue branch to applying patch
def newTestBranch = "test-branch-${jira}-${attachementURL}-${System.currentTimeMillis()}"
execGit "git checkout -b ${newTestBranch}"
execGit "git branch"
println "Trying to apply patch."
execGit "git am dev-tools/${patchFile.name}"
println "Patch was applied successfully."
}
catch (Throwable e) {
println "Patch was not applied successfully. Aborting patch applying."
tryGitAmAbort()
throw e;
}
}
def JIRA_xml = { jiranum ->
"https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$jiranum/${jiranum}.xml"
}
/**
* Gets all builds from TC project.
*/
def getTestBuilds = { ->
def tcURL = System.getenv('TC_URL')
def project = new XmlSlurper().parse("http://$tcURL:80/guestAuth/app/rest/projects/id:$TC_PROJECT_NAME")
def buildIds = []
def count = Integer.valueOf(project.buildTypes.@count as String)
for (int i = 0; i < count; i++) {
def id = project.buildTypes.buildType[i].@id
if (TC_BUILD_EXCLUDE_LIST == null || !TC_BUILD_EXCLUDE_LIST.contains(id))
buildIds.add(id)
}
buildIds
}
/**
* Adds comment to jira ticket.
*/
def addJiraComment = { jiraNum, comment ->
try {
println "Comment: $comment"
def jsonComment = "{\n \"body\": \"${comment}\"\n}";
def response = sendPostRequest(
"$JIRA_URL/jira/rest/api/2/issue/$jiraNum/comment" as String,
JIRA_USER,
JIRA_PWD,
jsonComment,
"application/json")
println "Response: $response"
}
catch (Exception e) {
e.printStackTrace()
}
}
/**
* Runs all given test builds to validate last patch from given jira.
*/
def runAllTestBuilds = {builds, jiraNum ->
def tcURL = System.getenv('TC_URL')
def user = System.getenv('TASK_RUNNER_USER')
def pwd = System.getenv('TASK_RUNNER_PWD')
def triggeredBuilds = [:]
builds.each {
try {
println "Triggering $it build for $jiraNum jira..."
String postData
if (jiraNum == 'null' || jiraNum == null) {
postData = "<build>" +
" <buildType id='$it'/>" +
"</build>";
}
else {
postData = "<build>" +
" <buildType id='$it'/>" +
" <comment>" +
" <text>Auto triggered build to validate last attached patch file at $jiraNum.</text>" +
" </comment>" +
" <properties>" +
" <property name='env.JIRA_NUM' value='$jiraNum'/>" +
" </properties>" +
"</build>";
}
println "Request: $postData"
def response = sendPostRequest(
"http://$tcURL:80/httpAuth/app/rest/buildQueue" as String,
user,
pwd,
postData,
"application/xml")
println "Response: $response"
def build = new XmlSlurper().parseText(response)
triggeredBuilds.put(build.buildType.@name, build.@webUrl)
}
catch (Exception e) {
e.printStackTrace()
}
}
// Format comment for jira.
def triggeredBuildsComment = "There was triggered next test builds for last attached patch-file:\\n"
def n = 1;
triggeredBuilds.each { name, url ->
def prefix = n < 10 ? "0" : ""
triggeredBuildsComment += "${prefix}${n}. ${url as String} - ${name as String}\\n"
n++
}
addJiraComment(jiraNum, triggeredBuildsComment)
}
/**
* Main.
*
* Modes:
* 1. "slurp" mode - triggers all TC test builds for all jiras with valid attachment
* (Jira in "patch availiable" state, there is attached file from approved user with "patch" extension)
* 2. "patchApply" mode - gets last valid patch file from given jira number and applies it.
* 3. "runAllBuilds" - triggers given jira number for all TC test builds.
*
* Main workflow:
* 1. run in "slurp" mode
* 2. each triggered build uses "patchApply" mode to apply latest valid patch.
*/
args.each {
println "Arg=$it"
def parameters = it.split(",")
println parameters
if (parameters.length >= 1 && parameters[0] == "slurp") {
println "Running in 'slurp' mode."
def builds = getTestBuilds()
println "Test builds to be triggered=$builds"
def attachments = findAttachments()
// For each ticket with new attachment, let's trigger remove build
attachments.each { k, v ->
// Trailing slash is important for download; only need to pass JIRA number
println "Triggering the test builds for: $k = $ATTACHMENT_URL/$v/"
runAllTestBuilds(builds, k)
addToHistory(k, v)
}
}
else if (parameters.length >= 1 && parameters[0] == "patchApply") {
if (parameters.length < 2 || parameters[1] == 'null') {
println "There is no jira number to apply. Exit."
return
}
def jiraNum = parameters[1]
println "Running in 'patch apply' mode with jira number '$jiraNum'"
// Extract JIRA rss from the and pass the ticket element into attachment extraction
def rss = new XmlSlurper().parse(JIRA_xml(parameters[1]))
String row = getLatestAttachment(rss.channel.item)
println "Got row: $row."
if (row != null) {
def pair = row.split(',')
def jira = pair[0]
def attachementURL = pair[1]
applyPatch(jira, attachementURL)
}
}
else if (parameters.length > 1 && parameters[0] == "runAllBuilds" ) {
def jiraNum = parameters[1]
println "Running in 'all builds' mode with jira number='$jiraNum'."
def builds = getTestBuilds()
println "Test builds to be triggered=$builds"
runAllTestBuilds(builds, jiraNum)
}
}