| 'use strict'; |
| |
| const fs = require('fs'); |
| const isString = require('is-string'); |
| const jsonfile = require('jsonfile'); |
| const mkdirp = require('mkdirp'); |
| const parseXml = require('xml2js').parseString; |
| const path = require('path'); |
| const process = require('process'); |
| const request = require('request'); |
| |
| const apiXml = 'api/xml'; |
| let baseDir; |
| |
| function init() { |
| if (!process.env.JENKINS_LOGS_DIR) { |
| console.log('WARNING: JENKINS_LOGS_DIR is not set.'); |
| } |
| baseDir = process.env.JENKINS_LOGS_DIR || __dirname; |
| console.log('Will write logs to', baseDir); |
| } |
| |
| |
| function crawlFromRootUrl(url) { |
| console.log(` |
| ================================================================================ |
| `); |
| console.log(new Date()); |
| init(); |
| request(url, function(err, message, body) { |
| if (err) { |
| return exitOnError(err); |
| } |
| |
| parseXml(body, function (err, result) { |
| if (err) { |
| return exitOnError(err); |
| } |
| |
| if (!result.matrixProject) { |
| return exitOnError(new Error('Not found: matrixProject')); |
| } |
| if (!result.matrixProject.build) { |
| return exitOnError(new Error('Not found: matrixProject.build')); |
| } |
| if (!Array.isArray(result.matrixProject.build)) { |
| return exitOnError(new Error('Not an array: matrixProject.build')); |
| } |
| result.matrixProject.build.forEach(build => { |
| fetchBuild(build); |
| }); |
| }); |
| }); |
| } |
| |
| |
| function fetchBuild(build) { |
| if (build.number == null) { |
| return printWarning(new Error('Not found: build.number')); |
| } |
| if (build.url == null) { |
| return printWarning(new Error('Not found: build.url')); |
| } |
| |
| const url = build.url + apiXml; |
| console.log('Retrieving meta data for build', build.number[0], 'from ', url); |
| request(url, function(err, message, body) { |
| if (err) { |
| return printWarning(err); |
| } |
| parseXml(body, function (err, result) { |
| if (err) { |
| return printWarning(err); |
| } |
| |
| if (!result.matrixBuild) { |
| return printWarning(new Error('Not found: matrixBuild')); |
| } |
| if (!result.matrixBuild.run) { |
| return printWarning(new Error('Not found: matrixBuild.run')); |
| } |
| if (!Array.isArray(result.matrixBuild.run)) { |
| return printWarning(new Error('Not an array: matrixBuild.run')); |
| } |
| result.matrixBuild.run.forEach(run => { |
| fetchRun(run); |
| }); |
| }); |
| }); |
| } |
| |
| |
| function fetchRun(run) { |
| if (run.number == null) { |
| return printWarning(new Error('Not found: run.number')); |
| } |
| if (run.url == null) { |
| return printWarning(new Error('Not found: run.url')); |
| } |
| |
| const url = run.url + apiXml; |
| console.log('Retrieving meta data for run', run.number[0], 'from ', url); |
| request(url, function(err, message, body) { |
| if (err) { |
| return printWarning(err); |
| } |
| parseXml(body, function (err, result) { |
| if (err) { |
| return printWarning(err); |
| } |
| |
| if (!result.matrixRun) { |
| return printWarning(new Error('Not found: matrixRun')); |
| } |
| if (!result.matrixRun.number) { |
| return printWarning(new Error('Not found: matrixRun.number')); |
| } |
| if (!result.matrixRun.fullDisplayName) { |
| return printWarning(new Error('Not found: matrixRun.fullDisplayName')); |
| } |
| if (!Array.isArray(result.matrixRun.fullDisplayName)) { |
| return printWarning(new Error('Not an array: matrixRun.fullDisplayName')); |
| |
| } |
| if (result.matrixRun.fullDisplayName.length !== 1) { |
| return printWarning(new Error('Not an array of length 1: matrixRun.fullDisplayName')); |
| } |
| if (!isString(result.matrixRun.fullDisplayName[0])) { |
| return printWarning(new Error('Not a string: matrixRun.fullDisplayName[0]')); |
| } |
| |
| fetchMatrixRunAndLog(result.matrixRun, run.url); |
| }); |
| }); |
| } |
| |
| |
| function fetchMatrixRunAndLog(metaData, runBaseUrl) { |
| const logUrl = runBaseUrl + 'consoleText'; |
| request(logUrl, function(err, message, body) { |
| if (err) { |
| return printWarning(err); |
| } |
| saveToDisk(metaData, body); |
| }); |
| } |
| |
| |
| function saveToDisk(metaData, buildLog) { |
| const dir = path.join(baseDir, String(metaData.number), metaData.fullDisplayName[0]); |
| mkdirp.sync(dir); |
| const metaDataFile = path.join(dir, 'metadata.json'); |
| jsonfile.writeFile(metaDataFile, metaData, { spaces: 2 }, function (err) { |
| if (err) { |
| return printWarning(err); |
| } |
| console.log('Written meta data to ', metaDataFile); |
| }); |
| const buildLogFile = path.join(dir, 'build.log'); |
| fs.writeFile(buildLogFile, buildLog, function(err) { |
| if (err) { |
| return printWarning(err); |
| } |
| console.log('Written build log to ', buildLogFile); |
| }); |
| } |
| |
| |
| function exitOnError(error) { |
| console.error(error); |
| process.exit(1); |
| } |
| |
| |
| function printWarning(error) { |
| console.error(error); |
| } |
| |
| |
| const rootUrl = 'https://builds.apache.org/job/CouchDB/api/xml'; |
| |
| crawlFromRootUrl(rootUrl); |