Repository: couchdb-ci Updated Branches: refs/heads/master fec584319 -> 045028834
Node.js scripts to download all Jenkins logs and analyze them. Project: http://git-wip-us.apache.org/repos/asf/couchdb-ci/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-ci/commit/04502883 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-ci/tree/04502883 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-ci/diff/04502883 Branch: refs/heads/master Commit: 045028834477dce9c4e26d6909896f0c64ef4d44 Parents: fec5843 Author: Bastian Krol <[email protected]> Authored: Thu Feb 18 14:24:48 2016 +0100 Committer: Bastian Krol <[email protected]> Committed: Sat Feb 27 17:48:28 2016 +0100 ---------------------------------------------------------------------- scraping-jenkins/.gitignore | 3 - scraping-jenkins/jobnames.txt | 13 -- scraping-jenkins/readme.markdown | 4 - scraping-jenkins/rename.sh | 19 -- scraping-jenkins/scrape.sh | 36 ---- utils/analyze-jenkins-logs/.gitignore | 1 + .../bin/analyze-jenkins-logs | 3 + utils/analyze-jenkins-logs/bin/run.sh | 2 + utils/analyze-jenkins-logs/ci-errors.markdown | 154 +++++++++++++++ utils/analyze-jenkins-logs/index.js | 197 +++++++++++++++++++ utils/analyze-jenkins-logs/package.json | 20 ++ utils/auto-download-jenkins-logs/.gitignore | 1 + .../bin/auto-download-jenkins-logs | 3 + utils/auto-download-jenkins-logs/index.js | 182 +++++++++++++++++ utils/auto-download-jenkins-logs/package.json | 24 +++ 15 files changed, 587 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/.gitignore ---------------------------------------------------------------------- diff --git a/scraping-jenkins/.gitignore b/scraping-jenkins/.gitignore deleted file mode 100644 index 4f7c67f..0000000 --- a/scraping-jenkins/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -jenkins-api-token -root.xml -jobs/ http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/jobnames.txt ---------------------------------------------------------------------- diff --git a/scraping-jenkins/jobnames.txt b/scraping-jenkins/jobnames.txt deleted file mode 100644 index e8a174d..0000000 --- a/scraping-jenkins/jobnames.txt +++ /dev/null @@ -1,13 +0,0 @@ -2.0-developer-preview -Apache-CouchDB-Matrix-Mac-OS-X-10-7-5-1.6.x -Apache-CouchDB-Matrix-Mac-OS-X-10-7-5-master -Apache-CouchDB-Matrix-Mac-OS-X-10-8-2-1.6.x -Apache-CouchDB-Matrix-Mac-OS-X-10-8.2-master -Apache-CouchDB-Matrix-Ubuntu-Server-64-1.6.x -Apache-CouchDB-Windows -Fauxton -Rebuild%20Docs%201.3.x -Rebuild%20Docs%201.4.x -Rebuild%20Docs%201.5.x -Rebuild%20Docs%201.6.x -Rebuild%20Docs%20master http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/readme.markdown ---------------------------------------------------------------------- diff --git a/scraping-jenkins/readme.markdown b/scraping-jenkins/readme.markdown deleted file mode 100644 index c44dc2e..0000000 --- a/scraping-jenkins/readme.markdown +++ /dev/null @@ -1,4 +0,0 @@ -Scraping the Old CouchDB CI Jobs -================================ - -Execute `./scrape.sh` to download all Jenkins jobs configurations from the old CI setup at ci.couchdb.org:8888. http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/rename.sh ---------------------------------------------------------------------- diff --git a/scraping-jenkins/rename.sh b/scraping-jenkins/rename.sh deleted file mode 100755 index 32314ba..0000000 --- a/scraping-jenkins/rename.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -e - -pushd `dirname $0`/jobs > /dev/null - -shopt -s nullglob # empty directory will return empty list - -# convert to all-lower-case -for dir in ./*/;do - mv "$dir" "${dir,,}" 2> /dev/null || true -done - -# replace %20% by hyphen -for dir in ./*/;do - mv "$dir" "${dir//%20/-}" 2> /dev/null || true -done - -popd > /dev/null - http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/scrape.sh ---------------------------------------------------------------------- diff --git a/scraping-jenkins/scrape.sh b/scraping-jenkins/scrape.sh deleted file mode 100755 index a354b8f..0000000 --- a/scraping-jenkins/scrape.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -e - -pushd `dirname $0` > /dev/null - -if [[ ! -f jenkins-api-token ]]; then - echo Please create a file named jenkins-api-token and put your Jenkins API token in this file. - exit 1 -fi - -JENKINS_API_TOKEN=$(<jenkins-api-token) -echo "using Jenkins API token: $JENKINS_API_TOKEN" - -############################################################################### -# root Jenkins config -############################################################################### -curl --show-error "http://basti1302:[email protected]:8888/api/xml"; > root.tmp.xml -xmllint --format root.tmp.xml > root.xml || mv root.tmp.xml root.xml -rm -f root.tmp.xml - -############################################################################### -# Job configs -############################################################################### -mkdir -p jobs - -while IFS='' read -r jobname || [[ -n $jobname ]]; do - echo "scraping job: $jobname" - mkdir -p "jobs/$jobname" - curl --show-error "http://basti1302:[email protected]:8888/job/$jobname/config.xml"; > "jobs/$jobname/config.xml" -done < jobnames.txt - -# rename jobs to a sane naming pattern -./rename.sh - -popd > /dev/null - http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/.gitignore ---------------------------------------------------------------------- diff --git a/utils/analyze-jenkins-logs/.gitignore b/utils/analyze-jenkins-logs/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/utils/analyze-jenkins-logs/.gitignore @@ -0,0 +1 @@ +node_modules/ http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs ---------------------------------------------------------------------- diff --git a/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs b/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs new file mode 100755 index 0000000..229356d --- /dev/null +++ b/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs @@ -0,0 +1,3 @@ +#!/usr/bin/env node + +require('..'); http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/bin/run.sh ---------------------------------------------------------------------- diff --git a/utils/analyze-jenkins-logs/bin/run.sh b/utils/analyze-jenkins-logs/bin/run.sh new file mode 100755 index 0000000..239870b --- /dev/null +++ b/utils/analyze-jenkins-logs/bin/run.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +JENKINS_LOGS_DIR=/home/bastian/projekte/couchdb/ci-logs /home/bastian/.nvm/versions/node/v4.2.6/bin/node `dirname $0`/analyze-jenkins-logs http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/ci-errors.markdown ---------------------------------------------------------------------- diff --git a/utils/analyze-jenkins-logs/ci-errors.markdown b/utils/analyze-jenkins-logs/ci-errors.markdown new file mode 100644 index 0000000..36ca487 --- /dev/null +++ b/utils/analyze-jenkins-logs/ci-errors.markdown @@ -0,0 +1,154 @@ +# Summary + +Successes: 38 Failures: 57 + + +# Uncategorized Failures + +* Number of failures: 6 + +## Builds +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/18/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/22/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/23/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/26/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/18/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/28/consoleText> + + +# Failures with reason "eunit_replicator" + +* Number of failures: 30 +## Regular Expressions +When one of these regular expression has a match in the build log, I assume this build failure falls into this category. + +* `/\\*\\*in function couch_replicator_filtered_tests:should_succeed/` +* `/\\*\\*error:\{assertion_failed,\[\{module,couch_replicator_compact_tests\}/` + +## Builds + +Links to the build logs: + +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/19/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/21/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/21/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/23/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/23/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/24/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/24/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/16/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/27/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/27/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/27/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/28/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/28/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/28/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/29/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/29/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/29/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/30/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/29/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/30/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/30/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/30/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/30/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/33/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/33/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/33/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/33/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/33/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/33/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/30/consoleText> + + +# Failures with reason "eunit_compression" + +* Number of failures: 10 +## Regular Expressions +When one of these regular expression has a match in the build log, I assume this build failure falls into this category. + +* `/couchdb_file_compression_tests:110: should_compare_compression_methods.*\\*failed\\*/` +* `/in call from couchdb_file_compression_tests:setup\/0 \(test\/couchdb_file_compression_tests.erl, line 38\)/` + +## Builds + +Links to the build logs: + +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/12/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/12/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/17/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/15/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/20/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/19/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/24/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/26/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/29/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/29/consoleText> + + +# Failures with reason "network" + +* Number of failures: 6 +## Regular Expressions +When one of these regular expression has a match in the build log, I assume this build failure falls into this category. + +* `/fatal: unable to access 'https:\/\/git-wip-us.apache.org/` +* `/fatal: read error: Connection reset by peer/` + +## Builds + +Links to the build logs: + +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/14/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/20/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/24/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/24/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/28/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/28/consoleText> + + +# Failures with reason "docker" + +* Number of failures: 3 +## Regular Expressions +When one of these regular expression has a match in the build log, I assume this build failure falls into this category. + +* `/Cannot connect to the Docker daemon. Is the docker daemon running on this host?/` + +## Builds + +Links to the build logs: + +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/27/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/27/consoleText> +* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/27/consoleText> + + +# Failures with reason "libdl" + +* Number of failures: 1 +## Regular Expressions +When one of these regular expression has a match in the build log, I assume this build failure falls into this category. + +* `/sed: error while loading shared libraries: libdl.so.2/` + +## Builds + +Links to the build logs: + +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/14/consoleText> + + +# Failures with reason "aborted" + +* Number of failures: 1 +## Regular Expressions +When one of these regular expression has a match in the build log, I assume this build failure falls into this category. + +* `/Build was aborted/` + +## Builds + +Links to the build logs: + +* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/19/consoleText> http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/index.js ---------------------------------------------------------------------- diff --git a/utils/analyze-jenkins-logs/index.js b/utils/analyze-jenkins-logs/index.js new file mode 100644 index 0000000..0656763 --- /dev/null +++ b/utils/analyze-jenkins-logs/index.js @@ -0,0 +1,197 @@ +'use strict'; + +const _ = require('lodash'); +const fs = require('fs'); +const jsonfile = require('jsonfile'); +const glob = require('glob'); +const path = require('path'); +const process = require('process'); + + +let baseDir; +let success = 0; +let failure = 0; +let unrecognized; + +// TODO Verify that the errors are indeed in the log file + +const reasons = {}; + +const regexes = { + aborted: [ /Build was aborted/ ], + network: [ + /fatal: unable to access 'https:\/\/git-wip-us.apache.org/, + /fatal: read error: Connection reset by peer/, + ], + docker: [ + /Cannot connect to the Docker daemon. Is the docker daemon running on this host?/ + ], + libdl: [ /sed: error while loading shared libraries: libdl.so.2/ ], + eunit_replicator: [ + /\\*\\*in function couch_replicator_filtered_tests:should_succeed/, + /\\*\\*error:\{assertion_failed,\[\{module,couch_replicator_compact_tests\}/, + ], + eunit_compression: [ + /couchdb_file_compression_tests:110: should_compare_compression_methods.*\\*failed\\*/, + /in call from couchdb_file_compression_tests:setup\/0 \(test\/couchdb_file_compression_tests.erl, line 38\)/ + ], + eunit: [ /XRROR: One or more eunit tests failed./ ], +}; + +// from https://gist.github.com/colingourlay/82506396503c05e2bb94 +_.mixin({ + 'sortKeysBy': function (obj, comparator) { + var keys = _.sortBy(_.keys(obj), function (key) { + return comparator ? comparator(obj[key], key) : key; + }); + + return _.zipObject(keys, _.map(keys, function (key) { + return obj[key]; + })); + } +}); + +function init() { + if (!process.env.JENKINS_LOGS_DIR) { + console.log('WARNING: JENKINS_LOGS_DIR is not set.\n'); + } + baseDir = process.env.JENKINS_LOGS_DIR || __dirname; + // console.log('Will read logs from', baseDir, '\n\n'); + + process.on('exit', function() { + + console.log('# Summary'); + console.log('\nSuccesses:', success, 'Failures:', failure); + if (reasons.unrecognized.counter > 0) { + console.log('\n\n# Uncategorized Failures'); + console.log('\n* Number of failures: ' + reasons.unrecognized.counter); + console.log('\n## Builds'); + reasons.unrecognized.urls.forEach( url => { + console.log('* <' + url + '>'); + }); + } + + // print results, most frequent errors first + _(reasons) + .omit(['unrecognized']) + .sortKeysBy((value, key) => { + return -value.counter; + }) + .forOwn((reasonObject, reasonKey) => { + console.log('\n\n# Failures with reason "' + reasonKey + '"'); + console.log('\n* Number of failures: ' + reasonObject.counter); + console.log('## Regular Expressions'); + console.log('When one of these regular expression has a match in the build log, I assume this build failure falls into this category.\n'); + regexes[reasonKey].forEach( regex => { + console.log('* `' + regex + '`'); + }); + console.log('\n## Builds\n'); + console.log('Links to the build logs:\n'); + reasonObject.urls.forEach( url => { + console.log('* <' + url + '>'); + }); + }); + }); +} + + +function initReasonObject() { + return { + counter: 0, + urls: [], + directories: [], + }; +} + + +function analyzeLogs() { + init(); + glob('+([0123456789])/', { cwd: baseDir, }, function (err, buildDirectories) { + if (err) { + exitOnError(err); + } + if (buildDirectories.length === 0) { + exitOnError(new Error('No matching directories found.')); + } + buildDirectories.forEach(buildDir => { + readBuild(path.join(baseDir, buildDir)); + }); + }); +} + + +function readBuild(buildDirectory) { + fs.readdir(buildDirectory, function(err, runDirectories) { + if (err) { + return printWarning(err); + } + runDirectories.forEach(runDirectory => { + readRun(path.join(buildDirectory, runDirectory)); + }); + }); +} + + +function readRun(directory) { + const metaDataFile = path.join(directory, 'metadata.json'); + const buildLogFile = path.join(directory, 'build.log'); + jsonfile.readFile(metaDataFile, 'utf-8', (err, metaData) => { + if (err) { + return printWarning(err); + } + const result = metaData.result[0]; + if (metaData.result[0] !== 'SUCCESS') { + // console.error('Failure:', directory, result); + failure++; + fs.readFile(buildLogFile, 'utf-8', (err, buildLog) => { + if (err) { + return printWarning(err); + } + + for (var key in regexes) { + for (var i = 0; i < regexes[key].length; i++) { + if (contains(buildLog, regexes[key][i])) { + appendToReason(key, metaData, directory); + return; + } + } + } + // console.error('Uncategorized failure', buildLogFile); + appendToReason('unrecognized', metaData, directory); + }); + } else { + // console.error('Success:', directory, result); + success++; + } + }); +} + + +function appendToReason(key, metaData, directory) { + let reasonObject = reasons[key]; + if (!reasonObject) { + reasonObject = reasons[key] = initReasonObject(); + } + reasonObject.counter++; + reasonObject.urls.push(metaData.url + 'consoleText'); + reasonObject.directories.push(directory); +} + + +function contains(buildLog, regex) { + return buildLog.search(regex) >= 0; +} + + +function exitOnError(error) { + console.error(error); + process.exit(1); +} + + +function printWarning(error) { + console.error(error); +} + + +analyzeLogs(); http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/package.json ---------------------------------------------------------------------- diff --git a/utils/analyze-jenkins-logs/package.json b/utils/analyze-jenkins-logs/package.json new file mode 100644 index 0000000..429ace7 --- /dev/null +++ b/utils/analyze-jenkins-logs/package.json @@ -0,0 +1,20 @@ +{ + "name": "analyze-jenkins-logs", + "version": "1.0.0", + "description": "Analyzes Jenkins logs for failures", + "main": "index.js", + "bin": { + "analyze-jenkins-logs": "./bin/analyze-jenkins-logs" + }, + "repository": { + "type": "git", + "url": "https://git-wip-us.apache.org/repos/asf/couchdb.git"; + }, + "author": "The Apache CouchDB contributors", + "license": "Apache-2.0", + "dependencies": { + "glob": "^7.0.0", + "jsonfile": "^2.2.3", + "lodash": "^4.5.1" + } +} http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/.gitignore ---------------------------------------------------------------------- diff --git a/utils/auto-download-jenkins-logs/.gitignore b/utils/auto-download-jenkins-logs/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/utils/auto-download-jenkins-logs/.gitignore @@ -0,0 +1 @@ +node_modules/ http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs ---------------------------------------------------------------------- diff --git a/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs b/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs new file mode 100755 index 0000000..229356d --- /dev/null +++ b/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs @@ -0,0 +1,3 @@ +#!/usr/bin/env node + +require('..'); http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/index.js ---------------------------------------------------------------------- diff --git a/utils/auto-download-jenkins-logs/index.js b/utils/auto-download-jenkins-logs/index.js new file mode 100644 index 0000000..862172a --- /dev/null +++ b/utils/auto-download-jenkins-logs/index.js @@ -0,0 +1,182 @@ +'use strict'; + +const fs = require('fs'); +const isString = require('is-string'); +const jsonfile = require('jsonfile'); +const mkdirp = require('mkdirp'); +const parseXml = require('xml2js').parseString; +const path = require('path'); +const process = require('process'); +const request = require('request'); + +const apiXml = 'api/xml'; +let baseDir; + +function init() { + if (!process.env.JENKINS_LOGS_DIR) { + console.log('WARNING: JENKINS_LOGS_DIR is not set.'); + } + baseDir = process.env.JENKINS_LOGS_DIR || __dirname; + console.log('Will write logs to', baseDir); +} + + +function crawlFromRootUrl(url) { + console.log(` +================================================================================ +`); + console.log(new Date()); + init(); + request(url, function(err, message, body) { + if (err) { + return exitOnError(err); + } + + parseXml(body, function (err, result) { + if (err) { + return exitOnError(err); + } + + if (!result.matrixProject) { + return exitOnError(new Error('Not found: matrixProject')); + } + if (!result.matrixProject.build) { + return exitOnError(new Error('Not found: matrixProject.build')); + } + if (!Array.isArray(result.matrixProject.build)) { + return exitOnError(new Error('Not an array: matrixProject.build')); + } + result.matrixProject.build.forEach(build => { + fetchBuild(build); + }); + }); + }); +} + + +function fetchBuild(build) { + if (build.number == null) { + return printWarning(new Error('Not found: build.number')); + } + if (build.url == null) { + return printWarning(new Error('Not found: build.url')); + } + + const url = build.url + apiXml; + console.log('Retrieving meta data for build', build.number[0], 'from ', url); + request(url, function(err, message, body) { + if (err) { + return printWarning(err); + } + parseXml(body, function (err, result) { + if (err) { + return printWarning(err); + } + + if (!result.matrixBuild) { + return printWarning(new Error('Not found: matrixBuild')); + } + if (!result.matrixBuild.run) { + return printWarning(new Error('Not found: matrixBuild.run')); + } + if (!Array.isArray(result.matrixBuild.run)) { + return printWarning(new Error('Not an array: matrixBuild.run')); + } + result.matrixBuild.run.forEach(run => { + fetchRun(run); + }); + }); + }); +} + + +function fetchRun(run) { + if (run.number == null) { + return printWarning(new Error('Not found: run.number')); + } + if (run.url == null) { + return printWarning(new Error('Not found: run.url')); + } + + const url = run.url + apiXml; + console.log('Retrieving meta data for run', run.number[0], 'from ', url); + request(url, function(err, message, body) { + if (err) { + return printWarning(err); + } + parseXml(body, function (err, result) { + if (err) { + return printWarning(err); + } + + if (!result.matrixRun) { + return printWarning(new Error('Not found: matrixRun')); + } + if (!result.matrixRun.number) { + return printWarning(new Error('Not found: matrixRun.number')); + } + if (!result.matrixRun.fullDisplayName) { + return printWarning(new Error('Not found: matrixRun.fullDisplayName')); + } + if (!Array.isArray(result.matrixRun.fullDisplayName)) { + return printWarning(new Error('Not an array: matrixRun.fullDisplayName')); + + } + if (result.matrixRun.fullDisplayName.length !== 1) { + return printWarning(new Error('Not an array of length 1: matrixRun.fullDisplayName')); + } + if (!isString(result.matrixRun.fullDisplayName[0])) { + return printWarning(new Error('Not a string: matrixRun.fullDisplayName[0]')); + } + + fetchMatrixRunAndLog(result.matrixRun, run.url); + }); + }); +} + + +function fetchMatrixRunAndLog(metaData, runBaseUrl) { + const logUrl = runBaseUrl + 'consoleText'; + request(logUrl, function(err, message, body) { + if (err) { + return printWarning(err); + } + saveToDisk(metaData, body); + }); +} + + +function saveToDisk(metaData, buildLog) { + const dir = path.join(baseDir, String(metaData.number), metaData.fullDisplayName[0]); + mkdirp.sync(dir); + const metaDataFile = path.join(dir, 'metadata.json'); + jsonfile.writeFile(metaDataFile, metaData, { spaces: 2 }, function (err) { + if (err) { + return printWarning(err); + } + console.log('Written meta data to ', metaDataFile); + }); + const buildLogFile = path.join(dir, 'build.log'); + fs.writeFile(buildLogFile, buildLog, function(err) { + if (err) { + return printWarning(err); + } + console.log('Written build log to ', buildLogFile); + }); +} + + +function exitOnError(error) { + console.error(error); + process.exit(1); +} + + +function printWarning(error) { + console.error(error); +} + + +const rootUrl = 'https://builds.apache.org/job/CouchDB/api/xml'; + +crawlFromRootUrl(rootUrl); http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/package.json ---------------------------------------------------------------------- diff --git a/utils/auto-download-jenkins-logs/package.json b/utils/auto-download-jenkins-logs/package.json new file mode 100644 index 0000000..5cbe0d3 --- /dev/null +++ b/utils/auto-download-jenkins-logs/package.json @@ -0,0 +1,24 @@ +{ + "name": "auto-download-jenkins-logs", + "version": "1.0.0", + "description": "Automatically downloads Jenkins logs", + "main": "index.js", + "bin": { + "auto-download-jenkins-logs": "./bin/auto-download-jenkins-logs" + }, + "repository": { + "type": "git", + "url": "https://git-wip-us.apache.org/repos/asf/couchdb.git"; + }, + "author": "The Apache CouchDB contributors", + "license": "Apache-2.0", + "dependencies": { + "jsonfile": "^2.2.3", + "mkdirp": "^0.5.1", + "request": "^2.69.0", + "xml2js": "^0.4.16" + }, + "devDependencies": { + "is-string": "^1.0.4" + } +}
