Arlolra has uploaded a new change for review. https://gerrit.wikimedia.org/r/163081
Change subject: Group latex errors ...................................................................... Group latex errors * Requires https://gerrit.wikimedia.org/r/#/c/163064 * Output of these[0] tests cases can be see here[1] [0] http://pastebin.com/MmYNe8Az [1] https://gist.github.com/arlolra/e1b18ccd4576951f66c1 Change-Id: I13b995d5471cc212e699f1a823e1dc3819e9bb78 --- M loadtest/loadtest.js 1 file changed, 54 insertions(+), 14 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/ocg-collection refs/changes/81/163081/1 diff --git a/loadtest/loadtest.js b/loadtest/loadtest.js index c0cbb0e..fc41ae6 100755 --- a/loadtest/loadtest.js +++ b/loadtest/loadtest.js @@ -23,7 +23,7 @@ 'Only queue the first N pages', null) .option('-o, --output <fileprefix>', 'Save results to <fileprefix>-*.txt', null) - .option('--parsoid <url>', + .option('--parsoid <url>', // the default only works if you're testing pages on public wp 'Parsoid API for article existence checks', 'http://parsoid-lb.eqiad.wikimedia.org/') @@ -51,7 +51,7 @@ titles = titles.filter(function(t) { return t.prefix === program.prefix; }); } if (+program.limit) { - titles.length = +program.limit; + titles.length = +program.limit; } // create output files @@ -65,14 +65,15 @@ var failedInject = mkout('failed-inject.txt'); var failedRender = mkout('failed-render.txt'); var passedRender = mkout('passed-render.txt'); +var failedGroups = mkout('failed-groups.txt'); -var doOne = Promise.guard(+program.j || 10, function(prefix, title) { - var collection_id; - console.log(prefix, title); - return bundler.metabook.fromArticles( +var doOne = Promise.guard(+program.j || 10, function(prefix, title, crashGroups) { + var collection_id; + console.log(prefix, title); + return bundler.metabook.fromArticles( [ { prefix: prefix, title: title } ], {} - ).then(function(metabook) { + ).then(function(metabook) { metabook.title = title; // submit it! return request({ @@ -100,7 +101,7 @@ e.error = error; throw e; }); - }).then(function(body) { + }).then(function(body) { collection_id = JSON.parse(body).collection_id; // check status until it's complete. var check = function() { @@ -118,6 +119,22 @@ } var state = JSON.parse(body).state; if (/^(failed|finished)$/.test(state)) { + if ( state === "failed" ) { + var status = JSON.parse(body).status, + group = notSure, // defined below + page = prefix + " " + title; + if ( status && status.status ) { + var match = status.status.match(/\n(! .*)\n/); + if ( match ) { + group = match[1]; + } + } + if ( Array.isArray(crashGroups[group]) ) { + crashGroups[group].push( page ); + } else { + crashGroups[group] = [ page ]; + } + } return state; } if (!/^(pending|progress)$/.test(state)) { @@ -127,7 +144,7 @@ }); }; return check(); - }).then(function(status) { + }).then(function(status) { // double check that failed articles actually exist: sometimes // the title list contains deleted articles. if (status==='failed' && program.parsoid) { @@ -151,7 +168,7 @@ failedRender.write(prefix + ':' + title + '\n'); } return status; - }, function(error) { + }, function(error) { failedInject.write(prefix + ':' + title + '\n'); if (error.message === 'failed inject') { if (program.debug) { console.log(error); } @@ -159,18 +176,41 @@ } console.error('Unusual error', error); return 'error'; - }); + }); }); +var crashGroups = {}; +var notSure = "~ Not sure ~"; + +function writeGroup(group) { + failedGroups.write(group + "\n\n"); + crashGroups[group].forEach(function(page) { + failedGroups.write(page + "\n"); + }); + failedGroups.write("\n\n"); +} + Promise.map(titles, function(article) { - return doOne(article.prefix, article.title); + return doOne(article.prefix, article.title, crashGroups); }).finally(function() { - return Promise.map([failedInject, failedRender, passedRender], function(s) { + var groups = Object.keys(crashGroups); + groups.sort(function(a, b) { + return crashGroups[b].length - crashGroups[a].length; + }) + groups.forEach(function(group) { + if ( group === notSure ) { return; } + writeGroup(group); + }); + if ( crashGroups[notSure] ) { + failedGroups.write("\n\n"); + writeGroup(notSure); + } + return Promise.map([failedInject, failedRender, passedRender, failedGroups], function(s) { return new Promise(function(resolve, reject) { return s.end(function(error) { if (error) { return reject(error); } resolve(); }); }); - }); + }); }).done(); -- To view, visit https://gerrit.wikimedia.org/r/163081 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I13b995d5471cc212e699f1a823e1dc3819e9bb78 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/ocg-collection Gerrit-Branch: master Gerrit-Owner: Arlolra <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
