DCausse has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/394612 )
Change subject: Process nodejs jobs in chunks
......................................................................
Process nodejs jobs in chunks
Change-Id: I22db3cf68db9c811060a5f24585227d080b6c898
---
M tests/integration/features/support/hooks.js
1 file changed, 54 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/12/394612/1
diff --git a/tests/integration/features/support/hooks.js
b/tests/integration/features/support/hooks.js
index beec9d8..772db52 100644
--- a/tests/integration/features/support/hooks.js
+++ b/tests/integration/features/support/hooks.js
@@ -35,7 +35,7 @@
const waitForBatch = Promise.coroutine( function* ( wiki, batchJobs ) {
let stepHelpers = this.stepHelpers.onWiki( wiki );
- const queue = [];
+ let queue = [];
if ( Array.isArray( batchJobs ) ) {
for ( let job of batchJobs ) {
queue.push( [ job[0], job[1] ] );
@@ -63,17 +63,62 @@
}
} );
+ const chunkifyBatch = ( batchJobs, chunkSize = 10 ) => {
+ let jobs = [];
+ if ( !Array.isArray(batchJobs) ) {
+ let flatJobs = [];
+ for ( let op in batchJobs ) {
+ let data = batchJobs[op];
+ let jobData = [ op ];
+ if ( Array.isArray(data) ) {
+ for ( let title of data ) {
+ flatJobs.push( jobData.concat(
Array.isArray( title ) ? title : [ title ] ) );
+ }
+ } else {
+ for ( let title in data ) {
+ let d = data[title];
+ flatJobs.push( jobData.concat(
[ title ] )
+ .concat(
Array.isArray(d) ? d : [ d ] ) );
+ }
+ }
+ }
+ batchJobs = flatJobs;
+ }
+ let chunk = [];
+ for ( let job of batchJobs ) {
+ chunk.push(job);
+ if ( chunk.length === chunkSize ) {
+ jobs.push(chunk);
+ chunk = [];
+ }
+ }
+ if ( chunk.length > 0 ) {
+ jobs.push(chunk);
+ }
+ return jobs;
+ };
+
// Run both in parallel so we don't have to wait for the batch to finish
// to start checking things. Hopefully they run in the same order...
const runBatch = Promise.coroutine( function* ( world, wiki, batchJobs
) {
wiki = wiki || world.config.wikis.default;
let client = yield world.onWiki( wiki );
- // TODO: If the batch operation fails the waitForBatch will
never complete,
- // it will just stick around forever ...
- yield Promise.all( [
- client.batch( batchJobs, 'CirrusSearch integration test
edit', 2 ),
- waitForBatch.call( world, wiki, batchJobs )
- ] );
+ // if the batch is too large we may wait too long for a page
+ // that will be added too late.
+ // This is perhaps because when we iterate over object
properties
+ // ordering may not be the same when running in mwbot and
waitForBatch
+ // We create small chunks to avoid this situation
+ let jobChunks = chunkifyBatch(batchJobs);
+ let i = 0;
+ for ( let chunk of jobChunks ) {
+ // TODO: If the batch operation fails the waitForBatch
will never complete,
+ // it will just stick around forever ...
+ MWBot.logStatus( '[=] ', ++i, jobChunks.length,
'BATCH', `Running ${chunk.length} jobs on ${wiki}` );
+ yield Promise.all([
+ client.batch(chunk, 'CirrusSearch integration
test edit', 2),
+ waitForBatch.call(world, wiki, chunk)
+ ]);
+ }
} );
const runBatchFn = ( wiki, batchJobs ) => Promise.coroutine( function*
() {
@@ -191,7 +236,7 @@
}
} ) );
- BeforeOnce( { tags: "@did_you_mean", timeout: 120000 }, function () {
+ BeforeOnce( { tags: "@did_you_mean", timeout: 240000 }, function () {
let edits = {
'Popular Culture': 'popular culture',
'Nobel Prize': 'nobel prize',
@@ -224,6 +269,7 @@
'suggest2 suggest3 suggest4': 'list of grammy awards
winners',
'suggest3 suggest4 suggest5': 'list of grammy awards
winners',
};
+ // TODO: investigate getting rid of this
for ( let i = 1; i <= 30; i++ ) {
edits['Grammy Awards ed. ' + i] = 'grammy awards';
}
--
To view, visit https://gerrit.wikimedia.org/r/394612
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I22db3cf68db9c811060a5f24585227d080b6c898
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: DCausse <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
