Mwalker has submitted this change and it was merged.
Change subject: Report status of bundler
......................................................................
Report status of bundler
Change-Id: I3995d86bb71d99463fe59cbeb576ae54f730552a
---
M bin/mw-bundler
M lib/index.js
A lib/status.js
3 files changed, 52 insertions(+), 0 deletions(-)
Approvals:
Mwalker: Looks good to me, approved
jenkins-bot: Verified
diff --git a/bin/mw-bundler b/bin/mw-bundler
index f08f9a9..9a4b380 100755
--- a/bin/mw-bundler
+++ b/bin/mw-bundler
@@ -12,6 +12,7 @@
var Api = require('../lib/api');
var P = require('../lib/p');
var Siteinfo = require('../lib/siteinfo');
+var status = require('../lib/status');
program
.version(bundler.version)
@@ -114,6 +115,9 @@
filerepos: undefined
};
+// TODO: Update this number if we ever add more bundling stages
+status.setNumStages(4);
+
// Load metabook.json
if (program.metabook) {
// Load from file and parse JSON
@@ -203,10 +207,12 @@
// add our extension fields, if missing
p = p.then(function() {
var pp = when.resolve();
+ status.createStage(metabook.wikis.length + 1 /* commons wiki */);
if (program.toc !== 'auto') {
metabook.toc = !/^(no|false|off)$/i.test(program.toc);
}
metabook.wikis.forEach(function(w, idx) {
+ status.report('Obtaining source wiki metadata', w.baseurl);
if (!w.parsoid) {
w.parsoid = program.api;
}
@@ -228,6 +234,7 @@
});
// fudge a pointer to commonswiki if needed
p = p.then(function() {
+ status.report('Loading commons metadata');
if (needsCommonsWiki) {
metabook.wikis.push(COMMONSWIKI);
return fetchFileRepos(metabook.wikis.length - 1);
diff --git a/lib/index.js b/lib/index.js
index 9728047..3a1ec0d 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -4,6 +4,7 @@
var path = require('path');
var rimraf = require('rimraf');
var when = require('when');
+var status = require('./status');
var P = require('./p');
@@ -57,6 +58,8 @@
// returns a promise which is resolved when the sourceMap has been
// filled with all the parsoid sources.
+ var articleCount = 0;
+ var imageCount = 0;
var fetchParsed = function() {
log('Fetching parsed article contents');
var parsoidDb = new Db(path.join(options.output, "parsoid.db"));
@@ -66,6 +69,7 @@
// a promise to parse a single item (from parsoid & php)
var doOneItem = function(item) {
item.wiki = item.wiki || 0;
+ status.report('Fetching articles', item.title);
return parsoid.fetch(item.wiki, item.title,
item.revision)
.then(function(result) {
var revid = result.getRevisionId();
@@ -73,6 +77,7 @@
item.revision = '' + revid;
sourceMap.set(revid, result);
result.getImages().forEach(function(img) {
+ imageCount += 1;
imageMap.set(img.resource, img);
});
return parsoidDb.put(item.revision,
result.text);
@@ -84,8 +89,19 @@
};
// recursively visit all items in the metabook info structure
+ // we do this twice; once to get the article count; and another
to push tasks
+ (function visitToCount(item) {
+ if (item.type === 'article') {
+ articleCount += 1;
+ } else if (item.type === 'collection' || item.type ===
'chapter') {
+ item.items.forEach(visitToCount);
+ }
+ })(metabook);
+
+ status.createStage(articleCount);
(function visit(item) {
if (item.type === 'article') {
+ articleCount += 1;
tasks.push(doOneItem(item));
} else if (item.type === 'collection' || item.type ===
'chapter') {
item.items.forEach(visit);
@@ -110,10 +126,12 @@
// are downloaded.
var fetchImages = function() {
log('Fetching images');
+ status.createStage(imageCount);
var imageDb = new Db(path.join(options.output, "imageinfo.db"));
var tasks = [];
imageMap.forEach(function(img) {
+ status.report('Fetching media resources', img.filename);
var p = imageloader.fetchMetadata(img).then(function() {
if (img.imageinfo.mediatype === 'BITMAP' ||
img.imageinfo.mediatype === 'DRAWING') {
@@ -197,6 +215,8 @@
// promise to create the desired bundle!
var createBundle = function() {
log('Creating bundle');
+ status.createStage(0);
+ status.report('Compositing data bundle for renderer');
if (options.nozip) {
// make the directory readable, then we're done.
return P.call(fs.chmod, fs, options.output, 0755);
diff --git a/lib/status.js b/lib/status.js
new file mode 100644
index 0000000..d5a4c04
--- /dev/null
+++ b/lib/status.js
@@ -0,0 +1,25 @@
+var percentComplete = 0;
+var currentStage = 0.0;
+var stagesInv = 1.0;
+var stageLen = 0;
+
+module.exports.setNumStages = function(num) {
+ stagesInv = 1.0 / num;
+};
+
+module.exports.createStage = function(len) {
+ percentComplete = currentStage * stagesInv;
+ currentStage += 1;
+ if (len) {
+ stageLen = 1.0 / len;
+ } else {
+ stageLen = 0;
+ }
+};
+
+module.exports.report = function(message, file) {
+ percentComplete += 100.0 * (stagesInv * stageLen);
+ if (process.send) {
+ process.send(JSON.stringify({status: message, file: file,
percent: percentComplete}));
+ }
+};
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/96969
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3995d86bb71d99463fe59cbeb576ae54f730552a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator/bundler
Gerrit-Branch: master
Gerrit-Owner: Mwalker <mwal...@wikimedia.org>
Gerrit-Reviewer: Mwalker <mwal...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
