Mwalker has submitted this change and it was merged.
Change subject: Improve status reporting.
......................................................................
Improve status reporting.
Add more granular status reporting; refactor the 'status' module.
Change-Id: I83457a8b5424d3bc035ee4870047c2c7531ea7c7
---
M lib/db.js
M lib/index.js
M lib/status.js
3 files changed, 112 insertions(+), 42 deletions(-)
Approvals:
Mwalker: Looks good to me, approved
jenkins-bot: Verified
diff --git a/lib/db.js b/lib/db.js
index d331321..3e0668d 100644
--- a/lib/db.js
+++ b/lib/db.js
@@ -28,6 +28,18 @@
}).then(function() { return db; });
};
+// Returns a promise for the number of keys (used to compute status
percentages)
+Db.prototype.count = function() {
+ return this.db.then(function(db) {
+ return P.call(
+ db.get, db,
+ "SELECT count() AS count FROM kv_table;"
+ );
+ }).then(function(row) {
+ return row.count;
+ });
+};
+
// Returns a promise for the value.
Db.prototype.get = function(key, nojson) {
return this.db.then(function(db) {
diff --git a/lib/index.js b/lib/index.js
index cb9cc45..c78c63e 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -8,7 +8,6 @@
var gammalatex = require('gammalatex');
var guard = require('when/guard');
var path = require('path');
-var status = require('./status');
var stream = require('stream');
var tmp = require('tmp');
var url = require('url');
@@ -23,6 +22,7 @@
var Db = require('./db');
var DomUtil = require('./domutil');
var P = require('./p');
+var StatusReporter = require('./status');
var STD_HEADER = [
"%!TEX TS-program = xelatex",
@@ -532,17 +532,15 @@
// return a promise for the builddir and control file contents
// (after the bundle has been unpacked)
var unpackBundle = function(options) {
- var metabook, builddir;
+ var metabook, builddir, status = options.status;
- status.createStage(0);
+ status.createStage(0, 'Unpacking content bundle');
// first create a temporary directory
return P.call(tmp.dir, tmp, {
prefix: json.name,
unsafeCleanup: !(options.debug || options.latex)
}).then(function(_builddir) {
- status.report('Reading data bundle for document construction');
-
builddir = _builddir;
// make bundle and latex subdirs
return when.join(
@@ -552,7 +550,6 @@
}).then(function() {
// now unpack the zip archive
var bundledir = path.join(builddir, 'bundle');
- options.log('Unpacking bundle in', bundledir);
return P.spawn('unzip', [ path.resolve( options.bundle ) ], {
cwd: bundledir
});
@@ -643,23 +640,27 @@
// return a promise for a map from file resource URLs to on-disk filenames
// (after image processing / renaming has been done)
var processImages = function(metabook, builddir, options) {
- options.log('Processing images');
+ var status = options.status;
var imagedir = path.join(builddir, 'bundle', 'images');
var imagemap = new Map();
var imagedb = new Db(
path.join(builddir, 'bundle', 'imageinfo.db'), { readonly: true
}
);
- var p = when.resolve();
-
- status.createStage(imagedb.length);
+ var p = imagedb.count().then(function(n) {
+ status.createStage(n, 'Processing media files');
+ });
return imagedb.forEach(function(key, val) {
- status.report('Processing media files for inclusion',
val.filename);
+ var filename = val.filename;
+ p = p.then(function() {
+ // status reporting is serialized
+ status.report(null, filename || '');
+ });
+ if (!filename) { return; }
if (!/^https?:\/\//.test(key)) {
// compatibility with pediapress format
key = val.resource;
}
- var filename = val.filename;
- if (!filename) { return; }
+ // conversion/rename happens in parallel (new promise 'pp')
var pp = when.resolve({ imagedir: imagedir, filename: filename
});
// convert gifs to pngs
if (val.mime === 'image/gif') {
@@ -687,8 +688,19 @@
});
};
+// count total # of items (used for status reporting)
+var countItems = function(item) {
+ return (item.items || []).reduce(function(sum, item) {
+ return sum + countItems(item);
+ }, 1);
+};
+
// Return an empty promise after the output.tex file has been written.
var generateLatex = function(metabook, builddir, imagemap, options) {
+ var status = options.status;
+ status.createStage(countItems(metabook), 'Processing collection');
+ status.report(null, metabook.title);
+
var output = fs.createWriteStream(path.join(builddir, 'output.tex'), {
encoding: 'utf8'
});
@@ -747,7 +759,7 @@
var write = {};
write.article = function(item) {
console.assert(item.type === 'article');
- options.log('Writing article', item.title);
+ status.report(null, item.title);
var revid = item.revision;
var document, base = '';
var key = (item.wiki ? (item.wiki+'|') : '') + revid;
@@ -779,6 +791,7 @@
};
write.chapter = function(item) {
console.assert(item.type === 'chapter');
+ status.report(null, item.title);
if ('columns' in item && columns !== item.columns) {
columns = item.columns;
output.write(columns === 1 ? '\\onecolumn\n' :
'\\twocolumn\n');
@@ -787,8 +800,6 @@
return P.forEachSeq(item.items, write.article);
};
- status.createStage(1 /* XXX This should be the total number of nodes
we're going to visit */);
- status.report('Traversing page DOM', metabook.title); /* XXX Call this
for every node */
return P.forEachSeq(metabook.items, function(item) {
return write[item.type](item);
}).then(function() {
@@ -799,9 +810,8 @@
// Return an empty promise after the latex has been either written or
// compiled to a PDF.
var compileLatex = function(builddir, options) {
- options.log('Compiling to PDF with xelatex');
- status.createStage(0);
- status.report('Compiling to PDF with xelatex');
+ var status = options.status;
+ status.createStage(0, 'Compiling PDF');
gammalatex.setCompileCommand({
command: "xelatex",
@@ -831,13 +841,11 @@
writeStream.on('close', function() { deferred.resolve(); });
if (options.latex) {
- options.log('Writing LaTeX');
writeStream.end(latexOutput, 'utf8');
} else {
P.call(gammalatex.parse, gammalatex, latexOutput).
then(function(args) {
var readStream = args[0];
- options.log('Saving PDF');
readStream.pipe(writeStream);
}).done();
}
@@ -847,7 +855,12 @@
// Return a promise for an exit status (0 for success) after the bundle
// specified in the options has been converted.
var convert = function(options) {
- status.setNumStages(4); // update this number if we add more stages
+ var status = options.status = new StatusReporter(4, function(msg) {
+ if (options.log) {
+ var file = msg.file ? (': ' + msg.file) : '';
+ options.log('['+msg.percent.toFixed()+'%]', msg.status
+ file);
+ }
+ });
var metabook, builddir, imagemap;
return when.resolve().then(function() {
// unpack the bundle
@@ -867,13 +880,14 @@
// compile it to PDF
return compileLatex(builddir, options);
}).then(function() {
- options.log('Done.');
+ status.createStage(0, 'Done');
return 0; // success!
}, function(err) {
// xxx clean up?
if (options.debug) {
throw err;
}
+ // xxx send this error to parent process?
console.error('Error:', err);
return 1;
});
diff --git a/lib/status.js b/lib/status.js
index d5a4c04..c3b0a54 100644
--- a/lib/status.js
+++ b/lib/status.js
@@ -1,25 +1,69 @@
-var percentComplete = 0;
-var currentStage = 0.0;
-var stagesInv = 1.0;
-var stageLen = 0;
+/* Progress reporting using the node IPC mechanism. */
-module.exports.setNumStages = function(num) {
- stagesInv = 1.0 / num;
-};
-
-module.exports.createStage = function(len) {
- percentComplete = currentStage * stagesInv;
- currentStage += 1;
- if (len) {
- stageLen = 1.0 / len;
- } else {
- stageLen = 0;
+var StatusReporter = module.exports = function(numStages, extraLog) {
+ this.extraLog = extraLog;
+ this.percentComplete = 0;
+ this.currentStage = 0.0;
+ this.stagesInv = 1.0;
+ this.stageLen = 0;
+ this.message = '';
+ this.file = undefined;
+ if (numStages) {
+ this._setNumStages(numStages);
}
};
-module.exports.report = function(message, file) {
- percentComplete += 100.0 * (stagesInv * stageLen);
+/** Send one report with current status (internal function). */
+StatusReporter.prototype._send = function() {
+ var msg = {
+ status: this.message,
+ file: this.file,
+ percent: this.percentComplete
+ };
+ if (this.extraLog) {
+ this.extraLog(msg);
+ }
if (process.send) {
- process.send(JSON.stringify({status: message, file: file,
percent: percentComplete}));
+ process.send(msg);
}
-};
\ No newline at end of file
+};
+
+/** Update current status with message/file, and send it. */
+StatusReporter.prototype._report = function(message, file) {
+ if (message || message==='') {
+ this.message = message;
+ this.file = file;
+ } else if (file) {
+ // update only the current file
+ this.file = file;
+ }
+ this._send();
+};
+
+/** Initialize StatusReporter with given # of stages. */
+StatusReporter.prototype._setNumStages = function(num) {
+ this.stagesInv = 1.0 / num;
+};
+
+/** Start a new stage, which will be `opt_len` steps long. */
+StatusReporter.prototype.createStage = function(opt_len, opt_message,
opt_file) {
+ this.percentComplete = 100.0 * this.currentStage * this.stagesInv;
+ this.currentStage += 1;
+ if (opt_len) {
+ this.stageLen = 1.0 / opt_len;
+ } else {
+ this.stageLen = 0;
+ }
+ this._report(opt_message || '', opt_file);
+};
+
+/** Advance the stage by N steps. */
+StatusReporter.prototype.reportN = function(n, opt_message, opt_file) {
+ this._report(opt_message, opt_file);
+ this.percentComplete += 100.0 * (this.stagesInv * this.stageLen * n);
+};
+
+/** Advance the stage by 1. */
+StatusReporter.prototype.report = function(opt_message, opt_file) {
+ this.reportN(1, opt_message, opt_file);
+};
--
To view, visit https://gerrit.wikimedia.org/r/97740
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I83457a8b5424d3bc035ee4870047c2c7531ea7c7
Gerrit-PatchSet: 1
Gerrit-Project:
mediawiki/extensions/Collection/OfflineContentGenerator/latex_renderer
Gerrit-Branch: master
Gerrit-Owner: Cscott <canan...@wikimedia.org>
Gerrit-Reviewer: Mwalker <mwal...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
