Mwalker has uploaded a new change for review.
https://gerrit.wikimedia.org/r/96723
Change subject: Adding in the render pipeline
......................................................................
Adding in the render pipeline
Change-Id: Ia1b70f1c1418137bae2673fef120ebe233bf8dde
---
M defaults.json
M lib/JobDetails.js
M lib/backend.js
3 files changed, 85 insertions(+), 9 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator
refs/changes/23/96723/1
diff --git a/defaults.json b/defaults.json
index bc32ac3..d46df83 100644
--- a/defaults.json
+++ b/defaults.json
@@ -19,11 +19,18 @@
"status_set_name": "job_status"
},
"backend": {
+ "bundler": {
+ "bin": "./bundler/bin/mw-bundler",
+ "parsoid_api": "http://localhost/";,
+ "parsoid_prefix": "localhost"
+ },
"writers": {
- "rdf2latex": "./latex_renderer/bin/mw-latexer"
+ "rdf2latex": {
+ "bin": "./latex_renderer/bin/mw-latexer",
+ "extension": ".pdf"
+ }
},
- "temp_dir": null,
- "bundler_bin": "./bundler/bin/mw-bundler"
+ "temp_dir": null
}
}
\ No newline at end of file
diff --git a/lib/JobDetails.js b/lib/JobDetails.js
index be720b7..c784888 100644
--- a/lib/JobDetails.js
+++ b/lib/JobDetails.js
@@ -128,6 +128,16 @@
jd.status.process = progress;
jd.status.page = page;
};
+exports.updateFinished = function(jd, file) {
+ jd.timestamp = new Date();
+ jd.state = 'finished';
+ jd.renderedFileLoc = file;
+
+ jd.status.status = '';
+ jd.status.article = null;
+ jd.status.process = 100;
+ jd.status.page = '';
+};
exports.updateError = function(jd, error) {
jd.timestamp = new Date();
jd.state = 'failed';
diff --git a/lib/backend.js b/lib/backend.js
index 4b9112f..eeeb960 100644
--- a/lib/backend.js
+++ b/lib/backend.js
@@ -130,13 +130,20 @@
function runBundler(jobDetails, metabookPath) {
var child;
- var bundle = path.join(config.get('backend:temp_dir'),
jobDetails.collectionId + '.zip');
+ var bundleFile = path.join(config.get('backend:temp_dir'),
jobDetails.collectionId + '.zip');
// TODO: Don't do this if we have a cached bundle
- console.debug('Forking child to run job id %s, %s',
jobDetails.collectionId, '-m ' + metabookPath);
- child = child_process.fork(config.get('backend:bundler_bin'), [
+ console.debug(
+ 'Forking child bundler to run job id %s (input: %s, output:
%s)',
+ jobDetails.collectionId,
+ metabookPath,
+ bundleFile
+ );
+ child = child_process.fork(config.get('backend:bundler:bin'), [
+ '-p', config.get('backend:bundler:parsoid_prefix'),
+ '-a', config.get('backend:bundler:parsoid_api'),
'-m', metabookPath,
- '-o', bundle
+ '-o', bundleFile
]);
child.on('error', function(err) {
console.error('Bundler child reported back with spawn error:
%s', err);
@@ -158,8 +165,9 @@
);
getNewItemFromQueue();
} else {
- // TODO: Actually launch off the next process
- getNewItemFromQueue();
+ // OK: Bundler completed, now launch the renderer
+ console.debug('Bundle completed successfully!');
+ runRenderer(jobDetails, bundleFile);
}
});
@@ -169,6 +177,57 @@
redisClient.hset(config.get('redis:status_set_name'),
jobDetails.collectionId, JSON.stringify(jobDetails));
}
+function runRenderer(jobDetails, bundleFile) {
+ // TODO: It isn't safe to not check the writer (aka, do we have
configuration for it)
+ var writer = jobDetails.writer;
+ var renderFile = path.join(
+ config.get('backend:temp_dir'),
+ jobDetails.collectionId + config.get('backend:writers:' +
writer + ':extension')
+ );
+
+ console.debug(
+ 'Forking child renderer to run job id %s (input: %s, output:
%s)',
+ jobDetails.collectionId,
+ bundleFile,
+ renderFile
+ );
+ child = child_process.fork(config.get('backend:writers:' + writer +
':bin'), [
+ '-t',
+ '-o', renderFile,
+ bundleFile
+ ]);
+ child.on('error', function(err) {
+ console.error('Renderer child reported back with spawn error:
%s', err);
+ jd.updateError(jobDetails, 'Could not launch rendering
process');
+ redisClient.hset(config.get('redis:status_set_name'),
jobDetails.collectionId, JSON.stringify(jobDetails));
+ getNewItemFromQueue();
+ });
+ child.on('message', function(message, handle) {
+ console.info('Received message from child! %s', message);
+ });
+ child.on('exit', function(code, signal) {
+ console.info('Rendering child exited with: %s', code);
+ if (code !== 0) {
+ jd.updateError(jobDetails, 'Render process died
unexpectedly');
+ redisClient.hset(
+ config.get('redis:status_set_name'),
+ jobDetails.collectionId,
+ JSON.stringify(jobDetails)
+ );
+ getNewItemFromQueue();
+ } else {
+ // Yay; process finished! mark as such and get new job
+ console.debug('Render completed successfully!');
+ jd.updateFinished(jobDetails, renderFile);
+ redisClient.hset(config.get('redis:status_set_name'),
jobDetails.collectionId, JSON.stringify(jobDetails));
+
+ getNewItemFromQueue();
+ }
+ });
+ jd.updateRendering(jobDetails, null, 'Launching renderer', 0);
+ redisClient.hset(config.get('redis:status_set_name'),
jobDetails.collectionId, JSON.stringify(jobDetails));
+}
+
exports.init = initBackend;
exports.start = startBackend;
exports.stop = stopBackend;
--
To view, visit https://gerrit.wikimedia.org/r/96723
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia1b70f1c1418137bae2673fef120ebe233bf8dde
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator
Gerrit-Branch: master
Gerrit-Owner: Mwalker <mwal...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
