Cscott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/159177
Change subject: Allow rendering from unzipped bundle
......................................................................
Allow rendering from unzipped bundle
Allow the renderer to consume an unzipped bundle file. This saves CPU
cycles when the bundle was created on the same machine. Due to the
use of hardlinks, the bundle directory in this case must be on the
same file system as the temp dir -- use the system environment
variable TMPDIR to specify an appropriate directory if necessary.
See change Iad77b766a12654876a7c140220df9a37dc25f6c6 for the
corresponding commit in the `mw-ocg-latexer` package.
Change-Id: Ieac2c6f4a53727f0162aa0fc10c5c6b2866bf0fa
---
M bin/mw-ocg-texter
M lib/index.js
2 files changed, 79 insertions(+), 18 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator/text_renderer
refs/changes/77/159177/1
diff --git a/bin/mw-ocg-texter b/bin/mw-ocg-texter
index 81cc312..3015fc4 100755
--- a/bin/mw-ocg-texter
+++ b/bin/mw-ocg-texter
@@ -7,7 +7,7 @@
program
.version(texter.version)
- .usage('[options] <bundle.zip>')
+ .usage('[options] <bundle_dir or bundle.zip>')
.option('-o, --output <filename>',
'Save text to the given <filename>', null)
.option('-w, --no-wrap',
@@ -24,7 +24,7 @@
program.parse(process.argv);
if (program.args.length === 0) {
- console.error('A bundle filename is required.');
+ console.error('A bundle filename or directory is required.');
return 1;
}
if (program.args.length > 1) {
diff --git a/lib/index.js b/lib/index.js
index a130771..dc36f90 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -203,7 +203,7 @@
this.buffer.push(text);
};
-/* Document node visitor class. Collects LaTeX output as it traverses the
+/* Document node visitor class. Collects plain text output as it traverses the
* document tree. */
var Visitor = function(document, format, options) {
this.document = document;
@@ -855,8 +855,26 @@
// ---------------------------------------------------------------------
// Bundle and file processing
-// return a promise for the builddir and control file contents
-// (after the bundle has been unpacked)
+// Helper: hard link a directory, recursively.
+var cprl = function(from, to) {
+ return P.call(fs.mkdir, fs, to).then(function() {
+ return P.call(fs.readdir, fs, from);
+ }).map(function(file) {
+ var pathfrom = path.join(from, file);
+ var pathto = path.join(to, file);
+ return P.call(fs.lstat, fs, pathfrom).then(function(stats) {
+ if (stats.isFile()) {
+ return P.call(fs.link, fs, pathfrom, pathto);
+ }
+ if (stats.isDirectory()) {
+ return cprl(pathfrom, pathto);
+ }
+ // ignore other file types (symlink, block device, etc)
+ });
+ });
+};
+
+// Step 1a: unpack a bundle, and return a promise for the builddir.
var unpackBundle = function(options) {
var metabook, builddir, status = options.status;
@@ -865,7 +883,7 @@
// first create a temporary directory
return P.call(tmp.dir, tmp, {
prefix: json.name,
- unsafeCleanup: !(options.debug || options.latex)
+ unsafeCleanup: !(options.debug)
}).then(function(_builddir) {
builddir = _builddir;
// make bundle and output subdirs
@@ -880,14 +898,42 @@
cwd: bundledir
});
}).then(function() {
- // now read in the main metabook.json file
- return P.call(
- fs.readFile, fs, path.join(builddir, 'bundle',
'metabook.json')
- ).then(function(data) {
- metabook = JSON.parse(data);
- });
+ return builddir;
+ });
+};
+
+// Step 1b: we were given a bundle directory. Create a tmpdir and then
+// hard link the bundle directory into it. Be sure your TMPDIR is
+// on the same filesystem as the provided bundle directory if you
+// want this to be fast.
+var hardlinkBundle = function(options) {
+ var builddir, status = options.status;
+
+ status.createStage(0, 'Creating work space');
+ // first create a temporary directory
+ return P.call(tmp.dir, tmp, {
+ prefix: json.name,
+ unsafeCleanup: !(options.debug)
+ }).then(function(_builddir) {
+ builddir = _builddir;
+ // make output subdir
+ return Promise.join(
+ // make latex subdir
+ P.call(fs.mkdir, fs, path.join(builddir, 'output')),
+ // hardlink bundledir into 'bundle'
+ cprl(path.resolve( options.bundle ),
path.join(builddir, 'bundle')).
+ catch(function(e) {
+ // slightly helpful diagnostics
+ if (e.code === 'EXDEV') {
+ throw new Error(
+ "TMPDIR must be on same
filesystem as bundle dir"
+ );
+ }
+ throw e;
+ })
+ );
}).then(function() {
- return { metabook: metabook, builddir: builddir };
+ return builddir;
});
};
@@ -1021,11 +1067,26 @@
});
var metabook, builddir;
return Promise.resolve().then(function() {
- // unpack the bundle
- return unpackBundle(options);
- }).then(function(args) {
- metabook = args.metabook;
- builddir = args.builddir;
+ // were we given a zip file or a directory?
+ return P.call(fs.stat, fs, options.bundle);
+ }).then(function(stat) {
+ if (stat.isDirectory()) {
+ // create a workspace and hard link the provided
directory
+ return hardlinkBundle(options);
+ } else {
+ // unpack the bundle
+ return unpackBundle(options);
+ }
+ }).then(function(_builddir) {
+ builddir = _builddir;
+ // read the main metabook.json file
+ return P.call(
+ fs.readFile, fs,
+ path.join(builddir, 'bundle', 'metabook.json'),
+ { encoding: 'utf8' }
+ ).then(function(data) {
+ metabook = JSON.parse(data);
+ });
}).then(function() {
// generate the plaintext
return generateOutput(metabook, builddir, options);
--
To view, visit https://gerrit.wikimedia.org/r/159177
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ieac2c6f4a53727f0162aa0fc10c5c6b2866bf0fa
Gerrit-PatchSet: 1
Gerrit-Project:
mediawiki/extensions/Collection/OfflineContentGenerator/text_renderer
Gerrit-Branch: master
Gerrit-Owner: Cscott <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
