Mwalker has uploaded a new change for review.
https://gerrit.wikimedia.org/r/95644
Change subject: WIP Node.JS Renderer
......................................................................
WIP Node.JS Renderer
Change-Id: I652de11abe0fd48a8a9d48572b85177cccb91560
---
A collectoid/README.md
A collectoid/collectoid.js
A collectoid/defaults.json
A collectoid/frontend.js
A collectoid/package.json
5 files changed, 309 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection
refs/changes/44/95644/1
diff --git a/collectoid/README.md b/collectoid/README.md
new file mode 100644
index 0000000..b5b1652
--- /dev/null
+++ b/collectoid/README.md
@@ -0,0 +1,2 @@
+Ubuntu Dependencies
+-- redis-server
\ No newline at end of file
diff --git a/collectoid/collectoid.js b/collectoid/collectoid.js
new file mode 100755
index 0000000..51a72d2
--- /dev/null
+++ b/collectoid/collectoid.js
@@ -0,0 +1,97 @@
+#!/usr/bin/env node
+
+/*
+ Startup file for the WMF renderer
+
+ There are three main components that this script will manage.
+ - The frontend is responsible for handling all HTTP bookcmd requests
+ - Spider clients are responsible for getting all content
+ - Render clients are responsible for rendering content
+
+ This script will cluster.fork() as many frontends as configured (default
+ is one) and then poll redis in the origin thread spawning clients as
+ required.
+*/
+
+var nconf = require('nconf');
+require('rconsole');
+var redis = require('redis');
+var sleep = require('sleep');
+
+var redisClient = null;
+
+/* --- Configuration Options & File ---------------------------------------- */
+nconf
+ .argv({
+ c: {
+ alias: 'config-file',
+ describe: 'Local configuration file',
+ default: '/etc/collectoid.json'
+ }
+ })
+ .file({file: nconf.get('config-file')})
+ .file({file: 'defaults.json'});
+
+/* --- Initial Logging ----------------------------------------------------- */
+console.set({
+ facility: 'local0',
+ title: 'collectoid'
+});
+
+/* --- Redis Connection ---------------------------------------------------- */
+function startRedis() {}
+console.debug(
+ 'Starting connection to redis on %s, port %s (using password: %s)',
+ nconf.get("redis:host"),
+ nconf.get("redis:port"),
+ nconf.get("redis:password") ? true : false
+);
+redisClient = redis.createClient(
+ nconf.get("redis:port"),
+ nconf.get("redis:host"),
+ {
+ enable_offline_queue: false,
+ retry_max_delay: nconf.get("redis:retry_max_delay"),
+ auth_pass: nconf.get("redis:password")
+ }
+);
+function redisConnectionClosed( err ) {
+ console.error(
+ 'Redis connection (now %s) had error: %s',
+ redisClient.connected ? 'connected': 'disconnected',
+ err
+ );
+}
+redisClient.on("error", redisConnectionClosed);
+redisClient.on("end", redisConnectionClosed);
+redisClient.on("ready", function() {
+ console.debug("Redis connection now ready");
+});
+
+/* --- Process management -------------------------------------------------- */
+var respawnThreads = true;
+
+function gracefulShutdown() {
+ respawnThreads = false;
+ console.info('Beginning graceful shutdown');
+ frontend.stopServer(process.exit);
+ //process.exit();
+}
+function immediateShutdown() {
+ respawnThreads = false;
+ console.info('Shutting down immediately');
+
+ process.exit();
+}
+
+process.on('SIGINT', gracefulShutdown);
+process.on('SIGTERM', gracefulShutdown);
+process.on('SIGHUP', immediateShutdown);
+
+/* --- Frontend thread startup --------------------------------------------- */
+var frontend = require('./frontend.js' );
+frontend.init(nconf, redisClient);
+frontend.startServer();
+
+/* --- Client management --------------------------------------------------- */
+//redisClient.pop();
diff --git a/collectoid/defaults.json b/collectoid/defaults.json
new file mode 100644
index 0000000..45ef2d7
--- /dev/null
+++ b/collectoid/defaults.json
@@ -0,0 +1,17 @@
+{
+ "redis": {
+ "host": "localhost",
+ "port": 6379,
+ "password": null,
+ "retry_max_delay": 60000,
+
+ "spider_queue_name": "spider_queue",
+ "render_queue_name": "render_queue",
+ "status_set_name": "job_status"
+ },
+ "frontend": {
+ "socket": null,
+ "port": 17080,
+ "address": null
+ }
+}
\ No newline at end of file
diff --git a/collectoid/frontend.js b/collectoid/frontend.js
new file mode 100644
index 0000000..d39c76a
--- /dev/null
+++ b/collectoid/frontend.js
@@ -0,0 +1,179 @@
+var http = require('http');
+var url = require('url');
+var crypto = require('crypto');
+
+var config = null;
+var server = null;
+var redisClient = null;
+
+exports.init = function(nconf, redis) {
+ config = nconf;
+ redisClient = redis;
+
+ server = http.createServer(handleRequest);
+};
+
+/**
+ * Starts the frontend server
+ * @public
+ */
+exports.startServer = function() {
+ var socket = config.get('frontend:socket');
+ var port = config.get('frontend:port');
+ var address = config.get('frontend:address');
+
+ if (socket) {
+ console.info('Frontend server reporting for duty on unix socket
"%s"', socket);
+ server.listen( socket );
+ } else if (port) {
+ console.info(
+ 'Frontend server reporting for duty on %s, port %s',
+ address ? address : 'all IP interfaces',
+ port
+ );
+ server.listen(port, address);
+ } else {
+ console.error('Frontend could not start, no server parameters
specified');
+ process.abort();
+ }
+};
+
+exports.stopServer = function(callbackFunc) {
+ server.close(callbackFunc);
+};
+
+function handleRequest(request, response) {
+ if (request.method !== "GET") {
+ response.writeHead(405, "Only GET requests are supported");
+ response.end();
+ } else {
+ var args = url.parse(request.url, true ).query;
+
+ try {
+ switch (args.bookcmd) {
+ case "render":
+ var retval = handleRender(args);
+ console.debug('handle render returned
%s', JSON.stringify(retval));
+ response.write(JSON.stringify(retval));
+ break;
+ case "render_status":
+
response.write(handleRenderStatus(args));
+ break;
+ case "download":
+ handleDownload(args, response);
+ break;
+ }
+ } catch(err) {
+ if (err instanceof FrontendError) {
+ console.error("Error whilst serving request:
%s", err.message);
+ response.writeHead(err.code || 500,
err.message);
+ } else {
+ console.trace('Unknown error: ' + err);
+ response.writeHead(500, "Unexpected server
error");
+ }
+ } finally {
+ response.end();
+ }
+ }
+}
+
+function handleRender(args) {
+ var collectionId = args.collection_id;
+ var metabook = args.metabook;
+ var metabookObj = {};
+ var writer = args.writer;
+ var language = args.language;
+ var forceRender = args.force_render || false;
+ var isCached = false;
+
+ if (!writer) {
+ throw new FrontendError('Writer not specified', 400);
+ }
+
+ if (!collectionId) {
+ // Create the SHA hash of this collection request
+ if (!metabook) {
+ throw new FrontendError('Metabook parameter not given',
400);
+ } else {
+ metabookObj = JSON.parse(metabook);
+ if (!metabookObj) {
+ throw new FrontendError('Metabook parameter was
not valid JSON', 400);
+ }
+ collectionId = createCollectionId(metabookObj);
+ }
+ }
+
+ if (!forceRender) {
+ // TODO: Check to see if we already have a copy of the rendered
content somewhere
+ }
+
+ if (!isCached || forceRender) {
+ // Shove a new job into redis
+ console.info('Adding job with id %s to redis', collectionId);
+ try {
+ redisClient.rpush(
+ config.get('redis:spider_queue_name'),
+ collectionId
+ );
+ redisClient.sadd(
+ config.get('redis:status_set_name'),
+ JSON.stringify({
+ id: collectionId,
+ metabook: metabookObj,
+ writer: writer,
+ language: language
+ })
+ )
+ } catch (err) {
+ console.error('Job insertion into redis failed for job
%s with error: %s', collectionId, err);
+ throw new FrontendError('Job insertion failed (redis
failure?)', 500);
+ }
+ }
+
+ return {
+ collection_id: collectionId,
+ writer: writer,
+ is_cached: isCached
+ };
+}
+
+function handleRenderStatus(args) {
+ var collectionId = args.collection_id;
+
+
+}
+
+function handleDownload(args, response) {
+
+}
+
+function createCollectionId(metabookObj) {
+ collectionId = crypto.createHash('sha1');
+ collectionId.update(writer);
+ collectionId.update(metabookObj.title);
+ collectionId.update(metabookObj.subtitle);
+
+ var updateChapter = function(items) {
+ items.forEach(function(item) {
+ if (item.type === 'chapter') {
+ collectionId.update(item.title);
+ updateChapter(item.items);
+ } else {
+ collectionId.update(item.url);
+ if (item.currentVersion) {
+ collectionId.update(item.latest);
+ } else {
+ collectionId.update(item.revision);
+ }
+ }
+ });
+ };
+ updateChapter(metabookObj.items);
+
+ return collectionId.digest('hex');
+}
+
+var FrontendError = function( message, code ) {
+ this.message = message;
+ this.code = code || 500;
+};
diff --git a/collectoid/package.json b/collectoid/package.json
new file mode 100644
index 0000000..15b7805
--- /dev/null
+++ b/collectoid/package.json
@@ -0,0 +1,14 @@
+{
+ "name": "Collectoid",
+ "description": "Backend render server for the collection extension",
+ "version": "0.0.1",
+ "dependencies": {
+ "cluster": "*",
+ "crypto": "*",
+ "nconf": "*",
+ "rconsole": "*",
+ "sleep": "*",
+ "redis": "*",
+ "hiredis": "*"
+ }
+}
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/95644
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I652de11abe0fd48a8a9d48572b85177cccb91560
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection
Gerrit-Branch: master
Gerrit-Owner: Mwalker <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
