Santhosh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/115132
Change subject: Redis data store for the CX data model
......................................................................
Redis data store for the CX data model
A very simple redis strore that stores the data model in json string
with key as the source page.
Change-Id: Idca80f88e42ccc2e385bd24ced0b61013ef84033
---
M server/ContentTranslationService.js
M server/models/dataModelManager.js
M server/package.json
3 files changed, 73 insertions(+), 35 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ContentTranslation
refs/changes/32/115132/1
diff --git a/server/ContentTranslationService.js
b/server/ContentTranslationService.js
index ea87685..9269c41 100644
--- a/server/ContentTranslationService.js
+++ b/server/ContentTranslationService.js
@@ -14,40 +14,51 @@
'use strict';
-var app, instanceName, server, io, port, context,
+var instanceName, context,
express = require( 'express' );
-port = process.argv[2] || 8000;
+var port = process.argv[2] || 8000;
-app = express();
-server = require( 'http' ).createServer( app );
-io = require( 'socket.io' ).listen( server );
+var app = express();
+var server = require( 'http' ).createServer( app );
+var io = require( 'socket.io' ).listen( server );
+var redis = require( 'redis' );
+
+instanceName = 'worker(' + process.pid + ')';
// socket.io connection establishment
io.sockets.on( 'connection', function ( socket ) {
var datamodelManager,
- PageLoader,
- CXDataModelManager;
- console.log( '[CX] Client connected to worker(' + process.pid + ').
Socket: ' + socket.id );
+ CXDataModelManager,
+ redisSub = redis.createClient();
+
+ console.log( '[CX] Client connected to ' + instanceName + '). Socket: '
+ socket.id );
+ redisSub.subscribe( 'cx' );
+ redisSub.on( 'message', function( channel, message ) {
+ socket.emit( 'cx.data.update', JSON.parse( message ) );
+ console.log('[CX] Received from channel #' + channel + ':' +
message );
+ } );
+
socket.on( 'cx.init', function ( data ) {
- var pageloader;
CXDataModelManager = require( __dirname +
'/models/dataModelManager.js').CXDataModelManager;
context = {
sourceLanguage: data.sourceLanguage,
targetLanguage: data.targetLanguage,
sourcePage: data.sourcePage,
- socket: socket
+ pub: redis.createClient(),
+ store: redis.createClient()
};
- PageLoader = require( __dirname +
'/pageloader/PageLoader.js').PageLoader;
- pageloader = new PageLoader( context.sourcePage );
- pageloader.load().done( function ( data ) {
- context.sourceText = data;
- // Inject the session context to dataModelManager
- // It should take care of managing the data model and
pushing
- // it to the client through socket.
- datamodelManager = new CXDataModelManager( context );
- } );
+ // Inject the session context to dataModelManager
+ // It should take care of managing the data model and pushing
+ // it to the client through socket.
+ datamodelManager = new CXDataModelManager( context );
} );
+
+ socket.on( 'disconnect', function () {
+ console.warn( '[CX] Dsconnecting from redis' );
+ redisSub.quit();
+ } );
+
} );
// Everything else goes through this.
diff --git a/server/models/dataModelManager.js
b/server/models/dataModelManager.js
index 346bd62..501af26 100644
--- a/server/models/dataModelManager.js
+++ b/server/models/dataModelManager.js
@@ -25,27 +25,53 @@
* Initialize
*/
CXDataModelManager.prototype.init = function () {
- var dataModelManager = this, segmenter;
+ var dataModelManager = this,
+ segmenter,
+ PageLoader, pageloader;
- segmenter = new CXSegmenter( this.context.sourceText );
- segmenter.segment();
- this.dataModel = {
- version: 0,
- sourceLanguage: this.context.sourceLanguage,
- targetLanguage: this.context.targetLanguage,
- sourceLocation: this.context.sourceTitle,
- segments: segmenter.getSegments(),
- segmentedContent: segmenter.getSegmentedContent(),
- links: segmenter.getLinks()
- };
- dataModelManager.refresh();
+ // TODO: refactor this
+ this.context.store.get( this.context.sourcePage, function ( err, data )
{
+ dataModelManager.dataModel = JSON.parse( data );
+ if ( dataModelManager.dataModel ) {
+ // data model present in redis store
+ dataModelManager.publish();
+ } else {
+ PageLoader = require( __dirname +
'/../pageloader/PageLoader.js').PageLoader;
+ pageloader = new PageLoader(
dataModelManager.context.sourcePage );
+ pageloader.load().done( function ( data ) {
+ dataModelManager.context.sourceText = data;
+ segmenter = new CXSegmenter(
dataModelManager.context.sourceText );
+ segmenter.segment();
+ dataModelManager.dataModel = {
+ version: 0,
+ sourceLanguage:
dataModelManager.context.sourceLanguage,
+ targetLanguage:
dataModelManager.context.targetLanguage,
+ sourcePage:
dataModelManager.context.sourcePage,
+ segments: segmenter.getSegments(),
+ segmentedContent:
segmenter.getSegmentedContent(),
+ links: segmenter.getLinks()
+ };
+ dataModelManager.publish();
+ // TODO: Dispatch the context to a number of
task runners
+ // Once each task runners finish, publish.
+ } );
+ }
+ } );
};
/**
- * Refresh the data model. Syncs the data with the socket, updates version
+ * Publish the data model. Syncs the data with the socket, updates version
*/
-CXDataModelManager.prototype.refresh = function () {
- this.context.socket.emit( 'cx.data.update', this.getDataModel() );
+CXDataModelManager.prototype.publish = function () {
+ var dataModelManager = this,
+ data = JSON.stringify( dataModelManager.getDataModel() );
+
+ // TODO: Make the key unique, language pair also should be considered
+ // TODO: Make the data model in the redis store more granular than
+ // a single json dump.
+ this.context.store.set( this.dataModel.sourcePage, data, function () {
+ dataModelManager.context.pub.publish( 'cx', data );
+ } );
console.log( '[CX] Sending data. Version: ' + this.dataModel.version );
this.updateVersion();
};
diff --git a/server/package.json b/server/package.json
index 2add13e..856fe19 100644
--- a/server/package.json
+++ b/server/package.json
@@ -6,6 +6,7 @@
"express": "3.x",
"socket.io": "0.9.x",
"q": "*",
+ "redis": "0.10",
"request": "*"
},
"devDependencies": {
--
To view, visit https://gerrit.wikimedia.org/r/115132
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Idca80f88e42ccc2e385bd24ced0b61013ef84033
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ContentTranslation
Gerrit-Branch: master
Gerrit-Owner: Santhosh <santhosh.thottin...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
