Santhosh has uploaded a new change for review. https://gerrit.wikimedia.org/r/115132
Change subject: Redis data store for the CX data model ...................................................................... Redis data store for the CX data model A very simple redis strore that stores the data model in json string with key as the source page. Change-Id: Idca80f88e42ccc2e385bd24ced0b61013ef84033 --- M server/ContentTranslationService.js M server/models/dataModelManager.js M server/package.json 3 files changed, 73 insertions(+), 35 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ContentTranslation refs/changes/32/115132/1 diff --git a/server/ContentTranslationService.js b/server/ContentTranslationService.js index ea87685..9269c41 100644 --- a/server/ContentTranslationService.js +++ b/server/ContentTranslationService.js @@ -14,40 +14,51 @@ 'use strict'; -var app, instanceName, server, io, port, context, +var instanceName, context, express = require( 'express' ); -port = process.argv[2] || 8000; +var port = process.argv[2] || 8000; -app = express(); -server = require( 'http' ).createServer( app ); -io = require( 'socket.io' ).listen( server ); +var app = express(); +var server = require( 'http' ).createServer( app ); +var io = require( 'socket.io' ).listen( server ); +var redis = require( 'redis' ); + +instanceName = 'worker(' + process.pid + ')'; // socket.io connection establishment io.sockets.on( 'connection', function ( socket ) { var datamodelManager, - PageLoader, - CXDataModelManager; - console.log( '[CX] Client connected to worker(' + process.pid + '). Socket: ' + socket.id ); + CXDataModelManager, + redisSub = redis.createClient(); + + console.log( '[CX] Client connected to ' + instanceName + '). Socket: ' + socket.id ); + redisSub.subscribe( 'cx' ); + redisSub.on( 'message', function( channel, message ) { + socket.emit( 'cx.data.update', JSON.parse( message ) ); + console.log('[CX] Received from channel #' + channel + ':' + message ); + } ); + socket.on( 'cx.init', function ( data ) { - var pageloader; CXDataModelManager = require( __dirname + '/models/dataModelManager.js').CXDataModelManager; context = { sourceLanguage: data.sourceLanguage, targetLanguage: data.targetLanguage, sourcePage: data.sourcePage, - socket: socket + pub: redis.createClient(), + store: redis.createClient() }; - PageLoader = require( __dirname + '/pageloader/PageLoader.js').PageLoader; - pageloader = new PageLoader( context.sourcePage ); - pageloader.load().done( function ( data ) { - context.sourceText = data; - // Inject the session context to dataModelManager - // It should take care of managing the data model and pushing - // it to the client through socket. - datamodelManager = new CXDataModelManager( context ); - } ); + // Inject the session context to dataModelManager + // It should take care of managing the data model and pushing + // it to the client through socket. + datamodelManager = new CXDataModelManager( context ); } ); + + socket.on( 'disconnect', function () { + console.warn( '[CX] Dsconnecting from redis' ); + redisSub.quit(); + } ); + } ); // Everything else goes through this. diff --git a/server/models/dataModelManager.js b/server/models/dataModelManager.js index 346bd62..501af26 100644 --- a/server/models/dataModelManager.js +++ b/server/models/dataModelManager.js @@ -25,27 +25,53 @@ * Initialize */ CXDataModelManager.prototype.init = function () { - var dataModelManager = this, segmenter; + var dataModelManager = this, + segmenter, + PageLoader, pageloader; - segmenter = new CXSegmenter( this.context.sourceText ); - segmenter.segment(); - this.dataModel = { - version: 0, - sourceLanguage: this.context.sourceLanguage, - targetLanguage: this.context.targetLanguage, - sourceLocation: this.context.sourceTitle, - segments: segmenter.getSegments(), - segmentedContent: segmenter.getSegmentedContent(), - links: segmenter.getLinks() - }; - dataModelManager.refresh(); + // TODO: refactor this + this.context.store.get( this.context.sourcePage, function ( err, data ) { + dataModelManager.dataModel = JSON.parse( data ); + if ( dataModelManager.dataModel ) { + // data model present in redis store + dataModelManager.publish(); + } else { + PageLoader = require( __dirname + '/../pageloader/PageLoader.js').PageLoader; + pageloader = new PageLoader( dataModelManager.context.sourcePage ); + pageloader.load().done( function ( data ) { + dataModelManager.context.sourceText = data; + segmenter = new CXSegmenter( dataModelManager.context.sourceText ); + segmenter.segment(); + dataModelManager.dataModel = { + version: 0, + sourceLanguage: dataModelManager.context.sourceLanguage, + targetLanguage: dataModelManager.context.targetLanguage, + sourcePage: dataModelManager.context.sourcePage, + segments: segmenter.getSegments(), + segmentedContent: segmenter.getSegmentedContent(), + links: segmenter.getLinks() + }; + dataModelManager.publish(); + // TODO: Dispatch the context to a number of task runners + // Once each task runners finish, publish. + } ); + } + } ); }; /** - * Refresh the data model. Syncs the data with the socket, updates version + * Publish the data model. Syncs the data with the socket, updates version */ -CXDataModelManager.prototype.refresh = function () { - this.context.socket.emit( 'cx.data.update', this.getDataModel() ); +CXDataModelManager.prototype.publish = function () { + var dataModelManager = this, + data = JSON.stringify( dataModelManager.getDataModel() ); + + // TODO: Make the key unique, language pair also should be considered + // TODO: Make the data model in the redis store more granular than + // a single json dump. + this.context.store.set( this.dataModel.sourcePage, data, function () { + dataModelManager.context.pub.publish( 'cx', data ); + } ); console.log( '[CX] Sending data. Version: ' + this.dataModel.version ); this.updateVersion(); }; diff --git a/server/package.json b/server/package.json index 2add13e..856fe19 100644 --- a/server/package.json +++ b/server/package.json @@ -6,6 +6,7 @@ "express": "3.x", "socket.io": "0.9.x", "q": "*", + "redis": "0.10", "request": "*" }, "devDependencies": { -- To view, visit https://gerrit.wikimedia.org/r/115132 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Idca80f88e42ccc2e385bd24ced0b61013ef84033 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/ContentTranslation Gerrit-Branch: master Gerrit-Owner: Santhosh <santhosh.thottin...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits