Santhosh has uploaded a new change for review. https://gerrit.wikimedia.org/r/132931
Change subject: Rewrite the server as stateless, define REST apis ...................................................................... Rewrite the server as stateless, define REST apis Removed redis, websocket based architecture Introduced simple express server with REST apis APIs included now: /version - to get the version information of the server /page/:language/:title - Fetch the page with given title and language. The output contains segment annotated page content Allowed CORS for now. Change-Id: I120cb20fe672565343d8223312947ee55e211a9c --- M ContentTranslationService.js D models/DataModelManager.js M package.json M public/index.html M public/js/main.js 5 files changed, 82 insertions(+), 285 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver refs/changes/31/132931/1 diff --git a/ContentTranslationService.js b/ContentTranslationService.js index c635f1f..32fb516 100644 --- a/ContentTranslationService.js +++ b/ContentTranslationService.js @@ -14,104 +14,68 @@ 'use strict'; -var instanceName, context, port, app, server, io, fs, redis, express, - RedisStore, logger, args, privateKey, certificate, credentials; +var instanceName, + server, + fs = require( 'fs' ), + express = require( 'express' ), + app = express(), + logger = require( __dirname + '/utils/Logger.js' ), + args = require( 'minimist' )( process.argv.slice( 2 ) ), + port = args.port || 8000, + privateKey, + certificate, + credentials, + pkg = require( __dirname + '/package.json' ); -logger = require( __dirname + '/utils/Logger.js' ); -express = require( 'express' ); -fs = require( 'fs' ); -args = require( 'minimist' )( process.argv.slice( 2 ) ); -port = args.port || 8000; app = express(); - // Starts https server only if all needed args provided, else starts http server. if ( args.secure && args.key && args.cert ) { - privateKey = fs.readFileSync( args.key, 'utf8' ); + privateKey = fs.readFileSync( args.key, 'utf8' ); certificate = fs.readFileSync( args.cert, 'utf8' ); - credentials = { key: privateKey, cert: certificate }; + credentials = { + key: privateKey, + cert: certificate + }; server = require( 'https' ).createServer( credentials, app ); } else { server = require( 'http' ).createServer( app ); } -io = require( 'socket.io' ).listen( server, { - logger: { - debug: logger.debug, - info: logger.info, - error: logger.error, - warn: logger.warn - } -} ); - -// Production log configuration. -io.configure( 'production', function () { - io.set( 'log level', 1 ); // reduce logging - io.enable( 'browser client minification' ); // send minified client - io.enable( 'browser client etag' ); // apply etag caching logic based on version number - io.enable( 'browser client gzip' ); // gzip the file - - // enable all transports - io.set( 'transports', [ - 'websocket', - 'flashsocket', - 'htmlfile', - 'xhr-polling', - 'jsonp-polling' - ] ); -} ); - -// Development log configuration. -io.configure( 'development', function () { - io.enable( 'browser client gzip' ); // gzip the file, reduce the log size - io.set( 'transports', [ 'websocket' ] ); -} ); - -redis = require( 'redis' ); -// Use Redis as the store for socket.io -RedisStore = require( 'socket.io/lib/stores/redis' ); -io.set( 'store', - new RedisStore( { - redisPub: redis.createClient(), - redisSub: redis.createClient(), - redisClient: redis.createClient() - } ) -); instanceName = 'worker(' + process.pid + ')'; -// socket.io connection establishment -io.sockets.on( 'connection', function ( socket ) { - var dataModelManager, - CXDataModelManager, - redisSub = redis.createClient(); - - logger.debug( 'Client connected to ' + instanceName + '. Socket: ' + socket.id ); - redisSub.subscribe( 'cx' ); - redisSub.on( 'message', function ( channel, message ) { - socket.emit( 'cx.data.update', JSON.parse( message ) ); - logger.debug( 'Received from channel #' + channel + ':' + message ); - } ); - - socket.on( 'cx.init', function ( data ) { - CXDataModelManager = require( __dirname + '/models/DataModelManager.js' ).CXDataModelManager; - context = { - sourceLanguage: data.sourceLanguage, - targetLanguage: data.targetLanguage, - sourcePage: data.sourcePage, - pub: redis.createClient(), - store: redis.createClient() - }; - // Inject the session context to dataModelManager - // It should take care of managing the data model and pushing - // it to the client through socket. - dataModelManager = new CXDataModelManager( context ); - } ); - - socket.on( 'disconnect', function () { - logger.debug( 'Disconnecting from redis' ); - redisSub.quit(); - } ); - +app.use( function ( req, res, next ) { + res.header( 'Access-Control-Allow-Origin', '*' ); + res.header( 'Access-Control-Allow-Headers', 'X-Requested-With' ); + next(); } ); +app.get( '/page/:language/:title', function ( req, res ) { + var sourceLanguage = req.params.language, + title = req.params.title, + CXSegmenter = require( __dirname + '/segmentation/CXSegmenter.js' ).CXSegmenter, + PageLoader = require( __dirname + '/pageloader/PageLoader.js' ).PageLoader, + pageloader = new PageLoader( title, sourceLanguage ); + + pageloader.load().then( function ( data ) { + var segmenter; + + logger.debug( 'Page fetched' ); + segmenter = new CXSegmenter( data ); + segmenter.segment(); + res.send( { + sourceLanguage: sourceLanguage, + title: title, + segmentedContent: segmenter.getSegmentedContent(), + } ); + } ); +} ); + +app.get( '/version', function ( req, res ) { + var version = { + name: pkg.name, + version: pkg.version + }; + res.json( version ); +} ); // Everything else goes through this. app.use( express.static( __dirname + '/public' ) ); logger.info( instanceName + ' ready. Listening on port: ' + port ); diff --git a/models/DataModelManager.js b/models/DataModelManager.js deleted file mode 100644 index 60391b6..0000000 --- a/models/DataModelManager.js +++ /dev/null @@ -1,102 +0,0 @@ -/** - * ContentTranslation Server - * - * @file - * @ingroup Extensions - * @copyright See AUTHORS.txt - * @license GPL-2.0+ - */ - -'use strict'; - -var CXSegmenter = require( __dirname + '/../segmentation/CXSegmenter.js' ).CXSegmenter, - logger = require( __dirname + '/../utils/Logger.js' ); - -/** - * CXDataModelManager - * @class - */ -function CXDataModelManager( context ) { - this.context = context; - this.dataModel = null; - this.init(); -} - -/** - * Initialize - */ -CXDataModelManager.prototype.init = function () { - var dataModelManager = this, - segmenter, - PageLoader, pageloader; - - // TODO: refactor this - this.context.store.get( this.context.sourcePage, function ( err, data ) { - dataModelManager.dataModel = JSON.parse( data ); - - if ( dataModelManager.dataModel ) { - // data model present in redis store - dataModelManager.publish(); - } else { - PageLoader = require( __dirname + '/../pageloader/PageLoader.js' ).PageLoader; - pageloader = new PageLoader( dataModelManager.context.sourcePage ); - - pageloader.load().then( function ( data ) { - logger.debug( 'Page fetched' ); - dataModelManager.context.sourceText = data; - segmenter = new CXSegmenter( dataModelManager.context.sourceText ); - segmenter.segment(); - - dataModelManager.dataModel = { - version: 0, - sourceLanguage: dataModelManager.context.sourceLanguage, - targetLanguage: dataModelManager.context.targetLanguage, - sourcePage: dataModelManager.context.sourcePage, - segments: segmenter.getSegments(), - segmentedContent: segmenter.getSegmentedContent(), - links: segmenter.getLinks() - }; - - dataModelManager.publish(); - }, function () { - logger.error( 'Error in retrieving the page ' + - dataModelManager.context.sourcePage ); - } ); - } - } ); -}; - -/** - * Publish the data model. Syncs the data with the socket, updates version - */ -CXDataModelManager.prototype.publish = function () { - var dataModelManager = this, - data = JSON.stringify( dataModelManager.getDataModel() ); - - // TODO: Make the key unique, language pair also should be considered - // TODO: Make the data model in the redis store more granular than - // a single json dump. - this.context.store.set( this.dataModel.sourcePage, data, function () { - dataModelManager.context.pub.publish( 'cx', data ); - } ); - - logger.debug( 'Sending data. Version: ' + this.dataModel.version ); - - this.incrementVersionNumber(); -}; - -/** - * Update the version number of the model - */ -CXDataModelManager.prototype.incrementVersionNumber = function () { - this.dataModel.version += 1; -}; - -/** - * Get the data model - */ -CXDataModelManager.prototype.getDataModel = function () { - return this.dataModel; -}; - -module.exports.CXDataModelManager = CXDataModelManager; diff --git a/package.json b/package.json index 6c72d7c..c6b910e 100644 --- a/package.json +++ b/package.json @@ -10,10 +10,8 @@ "jquery": "1.8.3", "minimist": "*", "q": "*", - "redis": "0.10", "request": "*", "sax": "0.6.0", - "socket.io": "0.9.x", "winston": "*" }, "devDependencies": { diff --git a/public/index.html b/public/index.html index 7a8351d..73b69f5 100644 --- a/public/index.html +++ b/public/index.html @@ -1,79 +1,35 @@ <html> + <head> - <script src='//code.jquery.com/jquery-1.10.2.min.js'></script> - <script src="/socket.io/socket.io.js"></script> - <title>Content Translation Server</title> - <style> - body { - width: 80%; - margin-left: 10%; - } - - label { - width: 20%; - } - - input { - width: 40%; - padding: 5px; - } - - .lang { - width: 5%; - } - - button { - width: 10%; - padding: 5px; - } - - .status { - position: fixed; - bottom: 0; - left: 0; - right: 0; - height: 16px; - padding: 5px; - background: #ccc; - color: green; - font-size: 0.8em; - } - - .cx-segment:hover { - background-color: #ccc; - } - - .cx-link:hover { - background-color: #aaa; - } - - .form { - border: 1px solid #000; - padding: 10px; - } - - .info { - color: #555; - } - </style> + <script src='//code.jquery.com/jquery-1.10.2.min.js'></script> + <title>Content Translation Server</title> + <link rel='stylesheet' href="css/main.css" type='text/css' /> </head> <body> - <h1>Content Translation Server</h1> - <div class='form'> - <label for="sourcePage">Translate</label> - <input name="sourcePage" value="https://en.wikipedia.org/wiki/Food" /> - <label for="sourceLanguage">from</label> - <input class="lang" name="sourceLanguage" value="en" /> - <label for="targetLanguage">to</label> - <input class="lang" name="targetLanguage" value="cy" /> - <button>Go</button> - <progress></progress> - </div> - <div contenteditable class="article"></div> - - <div class='status'>Server console. Keep your browser developer console open. Not connected to server.</div> - <script src="js/main.js"></script> + <h1>Content Translation Server</h1> + <div class='form '> + <label for="sourcePage">Translate the article</label> + <input name="sourcePage" value="Food" /> + <label for="sourceLanguage">from</label> + <input class="lang" name="sourceLanguage" value="en" /> + <label for="targetLanguage">to</label> + <input class="lang" name="targetLanguage" value="cy" /> + <button>Go</button> + <progress></progress> + </div> + <div class="article"></div> + <script src="js/main.js"></script> </body> -</html> \ No newline at end of file +</html> + + + + + + + + + + diff --git a/public/js/main.js b/public/js/main.js index 463d2fb..259b747 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -2,35 +2,16 @@ ( function ( $ ) { 'use strict'; - var cxdata; - $( '.article' ).on( 'click', '.cx-segment', function () { - var segment = cxdata.segments[ $( this ).data( 'segmentid' ) ]; - console.log( segment ); - } ); - - $( '.article' ).on( 'click', '.cx-link', function () { - var linkid = cxdata.links[ $( this ).data( 'linkid' ) ]; - console.log( linkid ); - } ); - - /* global io */ $( document ).ready( function () { - var socket = io.connect( '/' ); $( 'progress' ).hide(); - socket.on( 'cx.data.update', function ( data ) { - cxdata = data; - $( 'progress' ).hide(); - $( '.status' ).text( 'Received version ' + cxdata.version + '. Click on the content segments to inspect.' ); - $( '.article' ).html( cxdata.segmentedContent ); - console.log( cxdata ); - } ); $( 'button' ).click( function () { $( 'progress' ).show(); $( '.status' ).text( 'Connecting to server...' ); - socket.emit( 'cx.init', { - sourcePage: $( 'input[name=sourcePage]' ).val(), - sourceLanguage: $( 'input[name=sourceLanguage]' ).val(), - targetLanguage: $( 'input[name=targetLanguage]' ).val() + var sourcePage = $( 'input[name=sourcePage]' ).val(), + sourceLanguage = $( 'input[name=sourceLanguage]' ).val(); + $.get( 'page/' + sourceLanguage + '/' + sourcePage, function ( response ) { + $( '.article' ).html( response.segmentedContent ); + $( 'progress' ).hide(); } ); } ); } ); -- To view, visit https://gerrit.wikimedia.org/r/132931 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I120cb20fe672565343d8223312947ee55e211a9c Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/cxserver Gerrit-Branch: master Gerrit-Owner: Santhosh <santhosh.thottin...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits