Santhosh has uploaded a new change for review. https://gerrit.wikimedia.org/r/135016
Change subject: Pageloader: Use only parsoid and remove hardcoded enwiki host ...................................................................... Pageloader: Use only parsoid and remove hardcoded enwiki host * Remvoed the MediaWiki page loader because we need parsoid actually * Make the page loader work with any source language * Added a bit of error handling Change-Id: I6a5add64bf057575f42e2bb9635b0f2a506973bc --- M ContentTranslationService.js M config.example.js D pageloader/MediaWikiApiPageLoader.js M pageloader/PageLoader.js D pageloader/ParsoidPageLoader.js 5 files changed, 31 insertions(+), 121 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver refs/changes/16/135016/1 diff --git a/ContentTranslationService.js b/ContentTranslationService.js index 3789500..7723f6a 100644 --- a/ContentTranslationService.js +++ b/ContentTranslationService.js @@ -66,6 +66,10 @@ title: title, segmentedContent: segmenter.getSegmentedContent(), } ); + }, function ( error ) { + res.send( { + error: error + } ); } ); } ); diff --git a/config.example.js b/config.example.js index 7eacd64..1e3180e 100644 --- a/config.example.js +++ b/config.example.js @@ -2,13 +2,7 @@ module.exports = { port: 8080, - pageloaderservice: 'parsoid', - pageloaderservices: { - parsoid: { - api: 'http://parsoid-lb.eqiad.wikimedia.org' - }, - mediawiki: { - api: 'http://en.wikipedia.org/w/api.php' - } + parsoid: { + api: 'http://parsoid-lb.eqiad.wikimedia.org' } }; diff --git a/pageloader/MediaWikiApiPageLoader.js b/pageloader/MediaWikiApiPageLoader.js deleted file mode 100644 index 0311f6b..0000000 --- a/pageloader/MediaWikiApiPageLoader.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Loads html for pages to translate via MediaWiki API - * - * @file - * @copyright See AUTHORS.txt - * @license GPL-2.0+ - */ - -'use strict'; - -var request = require( 'request' ), - Q = require( 'q' ); - -/** - * @class MediaWikiApiPageLoader - */ -function MediaWikiApiPageLoader( api ) { - this.api = api; -} - -MediaWikiApiPageLoader.prototype.load = function ( documentId ) { - var deferred = Q.defer(); - - request( - this.api + '?action=parse&format=json&page=' + encodeURIComponent( documentId ), - function ( error, response, body ) { - if ( error ) { - deferred.reject( error ); - } - - if ( response.statusCode !== 200 ) { - deferred.reject( response.statusCode ); - } - - deferred.resolve( JSON.parse( body ).parse.text['*'] ); - } - ); - - return deferred.promise; -}; - -module.exports.MediaWikiApiPageLoader = MediaWikiApiPageLoader; diff --git a/pageloader/PageLoader.js b/pageloader/PageLoader.js index ecc99a3..d91bf3e 100644 --- a/pageloader/PageLoader.js +++ b/pageloader/PageLoader.js @@ -8,7 +8,9 @@ 'use strict'; -var config; +var config, + request = require( 'request' ), + Q = require( 'q' ); try { // TODO: Have an example configuration file. @@ -16,46 +18,41 @@ } catch ( e ) { // TODO: define this configuration in better way config = { - pageloaderservice: 'parsoid', - pageloaderservices: { - parsoid: { - api: 'http://parsoid.wmflabs.org' - }, - mediawiki: { - api: 'http://en.wikipedia.org/w/api.php' - } + parsoid: { + api: 'http://parsoid.wmflabs.org' } }; } /** * @class ParsoidPageLoader + * + * @param {string} page + * @param {string} sourceLanguage + * @retrun {Q.Promise} */ -function PageLoader( page ) { +function PageLoader( page, sourceLanguage ) { this.page = page; + this.sourceLanguage = sourceLanguage; } PageLoader.prototype.load = function () { - var loader, promise, ParsoidPageLoader, MediaWikiApiPageLoader, title; + var deferred = Q.defer(); - // FIXME This way of getting tile is not reliable - title = this.page.split( '/' ).pop(); + request( + config.parsoid.api + '/' + this.sourceLanguage + 'wiki/' + this.page, + function ( error, response, body ) { + if ( error ) { + deferred.reject( error ); + } + if ( response.statusCode !== 200 ) { + deferred.reject( response.statusCode ); + } + deferred.resolve( body ); + } + ); - if ( config.pageloaderservice === 'parsoid' ) { - ParsoidPageLoader = require( __dirname + '/ParsoidPageLoader.js' ).ParsoidPageLoader; - // FIXME It should be possible to fetch articles from any wiki. - loader = new ParsoidPageLoader( config.pageloaderservices.parsoid.api, 'enwiki' ); - promise = loader.load( title ); - } - - if ( config.pageloaderservice === 'mediawiki' ) { - MediaWikiApiPageLoader = require( __dirname + '/MediaWikiApiPageLoader.js' ) - .MediaWikiApiPageLoader; - loader = new MediaWikiApiPageLoader( config.pageloaderservices.mediawiki.api ); - promise = loader.load( title ); - } - - return promise; + return deferred.promise; }; module.exports.PageLoader = PageLoader; diff --git a/pageloader/ParsoidPageLoader.js b/pageloader/ParsoidPageLoader.js deleted file mode 100644 index 61eeb7d..0000000 --- a/pageloader/ParsoidPageLoader.js +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Loads html for pages to translate form Parsoid service - * - * @file - * @copyright See AUTHORS.txt - * @license GPL-2.0+ - */ - -'use strict'; - -var request = require( 'request' ), - Q = require( 'q' ); - -/** - * @class ParsoidPageLoader - */ -function ParsoidPageLoader( host, wikiId ) { - this.host = host; - this.wikiId = wikiId; -} - -ParsoidPageLoader.prototype.load = function ( documentId ) { - var deferred = Q.defer(); - - request( - this.host + '/' + this.wikiId + '/' + documentId, - function ( error, response, body ) { - if ( error ) { - deferred.reject( error ); - } - - if ( response.statusCode !== 200 ) { - deferred.reject( response.statusCode ); - } - - deferred.resolve( body ); - } - ); - - return deferred.promise; -}; - -module.exports.ParsoidPageLoader = ParsoidPageLoader; -- To view, visit https://gerrit.wikimedia.org/r/135016 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6a5add64bf057575f42e2bb9635b0f2a506973bc Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/cxserver Gerrit-Branch: master Gerrit-Owner: Santhosh <santhosh.thottin...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits