Santhosh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/135016
Change subject: Pageloader: Use only parsoid and remove hardcoded enwiki host
......................................................................
Pageloader: Use only parsoid and remove hardcoded enwiki host
* Remvoed the MediaWiki page loader because we need parsoid actually
* Make the page loader work with any source language
* Added a bit of error handling
Change-Id: I6a5add64bf057575f42e2bb9635b0f2a506973bc
---
M ContentTranslationService.js
M config.example.js
D pageloader/MediaWikiApiPageLoader.js
M pageloader/PageLoader.js
D pageloader/ParsoidPageLoader.js
5 files changed, 31 insertions(+), 121 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver
refs/changes/16/135016/1
diff --git a/ContentTranslationService.js b/ContentTranslationService.js
index 3789500..7723f6a 100644
--- a/ContentTranslationService.js
+++ b/ContentTranslationService.js
@@ -66,6 +66,10 @@
title: title,
segmentedContent: segmenter.getSegmentedContent(),
} );
+ }, function ( error ) {
+ res.send( {
+ error: error
+ } );
} );
} );
diff --git a/config.example.js b/config.example.js
index 7eacd64..1e3180e 100644
--- a/config.example.js
+++ b/config.example.js
@@ -2,13 +2,7 @@
module.exports = {
port: 8080,
- pageloaderservice: 'parsoid',
- pageloaderservices: {
- parsoid: {
- api: 'http://parsoid-lb.eqiad.wikimedia.org'
- },
- mediawiki: {
- api: 'http://en.wikipedia.org/w/api.php'
- }
+ parsoid: {
+ api: 'http://parsoid-lb.eqiad.wikimedia.org'
}
};
diff --git a/pageloader/MediaWikiApiPageLoader.js
b/pageloader/MediaWikiApiPageLoader.js
deleted file mode 100644
index 0311f6b..0000000
--- a/pageloader/MediaWikiApiPageLoader.js
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Loads html for pages to translate via MediaWiki API
- *
- * @file
- * @copyright See AUTHORS.txt
- * @license GPL-2.0+
- */
-
-'use strict';
-
-var request = require( 'request' ),
- Q = require( 'q' );
-
-/**
- * @class MediaWikiApiPageLoader
- */
-function MediaWikiApiPageLoader( api ) {
- this.api = api;
-}
-
-MediaWikiApiPageLoader.prototype.load = function ( documentId ) {
- var deferred = Q.defer();
-
- request(
- this.api + '?action=parse&format=json&page=' +
encodeURIComponent( documentId ),
- function ( error, response, body ) {
- if ( error ) {
- deferred.reject( error );
- }
-
- if ( response.statusCode !== 200 ) {
- deferred.reject( response.statusCode );
- }
-
- deferred.resolve( JSON.parse( body ).parse.text['*'] );
- }
- );
-
- return deferred.promise;
-};
-
-module.exports.MediaWikiApiPageLoader = MediaWikiApiPageLoader;
diff --git a/pageloader/PageLoader.js b/pageloader/PageLoader.js
index ecc99a3..d91bf3e 100644
--- a/pageloader/PageLoader.js
+++ b/pageloader/PageLoader.js
@@ -8,7 +8,9 @@
'use strict';
-var config;
+var config,
+ request = require( 'request' ),
+ Q = require( 'q' );
try {
// TODO: Have an example configuration file.
@@ -16,46 +18,41 @@
} catch ( e ) {
// TODO: define this configuration in better way
config = {
- pageloaderservice: 'parsoid',
- pageloaderservices: {
- parsoid: {
- api: 'http://parsoid.wmflabs.org'
- },
- mediawiki: {
- api: 'http://en.wikipedia.org/w/api.php'
- }
+ parsoid: {
+ api: 'http://parsoid.wmflabs.org'
}
};
}
/**
* @class ParsoidPageLoader
+ *
+ * @param {string} page
+ * @param {string} sourceLanguage
+ * @retrun {Q.Promise}
*/
-function PageLoader( page ) {
+function PageLoader( page, sourceLanguage ) {
this.page = page;
+ this.sourceLanguage = sourceLanguage;
}
PageLoader.prototype.load = function () {
- var loader, promise, ParsoidPageLoader, MediaWikiApiPageLoader, title;
+ var deferred = Q.defer();
- // FIXME This way of getting tile is not reliable
- title = this.page.split( '/' ).pop();
+ request(
+ config.parsoid.api + '/' + this.sourceLanguage + 'wiki/' +
this.page,
+ function ( error, response, body ) {
+ if ( error ) {
+ deferred.reject( error );
+ }
+ if ( response.statusCode !== 200 ) {
+ deferred.reject( response.statusCode );
+ }
+ deferred.resolve( body );
+ }
+ );
- if ( config.pageloaderservice === 'parsoid' ) {
- ParsoidPageLoader = require( __dirname +
'/ParsoidPageLoader.js' ).ParsoidPageLoader;
- // FIXME It should be possible to fetch articles from any wiki.
- loader = new ParsoidPageLoader(
config.pageloaderservices.parsoid.api, 'enwiki' );
- promise = loader.load( title );
- }
-
- if ( config.pageloaderservice === 'mediawiki' ) {
- MediaWikiApiPageLoader = require( __dirname +
'/MediaWikiApiPageLoader.js' )
- .MediaWikiApiPageLoader;
- loader = new MediaWikiApiPageLoader(
config.pageloaderservices.mediawiki.api );
- promise = loader.load( title );
- }
-
- return promise;
+ return deferred.promise;
};
module.exports.PageLoader = PageLoader;
diff --git a/pageloader/ParsoidPageLoader.js b/pageloader/ParsoidPageLoader.js
deleted file mode 100644
index 61eeb7d..0000000
--- a/pageloader/ParsoidPageLoader.js
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Loads html for pages to translate form Parsoid service
- *
- * @file
- * @copyright See AUTHORS.txt
- * @license GPL-2.0+
- */
-
-'use strict';
-
-var request = require( 'request' ),
- Q = require( 'q' );
-
-/**
- * @class ParsoidPageLoader
- */
-function ParsoidPageLoader( host, wikiId ) {
- this.host = host;
- this.wikiId = wikiId;
-}
-
-ParsoidPageLoader.prototype.load = function ( documentId ) {
- var deferred = Q.defer();
-
- request(
- this.host + '/' + this.wikiId + '/' + documentId,
- function ( error, response, body ) {
- if ( error ) {
- deferred.reject( error );
- }
-
- if ( response.statusCode !== 200 ) {
- deferred.reject( response.statusCode );
- }
-
- deferred.resolve( body );
- }
- );
-
- return deferred.promise;
-};
-
-module.exports.ParsoidPageLoader = ParsoidPageLoader;
--
To view, visit https://gerrit.wikimedia.org/r/135016
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6a5add64bf057575f42e2bb9635b0f2a506973bc
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Santhosh <santhosh.thottin...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
