Santhosh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/200516
Change subject: Youdao MT client
......................................................................
Youdao MT client
Proof of concept. Translates with a valid key, but cannot do
annotation mapping.
Change-Id: Ie904d29f5cb698a70aeb946e237ad5e345d33485
---
M config.defaults.js
M index.js
M mt/MTClient.js
A mt/Youdao.js
M mt/index.js
A tests/mt/Youdao.test.js
6 files changed, 170 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver
refs/changes/16/200516/4
diff --git a/config.defaults.js b/config.defaults.js
index ff5e2be..fb2330b 100644
--- a/config.defaults.js
+++ b/config.defaults.js
@@ -14,6 +14,8 @@
'mt.apertium.api': 'http://apertium.wmflabs.org',
'mt.yandex.api': 'https://translate.yandex.net',
'mt.yandex.key': null,
+ 'mt.youdao.api': 'https://fanyi.youdao.com/paidapi/fanyiapi',
+ 'mt.youdao.key': null,
// Use SSL?
secure: false,
// SSL key filename
@@ -22,8 +24,8 @@
cert: null,
// Service registry
registry: {
- source: [ 'af', 'an', 'ar', 'az', 'bg', 'bs', 'ca', 'cr', 'cy',
'en', 'eo', 'es', 'fr', 'gl', 'gu', 'hi', 'hr', 'id', 'ja', 'kk', 'km', 'kn',
'ky', 'kz', 'min', 'mk', 'ms', 'mt', 'nl', 'no', 'nn', 'oc', 'pa', 'pl', 'pt',
'ru', 'sh', 'simple', 'sl', 'tr', 'tt', 'uk', 'ur', 'uz', 'vi', 'xh', 'zh' ],
- target: [ 'af', 'an', 'ar', 'az', 'bg', 'bs', 'ca', 'cr', 'cy',
'eo', 'es', 'fr', 'gl', 'gu', 'hi', 'hr', 'id', 'ja', 'kk', 'km', 'kn', 'ky',
'kz', 'min', 'mk', 'ms', 'mt', 'nl', 'no', 'nn', 'oc', 'pa', 'pl', 'pt', 'ru',
'sh', 'simple', 'sl', 'tt', 'tr', 'uk', 'ur', 'uz', 'vi', 'xh', 'zh' ],
+ source: [ 'af', 'an', 'ar', 'az', 'bg', 'bs', 'ca', 'cr', 'cy',
'en', 'eo', 'es', 'fr', 'gl', 'gu', 'hi', 'hr', 'id', 'ja', 'kk', 'km', 'kn',
'ky', 'kz', 'min', 'mk', 'ms', 'mt', 'nl', 'no', 'nn', 'oc', 'pa', 'pl', 'pt',
'ru', 'sh', 'simple', 'sl', 'tr', 'tt', 'uk', 'ur', 'uz', 'vi', 'xh', 'zh',
'zh-Hans', 'zh-Hant' ],
+ target: [ 'af', 'an', 'ar', 'az', 'bg', 'bs', 'ca', 'cr', 'cy',
'eo', 'es', 'fr', 'gl', 'gu', 'hi', 'hr', 'id', 'ja', 'kk', 'km', 'kn', 'ky',
'kz', 'min', 'mk', 'ms', 'mt', 'nl', 'no', 'nn', 'oc', 'pa', 'pl', 'pt', 'ru',
'sh', 'simple', 'sl', 'tt', 'tr', 'uk', 'ur', 'uz', 'vi', 'xh', 'zh',
'zh-Hans', 'zh-Hant' ],
mt: {
Apertium: {
af: [ 'nl' ],
@@ -62,6 +64,14 @@
Yandex: {
en: [ 'ru' ]
},
+ Youdao: {
+ en: [ 'zh', 'zh-Hans', 'ja' ],
+ simple: [ 'zh', 'zh-Hans' ],
+ ja: [ 'zh-Hans', 'zh' ],
+ zh: [ 'en', 'ja' ],
+ 'zh-Hans': [ 'en', 'ja' ],
+ 'zh-Hant': [ 'en', 'ja' ]
+ },
defaults: {
'en-ru': 'Yandex'
}
diff --git a/index.js b/index.js
index 4051b72..e27bbfa 100644
--- a/index.js
+++ b/index.js
@@ -2,6 +2,7 @@
Segmenter: require( './segmentation/CXSegmenter.js' ).CXSegmenter,
Apertium: require( './mt/Apertium.js' ),
Yandex: require( './mt/Yandex.js' ),
+ Youdao: require( './mt/Youdao.js' ),
MTClient: require( './mt/MTClient.js' ),
LinearDoc: require( './lineardoc' ),
Dictionary: require( './dictionary' )
diff --git a/mt/MTClient.js b/mt/MTClient.js
index 3477a93..53c0ef6 100644
--- a/mt/MTClient.js
+++ b/mt/MTClient.js
@@ -177,18 +177,23 @@
*/
MTClient.prototype.translateLines = function ( sourceLang, targetLang,
sourceLines ) {
var sourceLinesText,
+ self = this,
deferred = Q.defer();
// Join lines into single string. Separator must break sentences and
pass through unchanged
// Using Devangari seperator Double Danda twice.
- sourceLinesText = sourceLines.join( '.॥॥.' );
+ sourceLinesText = sourceLines.join( this.getSentenceDelimiter() );
this.translateText(
sourceLang,
targetLang,
sourceLinesText
).then( function ( targetLinesText ) {
- var targetText = targetLinesText.split( /\.॥॥\./g );
+ var targetText;
+
+ targetText = Array.isArray( targetLinesText ) ?
+ targetLinesText :
+ targetLinesText.split( self.getSentenceDelimiter() );
deferred.resolve( targetText );
}, function ( error ) {
logger.error( error.toString() );
@@ -197,6 +202,10 @@
return deferred.promise;
};
+MTClient.prototype.getSentenceDelimiter = function () {
+ return '.॥॥.';
+};
+
/**
* Create variants of the text, with a different annotation uppercased in each.
* @param {string} lang Language code
diff --git a/mt/Youdao.js b/mt/Youdao.js
new file mode 100644
index 0000000..8b982dd
--- /dev/null
+++ b/mt/Youdao.js
@@ -0,0 +1,87 @@
+var languageCodes, Q = require( 'q' ),
+ util = require( 'util' ),
+ request = require( 'request' ),
+ conf = require( __dirname + '/../utils/Conf.js' ),
+ MTClient = require( './MTClient.js' );
+
+languageCodes = {
+ 'en>zh': 'EN2ZH_CN', // English to Chinese Simplified
+ 'en>ja': 'EN2JA', // English to Chinese Simplified
+ 'simple>zh': 'EN2ZH_CN', // Simple English to Chinese Simplified
+ 'en>zh-Hans': 'EN2ZH_CN', // English to Chinese Simplified
+ 'ja>zh-Hans': 'JA2ZH_CN', // Japanese to Chinese Simplified,
+ 'ja>zh': 'JA2ZH_CN', // Japanese to Chinese Simplified,
+ 'ko>zh-Hans': 'KR2ZH_CN', // Korean to Chinese Simplified
+ 'fr>zh-Hans': 'FR2ZH_CN', // Korean to Chinese Simplified
+ 'ru>zh-Hans': 'RU2ZH_CN', // Russian to Chinese Simplified
+ 'es>zh-Hans': 'SP2ZH_CN', // Spanish to Chinese Simplified
+ 'zh-Hans>en': 'ZH_CN2EN', // Chinese Simplified to English
+ 'zh-Hant>en': 'ZH_CN2EN', // Chinese Traditional to English
+ 'zh>ja': 'ZH_CN2JA' // Chinese Traditional to English
+};
+// TODO: There are some more languages. See Youdao API documentation
+
+function Youdao() {
+
+}
+
+util.inherits( Youdao, MTClient );
+
+/**
+ * Youdao expect multiple sentences delimited by new line character
+ */
+Youdao.prototype.getSentenceDelimiter = function () {
+ return '\n';
+};
+
+/**
+ * Translate plain text with Youdao API
+ * Youdao is not capable of HTML translation with all annotation
+ * mapping. For translating HTML, It use CX's annotation mapping on top
+ * of the plaintext translation. Hence it inherits translateHTML method
+ * of MTClient.
+ * @param {string} sourceLang Source language code
+ * @param {string} targetLang Target language code
+ * @param {string} sourceText Source language text
+ * @return {Object} Deferred promise: Target language text
+ */
+Youdao.prototype.translateText = function ( sourceLang, targetLang, sourceText
) {
+ var deferred = Q.defer(),
+ postData;
+
+ postData = {
+ url: conf( 'mt.youdao.api' ),
+ form: {
+ key: conf( 'mt.youdao.key' ),
+ doctype: 'json',
+ type: 'data',
+ q: sourceText,
+ l: languageCodes[ sourceLang + '>' + targetLang ],
+ transtype: 'translate'
+ }
+ };
+
+ // Youdao paid api accept POST requests, open API accepts GET requests.
+ // url = postData.url + '?' + querystring.stringify( postData.form );
+ request.post( postData,
+ function ( error, response, body ) {
+ var message;
+
+ if ( error ) {
+ deferred.reject( new Error( error ) );
+ return;
+ }
+ if ( response.statusCode !== 200 ) {
+ message = 'Error ' + response.statusCode;
+ message += ' sourceText={' + sourceText + '},
body={' + body + '}';
+ deferred.reject( new Error( message ) );
+ return;
+ }
+
+ deferred.resolve( JSON.parse( body ).translation[ 0 ] );
+ }
+ );
+ return deferred.promise;
+};
+
+module.exports = Youdao;
diff --git a/mt/index.js b/mt/index.js
index acf5730..b6b20d6 100644
--- a/mt/index.js
+++ b/mt/index.js
@@ -1,4 +1,5 @@
module.exports = {
Apertium: require( './Apertium.js' ),
- Yandex: require( './Yandex.js' )
+ Yandex: require( './Yandex.js' ),
+ Youdao: require( './Youdao.js' )
};
diff --git a/tests/mt/Youdao.test.js b/tests/mt/Youdao.test.js
new file mode 100644
index 0000000..65bf988
--- /dev/null
+++ b/tests/mt/Youdao.test.js
@@ -0,0 +1,57 @@
+QUnit.module( 'Youdao' );
+
+var tests;
+
+tests = [
+ {
+ sourceLang: 'en',
+ targetLang: 'zh-Hans',
+ title: 'English to Chinese',
+ source: '<p>A <b>Japanese</b> <i>BBC</i> article</p>',
+ target: '<p>日本广播公司的一篇文章</p>',
+ },
+ {
+ sourceLang: 'zh-Hant',
+ targetLang: 'en',
+ title: 'Chinese Traditional to English',
+ source: '<p>我們寫字</p>',
+ target: '<p>We write</p>'
+ },
+ {
+ sourceLang: 'zh-Hans',
+ targetLang: 'en',
+ title: 'Chinese Simplified to English',
+ source: '<p>我们写字</p>',
+ target: '<p>We write</p>'
+ }
+];
+
+QUnit.test( 'Youdao wrapper tests', function ( assert ) {
+ QUnit.expect( tests.length );
+
+ function resumeTests( i ) {
+ var test,
+ youdao = new CX.Youdao();
+
+ if ( i >= tests.length ) {
+ return;
+ }
+ test = tests[ i ];
+
+ QUnit.stop();
+ youdao.translate( test.sourceLang, test.targetLang, test.source
).then( function ( target ) {
+ assert.strictEqual(
+ target,
+ test.target,
+ test.title
+ );
+ QUnit.start();
+ resumeTests( i + 1 );
+ }, function ( error ) {
+ assert.ok( false, test.title + ': ' + error );
+ QUnit.start();
+ resumeTests( i + 1 );
+ } );
+ }
+ resumeTests( 0 );
+} );
--
To view, visit https://gerrit.wikimedia.org/r/200516
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie904d29f5cb698a70aeb946e237ad5e345d33485
Gerrit-PatchSet: 4
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Santhosh <santhosh.thottin...@gmail.com>
Gerrit-Reviewer: Amire80 <amir.ahar...@mail.huji.ac.il>
Gerrit-Reviewer: Arrbee <run...@gmail.com>
Gerrit-Reviewer: Divec <da...@troi.org>
Gerrit-Reviewer: KartikMistry <kartik.mis...@gmail.com>
Gerrit-Reviewer: Nikerabbit <niklas.laxst...@gmail.com>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
