Arlolra has uploaded a new change for review.
https://gerrit.wikimedia.org/r/151068
Change subject: Use a map for the interwikiMap
......................................................................
Use a map for the interwikiMap
* Avoid building it on each request.
Bug: 63370
Change-Id: I0e0cae162d6448799a1ca2c1c44ec08aff753fac
---
M lib/ext.core.LinkHandler.js
M lib/linter.js
M lib/mediawiki.ApiRequest.js
M lib/mediawiki.ParsoidConfig.js
M lib/mediawiki.WikiConfig.js
M lib/mediawiki.parser.environment.js
M lib/wts.LinkHandler.js
M tests/parserTests.js
8 files changed, 93 insertions(+), 68 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/68/151068/1
diff --git a/lib/ext.core.LinkHandler.js b/lib/ext.core.LinkHandler.js
index 01f9e38..e734377 100644
--- a/lib/ext.core.LinkHandler.js
+++ b/lib/ext.core.LinkHandler.js
@@ -81,7 +81,7 @@
var nsPrefix = hrefBits[1];
info.prefix = nsPrefix;
var nnn = Util.normalizeNamespaceName( nsPrefix.trim() ),
- interwikiInfo = env.conf.wiki.interwikiMap[nnn],
+ interwikiInfo = env.conf.wiki.interwikiMap.get(nnn),
// check for interwiki / language links
ns = env.conf.wiki.namespaceIds[nnn];
// console.warn( nsPrefix, ns, interwikiInfo );
diff --git a/lib/linter.js b/lib/linter.js
index fa8a669..d0cf5d2 100644
--- a/lib/linter.js
+++ b/lib/linter.js
@@ -56,7 +56,7 @@
msg.wiki = wiki;
msg.page = logData._env.page.name;
msg.revision = logData._env.page.meta.revision.revid;
- msg.wikiurl =
logData._env.conf.parsoid.interwikiMap[wiki].split('/w/')[0];
+ msg.wikiurl =
logData._env.conf.parsoid.interwikiMap.get(wiki).split('/w/')[0];
if (dsr) {
msg.dsr = dsr;
diff --git a/lib/mediawiki.ApiRequest.js b/lib/mediawiki.ApiRequest.js
index 78c56dc..a41c7a0 100644
--- a/lib/mediawiki.ApiRequest.js
+++ b/lib/mediawiki.ApiRequest.js
@@ -297,8 +297,8 @@
title = interwiki.title;
regex = new RegExp( '^' + interwiki.iw + ':' );
title = title.replace( regex, '' );
- iwstr =
this.env.conf.wiki.interwikiMap[interwiki.iw].url ||
-
this.env.conf.parsoid.interwikiMap[interwiki.iw].url ||
+ iwstr =
this.env.conf.wiki.interwikiMap.get(interwiki.iw).url ||
+
this.env.conf.parsoid.interwikiMap.get(interwiki.iw).url ||
'/' + interwiki.iw + '/' + '$1';
location = iwstr.replace( '$1', title );
error = new DoesNotExistError( 'The page at ' +
this.title +
diff --git a/lib/mediawiki.ParsoidConfig.js b/lib/mediawiki.ParsoidConfig.js
index 886544b..dff8889 100644
--- a/lib/mediawiki.ParsoidConfig.js
+++ b/lib/mediawiki.ParsoidConfig.js
@@ -4,10 +4,60 @@
*/
"use strict";
+require('./core-upgrade.js');
var Cite = require('./ext.Cite.js').Cite;
var Util = require('./mediawiki.Util.js').Util;
var wikipedias =
"en|de|fr|nl|it|pl|es|ru|ja|pt|zh|sv|vi|uk|ca|no|fi|cs|hu|ko|fa|id|tr|ro|ar|sk|eo|da|sr|lt|ms|eu|he|sl|bg|kk|vo|war|hr|hi|et|az|gl|simple|nn|la|th|el|new|roa-rup|oc|sh|ka|mk|tl|ht|pms|te|ta|be-x-old|ceb|br|be|lv|sq|jv|mg|cy|lb|mr|is|bs|yo|an|hy|fy|bpy|lmo|pnb|ml|sw|bn|io|af|gu|zh-yue|ne|nds|ku|ast|ur|scn|su|qu|diq|ba|tt|my|ga|cv|ia|nap|bat-smg|map-bms|wa|kn|als|am|bug|tg|gd|zh-min-nan|yi|vec|hif|sco|roa-tara|os|arz|nah|uz|sah|mn|sa|mzn|pam|hsb|mi|li|ky|si|co|gan|glk|ckb|bo|fo|bar|bcl|ilo|mrj|fiu-vro|nds-nl|tk|vls|se|gv|ps|rue|dv|nrm|pag|koi|pa|rm|km|kv|udm|csb|mhr|fur|mt|wuu|lij|ug|lad|pi|zea|sc|bh|zh-classical|nov|ksh|or|ang|kw|so|nv|xmf|stq|hak|ay|frp|frr|ext|szl|pcd|ie|gag|haw|xal|ln|rw|pdc|pfl|krc|crh|eml|ace|gn|to|ce|kl|arc|myv|dsb|vep|pap|bjn|as|tpi|lbe|wo|mdf|jbo|kab|av|sn|cbk-zam|ty|srn|kbd|lo|ab|lez|mwl|ltg|ig|na|kg|tet|za|kaa|nso|zu|rmy|cu|tn|chr|got|sm|bi|mo|bm|iu|chy|ik|pih|ss|sd|pnt|cdo|ee|ha|ti|bxr|om|ks|ts|ki|ve|sg|rn|dz|cr|lg|ak|tum|fj|st|tw|ch|ny|ff|xh|ng|ii|cho|mh|aa|kj|ho|mus|kr|hz|tyv|min";
+
+var interwikiMap = new Map();
+wikipedias.split('|').forEach(function(lang) {
+ var dbLangPrefix = lang.replace(/-/g, '_');
+ [ 'wikipedia', 'wikivoyage', 'wikibooks', 'wikisource', 'wikinews',
+ 'wikiquote', 'wikiversity', 'wiktionary'
+ ].forEach(function(suffix) {
+ interwikiMap.set(
+ dbLangPrefix + suffix.replace('pedia', ''),
+ 'http://' + lang + '.' + suffix + '.org/w/api.php'
+ );
+ });
+});
+
+// Add mediawiki.org, commons and localhost too
+interwikiMap.set('mediawikiwiki', 'http://www.mediawiki.org/w/api.php');
+interwikiMap.set('commonswiki', 'http://commons.wikimedia.org/w/api.php');
+interwikiMap.set('localhost', 'http://localhost/wiki/api.php');
+
+// Subclass a Map to avoid overwriting defaults
+// Maybe just let that happen ... it only seems to occur
+// from localSettings
+function IMap() { Map.call(this); }
+Object.setPrototypeOf(IMap, Map);
+IMap.prototype = Object.create(Map.prototype, {
+ constructor: { value: IMap },
+ get: {
+ writeable: false,
+ value: function(key) {
+ return this.has(key)
+ ? Map.prototype.get.call(this, key)
+ : interwikiMap.get(key);
+ }
+ },
+ keys: {
+ writeable: false,
+ value: function() {
+ // This return an array because the es6-shim doesn't
+ // expose Iterators.
+ var keys = Array.from(Map.prototype.keys.call(this));
+ interwikiMap.forEach(function(val, key) {
+ if ( keys.indexOf(key) < 0 ) {
+ keys.push(key);
+ }
+ });
+ return keys;
+ }
+ }
+});
/**
* @class
@@ -22,44 +72,20 @@
* @param {Object} options Any options we want to set over the defaults. Will
not overwrite things set by the localSettings.setup function. See the class
properties for more information.
*/
function ParsoidConfig( localSettings, options ) {
- var self = this;
- this.interwikiMap = {};
-
// 'enwiki' etc for per-wiki proxy. Used by setInterwiki.
- this.apiProxyURIs = {};
+ this.apiProxyURIs = new Map();
// The default api proxy, overridden by apiProxyURIs entries
this.defaultAPIProxyURI = undefined;
- // XXX: move to prototype to avoid reconstructing this for each request?
- wikipedias.split( '|' ).forEach(function(lang) {
- // Wikipedia
- var dbLangPrefix = lang.replace(/-/g, '_');
- self.interwikiMap[dbLangPrefix + 'wiki'] = 'http://' + lang +
- '.wikipedia.org/w/api.php';
-
- // Wiktionary
- self.interwikiMap[dbLangPrefix + 'wiktionary'] = 'http://' +
lang +
- '.wiktionary.org/w/api.php';
-
- // Wikivoyage, Wikibooks, Wikisource, Wikinews, Wikiquote &
Wikiversity
- // all follow the same pattern
- ['voyage', 'books', 'source', 'news', 'quote', 'versity']
- .forEach(function(suffix) {
- self.interwikiMap[dbLangPrefix + 'wiki' +
suffix] = 'http://' +
- lang + '.wiki' + suffix +
'.org/w/api.php';
- });
- });
-
- // Add mediawiki.org too
- this.interwikiMap.mediawikiwiki = 'http://www.mediawiki.org/w/api.php';
- // Also commons
- this.interwikiMap.commonswiki =
'http://commons.wikimedia.org/w/api.php';
-
- // Add localhost too
- this.interwikiMap.localhost = 'http://localhost/wiki/api.php';
-
- this.interwikiRegexp = Object.keys( this.interwikiMap ).join( '|' );
+ this.interwikiMap = new IMap();
+ var it = interwikiMap.keys();
+ var key = it.next();
+ this.interwikiRegexp = key.value;
+ while ( !key.done ) {
+ this.interwikiRegexp += "|" + key.value;
+ key = it.next();
+ }
if ( localSettings && localSettings.setup ) {
localSettings.setup( this );
@@ -93,11 +119,10 @@
* back to ParsoidConfig.defaultAPIProxyURI if undefined (default value).
*/
ParsoidConfig.prototype.setInterwiki = function ( prefix, apiURI, apiProxyURI
) {
- this.interwikiMap[prefix] = apiURI;
- if (apiProxyURI !== undefined) {
- this.apiProxyURIs[prefix] = apiProxyURI;
+ this.interwikiMap.set(prefix, apiURI);
+ if ( apiProxyURI !== undefined ) {
+ this.apiProxyURIs.set(prefix, apiProxyURI);
}
-
if ( this.interwikiRegexp.match( '\\|' + prefix + '\\|' ) === null ) {
this.interwikiRegexp += '|' + prefix;
}
@@ -111,7 +136,7 @@
* @param {string} prefix
*/
ParsoidConfig.prototype.removeInterwiki = function ( prefix ) {
- delete this.interwikiMap[prefix];
+ this.interwikiMap.delete(prefix, null);
this.interwikiRegexp = this.interwikiRegexp.replace(
new RegExp( '(^|\\|)' + prefix + '(\\||$)' ), function() {
return arguments[0] === ("|" + prefix + "|") ? "|" : '';
diff --git a/lib/mediawiki.WikiConfig.js b/lib/mediawiki.WikiConfig.js
index 5e1299e..b44c2e4 100644
--- a/lib/mediawiki.WikiConfig.js
+++ b/lib/mediawiki.WikiConfig.js
@@ -121,32 +121,32 @@
// The interwikiMap maps prefixes to the corresponding information
// gathered from the api query (prefix, language, url, local)
- this.interwikiMap = {};
+ this.interwikiMap = new Map();
var interwikimap = resultConf.interwikimap;
- var interwikiKeys = Object.keys(interwikimap);
- for ( var index = 0; index < interwikiKeys.length; index++ ) {
- var key = interwikimap[index].prefix;
- this.interwikiMap[key] = interwikimap[index];
- if (!/\$1/.test(this.interwikiMap[key].url)) {
+ Object.keys(interwikimap).forEach(function(key) {
+ var interwiki = interwikimap[key];
+ conf.interwikiMap.set(interwiki.prefix, interwiki);
+ if ( !/\$1/.test(interwiki.url) ) {
// Fix up broken interwiki hrefs that are missing a $1
placeholder
// Just append the placeholder at the end.
// This makes sure that the InterWikiMatcher below adds
one match
// group per URI, and that interwiki links work as
expected.
- this.interwikiMap[key].url += '$1';
+ interwiki.url += '$1';
}
- }
+ });
var cachedMatcher = null;
this.InterWikiMatcher = function () {
if (cachedMatcher) {
return cachedMatcher;
}
- var keys = Object.keys(conf.interwikiMap),
+ var keys = [],
patterns = [];
- keys.forEach(function(key) {
- var url = conf.interwikiMap[key].url;
+ conf.interwikiMap.forEach(function(val, key) {
+ keys.push(key);
+ var url = val.url;
var protocolRelative = url.startsWith('//');
- if (conf.interwikiMap[key].protorel !== undefined) {
+ if (val.protorel !== undefined) {
url = url.replace( /^https?:/, '' );
protocolRelative = true;
}
@@ -169,7 +169,7 @@
if (groups[i] !== undefined) {
// The interwiki prefix: 'en',
'de' etc
var key = keys[i];
- if
(conf.interwikiMap[key].language) {
+ if
(conf.interwikiMap.get(key).language) {
// Escape language
interwikis with a colon
key = ':' + key;
}
diff --git a/lib/mediawiki.parser.environment.js
b/lib/mediawiki.parser.environment.js
index 69b78b1..24927ca 100644
--- a/lib/mediawiki.parser.environment.js
+++ b/lib/mediawiki.parser.environment.js
@@ -277,7 +277,7 @@
* Figure out the proxy URI to use for API requests for a given wiki
*/
MWParserEnvironment.prototype.getAPIProxyURI = function (prefix) {
- var apiProxyURI = this.conf.parsoid.apiProxyURIs[prefix];
+ var apiProxyURI = this.conf.parsoid.apiProxyURIs.get(prefix);
if (apiProxyURI === undefined) {
// No specific api proxy set. Fall back to generic API proxy.
apiProxyURI = this.conf.parsoid.defaultAPIProxyURI;
@@ -315,7 +315,7 @@
return;
}
- var uri = this.conf.parsoid.interwikiMap[prefix];
+ var uri = this.conf.parsoid.interwikiMap.get(prefix);
if (!uri) {
// SSS: Ugh! Looks like parser tests use a prefix
// that is not part of the interwikiMap -- so we
@@ -324,7 +324,7 @@
// interwikiMap need updating or is this use-case
// valid outside of parserTests??
this.log("error","Did not find api uri for " + prefix + ";
defaulting to en");
- uri = this.conf.parsoid.interwikiMap.en;
+ uri = this.conf.parsoid.interwikiMap.get("en");
}
this.conf.parsoid.apiURI = uri;
@@ -387,7 +387,7 @@
var nsMatch = name.match( /^([a-zA-Z\-]+):/ ),
ns = nsMatch && nsMatch[1] || '';
if( ns !== '' && ns !== name ) {
- if ( self.conf.parsoid.interwikiMap[ns.toLowerCase()] )
{
+ if (
self.conf.parsoid.interwikiMap.get(ns.toLowerCase()) ) {
forceNS += ns + ':';
name = name.substr( nsMatch[0].length );
splitNS();
diff --git a/lib/wts.LinkHandler.js b/lib/wts.LinkHandler.js
index f2f6484..1707997 100644
--- a/lib/wts.LinkHandler.js
+++ b/lib/wts.LinkHandler.js
@@ -284,8 +284,8 @@
// Check if the old prefix
mapped to the same URL as
// the new one. Use the old one
if that's the case.
// Example: [[w:Foo]] vs.
[[:en:Foo]]
-
(env.conf.wiki.interwikiMap[oldPrefix[1].toLowerCase().trim().replace(/^:/,
'')] || {}).url ===
-
(env.conf.wiki.interwikiMap[interWikiMatch[0].toLowerCase().replace(/^:/, '')]
|| {}).url
+
(env.conf.wiki.interwikiMap.get(oldPrefix[1].toLowerCase().trim().replace(/^:/,
'')) || {}).url ===
+
(env.conf.wiki.interwikiMap.get(interWikiMatch[0].toLowerCase().replace(/^:/,
'')) || {}).url
))
{
// Reuse old prefix capitalization
diff --git a/tests/parserTests.js b/tests/parserTests.js
index 2bc90fd..b6b06ba 100755
--- a/tests/parserTests.js
+++ b/tests/parserTests.js
@@ -1617,7 +1617,7 @@
var i, key,
parsoidConfig = new ParsoidConfig( null, options ),
- iwmap = Object.keys( parsoidConfig.interwikiMap );
+ iwmap = parsoidConfig.interwikiMap.keys();
// Set tracing and debugging before the env. object is
// constructed since tracing backends are registered there.
@@ -1628,7 +1628,7 @@
for ( i = 0; i < iwmap.length; i++ ) {
key = iwmap[i];
- parsoidConfig.interwikiMap[key] = mockAPIServerURL;
+ parsoidConfig.interwikiMap.set(key, mockAPIServerURL);
}
// Create a new parser environment
@@ -1917,12 +1917,12 @@
}
};
Object.keys(iwl).forEach(function(key) {
-
wikiConf.interwikiMap[key] =
-
wikiConf.interwikiMap[key] ?
-
Util.clone(wikiConf.interwikiMap[key]) :
- {};
+
wikiConf.interwikiMap.set(key,
+
wikiConf.interwikiMap.has(key) ?
+
Util.clone(wikiConf.interwikiMap.get(key)) :
+ {});
Object.keys(iwl[key]).forEach(function(f) {
-
wikiConf.interwikiMap[key][f] = iwl[key][f];
+
wikiConf.interwikiMap.get(key)[f] = iwl[key][f];
});
});
// Add 'MemoryAlpha'
namespace (bug 51680)
--
To view, visit https://gerrit.wikimedia.org/r/151068
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0e0cae162d6448799a1ca2c1c44ec08aff753fac
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra <abrea...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
