https://www.mediawiki.org/wiki/Special:Code/MediaWiki/109604
Revision: 109604
Author: gwicke
Date: 2012-01-20 02:36:18 +0000 (Fri, 20 Jan 2012)
Log Message:
-----------
Collapse all requests per template into a single outstanding request using an
event-emitting TemplateRequest object and a request queue.
Modified Paths:
--------------
trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMConverter.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
trunk/extensions/VisualEditor/modules/parser/parse.js
Modified:
trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
2012-01-20 02:36:05 UTC (rev 109603)
+++ trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
2012-01-20 02:36:18 UTC (rev 109604)
@@ -12,6 +12,7 @@
*/
var $ = require('jquery'),
request = require('request'),
+ events = require('events'),
qs = require('querystring'),
AttributeTransformManager =
require('./mediawiki.TokenTransformManager.js')
.AttributeTransformManager;
@@ -290,7 +291,7 @@
TemplateHandler.prototype._fetchTemplateAndTitle = function ( title, callback,
tplExpandData ) {
// @fixme normalize name?
var self = this;
- if (title in this.manager.env.pageCache) {
+ if (this.manager.env.pageCache[title]) {
// Unroll the stack here
process.nextTick(
function () {
@@ -300,115 +301,20 @@
} else if ( ! this.manager.env.fetchTemplates ) {
callback('Page/template fetching disabled, and no cache for ' +
title);
} else {
- // Not found in the pageCache, so go fetch the source using the
- // MediaWiki API.
-
+
// We are about to start an async request for a template, so
mark this
// template expansion as such.
tplExpandData.overallAsync = true;
this.manager.env.dp( 'trying to fetch ' + title );
- //console.log(this.manager.env.pageCache);
- var url = this.manager.env.wgScriptPath + '/api' +
- this.manager.env.wgScriptExtension +
- '?' +
- qs.stringify( {
- format: 'json',
- action: 'query',
- prop: 'revisions',
- rvprop: 'content',
- titles: title
- } );
-
//'?format=json&action=query&prop=revisions&rvprop=content&titles=' + title;
- request({
- method: 'GET',
- followRedirect: true,
- url: url,
- headers: {
- 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64;
rv:9.0.1) ' +
- 'Gecko/20100101
Firefox/9.0.1 Iceweasel/9.0.1'
- }
- },
- function (error, response, body) {
- //console.log( 'response for ' + title + ' :' + body +
':' );
- if(error) {
- self.manager.env.dp(error);
- callback('Page/template fetch failure for title
' + title, title);
- return ;
- }
+ // Start a new request if none is outstanding
+ this.manager.env.dp( 'requestQueue: ',
this.manager.env.requestQueue)
+ if ( this.manager.env.requestQueue[title] === undefined ) {
+ this.manager.env.requestQueue[title] = new
TemplateRequest( this.manager, title );
+ }
+ // Append a listener to the request
+ this.manager.env.requestQueue[title].addListener( 'src',
callback );
- if(response.statusCode == 200) {
- var src = '';
- try {
- //console.log( 'body: ' + body );
- var data = JSON.parse( body );
- } catch(e) {
- console.log( "Error: while parsing
result. Error was: " );
- console.log( e );
- console.log( "Response that didn't
parse was:");
- console.log(
"------------------------------------------\n" + body );
- console.log(
"------------------------------------------" );
- }
- try {
- $.each(data.query.pages, function(i,
page) {
- if (page.revisions &&
page.revisions.length) {
- src =
page.revisions[0]['*'];
- title = page.title;
- }
- });
- } catch ( e ) {
- console.log( 'Did not find page
revisions in the returned body:' + body );
- src = '';
- }
- //console.log( 'Page ' + title + ': got ' + src
);
- self.manager.env.dp( 'Success for ' + title + '
:' + body + ':' );
- self.manager.env.pageCache[title] = src;
- callback(src, title);
- self.manager.env.dp(data);
- }
- });
-
- /*
- * XXX: The jQuery version does not quite work with node, but
we keep
- * it around for now.
- $.ajax({
- url: url,
- data: {
- format: 'json',
- action: 'query',
- prop: 'revisions',
- rvprop: 'content',
- titles: title
- },
- success: function(data, statusString, xhr) {
- console.log( 'Page ' + title + ' success ' +
JSON.stringify( data ) );
- var src = null, title = null;
- $.each(data.query.pages, function(i, page) {
- if (page.revisions &&
page.revisions.length) {
- src = page.revisions[0]['*'];
- title = page.title;
- }
- });
- if (typeof src !== 'string') {
- console.log( 'Page ' + title + 'not
found! Got ' + src );
- callback( 'Page ' + title + ' not
found' );
- } else {
- // Add to cache
- console.log( 'Page ' + title + ': got '
+ src );
- this.manager.env.pageCache[title] = src;
- callback(src, title);
- }
- },
- error: function(xhr, msg, err) {
- console.log( 'Page/template fetch failure for
title ' +
- title + ', url=' + url +
JSON.stringify(xhr) + ', err=' + err );
- callback('Page/template fetch failure for title
' + title);
- },
- dataType: 'json',
- cache: false, // @fixme caching, versions etc?
- crossDomain: true
- });
- */
}
};
@@ -468,6 +374,123 @@
}
};
+
+/***************** Template fetch request helper class ********/
+
+function TemplateRequest ( manager, title ) {
+ // Increase the number of maximum listeners a bit..
+ this.setMaxListeners( 1000 );
+ var self = this,
+ url = manager.env.wgScriptPath + '/api' +
+ manager.env.wgScriptExtension +
+ '?' +
+ qs.stringify( {
+ format: 'json',
+ action: 'query',
+ prop: 'revisions',
+ rvprop: 'content',
+ titles: title
+ } );
+
//'?format=json&action=query&prop=revisions&rvprop=content&titles=' + title;
+
+ request({
+ method: 'GET',
+ followRedirect: true,
+ url: url,
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64;
rv:9.0.1) ' +
+ 'Gecko/20100101
Firefox/9.0.1 Iceweasel/9.0.1'
+ }
+ },
+ function (error, response, body) {
+ //console.log( 'response for ' + title + ' :' + body + ':' );
+ if(error) {
+ manager.env.dp(error);
+ callback('Page/template fetch failure for title ' +
title, title);
+ return ;
+ }
+
+ if(response.statusCode == 200) {
+ var src = '';
+ try {
+ //console.log( 'body: ' + body );
+ var data = JSON.parse( body );
+ } catch(e) {
+ console.log( "Error: while parsing result.
Error was: " );
+ console.log( e );
+ console.log( "Response that didn't parse was:");
+ console.log(
"------------------------------------------\n" + body );
+ console.log(
"------------------------------------------" );
+ }
+ try {
+ $.each(data.query.pages, function(i, page) {
+ if (page.revisions &&
page.revisions.length) {
+ src = page.revisions[0]['*'];
+ title = page.title;
+ }
+ });
+ } catch ( e ) {
+ console.log( 'Did not find page revisions in
the returned body:' + body );
+ src = '';
+ }
+ //console.log( 'Page ' + title + ': got ' + src );
+ manager.env.dp( 'Success for ' + title + ' :' + body +
':' );
+ manager.env.pageCache[title] = src;
+ manager.env.dp(data);
+ self.emit( 'src', src, title );
+ // Remove self from request queue
+ delete manager.env.requestQueue[title];
+ }
+ });
+}
+
+ /*
+ * XXX: The jQuery version does not quite work with node, but
we keep
+ * it around for now.
+ $.ajax({
+ url: url,
+ data: {
+ format: 'json',
+ action: 'query',
+ prop: 'revisions',
+ rvprop: 'content',
+ titles: title
+ },
+ success: function(data, statusString, xhr) {
+ console.log( 'Page ' + title + ' success ' +
JSON.stringify( data ) );
+ var src = null, title = null;
+ $.each(data.query.pages, function(i, page) {
+ if (page.revisions &&
page.revisions.length) {
+ src = page.revisions[0]['*'];
+ title = page.title;
+ }
+ });
+ if (typeof src !== 'string') {
+ console.log( 'Page ' + title + 'not
found! Got ' + src );
+ callback( 'Page ' + title + ' not
found' );
+ } else {
+ // Add to cache
+ console.log( 'Page ' + title + ': got '
+ src );
+ this.manager.env.pageCache[title] = src;
+ callback(src, title);
+ }
+ },
+ error: function(xhr, msg, err) {
+ console.log( 'Page/template fetch failure for
title ' +
+ title + ', url=' + url +
JSON.stringify(xhr) + ', err=' + err );
+ callback('Page/template fetch failure for title
' + title);
+ },
+ dataType: 'json',
+ cache: false, // @fixme caching, versions etc?
+ crossDomain: true
+ });
+ */
+
+// Inherit from EventEmitter
+TemplateRequest.prototype = new events.EventEmitter();
+TemplateHandler.prototype.constructor = TemplateRequest;
+
+
if (typeof module == "object") {
module.exports.TemplateHandler = TemplateHandler;
}
Modified: trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMConverter.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMConverter.js
2012-01-20 02:36:05 UTC (rev 109603)
+++ trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMConverter.js
2012-01-20 02:36:18 UTC (rev 109604)
@@ -1,5 +1,7 @@
/**
* Conversions between HTML DOM and WikiDom
+ *
+ * @author Gabriel Wicke <gwi...@wikimedia.org>
*
* @class
* @constructor
Modified:
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
===================================================================
---
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
2012-01-20 02:36:05 UTC (rev 109603)
+++
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
2012-01-20 02:36:18 UTC (rev 109604)
@@ -12,6 +12,10 @@
$.extend(this, options);
};
+// Outstanding page requests (for templates etc)
+// Class-static
+MWParserEnvironment.prototype.requestQueue = {};
+
MWParserEnvironment.prototype.lookupKV = function ( kvs, key ) {
if ( ! kvs ) {
return null;
Modified: trunk/extensions/VisualEditor/modules/parser/parse.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/parse.js 2012-01-20
02:36:05 UTC (rev 109603)
+++ trunk/extensions/VisualEditor/modules/parser/parse.js 2012-01-20
02:36:18 UTC (rev 109604)
@@ -1,6 +1,9 @@
/**
* Command line wikidom parse utility.
* Read from STDIN, write to STDOUT.
+ *
+ * @author Neil Kandalgaonkar <ne...@wikimedia.org>
+ * @author Gabriel Wicke <gwi...@wikimedia.org>
*/
@@ -12,7 +15,11 @@
optimist = require('optimist');
var env = new ParserEnv( {
+ // fetch templates from enwiki
by default..
+ wgScriptPath:
"http://en.wikipedia.org/w";,
+ wgScriptExtension: ".php",
fetchTemplates: true,
+ // enable/disable debug output
using this switch
debug: false
} ),
parser = new ParserPipeline( env );
_______________________________________________
MediaWiki-CVS mailing list
MediaWiki-CVS@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
