Mvolz has uploaded a new change for review. https://gerrit.wikimedia.org/r/193134
Change subject: Add ability to set requesting User-Agent ...................................................................... Add ability to set requesting User-Agent Modify localsettings.js.sample to contain a userAgent string. Pass this string to the Scraper.js object and use this string as the User-Agent in all outbound requests. Bug: T89757 Change-Id: I7dfa933181118527783da12cdb5b277329aef7dd --- M lib/CitoidService.js M lib/Scraper.js M lib/ZoteroService.js M localsettings.js.sample 4 files changed, 19 insertions(+), 13 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid refs/changes/34/193134/1 diff --git a/lib/CitoidService.js b/lib/CitoidService.js index f35b17a..dfb9ee8 100644 --- a/lib/CitoidService.js +++ b/lib/CitoidService.js @@ -3,8 +3,7 @@ */ /* Import Modules */ -var urlParse = require('url'), - util = require('util'); +var urlParse = require('url'); /* Import Local Modules */ var unshorten = require('./unshorten.js'), @@ -17,13 +16,10 @@ * @param {Object} CitoidConfig configuration object * @param {Object} logger bunyan logger object */ -function CitoidService(CitoidConfig, logger){ - this.CitoidConfig = CitoidConfig; +function CitoidService(citoidConfig, logger){ this.log = logger; - this.zoteroURL = util.format('http://%s:%s/', - CitoidConfig.zoteroInterface, CitoidConfig.zoteroPort.toString()); - this.zoteroService = new ZoteroService(this.zoteroURL, logger); - this.scraper = new Scraper(CitoidConfig, logger); + this.zoteroService = new ZoteroService(citoidConfig, logger); + this.scraper = new Scraper(citoidConfig, logger); } /** diff --git a/lib/Scraper.js b/lib/Scraper.js index f2e6c86..8e041af 100644 --- a/lib/Scraper.js +++ b/lib/Scraper.js @@ -16,6 +16,7 @@ function Scraper(citoidConfig, logger){ this.log = logger; + this.userAgent = citoidConfig.userAgent; } /** @@ -30,6 +31,7 @@ var chtml, log = this.log, scraper = this, + userAgent = this.userAgent, citation = {url: url, title: url}; log.info("Using native scraper on " + url); @@ -38,6 +40,9 @@ { url: url, followAllRedirects: true, + headers: { + 'User-Agent': userAgent + } }, function(error, response, html){ if (error || !response || response.statusCode !== 200) { diff --git a/lib/ZoteroService.js b/lib/ZoteroService.js index 62bf927..d00e1ee 100644 --- a/lib/ZoteroService.js +++ b/lib/ZoteroService.js @@ -9,13 +9,15 @@ var async = require('async'), crypto = require('crypto'), request = require('request'), + util = require('util'), pubMedRequest = require('./pubMedRequest.js'); -function ZoteroService(zoteroURL, logger){ +function ZoteroService(citoidConfig, logger){ this.log = logger; - this.baseURL = zoteroURL; - this.webURL = zoteroURL + 'web'; - this.exportURL = zoteroURL + 'export'; + var baseURL = util.format('http://%s:%s/', + citoidConfig.zoteroInterface, citoidConfig.zoteroPort.toString()); + this.webURL = baseURL + 'web'; + this.exportURL = baseURL + 'export'; } /** diff --git a/localsettings.js.sample b/localsettings.js.sample index f503415..23e60ba 100644 --- a/localsettings.js.sample +++ b/localsettings.js.sample @@ -21,7 +21,10 @@ citoidPort : 1970, citoidInterface : 'localhost', - //Settings for Zotero server + // userAgent string for lib/Scraper.js + userAgent : null, + + // Settings for Zotero server zoteroPort : 1969, zoteroInterface : 'localhost', }; -- To view, visit https://gerrit.wikimedia.org/r/193134 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I7dfa933181118527783da12cdb5b277329aef7dd Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/citoid Gerrit-Branch: master Gerrit-Owner: Mvolz <mv...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits