Mholloway has uploaded a new change for review.
https://gerrit.wikimedia.org/r/295606
Change subject: In the News endpoint
......................................................................
In the News endpoint
Bug: T132767
Change-Id: I78eb6ba1aab9a5bc8edcaf20454db76f4a6b0f68
---
M lib/feed/most-read.js
A lib/feed/news.js
M lib/mobile-util.js
M lib/mwapi.js
M lib/parseDefinition.js
M lib/parsoid-access.js
M routes/aggregated.js
A routes/news.js
M spec.yaml
A test/features/news/news.js
10 files changed, 196 insertions(+), 14 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/mobileapps
refs/changes/06/295606/1
diff --git a/lib/feed/most-read.js b/lib/feed/most-read.js
index bb644ce..a132fd4 100644
--- a/lib/feed/most-read.js
+++ b/lib/feed/most-read.js
@@ -52,7 +52,7 @@
goodTitles = blacklist.filter(rankedTitles)
.slice(0, mwapi.API_QUERY_MAX_TITLES);
queryTitlesList = constructQueryListFrom(goodTitles);
- return mwapi.requestMostReadMetadata(app, req,
queryTitlesList.join('|'));
+ return mwapi.getFeedPageListMetadata(app, req,
queryTitlesList.join('|'));
}).then(function (response) {
api.checkResponseStatus(response);
diff --git a/lib/feed/news.js b/lib/feed/news.js
new file mode 100644
index 0000000..7ac88a1
--- /dev/null
+++ b/lib/feed/news.js
@@ -0,0 +1,80 @@
+'use strict';
+
+var domino = require('domino');
+var api = require('../api-util');
+var mUtil = require('../mobile-util');
+var mwapi = require('../mwapi');
+var parsoid = require('../parsoid-access');
+
+function promise(app, req) {
+ req.params.title = 'Template:In_the_news';
+ var result = {
+ payload: [],
+ meta: {}
+ };
+ return parsoid.getParsoidHtml(app, req)
+ .then(function (response) {
+ result.meta.etag = parsoid.getRevisionFromEtag(response.headers);
+
+ var linkTitles = [];
+ var doc = domino.createDocument(response.body);
+ var newsList = doc.getElementsByTagName('ul')[0];
+ var stories = newsList.getElementsByTagName('li');
+
+ for (var j = 0, m = stories.length; j < m; j++) {
+ var anchors = stories[j].getElementsByTagName('a');
+ var story = {
+ blurb: mUtil.stripMarkup(stories[j].innerHTML),
+ links: {}
+ };
+
+ for (var i = 0, n = anchors.length; i < n; i++) {
+ var anchor = anchors[i];
+ var title = anchor.href.slice(1);
+ story.links[title] = {};
+ linkTitles.push(title);
+ }
+
+ result.payload.push(story);
+ }
+ return mwapi.getFeedPageListMetadata(app, req, linkTitles.join('|'));
+ }).then(function(response) {
+ api.checkResponseStatus(response);
+
+ var query = response.body && response.body.query;
+ var normalizations = query && query.normalized;
+ var pages = query && query.pages;
+
+ mUtil.adjustMemberKeys(normalizations, [['title', 'from'],
+ ['normalizedtitle', 'to']]);
+ mUtil.adjustMemberKeys(pages, [['normalizedtitle', 'title']]);
+ mUtil.mergeByProp(pages, normalizations, 'normalizedtitle');
+ mUtil.fillInMemberKeys(pages, [['title', 'normalizedtitle']]);
+
+ var pageResults = {};
+
+ pages.forEach(function(page) {
+ pageResults[page.title] = Object.assign(page, {
+ description: page.terms
+ && page.terms.description
+ && page.terms.description[0],
+ terms: undefined
+ });
+ });
+
+ result.payload.forEach(function(story) {
+ var linkTitles = story.links;
+ for (var title in linkTitles) {
+ if (linkTitles.hasOwnProperty(title)) {
+ linkTitles[title] = pageResults[title];
+ }
+ }
+ });
+
+ return result;
+ });
+}
+
+module.exports = {
+ promise: promise
+};
\ No newline at end of file
diff --git a/lib/mobile-util.js b/lib/mobile-util.js
index 7730343..2860537 100644
--- a/lib/mobile-util.js
+++ b/lib/mobile-util.js
@@ -151,6 +151,13 @@
return dateString + '/' + uuid.now().toString();
}
+/**
+ * Strip HTML markup from a string.
+ */
+function stripMarkup(text) {
+ return text.replace(/<[^>]*>/g, '');
+}
+
function throw404(message) {
throw new HTTPError({
status: 404,
@@ -171,5 +178,6 @@
mergeByProp: mergeByProp,
adjustMemberKeys: adjustMemberKeys,
fillInMemberKeys: fillInMemberKeys,
+ stripMarkup: stripMarkup,
throw404: throw404
};
diff --git a/lib/mwapi.js b/lib/mwapi.js
index ba552d2..4645462 100644
--- a/lib/mwapi.js
+++ b/lib/mwapi.js
@@ -172,7 +172,7 @@
};
}
-function requestMostReadMetadata(app, req, titlesList) {
+function getFeedPageListMetadata(app, req, titlesList) {
var query = {
action: 'query',
format: 'json',
@@ -182,8 +182,8 @@
pilimit: API_QUERY_MAX_TITLES,
pithumbsize: CARD_THUMB_LIST_ITEM_SIZE,
wbptterms: 'description',
- meta: 'siteinfo',
- siprop: 'general',
+ meta: req.url.indexOf('most-read') > -1 ? 'siteinfo' : undefined,
+ siprop: req.url.indexOf('most-read') > -1 ? 'general' : undefined,
titles: titlesList
};
return api.mwApiGet(app, req.params.domain, query);
@@ -242,7 +242,7 @@
requestExtractAndDescription: requestExtractAndDescription,
getRevisionFromExtract: getRevisionFromExtract,
buildSummaryResponse: buildSummaryResponse,
- requestMostReadMetadata: requestMostReadMetadata,
+ getFeedPageListMetadata: getFeedPageListMetadata,
API_QUERY_MAX_TITLES: API_QUERY_MAX_TITLES,
// VisibleForTesting
diff --git a/lib/parseDefinition.js b/lib/parseDefinition.js
index 17b7808..98efafe 100644
--- a/lib/parseDefinition.js
+++ b/lib/parseDefinition.js
@@ -9,6 +9,7 @@
var domino = require('domino');
var sUtil = require('./util');
+var mUtil = require('./mobile-util');
var transforms = require('./transforms');
var parseSection = require('./parseSection');
var languageList = require('../static/languages_list.json');
@@ -54,10 +55,6 @@
'Conjunction']
*/
};
-
-function stripMarkup(text) {
- return text.replace(/<[^>]*>/g, '');
-}
function stripSpanTags(text) {
return text.replace(/<\/?span[^>]*>/g, '');
@@ -154,7 +151,7 @@
for (j = 0; j < sectionDivs.length; j++) {
currentSectionDiv = sectionDivs[j];
- header = stripMarkup(currentSectionDiv.title);
+ header = mUtil.stripMarkup(currentSectionDiv.title);
/* Get the language from the first H2 header, and begin iterating over
sections.
Per the English Wiktionary style guide (linked in header above), H2
headings
diff --git a/lib/parsoid-access.js b/lib/parsoid-access.js
index 0fbe743..26fc4b8 100644
--- a/lib/parsoid-access.js
+++ b/lib/parsoid-access.js
@@ -206,6 +206,8 @@
module.exports = {
pageContentPromise: pageContentPromise,
definitionPromise: definitionPromise,
+ getParsoidHtml: getParsoidHtml,
+ getRevisionFromEtag: getRevisionFromEtag,
// VisibleForTesting
_addSectionDivs: addSectionDivs,
diff --git a/routes/aggregated.js b/routes/aggregated.js
index cbf524f..8bcfb1e 100644
--- a/routes/aggregated.js
+++ b/routes/aggregated.js
@@ -13,6 +13,7 @@
var mostRead = require('../lib/feed/most-read');
var featured = require('../lib/feed/featured');
var random = require('../lib/feed/random');
+var news = require('../lib/feed/news');
/**
* The main router object
@@ -33,8 +34,8 @@
return BBPromise.props({
tfa: featured.promise(app, req),
mostread: mostRead.promise(app, dateUtil.yesterday(req)),
- random: random.promise(app, req)
- //news: news.promise(app, req),
+ random: random.promise(app, req),
+ news: news.promise(app, req),
//image: media.featuredImagePromise(app, req),
//video: media.featuredVideoPromise(app, req)
}) .then(function (response) {
@@ -42,7 +43,7 @@
tfa: response.tfa.payload,
random: response.random.payload,
mostread: response.mostread.payload,
- news: 'Articles in the news here',
+ news: response.news.payload,
image: 'Today\'s featured image here',
video: 'Today\'s featured video here'
};
diff --git a/routes/news.js b/routes/news.js
new file mode 100644
index 0000000..72dd15a
--- /dev/null
+++ b/routes/news.js
@@ -0,0 +1,39 @@
+'use strict';
+
+var sUtil = require('../lib/util');
+var mUtil = require('../lib/mobile-util');
+var news = require('../lib/feed/news');
+
+/**
+ * The main router object
+ */
+var router = sUtil.router();
+
+/**
+ * The main application object reported when this module is require()d
+ */
+var app;
+
+/**
+ * GET {domain}/api/rest_v1/page/news
+ *
+ * Get descriptions of current events and related article links.
+ * Experimental and English-only.
+ */
+router.get('/news', function (req, res) {
+ return news.promise(app, req)
+ .then(function (response) {
+ res.status(200);
+ mUtil.setETagToValue(res, response.meta.etag);
+ res.json(response.payload).end();
+ });
+});
+
+module.exports = function (appObj) {
+ app = appObj;
+ return {
+ path: '/page',
+ api_version: 1,
+ router: router
+ };
+};
\ No newline at end of file
diff --git a/spec.yaml b/spec.yaml
index 6e719d1..7950617 100644
--- a/spec.yaml
+++ b/spec.yaml
@@ -110,7 +110,7 @@
articles: [ /.+/ ]
random:
title: /.+/
- news: /.+/
+ news: [ /.+/ ]
image: /.+/
video: /.+/
# from routes/featured.js
@@ -222,6 +222,21 @@
content-type: application/json
body:
title: /.+/
+ # from routes/news.js
+ /{domain}/v1/page/news:
+ get:
+ tags:
+ - News-related content
+ description: Gets content related to the current In the News template
(experimental, English-only).
+ produces:
+ - application/json
+ x-amples:
+ - title: get 'In the News' content
+ response:
+ status: 200
+ headers:
+ content-type: application/json
+ body: [ /.+/ ]
# from routes/media.js
/{domain}/v1/page/media/{title}:
get:
diff --git a/test/features/news/news.js b/test/features/news/news.js
new file mode 100644
index 0000000..0af833d
--- /dev/null
+++ b/test/features/news/news.js
@@ -0,0 +1,40 @@
+'use strict';
+
+var preq = require('preq');
+var assert = require('../../utils/assert');
+var server = require('../../utils/server');
+var headers = require('../../utils/headers');
+
+describe('in the news', function() {
+ this.timeout(20000);
+
+ before(function () { return server.start(); });
+
+ it('should respond to GET request with expected headers, incl. CORS and
CSP headers', function() {
+ return headers.checkHeaders(server.config.uri +
'en.wikipedia.org/v1/page/news',
+ 'application/json');
+ });
+ it('results list should have expected properties', function() {
+ return preq.get({ uri: server.config.uri +
'en.wikipedia.org/v1/page/news' })
+ .then(function(res) {
+ assert.deepEqual(res.status, 200);
+ assert.ok(res.body.length);
+ res.body.forEach(function (elem) {
+ assert.ok(elem.blurb, 'blurb should be present');
+ assert.ok(elem.links, 'links should be present');
+
+ for (var link in elem.links) {
+ if (elem.links.hasOwnProperty(link)) {
+ assert.ok(elem.links[link].pageid, 'page id should
be present');
+ assert.ok(elem.links[link].ns !== undefined,
'namespace should be present'); // 0 is falsey but good
+ assert.ok(elem.links[link].title, 'title should be
present');
+ assert.ok(elem.links[link].normalizedtitle,
'normalized title should be present');
+ if (link.thumbnail) {
+ assert.ok(elem.links[link].thumbnail.source,
'thumbnail should have source URL');
+ }
+ }
+ }
+ });
+ });
+ });
+});
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/295606
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I78eb6ba1aab9a5bc8edcaf20454db76f4a6b0f68
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/mobileapps
Gerrit-Branch: master
Gerrit-Owner: Mholloway <mhollo...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
