Dbrant has submitted this change and it was merged. Change subject: Wikidata descriptions retrieved via new PageTerms API ......................................................................
Wikidata descriptions retrieved via new PageTerms API This patch affects retrieving descriptions for - Full text search items - Lead image - Disambig items Had to change the WikidataCache to use PageTitles instead of Wikidata IDs as keys. Always add the description of the clicked on item in full text search to the cache so we don't even try to get it again when the page is opened. Change-Id: I6b308f57de805b6d1193cf6b55e5d74307f1aaa0 --- M wikipedia-it/src/main/java/org/wikipedia/test/FullSearchTaskTests.java A wikipedia-it/src/main/java/org/wikipedia/test/GetDescriptionsTaskTests.java M wikipedia-it/src/main/java/org/wikipedia/test/PageTests.java M wikipedia-it/src/main/java/org/wikipedia/test/ParcelableTest.java M wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionFeederTests.java D wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionsTaskTests.java M wikipedia/src/main/java/org/wikipedia/page/DisambigListAdapter.java M wikipedia/src/main/java/org/wikipedia/page/PageProperties.java M wikipedia/src/main/java/org/wikipedia/page/PageViewFragmentInternal.java M wikipedia/src/main/java/org/wikipedia/page/leadimages/LeadImagesHandler.java M wikipedia/src/main/java/org/wikipedia/search/FullSearchArticlesTask.java M wikipedia/src/main/java/org/wikipedia/search/FullSearchFragment.java M wikipedia/src/main/java/org/wikipedia/search/FullSearchResult.java M wikipedia/src/main/java/org/wikipedia/search/TitleSearchFragment.java M wikipedia/src/main/java/org/wikipedia/search/TitleSearchTask.java A wikipedia/src/main/java/org/wikipedia/wikidata/GetDescriptionsTask.java M wikipedia/src/main/java/org/wikipedia/wikidata/WikidataCache.java M wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionFeeder.java D wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionsTask.java D wikipedia/src/main/java/org/wikipedia/wikidata/WikidataIdsTask.java D wikipedia/src/main/java/org/wikipedia/wikidata/WikidataSite.java 21 files changed, 226 insertions(+), 363 deletions(-) Approvals: Dbrant: Looks good to me, approved jenkins-bot: Verified diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/FullSearchTaskTests.java b/wikipedia-it/src/main/java/org/wikipedia/test/FullSearchTaskTests.java index f3322d1..ad7b104 100644 --- a/wikipedia-it/src/main/java/org/wikipedia/test/FullSearchTaskTests.java +++ b/wikipedia-it/src/main/java/org/wikipedia/test/FullSearchTaskTests.java @@ -16,11 +16,13 @@ private static final int TASK_COMPLETION_TIMEOUT = 20000; private static final int BATCH_SIZE = 12; private static final Site SITE = new Site("test.wikipedia.org"); + private static final Site EN_SITE = new Site("en.wikipedia.org"); public FullSearchTaskTests() { super(TestDummyActivity.class); } + /** Have to use enwiki since I don't think there are any Wikidata descriptions for testwiki. */ public void testFullTextSearchWithResults() throws Throwable { startActivity(new Intent(), null, null); final CountDownLatch completionLatch = new CountDownLatch(1); @@ -28,7 +30,7 @@ @Override public void run() { final WikipediaApp app = (WikipediaApp) getInstrumentation().getTargetContext().getApplicationContext(); - new FullSearchArticlesTask(app.getAPIForSite(SITE), SITE, "test", BATCH_SIZE, null) { + new FullSearchArticlesTask(app.getAPIForSite(EN_SITE), EN_SITE, "test", BATCH_SIZE, null) { @Override public void onFinish(FullSearchResults results) { assertNotNull(results); @@ -38,7 +40,7 @@ for (FullSearchResult result : results.getResults()) { if (result.getTitle().getPrefixedText().equals("Test")) { - assertEquals(result.getWikiBaseId(), "Q377"); + assertEquals(result.getDescription(), "Wikipedia disambiguation page"); } } completionLatch.countDown(); @@ -49,25 +51,27 @@ assertTrue(completionLatch.await(TASK_COMPLETION_TIMEOUT, TimeUnit.MILLISECONDS)); } - public void testFullTextSearchWithSuggestion() throws Throwable { - startActivity(new Intent(), null, null); - final CountDownLatch completionLatch = new CountDownLatch(1); - runTestOnUiThread(new Runnable() { - @Override - public void run() { - final WikipediaApp app = (WikipediaApp) getInstrumentation().getTargetContext().getApplicationContext(); - new FullSearchArticlesTask(app.getAPIForSite(SITE), SITE, "teest", BATCH_SIZE, null) { // small typo should produce a suggestion - @Override - public void onFinish(FullSearchResults results) { - assertNotNull(results); - assertEquals(results.getSuggestion(), "test"); - completionLatch.countDown(); - } - }.execute(); - } - }); - assertTrue(completionLatch.await(TASK_COMPLETION_TIMEOUT, TimeUnit.MILLISECONDS)); - } + // can't seem to get suggestions anymore since search has changed + +// public void testFullTextSearchWithSuggestion() throws Throwable { +// startActivity(new Intent(), null, null); +// final CountDownLatch completionLatch = new CountDownLatch(1); +// runTestOnUiThread(new Runnable() { +// @Override +// public void run() { +// final WikipediaApp app = (WikipediaApp) getInstrumentation().getTargetContext().getApplicationContext(); +// new FullSearchArticlesTask(app.getAPIForSite(SITE), SITE, "teest", BATCH_SIZE, null) { // small typo should produce a suggestion +// @Override +// public void onFinish(FullSearchResults results) { +// assertNotNull(results); +// assertEquals(results.getSuggestion(), "test"); +// completionLatch.countDown(); +// } +// }.execute(); +// } +// }); +// assertTrue(completionLatch.await(TASK_COMPLETION_TIMEOUT, TimeUnit.MILLISECONDS)); +// } public void testEmptyResults() throws Throwable { startActivity(new Intent(), null, null); diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/GetDescriptionsTaskTests.java b/wikipedia-it/src/main/java/org/wikipedia/test/GetDescriptionsTaskTests.java new file mode 100644 index 0000000..e9e62a1 --- /dev/null +++ b/wikipedia-it/src/main/java/org/wikipedia/test/GetDescriptionsTaskTests.java @@ -0,0 +1,61 @@ +package org.wikipedia.test; + +import org.wikipedia.PageTitle; +import org.wikipedia.Site; +import org.wikipedia.WikipediaApp; +import org.wikipedia.wikidata.GetDescriptionsTask; +import android.content.Intent; +import android.test.ActivityUnitTestCase; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * Tests retrieval of Wikidata descriptions through enwiki. + */ +public class GetDescriptionsTaskTests extends ActivityUnitTestCase<TestDummyActivity> { + private static final int TASK_COMPLETION_TIMEOUT = 200000; + private static final Site SITE = Site.forLang("en"); + + public GetDescriptionsTaskTests() { + super(TestDummyActivity.class); + } + + public void testOneTitle() throws Throwable { + getWikidataDescriptions(new PageTitle[] { + new PageTitle("Test", SITE)} + ); + } + + public void testThreeTitles() throws Throwable { + getWikidataDescriptions(new PageTitle[] { + new PageTitle("SAT", SITE), + new PageTitle("Miller–Rabin primality test", SITE), + new PageTitle("Radiocarbon dating", SITE) + }); + } + + public void getWikidataDescriptions(final PageTitle[] ids) throws Throwable { + final ArrayList<PageTitle> idList = new ArrayList<PageTitle>(Arrays.asList(ids)); + startActivity(new Intent(), null, null); + final CountDownLatch completionLatch = new CountDownLatch(1); + runTestOnUiThread(new Runnable() { + @Override + public void run() { + final WikipediaApp app = (WikipediaApp) getInstrumentation().getTargetContext().getApplicationContext(); + new GetDescriptionsTask(app.getAPIForSite(SITE), SITE, idList) { + @Override + public void onFinish(Map<PageTitle, String> descriptionsMap) { + assertNotNull(descriptionsMap); + assertEquals(descriptionsMap.size(), idList.size()); + completionLatch.countDown(); + } + }.execute(); + } + }); + assertTrue(completionLatch.await(TASK_COMPLETION_TIMEOUT, TimeUnit.MILLISECONDS)); + } +} + diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/PageTests.java b/wikipedia-it/src/main/java/org/wikipedia/test/PageTests.java index f2cf666..bb8d4da 100644 --- a/wikipedia-it/src/main/java/org/wikipedia/test/PageTests.java +++ b/wikipedia-it/src/main/java/org/wikipedia/test/PageTests.java @@ -21,7 +21,7 @@ sections.add(new Section(i, 1, "Something " + i, "Something_" + i, "Content Something" + i)); } PageTitle title = new PageTitle(null, "Test", new Site("en.wikipedia.org")); - PageProperties props = new PageProperties("2001-02-03T04:00:00Z", "Test", null, true, true, null, null); + PageProperties props = new PageProperties("2001-02-03T04:00:00Z", "Test", null, true, true, null); Page page = new Page(title, sections, props); assertEquals(page, new Page(page.toJSON())); } diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/ParcelableTest.java b/wikipedia-it/src/main/java/org/wikipedia/test/ParcelableTest.java index 6d830f6..a5ccdca 100644 --- a/wikipedia-it/src/main/java/org/wikipedia/test/ParcelableTest.java +++ b/wikipedia-it/src/main/java/org/wikipedia/test/ParcelableTest.java @@ -51,7 +51,7 @@ sections.add(new Section(i, 1, "Something " + i, "Something_" + i, "Content Something" + i)); } PageTitle title = new PageTitle(null, "Test", new Site("en.wikipedia.org")); - PageProperties props = new PageProperties("", "Something", "autoconfirmed", false, false, null, null); + PageProperties props = new PageProperties("", "Something", "autoconfirmed", false, false, null); parcelAndTestObjects(title); parcelAndTestObjects(props); } diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionFeederTests.java b/wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionFeederTests.java index c58c7a9..c557c4a 100644 --- a/wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionFeederTests.java +++ b/wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionFeederTests.java @@ -15,11 +15,12 @@ import java.util.concurrent.TimeUnit; /** - * Tests retrieval of Wikidata descriptions. Talks to wikidata.org. + * Tests retrieval of Wikidata descriptions. Talks to enwiki since there a probably no Wikidata + * items related to pages on testwiki. */ public final class WikidataDescriptionFeederTests extends ActivityUnitTestCase<TestDummyActivity> { private static final int TASK_COMPLETION_TIMEOUT = 200000; - private static final Site SITE = new Site("test.wikipedia.org"); + private static final Site SITE = new Site("en.wikipedia.org"); private String originalLanguage; public WikidataDescriptionFeederTests() { @@ -46,18 +47,18 @@ public void testOneIDTwice() throws Throwable { getWikidataDescriptions(new FullSearchResult[] { - new FullSearchResult(new PageTitle("p1", SITE), null, "Q42") + new FullSearchResult(new PageTitle("Test", SITE), null, null) }); getWikidataDescriptions(new FullSearchResult[] { - new FullSearchResult(new PageTitle("p1", SITE), null, "Q42") + new FullSearchResult(new PageTitle("Test", SITE), null, null) }); } public void testThreeIDs() throws Throwable { getWikidataDescriptions(new FullSearchResult[] { - new FullSearchResult(new PageTitle("p1", SITE), null, "Q1"), - new FullSearchResult(new PageTitle("p2", SITE), null, "Q2"), - new FullSearchResult(new PageTitle("p3", SITE), null, "Q3") + new FullSearchResult(new PageTitle("SAT", SITE), null, null), + new FullSearchResult(new PageTitle("Miller–Rabin primality test", SITE), null, null), + new FullSearchResult(new PageTitle("Radiocarbon dating", SITE), null, null) }); } @@ -75,9 +76,10 @@ WikidataDescriptionFeeder.retrieveWikidataDescriptions(inputList, app, new WikidataCache.OnWikidataReceiveListener() { @Override - public void onWikidataReceived(Map<String, String> descriptions) { + public void onWikidataReceived(Map<PageTitle, String> descriptions) { + assertEquals(input.length, descriptions.size()); for (FullSearchResult res : input) { - assertFalse(descriptions.get(res.getWikiBaseId()).isEmpty()); + assertFalse(descriptions.get(res.getTitle()).isEmpty()); } completionLatch.countDown(); } diff --git a/wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionsTaskTests.java b/wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionsTaskTests.java deleted file mode 100644 index 7c3feaf..0000000 --- a/wikipedia-it/src/main/java/org/wikipedia/test/WikidataDescriptionsTaskTests.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.wikipedia.test; - -import org.wikipedia.Site; -import org.wikipedia.WikipediaApp; -import org.wikipedia.wikidata.WikidataDescriptionsTask; -import org.wikipedia.wikidata.WikidataSite; -import android.content.Intent; -import android.test.ActivityUnitTestCase; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -/** - * Tests retrieval of Wikidata descriptions. Talks to wikidata.org. - */ -public class WikidataDescriptionsTaskTests extends ActivityUnitTestCase<TestDummyActivity> { - private static final int TASK_COMPLETION_TIMEOUT = 200000; - private static final Site SITE = new WikidataSite(); - - public WikidataDescriptionsTaskTests() { - super(TestDummyActivity.class); - } - - public void testOneID() throws Throwable { - getWikidataDescriptions(new String[] {"Q42"}); - } - - public void testThreeIDs() throws Throwable { - getWikidataDescriptions(new String[] {"Q1", "Q2", "Q3"}); - } - - public void getWikidataDescriptions(final String[] ids) throws Throwable { - final ArrayList<String> idList = new ArrayList<String>(Arrays.asList(ids)); - startActivity(new Intent(), null, null); - final CountDownLatch completionLatch = new CountDownLatch(1); - runTestOnUiThread(new Runnable() { - @Override - public void run() { - final WikipediaApp app = (WikipediaApp) getInstrumentation().getTargetContext().getApplicationContext(); - new WikidataDescriptionsTask(app.getAPIForSite(SITE), "de", idList) { - @Override - public void onFinish(Map<String, String> descriptionsMap) { - assertNotNull(descriptionsMap); - assertEquals(descriptionsMap.size(), idList.size()); - completionLatch.countDown(); - } - }.execute(); - } - }); - assertTrue(completionLatch.await(TASK_COMPLETION_TIMEOUT, TimeUnit.MILLISECONDS)); - } -} - diff --git a/wikipedia/src/main/java/org/wikipedia/page/DisambigListAdapter.java b/wikipedia/src/main/java/org/wikipedia/page/DisambigListAdapter.java index 973edd9..23cc7a8 100644 --- a/wikipedia/src/main/java/org/wikipedia/page/DisambigListAdapter.java +++ b/wikipedia/src/main/java/org/wikipedia/page/DisambigListAdapter.java @@ -7,7 +7,6 @@ import org.wikipedia.WikipediaApp; import org.wikipedia.pageimages.PageImagesTask; import org.wikipedia.wikidata.WikidataCache; -import org.wikipedia.wikidata.WikidataIdsTask; import com.squareup.picasso.Picasso; import android.app.Activity; import android.view.LayoutInflater; @@ -17,7 +16,6 @@ import android.widget.ImageView; import android.widget.TextView; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,7 +30,6 @@ private final DisambigResult[] items; private final WikipediaApp app; private final Site site; - private final Map<PageTitle, String> titleWikidataIdMap = new HashMap<PageTitle, String>(); private final WikidataCache wikidataCache; /** @@ -48,7 +45,7 @@ site = app.getPrimarySite(); requestPageImages(); wikidataCache = app.getWikidataCache(); - fetchWikiDataIds(); + fetchDescriptions(); } private void requestPageImages() { @@ -89,9 +86,9 @@ } /** - * Start getting Wikidata ID, so that we can request Wikidata descriptions. + * Start getting Wikidata descriptions (directly from the current Wikipedia site). */ - private void fetchWikiDataIds() { + private void fetchDescriptions() { List<PageTitle> titleList = new ArrayList<PageTitle>(); for (DisambigResult r : items) { titleList.add(r.getTitle()); @@ -100,52 +97,17 @@ return; } - WikidataIdsTask wikidataIdsTask = new WikidataIdsTask( - app.getAPIForSite(site), - site, - titleList) { + wikidataCache.get(titleList, new WikidataCache.OnWikidataReceiveListener() { @Override - public void onFinish(Map<PageTitle, String> result) { - List<String> wikidataIds = new ArrayList<String>(result.size()); - for (Map.Entry<PageTitle, String> entry : result.entrySet()) { - if (entry.getValue() == null) { - continue; - } - titleWikidataIdMap.put(entry.getKey(), entry.getValue()); - wikidataIds.add(entry.getValue()); - } - fetchWikiDataDescription(wikidataIds); + public void onWikidataReceived(Map<PageTitle, String> result) { + notifyDataSetChanged(); } @Override - public void onCatch(Throwable caught) { - // Don't actually do anything. - // Thumbnails are expendable + public void onWikidataFailed(Throwable caught) { + // descriptions are expendable } - }; - wikidataIdsTask.execute(); - } - - /** - * Start the task of fetching the WikiData description for our page, if it has one. - * This should be done after the lead image view is laid out, but can be done independently - * of loading the WebView contents. - */ - private void fetchWikiDataDescription(final List<String> wikiDataIds) { - if (!wikiDataIds.isEmpty()) { - wikidataCache.get(wikiDataIds, - new WikidataCache.OnWikidataReceiveListener() { - @Override - public void onWikidataReceived(Map<String, String> result) { - notifyDataSetChanged(); - } - - @Override - public void onWikidataFailed(Throwable caught) { - // don't care - } - }); - } + }); } class ViewHolder { @@ -172,12 +134,7 @@ final DisambigResult item = items[position]; holder.title.setText(item.getTitle().getPrefixedText()); - String description = null; - String wikidataId = titleWikidataIdMap.get(item.getTitle()); - if (wikidataId != null) { - description = wikidataCache.get(wikidataId); - } - holder.description.setText(description); + holder.description.setText(wikidataCache.get(item.getTitle())); String thumbnail = pageImagesCache.get(item.getTitle().getPrefixedText()); if (thumbnail == null) { diff --git a/wikipedia/src/main/java/org/wikipedia/page/PageProperties.java b/wikipedia/src/main/java/org/wikipedia/page/PageProperties.java index 24c7440..d448b3b 100644 --- a/wikipedia/src/main/java/org/wikipedia/page/PageProperties.java +++ b/wikipedia/src/main/java/org/wikipedia/page/PageProperties.java @@ -22,7 +22,6 @@ private final String displayTitleText; private final String editProtectionStatus; private final boolean isMainPage; - private final String wikiDataId; private final String leadImageUrl; private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT); @@ -41,7 +40,7 @@ */ public PageProperties(String lastModifiedText, String displayTitleText, String editProtectionStatus, boolean canEdit, boolean isMainPage, - String wikiDataId, String leadImageUrl) { + String leadImageUrl) { lastModified = new Date(); sdf.setTimeZone(TimeZone.getTimeZone("UTC")); try { @@ -53,7 +52,6 @@ this.editProtectionStatus = editProtectionStatus; this.canEdit = canEdit; this.isMainPage = isMainPage; - this.wikiDataId = wikiDataId; this.leadImageUrl = leadImageUrl; } @@ -77,10 +75,6 @@ return isMainPage; } - public String getWikiDataId() { - return wikiDataId; - } - public String getLeadImageUrl() { return leadImageUrl; } @@ -97,7 +91,6 @@ parcel.writeString(editProtectionStatus); parcel.writeInt(canEdit ? 1 : 0); parcel.writeInt(isMainPage ? 1 : 0); - parcel.writeString(wikiDataId); parcel.writeString(leadImageUrl); } @@ -108,7 +101,6 @@ editProtectionStatus = in.readString(); canEdit = in.readInt() == 1; isMainPage = in.readInt() == 1; - wikiDataId = in.readString(); leadImageUrl = in.readString(); } @@ -139,7 +131,6 @@ && canEdit == that.canEdit && isMainPage == that.isMainPage && TextUtils.equals(editProtectionStatus, that.editProtectionStatus) - && TextUtils.equals(wikiDataId, that.wikiDataId) && TextUtils.equals(leadImageUrl, that.leadImageUrl); } @@ -152,9 +143,6 @@ } if (leadImageUrl != null) { result = 127 * result + leadImageUrl.hashCode(); - } - if (wikiDataId != null) { - result = 255 * result + wikiDataId.hashCode(); } return result; } @@ -181,11 +169,6 @@ json.put("editable", canEdit); if (isMainPage) { json.put("mainpage", ""); - } - if (wikiDataId != null) { - JSONObject pagePropsObject = new JSONObject(); - pagePropsObject.put("wikibase_item", wikiDataId); - json.put("pageprops", pagePropsObject); } if (leadImageUrl != null) { JSONObject thumbObject = new JSONObject(); @@ -214,12 +197,7 @@ ) { editProtection = json.optJSONObject("protection").optJSONArray("edit").optString(0); } - String wikiDataId = null; String leadImageUrl = null; - JSONObject pageProps = json.optJSONObject("pageprops"); - if (pageProps != null) { - wikiDataId = pageProps.optString("wikibase_item"); - } JSONObject thumb = json.optJSONObject("thumb"); if (thumb != null) { leadImageUrl = thumb.optString("url"); @@ -230,7 +208,6 @@ editProtection, json.optBoolean("editable"), json.has("mainpage"), - wikiDataId, leadImageUrl ); } diff --git a/wikipedia/src/main/java/org/wikipedia/page/PageViewFragmentInternal.java b/wikipedia/src/main/java/org/wikipedia/page/PageViewFragmentInternal.java index 3e43df9..934db21 100644 --- a/wikipedia/src/main/java/org/wikipedia/page/PageViewFragmentInternal.java +++ b/wikipedia/src/main/java/org/wikipedia/page/PageViewFragmentInternal.java @@ -733,7 +733,7 @@ @Override public RequestBuilder buildRequest(Api api) { RequestBuilder builder = super.buildRequest(api); - builder.param("prop", builder.getParams().get("prop") + "|pageprops|thumb|" + builder.param("prop", builder.getParams().get("prop") + "|thumb|" + Page.API_REQUEST_PROPS); builder.param("thumbsize", Integer.toString((int)(getResources().getDimension(R.dimen.leadImageWidth) / getResources().getDisplayMetrics().density))); diff --git a/wikipedia/src/main/java/org/wikipedia/page/leadimages/LeadImagesHandler.java b/wikipedia/src/main/java/org/wikipedia/page/leadimages/LeadImagesHandler.java index 37f64e1..8401316 100644 --- a/wikipedia/src/main/java/org/wikipedia/page/leadimages/LeadImagesHandler.java +++ b/wikipedia/src/main/java/org/wikipedia/page/leadimages/LeadImagesHandler.java @@ -5,6 +5,7 @@ import android.graphics.Point; import android.os.Build; import android.text.Html; +import android.text.TextUtils; import android.util.TypedValue; import android.graphics.PointF; import android.view.View; @@ -22,6 +23,7 @@ import org.json.JSONException; import org.json.JSONObject; +import org.wikipedia.PageTitle; import org.wikipedia.R; import org.wikipedia.Utils; import org.wikipedia.ViewAnimations; @@ -444,19 +446,22 @@ * of loading the WebView contents. */ private void fetchWikiDataDescription() { - final String wikiDataId = parentFragment.getFragment().getPage().getPageProperties().getWikiDataId(); - final String language = parentFragment.getFragment().getTitle().getSite().getLanguage(); + final PageTitle pageTitle = parentFragment.getFragment().getPage().getTitle(); - if (wikiDataId != null) { - WikipediaApp.getInstance().getWikidataCache().get(wikiDataId, language, + if (pageTitle != null) { + final String language = pageTitle.getSite().getLanguage(); + WikipediaApp.getInstance().getWikidataCache().get(pageTitle, language, new WikidataCache.OnWikidataReceiveListener() { @Override - public void onWikidataReceived(Map<String, String> result) { + public void onWikidataReceived(Map<PageTitle, String> result) { if (!parentFragment.isAdded()) { return; } - if (result.containsKey(wikiDataId)) { - layoutWikiDataDescription(result.get(wikiDataId)); + if (result.containsKey(pageTitle)) { + final String description = result.get(pageTitle); + if (!TextUtils.isEmpty(description)) { + layoutWikiDataDescription(description); + } } } @Override diff --git a/wikipedia/src/main/java/org/wikipedia/search/FullSearchArticlesTask.java b/wikipedia/src/main/java/org/wikipedia/search/FullSearchArticlesTask.java index 1f29f4a..0f4a07f 100644 --- a/wikipedia/src/main/java/org/wikipedia/search/FullSearchArticlesTask.java +++ b/wikipedia/src/main/java/org/wikipedia/search/FullSearchArticlesTask.java @@ -1,5 +1,6 @@ package org.wikipedia.search; +import org.json.JSONArray; import org.wikipedia.ApiTask; import org.wikipedia.PageTitle; import org.wikipedia.Site; @@ -35,8 +36,8 @@ public RequestBuilder buildRequest(Api api) { final String maxResultsString = Integer.toString(maxResults); final RequestBuilder req = api.action("query") - .param("prop", "pageprops|pageimages") - .param("ppprop", "wikibase_item") // only interested in wikibase_item + .param("prop", "pageterms|pageimages") + .param("wbptterms", "description") // only interested in Wikidata description .param("generator", "search") .param("gsrsearch", searchTerm) .param("gsrnamespace", "0") @@ -128,11 +129,14 @@ if (item.has("thumbnail")) { thumbUrl = item.getJSONObject("thumbnail").optString("source", null); } - String wikiBaseId = null; - if (item.has("pageprops")) { - wikiBaseId = item.getJSONObject("pageprops").optString("wikibase_item", null); + String description = null; + if (item.has("terms")) { + JSONArray arr = item.getJSONObject("terms").optJSONArray("description"); + if (arr != null && arr.length() > 0) { + description = arr.getString(0); + } } - resultList.add(new FullSearchResult(pageTitle, thumbUrl, wikiBaseId)); + resultList.add(new FullSearchResult(pageTitle, thumbUrl, description)); } return new FullSearchResults(resultList, nextContinueOffset, suggestion); } diff --git a/wikipedia/src/main/java/org/wikipedia/search/FullSearchFragment.java b/wikipedia/src/main/java/org/wikipedia/search/FullSearchFragment.java index 9ba88ae..d8da20d 100644 --- a/wikipedia/src/main/java/org/wikipedia/search/FullSearchFragment.java +++ b/wikipedia/src/main/java/org/wikipedia/search/FullSearchFragment.java @@ -1,18 +1,14 @@ package org.wikipedia.search; -import org.wikipedia.PageTitle; import org.wikipedia.R; import org.wikipedia.WikipediaApp; import org.wikipedia.page.PageActivity; -import org.wikipedia.wikidata.WikidataCache; -import org.wikipedia.wikidata.WikidataDescriptionFeeder; import com.squareup.picasso.Picasso; import android.os.Bundle; import android.os.Handler; import android.os.Message; import android.support.v4.app.Fragment; import android.text.Html; -import android.text.TextUtils; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; @@ -24,7 +20,6 @@ import android.widget.Toast; import java.util.ArrayList; import java.util.List; -import java.util.Map; public class FullSearchFragment extends Fragment { private static final int BATCH_SIZE = 12; @@ -69,8 +64,10 @@ searchResultsList.setOnItemClickListener(new AdapterView.OnItemClickListener() { @Override public void onItemClick(AdapterView<?> parent, View view, int position, long id) { - PageTitle title = ((FullSearchResult) searchResultsList.getAdapter().getItem(position)).getTitle(); - searchFragment.navigateToTitle(title); + FullSearchResult item = (FullSearchResult) searchResultsList.getAdapter().getItem(position); + // always add the description of the item to the cache so we don't even try to get it again + app.getWikidataCache().put(item.getTitle().toString(), item.getDescription()); + searchFragment.navigateToTitle(item.getTitle()); } }); @@ -189,20 +186,6 @@ searchResultsList.setVisibility(View.GONE); } else { searchResultsList.setVisibility(View.VISIBLE); - WikidataDescriptionFeeder.retrieveWikidataDescriptions(lastResults.getResults(), app, - new WikidataCache.OnWikidataReceiveListener() { - @Override - public void onWikidataReceived(Map<String, String> result) { - ((BaseAdapter) searchResultsList.getAdapter()) - .notifyDataSetChanged(); - } - - @Override - public void onWikidataFailed(Throwable caught) { - // Don't actually do anything. - // Descriptions are expendable - } - }); } if (continueOffset == null) { @@ -285,11 +268,8 @@ FullSearchResult result = (FullSearchResult) getItem(position); pageTitleText.setText(result.getTitle().getDisplayText()); - String wikidataId = result.getWikiBaseId(); - if (!TextUtils.isEmpty(wikidataId)) { - TextView descriptionText = (TextView) convertView.findViewById(R.id.result_description); - descriptionText.setText(app.getWikidataCache().get(wikidataId)); - } + TextView descriptionText = (TextView) convertView.findViewById(R.id.result_description); + descriptionText.setText(result.getDescription()); ImageView imageView = (ImageView) convertView.findViewById(R.id.result_image); String thumbnail = result.getThumbUrl(); diff --git a/wikipedia/src/main/java/org/wikipedia/search/FullSearchResult.java b/wikipedia/src/main/java/org/wikipedia/search/FullSearchResult.java index c84a104..2de4bcc 100644 --- a/wikipedia/src/main/java/org/wikipedia/search/FullSearchResult.java +++ b/wikipedia/src/main/java/org/wikipedia/search/FullSearchResult.java @@ -5,11 +5,11 @@ public class FullSearchResult { private final PageTitle title; private final String thumbUrl; - private final String wikiBaseId; + private final String description; - public FullSearchResult(PageTitle title, String thumbUrl, String wikiBaseId) { + public FullSearchResult(PageTitle title, String thumbUrl, String description) { this.thumbUrl = thumbUrl; - this.wikiBaseId = wikiBaseId; + this.description = description; this.title = title; } @@ -21,7 +21,7 @@ return thumbUrl; } - public String getWikiBaseId() { - return wikiBaseId; + public String getDescription() { + return description; } } diff --git a/wikipedia/src/main/java/org/wikipedia/search/TitleSearchFragment.java b/wikipedia/src/main/java/org/wikipedia/search/TitleSearchFragment.java index d3660d3..bd54b98 100644 --- a/wikipedia/src/main/java/org/wikipedia/search/TitleSearchFragment.java +++ b/wikipedia/src/main/java/org/wikipedia/search/TitleSearchFragment.java @@ -263,7 +263,7 @@ public boolean handleMessage(Message msg) { final String mySearchTerm = (String) msg.obj; final long startMillis = System.currentTimeMillis(); - TitleSearchTask searchTask = new TitleSearchTask(app, app.getAPIForSite(app.getPrimarySite()), app.getPrimarySite(), mySearchTerm) { + TitleSearchTask searchTask = new TitleSearchTask(app.getAPIForSite(app.getPrimarySite()), app.getPrimarySite(), mySearchTerm) { @Override public void onBeforeExecute() { ((PageActivity)getActivity()).updateProgressBar(true, true, 0); diff --git a/wikipedia/src/main/java/org/wikipedia/search/TitleSearchTask.java b/wikipedia/src/main/java/org/wikipedia/search/TitleSearchTask.java index a9ed16e..b0bb505 100644 --- a/wikipedia/src/main/java/org/wikipedia/search/TitleSearchTask.java +++ b/wikipedia/src/main/java/org/wikipedia/search/TitleSearchTask.java @@ -10,7 +10,6 @@ import org.mediawiki.api.json.RequestBuilder; import org.json.JSONException; import org.json.JSONObject; -import android.content.Context; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -23,7 +22,7 @@ private static final String NUM_RESULTS_PER_QUERY = "12"; - public TitleSearchTask(Context context, Api api, Site site, String prefix) { + public TitleSearchTask(Api api, Site site, String prefix) { super(HIGH_CONCURRENCY, api); this.prefix = prefix; this.site = site; diff --git a/wikipedia/src/main/java/org/wikipedia/wikidata/GetDescriptionsTask.java b/wikipedia/src/main/java/org/wikipedia/wikidata/GetDescriptionsTask.java new file mode 100644 index 0000000..99b489d --- /dev/null +++ b/wikipedia/src/main/java/org/wikipedia/wikidata/GetDescriptionsTask.java @@ -0,0 +1,41 @@ +package org.wikipedia.wikidata; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.mediawiki.api.json.Api; +import org.mediawiki.api.json.RequestBuilder; +import org.wikipedia.PageQueryTask; +import org.wikipedia.PageTitle; +import org.wikipedia.Site; + +import java.util.List; + +/** + * Retrieves Wikidata descriptions via a Wikipedia site. + */ +public class GetDescriptionsTask extends PageQueryTask<String> { + + public GetDescriptionsTask(Api api, Site site, List<PageTitle> titles) { + super(LOW_CONCURRENCY, api, site, titles); + } + + @Override + public void buildQueryParams(RequestBuilder builder) { + builder.param("prop", "pageterms") + .param("wbptterm", "description"); + } + + @Override + public String processPage(int pageId, PageTitle pageTitle, JSONObject pageData) throws Throwable { + JSONObject terms = pageData.optJSONObject("terms"); + if (terms != null) { + final JSONArray array = terms.optJSONArray("description"); + if (array != null && array.length() > 0) { + String value = array.getString(0); + //Capitalise the first letter of the description, for style + return value.substring(0, 1).toUpperCase() + value.substring(1); + } + } + return null; + } +} diff --git a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataCache.java b/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataCache.java index 2813a30..aff10d1 100644 --- a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataCache.java +++ b/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataCache.java @@ -1,6 +1,8 @@ package org.wikipedia.wikidata; +import org.wikipedia.PageTitle; import org.wikipedia.ParcelableLruCache; +import org.wikipedia.Site; import org.wikipedia.WikipediaApp; import java.util.ArrayList; @@ -9,7 +11,7 @@ import java.util.Map; public class WikidataCache { - private static final int MAX_CACHE_SIZE_DESCRIPTIONS = 96; + private static final int MAX_CACHE_SIZE_DESCRIPTIONS = 48; private WikipediaApp app; private ParcelableLruCache<String> descriptionCache = new ParcelableLruCache<String>(MAX_CACHE_SIZE_DESCRIPTIONS, String.class); @@ -36,6 +38,9 @@ * @param value Wikidata description of the page associated with the Wikidata ID. */ public void put(String key, String value) { + if (value == null) { + value = ""; + } descriptionCache.put(key, value); } @@ -46,73 +51,71 @@ * cache consider using the get method instead, which will (if necessary) retrieve descriptions * that the cache currently does not contain and cache them itself if appropriate. * - * @param id Wikidata ID of the page for which the description is required. + * @param title Page title for which the description is required. * @return The Wikidata description that was cached, or null if the description is not cached. */ - public String get(String id) { - return descriptionCache.get(id); + public String get(PageTitle title) { + return descriptionCache.get(title.toString()); } /** * Retrieves Wikidata description for a page in the app's current primary language. * - * @param id Wikidata ID of the page for which the description is required. + * @param title Page title for which the description is required. * @param listener Listener that will receive the description retrieved from the cache. */ - public void get(String id, OnWikidataReceiveListener listener) { - get(id, app.getPrimaryLanguage(), listener); + public void get(PageTitle title, OnWikidataReceiveListener listener) { + get(title, app.getPrimaryLanguage(), listener); } /** * Retrieves Wikidata description for a page in the specified language. * - * @param id Wikidata ID of the page for which the description is required. + * @param title Page title for which the description is required. * @param language The language in which the description is required. * @param listener Listener that will receive the description retrieved from the cache. */ - public void get(String id, String language, OnWikidataReceiveListener listener) { - List<String> idList = new ArrayList<String>(); - idList.add(id); + public void get(PageTitle title, String language, OnWikidataReceiveListener listener) { + List<PageTitle> idList = new ArrayList<PageTitle>(); + idList.add(title); get(idList, language, listener); } /** * Retrieves Wikidata descriptions for a list of pages in the app's current primary language. * - * @param ids Wikidata IDs of the pages for which the descriptions are required. + * @param titles Page titles for which the descriptions are required. * @param listener Listener that will receive the descriptions retrieved from the cache. */ - public void get(List<String> ids, final OnWikidataReceiveListener listener) { - get(ids, app.getPrimaryLanguage(), listener); + public void get(List<PageTitle> titles, final OnWikidataReceiveListener listener) { + get(titles, app.getPrimaryLanguage(), listener); } /** * Retrieves Wikidata descriptions for a list of pages in the specified language. * - * @param ids Wikidata IDs of the pages for which the descriptions are required. + * @param titles Page titles for which the descriptions are required. * @param language The language in which the descriptions are required. * @param listener Listener that will receive the descriptions retrieved from the cache. */ - public void get(List<String> ids, final String language, final OnWikidataReceiveListener listener) { - final Map<String, String> results = new HashMap<String, String>(); - List<String> idsToFetch = new ArrayList<String>(); - for (String id : ids) { - if (descriptionCache.get(id) == null || !language.equals(app.getPrimaryLanguage())) { + public void get(List<PageTitle> titles, final String language, final OnWikidataReceiveListener listener) { + final Map<PageTitle, String> results = new HashMap<PageTitle, String>(); + List<PageTitle> titlesToFetch = new ArrayList<PageTitle>(); + for (PageTitle title : titles) { + if (descriptionCache.get(title.toString()) == null || !language.equals(app.getPrimaryLanguage())) { // not in our cache yet, or we want a result in a different language from the one // the cache is currently storing descriptions in - idsToFetch.add(id); + titlesToFetch.add(title); } else { - results.put(id, descriptionCache.get(id)); + results.put(title, descriptionCache.get(title.toString())); } } - if (idsToFetch.size() > 0) { - (new WikidataDescriptionsTask( - app.getAPIForSite(new WikidataSite()), - language, - idsToFetch) { + if (titlesToFetch.size() > 0) { + final Site site = Site.forLang(language); + (new GetDescriptionsTask(app.getAPIForSite(site), site, titlesToFetch) { @Override - public void onFinish(Map<String, String> result) { - for (Map.Entry<String, String> entry : result.entrySet()) { + public void onFinish(Map<PageTitle, String> result) { + for (Map.Entry<PageTitle, String> entry : result.entrySet()) { if (entry.getValue() == null) { continue; } @@ -120,7 +123,7 @@ // currently using; that way if the user chose "Read in another language" // we're not caching and displaying results from the incorrect language if (language.equals(app.getPrimaryLanguage())) { - descriptionCache.put(entry.getKey(), entry.getValue()); + descriptionCache.put(entry.getKey().toString(), entry.getValue()); } results.put(entry.getKey(), entry.getValue()); } @@ -136,7 +139,7 @@ } } public interface OnWikidataReceiveListener { - void onWikidataReceived(Map<String, String> result); + void onWikidataReceived(Map<PageTitle, String> result); void onWikidataFailed(Throwable caught); } } diff --git a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionFeeder.java b/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionFeeder.java index bb5a25f..2b12a5a 100644 --- a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionFeeder.java +++ b/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionFeeder.java @@ -1,8 +1,8 @@ package org.wikipedia.wikidata; +import org.wikipedia.PageTitle; import org.wikipedia.WikipediaApp; import org.wikipedia.search.FullSearchResult; -import android.text.TextUtils; import java.util.ArrayList; import java.util.List; @@ -19,13 +19,13 @@ */ public static void retrieveWikidataDescriptions(List<FullSearchResult> results, final WikipediaApp app, final WikidataCache.OnWikidataReceiveListener listener) { - List<String> wikiDataIds = new ArrayList<String>(); + List<PageTitle> pageTitles = new ArrayList<PageTitle>(); final WikidataCache wikidataCache = app.getWikidataCache(); for (FullSearchResult r : results) { - if (!TextUtils.isEmpty(r.getWikiBaseId())) { - wikiDataIds.add(r.getWikiBaseId()); + if (r.getDescription() == null) { + pageTitles.add(r.getTitle()); } } - wikidataCache.get(wikiDataIds, listener); + wikidataCache.get(pageTitles, listener); } } diff --git a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionsTask.java b/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionsTask.java deleted file mode 100644 index 1c4b022..0000000 --- a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataDescriptionsTask.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.wikipedia.wikidata; - -import org.wikipedia.ApiTask; -import org.mediawiki.api.json.Api; -import org.mediawiki.api.json.ApiResult; -import org.mediawiki.api.json.RequestBuilder; -import org.json.JSONObject; -import android.text.TextUtils; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * Retrieves descriptions for given language and IDs from Wikidata. - */ -public class WikidataDescriptionsTask extends ApiTask<Map<String, String>> { - - private final String language; - private final List<String> idList; - - public WikidataDescriptionsTask(Api api, String language, List<String> idList) { - super(LOW_CONCURRENCY, api); - this.language = language; - this.idList = idList; - } - - @Override - public RequestBuilder buildRequest(Api api) { - return api.action("wbgetentities") - .param("props", "descriptions") - .param("languages", language) - .param("ids", TextUtils.join("|", idList)); - } - - @Override - public Map<String, String> processResult(ApiResult result) throws Throwable { - Map<String, String> map = new HashMap<String, String>(); - JSONObject data = result.asObject(); - JSONObject entities = data.getJSONObject("entities"); - - Iterator<String> keys = entities.keys(); - while (keys.hasNext()) { - String key = keys.next(); - JSONObject entity = entities.getJSONObject(key); - JSONObject descriptions = entity.optJSONObject("descriptions"); - if (descriptions != null && descriptions.has(language)) { - JSONObject langEntry = descriptions.getJSONObject(language); - String value = langEntry.optString("value"); - //Capitalise the first letter of the description, for style - value = value.substring(0, 1).toUpperCase() + value.substring(1); - map.put(key, value); - } - } - - return map; - } -} diff --git a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataIdsTask.java b/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataIdsTask.java deleted file mode 100644 index c606c85..0000000 --- a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataIdsTask.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.wikipedia.wikidata; - -import org.wikipedia.PageQueryTask; -import org.wikipedia.PageTitle; -import org.wikipedia.Site; -import org.mediawiki.api.json.Api; -import org.mediawiki.api.json.RequestBuilder; -import org.json.JSONObject; -import java.util.List; - -/** - * Retrieves Wikidata IDs from Wikipedia: PageTitles -> Wikidata IDs. - */ -public class WikidataIdsTask extends PageQueryTask<String> { - - public WikidataIdsTask(Api api, Site site, List<PageTitle> titles) { - super(LOW_CONCURRENCY, api, site, titles); - } - - @Override - public void buildQueryParams(RequestBuilder builder) { - builder.param("prop", "pageprops") - .param("ppprop", "wikibase_item"); - } - - @Override - public String processPage(int pageId, PageTitle pageTitle, JSONObject pageData) throws Throwable { - JSONObject pageProps = pageData.optJSONObject("pageprops"); - if (pageProps == null) { - return null; - } else { - return pageProps.optString("wikibase_item"); - } - } -} diff --git a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataSite.java b/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataSite.java deleted file mode 100644 index 8ac2ab8..0000000 --- a/wikipedia/src/main/java/org/wikipedia/wikidata/WikidataSite.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.wikipedia.wikidata; - -import org.wikipedia.Site; - -/** - * A Site for Wikidata API calls. One site for all languages. - */ -public class WikidataSite extends Site { - public WikidataSite() { - super("www.wikidata.org"); - } - - @Override - public String getApiDomain() { - return getDomain(); - } - - @Override - public String getLanguage() { - throw new UnsupportedOperationException("getLanguage not supported"); - } -} -- To view, visit https://gerrit.wikimedia.org/r/177740 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6b308f57de805b6d1193cf6b55e5d74307f1aaa0 Gerrit-PatchSet: 6 Gerrit-Project: apps/android/wikipedia Gerrit-Branch: master Gerrit-Owner: BearND <bsitzm...@wikimedia.org> Gerrit-Reviewer: Brion VIBBER <br...@wikimedia.org> Gerrit-Reviewer: Dbrant <dbr...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits