Mholloway has uploaded a new change for review. https://gerrit.wikimedia.org/r/271925
Change subject: WIP: Implement background saved page syncing ...................................................................... WIP: Implement background saved page syncing This isn't yet working but should give you some idea of where I'm going with this. Bug: T126753 Change-Id: Iae8e9e19fa4d774f590a85f238b1510bb5642878 --- M app/src/main/AndroidManifest.xml M app/src/main/java/org/wikipedia/WikipediaApp.java M app/src/main/java/org/wikipedia/database/DatabaseClient.java M app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java M app/src/main/java/org/wikipedia/savedpages/SavePageTask.java M app/src/main/java/org/wikipedia/savedpages/SavedPage.java M app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java A app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java M app/src/main/java/org/wikipedia/server/PageService.java M app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java M app/src/main/java/org/wikipedia/server/restbase/RbContentService.java M app/src/main/java/org/wikipedia/util/FileUtil.java 12 files changed, 245 insertions(+), 23 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/apps/android/wikipedia refs/changes/25/271925/1 diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 86d4685..47d19df 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -210,6 +210,10 @@ <action android:name="com.android.vending.INSTALL_REFERRER" /> </intent-filter> </receiver> + + <service + android:name=".savedpages.SavedPageSyncService" + android:exported="false"/> </application> </manifest> diff --git a/app/src/main/java/org/wikipedia/WikipediaApp.java b/app/src/main/java/org/wikipedia/WikipediaApp.java index 83f10a1..ddee0a1 100644 --- a/app/src/main/java/org/wikipedia/WikipediaApp.java +++ b/app/src/main/java/org/wikipedia/WikipediaApp.java @@ -38,6 +38,7 @@ import org.wikipedia.page.PageCache; import org.wikipedia.pageimages.PageImage; import org.wikipedia.savedpages.SavedPage; +import org.wikipedia.savedpages.SavedPageSyncService; import org.wikipedia.search.RecentSearch; import org.wikipedia.settings.Prefs; import org.wikipedia.theme.Theme; @@ -167,6 +168,7 @@ Fresco.initialize(this); bus = new Bus(); + SavedPageSyncService.start(); final Resources resources = getResources(); ViewAnimations.init(resources); diff --git a/app/src/main/java/org/wikipedia/database/DatabaseClient.java b/app/src/main/java/org/wikipedia/database/DatabaseClient.java index 3746566..a10b6fc 100644 --- a/app/src/main/java/org/wikipedia/database/DatabaseClient.java +++ b/app/src/main/java/org/wikipedia/database/DatabaseClient.java @@ -41,6 +41,10 @@ } } + public Cursor selectAll() { + return select("", new String[] {}, null); + } + public void deleteAll() { deleteWhere("", new String[] {}); } diff --git a/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java b/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java index cffa02d..8182849 100644 --- a/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java +++ b/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java @@ -7,11 +7,9 @@ import org.wikipedia.util.UriUtil; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import static org.wikipedia.util.FileUtil.copyStreams; +import static org.wikipedia.util.FileUtil.writeFile; /** */ public class DownloadImageTask extends SaneAsyncTask<Boolean> { @@ -50,15 +48,6 @@ } else { Log.e(TAG, "could not download image " + url + " to " + file.getAbsolutePath()); return false; - } - } - - private void writeFile(InputStream inputStream, File file) throws IOException { - FileOutputStream outputStream = new FileOutputStream(file); - try { - copyStreams(inputStream, outputStream); - } finally { - outputStream.close(); } } } diff --git a/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java b/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java index 817d694..3adeb85 100644 --- a/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java +++ b/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java @@ -29,17 +29,22 @@ @Override public Boolean performTask() throws Throwable { SavedPage savedPage = new SavedPage(title); - savedPage.writeToFileSystem(page); + + //savedPage.writeToFileSystem(page); + DatabaseClient<SavedPage> client = app.getDatabaseClient(SavedPage.class); client.upsert(savedPage, SavedPageDatabaseTable.SELECTION_KEYS); - final ImageUrlMap imageUrlMap = new ImageUrlMap.Builder(savedPage.getBaseDir()).extractUrls(page).build(); - final int numImagesAttempts = imageUrlMap.size(); + //final ImageUrlMap imageUrlMap = new ImageUrlMap.Builder(savedPage.getBaseDir()).extractUrls(page).build(); + //final int numImagesAttempts = imageUrlMap.size(); - parallelDownload(imageUrlMap); + //parallelDownload(imageUrlMap); - savedPage.writeUrlMap(imageUrlMap.toJSON()); - return numImagesAttempts == imageUrlMap.size(); + //savedPage.writeUrlMap(imageUrlMap.toJSON()); + //return numImagesAttempts == imageUrlMap.size(); + + //TODO: remove + return Boolean.TRUE; } /** diff --git a/app/src/main/java/org/wikipedia/savedpages/SavedPage.java b/app/src/main/java/org/wikipedia/savedpages/SavedPage.java index 139e914..25a8d61 100644 --- a/app/src/main/java/org/wikipedia/savedpages/SavedPage.java +++ b/app/src/main/java/org/wikipedia/savedpages/SavedPage.java @@ -115,7 +115,7 @@ * Gets a File object that represents the JSON contents of this page. * @return File object used for reading/writing page contents. */ - private File getContentsFile() { + protected File getContentsFile() { return new File(getBaseDir() + "/content.json"); } diff --git a/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java b/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java index 2022a75..eb968ba 100644 --- a/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java +++ b/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java @@ -23,10 +23,10 @@ private static final int DB_VER_INTRODUCED = 4; private static final int DB_VER_NAMESPACE_ADDED = 6; - private static final String COL_SITE = "site"; - private static final String COL_TITLE = "title"; - private static final String COL_NAMESPACE = "namespace"; - private static final String COL_TIMESTAMP = "timestamp"; + protected static final String COL_SITE = "site"; + protected static final String COL_TITLE = "title"; + protected static final String COL_NAMESPACE = "namespace"; + protected static final String COL_TIMESTAMP = "timestamp"; public static final String[] SELECTION_KEYS = { COL_SITE, diff --git a/app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java b/app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java new file mode 100644 index 0000000..d48db90 --- /dev/null +++ b/app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java @@ -0,0 +1,169 @@ +package org.wikipedia.savedpages; + +import android.app.IntentService; +import android.content.Intent; +import android.database.Cursor; +import android.support.annotation.NonNull; + +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; + +import org.wikipedia.Site; +import org.wikipedia.WikipediaApp; +import org.wikipedia.database.DatabaseClient; +import org.wikipedia.page.Page; +import org.wikipedia.page.PageTitle; +import org.wikipedia.server.ContentServiceFactory; +import org.wikipedia.server.PageCombo; +import org.wikipedia.server.PageService; +import org.wikipedia.util.log.L; + +import java.io.File; +import java.io.IOException; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.wikipedia.util.FileUtil.writeFile; + +public class SavedPageSyncService extends IntentService { + private List<PageTitle> queue = new LinkedList<>(); + + public SavedPageSyncService() { + super("SavedPageSyncService"); + + // Redeliver the intent if our process dies, so our sync completes. + setIntentRedelivery(true); + } + + @SuppressWarnings("MagicNumber") + public static void start() { + ScheduledExecutorService syncRunnerService = Executors.newScheduledThreadPool(1); + + //TODO: MAKE INTERVAL 15 *MINUTES* WHEN DONE DEBUGGING + syncRunnerService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + WikipediaApp.getInstance().startService( + new Intent(WikipediaApp.getInstance(), SavedPageSyncService.class) + ); + } + }, 0, 15, TimeUnit.SECONDS); + + L.i("Starting saved page sync service"); + } + + @Override + protected void onHandleIntent(@NonNull Intent intent) { + L.i("Checking DB for new saved page entries..."); + boolean newEntries = enqueueNewDbEntries(); + if (newEntries) { + saveNewEntries(); + } + } + + private boolean enqueueNewDbEntries() { + Cursor cursor = getEntriesFromDB(); + + if (cursor.getCount() == 0) { + L.i("No new saved page DB entries found; exiting."); + return false; + } + + while (cursor.moveToNext()) { + //TODO: add namespace (if possible?) + Site site = new Site(cursor.getString(cursor.getColumnIndexOrThrow(SavedPageDatabaseTable.COL_SITE))); + PageTitle title = new PageTitle(null, cursor.getString(cursor.getColumnIndexOrThrow(SavedPageDatabaseTable.COL_TITLE)), site); + Date timestamp = new Date(cursor.getLong(cursor.getColumnIndexOrThrow(SavedPageDatabaseTable.COL_TIMESTAMP))); + SavedPage entry = new SavedPage(title, timestamp); + if (!entry.getContentsFile().exists()) { + queue.add(title); + L.i("Added new saved page entry " + title.getDisplayText() + " to download queue."); + } + } + cursor.close(); + return true; + } + + @NonNull + private Cursor getEntriesFromDB() { + Cursor savedPagesDbEntries = WikipediaApp.getInstance().getDatabaseClient(SavedPage.class) + .selectAll(); + //TODO: remove logging and inline function before pushing + L.d("Found " + savedPagesDbEntries.getCount() + " saved pages"); + return savedPagesDbEntries; + } + + private void saveNewEntries() { + for (PageTitle entry : queue) { + PageCombo pageCombo = getPageContentSeq(entry); + + if (pageCombo.hasError()) { + L.e("Error loading page " + entry.getDisplayText() + " from network", + (Throwable) pageCombo.getError()); + continue; + } + + Page page = pageCombo.toPage(entry); + SavedPage savedPage = new SavedPage(entry); + + try { + savedPage.writeToFileSystem(page); + } catch (IOException e) { + L.e("Error writing page " + "to file system", e); + continue; + } + + DatabaseClient<SavedPage> client = + WikipediaApp.getInstance().getDatabaseClient(SavedPage.class); + client.upsert(savedPage, SavedPageDatabaseTable.SELECTION_KEYS); + + final ImageUrlMap imageUrlMap = new ImageUrlMap.Builder(savedPage.getBaseDir()) + .extractUrls(page).build(); + + getImagesSeq(imageUrlMap); + + try { + savedPage.writeUrlMap(imageUrlMap.toJSON()); + } catch (IOException e) { + L.e("Error writing image URL map", e); + } + } + } + + // Issues a *synchronous* PageCombo request to Retrofit. + @NonNull + private PageCombo getPageContentSeq(@NonNull final PageTitle title) { + return getApiService(title).pageComboSync(title.getPrefixedText(), + !WikipediaApp.getInstance().isImageDownloadEnabled()); + } + + private void getImagesSeq(@NonNull final ImageUrlMap imageUrlMap) { + OkHttpClient client = new OkHttpClient(); + + for (Map.Entry<String, String> entry : imageUrlMap.entrySet()) { + final String url = entry.getKey(); + final File file = new File(entry.getValue()); + + Request request = new Request.Builder().url(url).build(); + + try { + Response response = client.newCall(request).execute(); + writeFile(response.body().byteStream(), file); + L.i("Image downloaded to file: " + file.getAbsolutePath()); + } catch (IOException e) { + L.e("Image download failed: " + url); + } + } + } + + @NonNull + private PageService getApiService(@NonNull PageTitle title) { + return ContentServiceFactory.create(title.getSite()); + } +} diff --git a/app/src/main/java/org/wikipedia/server/PageService.java b/app/src/main/java/org/wikipedia/server/PageService.java index ff5826e..57ddc56 100644 --- a/app/src/main/java/org/wikipedia/server/PageService.java +++ b/app/src/main/java/org/wikipedia/server/PageService.java @@ -41,4 +41,12 @@ * @param cb a Retrofit callback which provides the populated PageCombo object in #success */ void pageCombo(String title, boolean noImages, PageCombo.Callback cb); + + /** + * Gets all page content of a given title. Used in the saved page sync background service. + * + * @param title the page title to be used including prefix + * @param noImages add the noimages flag to the request if true + */ + PageCombo pageComboSync(String title, boolean noImages); } diff --git a/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java b/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java index b4a52db..27c3453 100644 --- a/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java +++ b/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java @@ -92,6 +92,11 @@ }); } + @Override + public PageCombo pageComboSync(String title, boolean noImages) { + return webService.pageComboSync(title, noImages); + } + /** * Optional boolean Retrofit parameter. * We don't want to send the query parameter at all when it's false since the presence of the @@ -176,5 +181,19 @@ + "&noheadings=true") void pageCombo(@Query("page") String title, @Query("noimages") Boolean noImages, Callback<MwPageCombo> cb); + + /** + * Gets all page content of a given title -- for refreshing a saved page + * Note: the only difference in the URL from #pageLead is the sections=all instead of 0. + * + * @param title the page title to be used including prefix + * @param noImages add the noimages flag to the request if true + */ + @GET("/w/api.php?action=mobileview&format=json&formatversion=2&prop=" + + "text%7Csections%7Clanguagecount%7Cthumb%7Cimage%7Cid%7Crevision%7Cdescription" + + "%7Clastmodified%7Cnormalizedtitle%7Cdisplaytitle%7Cprotection%7Ceditable" + + "&onlyrequestedsections=1§ions=all§ionprop=toclevel%7Cline%7Canchor" + + "&noheadings=true") + PageCombo pageComboSync(@Query("page") String title, @Query("noimages") Boolean noImages); } } diff --git a/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java b/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java index 0fdb254..5682282 100644 --- a/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java +++ b/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java @@ -99,6 +99,11 @@ }); } + @Override + public PageCombo pageComboSync(String title, boolean noImages) { + return webService.pageComboSync(title, noImages); + } + /* Not defined in the PageService interface since the Wiktionary definition endpoint exists only * in the mobile content service, and does not concern the wholesale retrieval of the contents * of a wiki page. @@ -178,6 +183,14 @@ void pageCombo(@Path("title") String title, @Query("noimages") Boolean noImages, Callback<RbPageCombo> cb); + /** + * Gets all page content of a given title. Used in the saved page sync background service. + * + * @param title the page title to be used including prefix + * @param noImages add the noimages flag to the request if true + */ + @GET("/page/mobile-sections/{title}") + PageCombo pageComboSync(@Path("title") String title, @Query("noimages") Boolean noImages); /** * Gets selected Wiktionary content for a given title derived from user-selected text diff --git a/app/src/main/java/org/wikipedia/util/FileUtil.java b/app/src/main/java/org/wikipedia/util/FileUtil.java index 29e674b..19dc685 100644 --- a/app/src/main/java/org/wikipedia/util/FileUtil.java +++ b/app/src/main/java/org/wikipedia/util/FileUtil.java @@ -67,6 +67,15 @@ path.delete(); } + public static void writeFile(InputStream inputStream, File file) throws IOException { + FileOutputStream outputStream = new FileOutputStream(file); + try { + copyStreams(inputStream, outputStream); + } finally { + outputStream.close(); + } + } + /** * Utility method to copy a stream into another stream. * -- To view, visit https://gerrit.wikimedia.org/r/271925 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iae8e9e19fa4d774f590a85f238b1510bb5642878 Gerrit-PatchSet: 1 Gerrit-Project: apps/android/wikipedia Gerrit-Branch: master Gerrit-Owner: Mholloway <mhollo...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits