Mholloway has uploaded a new change for review.
https://gerrit.wikimedia.org/r/271925
Change subject: WIP: Implement background saved page syncing
......................................................................
WIP: Implement background saved page syncing
This isn't yet working but should give you some idea of where I'm going
with this.
Bug: T126753
Change-Id: Iae8e9e19fa4d774f590a85f238b1510bb5642878
---
M app/src/main/AndroidManifest.xml
M app/src/main/java/org/wikipedia/WikipediaApp.java
M app/src/main/java/org/wikipedia/database/DatabaseClient.java
M app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java
M app/src/main/java/org/wikipedia/savedpages/SavePageTask.java
M app/src/main/java/org/wikipedia/savedpages/SavedPage.java
M app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java
A app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java
M app/src/main/java/org/wikipedia/server/PageService.java
M app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java
M app/src/main/java/org/wikipedia/server/restbase/RbContentService.java
M app/src/main/java/org/wikipedia/util/FileUtil.java
12 files changed, 245 insertions(+), 23 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/apps/android/wikipedia
refs/changes/25/271925/1
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 86d4685..47d19df 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -210,6 +210,10 @@
<action android:name="com.android.vending.INSTALL_REFERRER" />
</intent-filter>
</receiver>
+
+ <service
+ android:name=".savedpages.SavedPageSyncService"
+ android:exported="false"/>
</application>
</manifest>
diff --git a/app/src/main/java/org/wikipedia/WikipediaApp.java
b/app/src/main/java/org/wikipedia/WikipediaApp.java
index 83f10a1..ddee0a1 100644
--- a/app/src/main/java/org/wikipedia/WikipediaApp.java
+++ b/app/src/main/java/org/wikipedia/WikipediaApp.java
@@ -38,6 +38,7 @@
import org.wikipedia.page.PageCache;
import org.wikipedia.pageimages.PageImage;
import org.wikipedia.savedpages.SavedPage;
+import org.wikipedia.savedpages.SavedPageSyncService;
import org.wikipedia.search.RecentSearch;
import org.wikipedia.settings.Prefs;
import org.wikipedia.theme.Theme;
@@ -167,6 +168,7 @@
Fresco.initialize(this);
bus = new Bus();
+ SavedPageSyncService.start();
final Resources resources = getResources();
ViewAnimations.init(resources);
diff --git a/app/src/main/java/org/wikipedia/database/DatabaseClient.java
b/app/src/main/java/org/wikipedia/database/DatabaseClient.java
index 3746566..a10b6fc 100644
--- a/app/src/main/java/org/wikipedia/database/DatabaseClient.java
+++ b/app/src/main/java/org/wikipedia/database/DatabaseClient.java
@@ -41,6 +41,10 @@
}
}
+ public Cursor selectAll() {
+ return select("", new String[] {}, null);
+ }
+
public void deleteAll() {
deleteWhere("", new String[] {});
}
diff --git a/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java
b/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java
index cffa02d..8182849 100644
--- a/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java
+++ b/app/src/main/java/org/wikipedia/savedpages/DownloadImageTask.java
@@ -7,11 +7,9 @@
import org.wikipedia.util.UriUtil;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import static org.wikipedia.util.FileUtil.copyStreams;
+import static org.wikipedia.util.FileUtil.writeFile;
/** */
public class DownloadImageTask extends SaneAsyncTask<Boolean> {
@@ -50,15 +48,6 @@
} else {
Log.e(TAG, "could not download image " + url + " to " +
file.getAbsolutePath());
return false;
- }
- }
-
- private void writeFile(InputStream inputStream, File file) throws
IOException {
- FileOutputStream outputStream = new FileOutputStream(file);
- try {
- copyStreams(inputStream, outputStream);
- } finally {
- outputStream.close();
}
}
}
diff --git a/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java
b/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java
index 817d694..3adeb85 100644
--- a/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java
+++ b/app/src/main/java/org/wikipedia/savedpages/SavePageTask.java
@@ -29,17 +29,22 @@
@Override
public Boolean performTask() throws Throwable {
SavedPage savedPage = new SavedPage(title);
- savedPage.writeToFileSystem(page);
+
+ //savedPage.writeToFileSystem(page);
+
DatabaseClient<SavedPage> client =
app.getDatabaseClient(SavedPage.class);
client.upsert(savedPage, SavedPageDatabaseTable.SELECTION_KEYS);
- final ImageUrlMap imageUrlMap = new
ImageUrlMap.Builder(savedPage.getBaseDir()).extractUrls(page).build();
- final int numImagesAttempts = imageUrlMap.size();
+ //final ImageUrlMap imageUrlMap = new
ImageUrlMap.Builder(savedPage.getBaseDir()).extractUrls(page).build();
+ //final int numImagesAttempts = imageUrlMap.size();
- parallelDownload(imageUrlMap);
+ //parallelDownload(imageUrlMap);
- savedPage.writeUrlMap(imageUrlMap.toJSON());
- return numImagesAttempts == imageUrlMap.size();
+ //savedPage.writeUrlMap(imageUrlMap.toJSON());
+ //return numImagesAttempts == imageUrlMap.size();
+
+ //TODO: remove
+ return Boolean.TRUE;
}
/**
diff --git a/app/src/main/java/org/wikipedia/savedpages/SavedPage.java
b/app/src/main/java/org/wikipedia/savedpages/SavedPage.java
index 139e914..25a8d61 100644
--- a/app/src/main/java/org/wikipedia/savedpages/SavedPage.java
+++ b/app/src/main/java/org/wikipedia/savedpages/SavedPage.java
@@ -115,7 +115,7 @@
* Gets a File object that represents the JSON contents of this page.
* @return File object used for reading/writing page contents.
*/
- private File getContentsFile() {
+ protected File getContentsFile() {
return new File(getBaseDir() + "/content.json");
}
diff --git
a/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java
b/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java
index 2022a75..eb968ba 100644
--- a/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java
+++ b/app/src/main/java/org/wikipedia/savedpages/SavedPageDatabaseTable.java
@@ -23,10 +23,10 @@
private static final int DB_VER_INTRODUCED = 4;
private static final int DB_VER_NAMESPACE_ADDED = 6;
- private static final String COL_SITE = "site";
- private static final String COL_TITLE = "title";
- private static final String COL_NAMESPACE = "namespace";
- private static final String COL_TIMESTAMP = "timestamp";
+ protected static final String COL_SITE = "site";
+ protected static final String COL_TITLE = "title";
+ protected static final String COL_NAMESPACE = "namespace";
+ protected static final String COL_TIMESTAMP = "timestamp";
public static final String[] SELECTION_KEYS = {
COL_SITE,
diff --git
a/app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java
b/app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java
new file mode 100644
index 0000000..d48db90
--- /dev/null
+++ b/app/src/main/java/org/wikipedia/savedpages/SavedPageSyncService.java
@@ -0,0 +1,169 @@
+package org.wikipedia.savedpages;
+
+import android.app.IntentService;
+import android.content.Intent;
+import android.database.Cursor;
+import android.support.annotation.NonNull;
+
+import com.squareup.okhttp.OkHttpClient;
+import com.squareup.okhttp.Request;
+import com.squareup.okhttp.Response;
+
+import org.wikipedia.Site;
+import org.wikipedia.WikipediaApp;
+import org.wikipedia.database.DatabaseClient;
+import org.wikipedia.page.Page;
+import org.wikipedia.page.PageTitle;
+import org.wikipedia.server.ContentServiceFactory;
+import org.wikipedia.server.PageCombo;
+import org.wikipedia.server.PageService;
+import org.wikipedia.util.log.L;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.wikipedia.util.FileUtil.writeFile;
+
+public class SavedPageSyncService extends IntentService {
+ private List<PageTitle> queue = new LinkedList<>();
+
+ public SavedPageSyncService() {
+ super("SavedPageSyncService");
+
+ // Redeliver the intent if our process dies, so our sync completes.
+ setIntentRedelivery(true);
+ }
+
+ @SuppressWarnings("MagicNumber")
+ public static void start() {
+ ScheduledExecutorService syncRunnerService =
Executors.newScheduledThreadPool(1);
+
+ //TODO: MAKE INTERVAL 15 *MINUTES* WHEN DONE DEBUGGING
+ syncRunnerService.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ WikipediaApp.getInstance().startService(
+ new Intent(WikipediaApp.getInstance(),
SavedPageSyncService.class)
+ );
+ }
+ }, 0, 15, TimeUnit.SECONDS);
+
+ L.i("Starting saved page sync service");
+ }
+
+ @Override
+ protected void onHandleIntent(@NonNull Intent intent) {
+ L.i("Checking DB for new saved page entries...");
+ boolean newEntries = enqueueNewDbEntries();
+ if (newEntries) {
+ saveNewEntries();
+ }
+ }
+
+ private boolean enqueueNewDbEntries() {
+ Cursor cursor = getEntriesFromDB();
+
+ if (cursor.getCount() == 0) {
+ L.i("No new saved page DB entries found; exiting.");
+ return false;
+ }
+
+ while (cursor.moveToNext()) {
+ //TODO: add namespace (if possible?)
+ Site site = new
Site(cursor.getString(cursor.getColumnIndexOrThrow(SavedPageDatabaseTable.COL_SITE)));
+ PageTitle title = new PageTitle(null,
cursor.getString(cursor.getColumnIndexOrThrow(SavedPageDatabaseTable.COL_TITLE)),
site);
+ Date timestamp = new
Date(cursor.getLong(cursor.getColumnIndexOrThrow(SavedPageDatabaseTable.COL_TIMESTAMP)));
+ SavedPage entry = new SavedPage(title, timestamp);
+ if (!entry.getContentsFile().exists()) {
+ queue.add(title);
+ L.i("Added new saved page entry " + title.getDisplayText() + "
to download queue.");
+ }
+ }
+ cursor.close();
+ return true;
+ }
+
+ @NonNull
+ private Cursor getEntriesFromDB() {
+ Cursor savedPagesDbEntries =
WikipediaApp.getInstance().getDatabaseClient(SavedPage.class)
+ .selectAll();
+ //TODO: remove logging and inline function before pushing
+ L.d("Found " + savedPagesDbEntries.getCount() + " saved pages");
+ return savedPagesDbEntries;
+ }
+
+ private void saveNewEntries() {
+ for (PageTitle entry : queue) {
+ PageCombo pageCombo = getPageContentSeq(entry);
+
+ if (pageCombo.hasError()) {
+ L.e("Error loading page " + entry.getDisplayText() + " from
network",
+ (Throwable) pageCombo.getError());
+ continue;
+ }
+
+ Page page = pageCombo.toPage(entry);
+ SavedPage savedPage = new SavedPage(entry);
+
+ try {
+ savedPage.writeToFileSystem(page);
+ } catch (IOException e) {
+ L.e("Error writing page " + "to file system", e);
+ continue;
+ }
+
+ DatabaseClient<SavedPage> client =
+
WikipediaApp.getInstance().getDatabaseClient(SavedPage.class);
+ client.upsert(savedPage, SavedPageDatabaseTable.SELECTION_KEYS);
+
+ final ImageUrlMap imageUrlMap = new
ImageUrlMap.Builder(savedPage.getBaseDir())
+ .extractUrls(page).build();
+
+ getImagesSeq(imageUrlMap);
+
+ try {
+ savedPage.writeUrlMap(imageUrlMap.toJSON());
+ } catch (IOException e) {
+ L.e("Error writing image URL map", e);
+ }
+ }
+ }
+
+ // Issues a *synchronous* PageCombo request to Retrofit.
+ @NonNull
+ private PageCombo getPageContentSeq(@NonNull final PageTitle title) {
+ return getApiService(title).pageComboSync(title.getPrefixedText(),
+ !WikipediaApp.getInstance().isImageDownloadEnabled());
+ }
+
+ private void getImagesSeq(@NonNull final ImageUrlMap imageUrlMap) {
+ OkHttpClient client = new OkHttpClient();
+
+ for (Map.Entry<String, String> entry : imageUrlMap.entrySet()) {
+ final String url = entry.getKey();
+ final File file = new File(entry.getValue());
+
+ Request request = new Request.Builder().url(url).build();
+
+ try {
+ Response response = client.newCall(request).execute();
+ writeFile(response.body().byteStream(), file);
+ L.i("Image downloaded to file: " + file.getAbsolutePath());
+ } catch (IOException e) {
+ L.e("Image download failed: " + url);
+ }
+ }
+ }
+
+ @NonNull
+ private PageService getApiService(@NonNull PageTitle title) {
+ return ContentServiceFactory.create(title.getSite());
+ }
+}
diff --git a/app/src/main/java/org/wikipedia/server/PageService.java
b/app/src/main/java/org/wikipedia/server/PageService.java
index ff5826e..57ddc56 100644
--- a/app/src/main/java/org/wikipedia/server/PageService.java
+++ b/app/src/main/java/org/wikipedia/server/PageService.java
@@ -41,4 +41,12 @@
* @param cb a Retrofit callback which provides the populated PageCombo
object in #success
*/
void pageCombo(String title, boolean noImages, PageCombo.Callback cb);
+
+ /**
+ * Gets all page content of a given title. Used in the saved page sync
background service.
+ *
+ * @param title the page title to be used including prefix
+ * @param noImages add the noimages flag to the request if true
+ */
+ PageCombo pageComboSync(String title, boolean noImages);
}
diff --git a/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java
b/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java
index b4a52db..27c3453 100644
--- a/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java
+++ b/app/src/main/java/org/wikipedia/server/mwapi/MwPageService.java
@@ -92,6 +92,11 @@
});
}
+ @Override
+ public PageCombo pageComboSync(String title, boolean noImages) {
+ return webService.pageComboSync(title, noImages);
+ }
+
/**
* Optional boolean Retrofit parameter.
* We don't want to send the query parameter at all when it's false since
the presence of the
@@ -176,5 +181,19 @@
+ "&noheadings=true")
void pageCombo(@Query("page") String title, @Query("noimages") Boolean
noImages,
Callback<MwPageCombo> cb);
+
+ /**
+ * Gets all page content of a given title -- for refreshing a saved
page
+ * Note: the only difference in the URL from #pageLead is the
sections=all instead of 0.
+ *
+ * @param title the page title to be used including prefix
+ * @param noImages add the noimages flag to the request if true
+ */
+ @GET("/w/api.php?action=mobileview&format=json&formatversion=2&prop="
+ +
"text%7Csections%7Clanguagecount%7Cthumb%7Cimage%7Cid%7Crevision%7Cdescription"
+ +
"%7Clastmodified%7Cnormalizedtitle%7Cdisplaytitle%7Cprotection%7Ceditable"
+ +
"&onlyrequestedsections=1§ions=all§ionprop=toclevel%7Cline%7Canchor"
+ + "&noheadings=true")
+ PageCombo pageComboSync(@Query("page") String title,
@Query("noimages") Boolean noImages);
}
}
diff --git
a/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java
b/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java
index 0fdb254..5682282 100644
--- a/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java
+++ b/app/src/main/java/org/wikipedia/server/restbase/RbContentService.java
@@ -99,6 +99,11 @@
});
}
+ @Override
+ public PageCombo pageComboSync(String title, boolean noImages) {
+ return webService.pageComboSync(title, noImages);
+ }
+
/* Not defined in the PageService interface since the Wiktionary
definition endpoint exists only
* in the mobile content service, and does not concern the wholesale
retrieval of the contents
* of a wiki page.
@@ -178,6 +183,14 @@
void pageCombo(@Path("title") String title, @Query("noimages") Boolean
noImages,
Callback<RbPageCombo> cb);
+ /**
+ * Gets all page content of a given title. Used in the saved page
sync background service.
+ *
+ * @param title the page title to be used including prefix
+ * @param noImages add the noimages flag to the request if true
+ */
+ @GET("/page/mobile-sections/{title}")
+ PageCombo pageComboSync(@Path("title") String title,
@Query("noimages") Boolean noImages);
/**
* Gets selected Wiktionary content for a given title derived from
user-selected text
diff --git a/app/src/main/java/org/wikipedia/util/FileUtil.java
b/app/src/main/java/org/wikipedia/util/FileUtil.java
index 29e674b..19dc685 100644
--- a/app/src/main/java/org/wikipedia/util/FileUtil.java
+++ b/app/src/main/java/org/wikipedia/util/FileUtil.java
@@ -67,6 +67,15 @@
path.delete();
}
+ public static void writeFile(InputStream inputStream, File file) throws
IOException {
+ FileOutputStream outputStream = new FileOutputStream(file);
+ try {
+ copyStreams(inputStream, outputStream);
+ } finally {
+ outputStream.close();
+ }
+ }
+
/**
* Utility method to copy a stream into another stream.
*
--
To view, visit https://gerrit.wikimedia.org/r/271925
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iae8e9e19fa4d774f590a85f238b1510bb5642878
Gerrit-PatchSet: 1
Gerrit-Project: apps/android/wikipedia
Gerrit-Branch: master
Gerrit-Owner: Mholloway <mhollo...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
