Brion VIBBER has submitted this change and it was merged. Change subject: Periodically remove old / unused article and associated data. ......................................................................
Periodically remove old / unused article and associated data. Change-Id: I82711860b0017f95d250e86d7a3a2e0b4b0bc5ac --- M Wikipedia.xcodeproj/project.pbxproj M wikipedia/AppDelegate.m M wikipedia/Categories/Article+Convenience.h M wikipedia/Categories/Article+Convenience.m A wikipedia/Housekeeping/CoreDataHousekeeping.h A wikipedia/Housekeeping/CoreDataHousekeeping.m M wikipedia/View Controllers/History/HistoryViewController.m M wikipedia/View Controllers/WebView/WebViewController.m 8 files changed, 201 insertions(+), 25 deletions(-) Approvals: Brion VIBBER: Verified; Looks good to me, approved diff --git a/Wikipedia.xcodeproj/project.pbxproj b/Wikipedia.xcodeproj/project.pbxproj index e1f4ac3..2b503ed 100644 --- a/Wikipedia.xcodeproj/project.pbxproj +++ b/Wikipedia.xcodeproj/project.pbxproj @@ -81,6 +81,7 @@ 048A26771906268100395F53 /* PaddedLabel.m in Sources */ = {isa = PBXBuildFile; fileRef = 048A26761906268100395F53 /* PaddedLabel.m */; }; 0493C2D419526A0100EBB973 /* WikiFont-Glyphs.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 0493C2D319526A0100EBB973 /* WikiFont-Glyphs.ttf */; }; 0493C2D619526FFE00EBB973 /* WikiFont-Glyphs-iOS.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 0493C2D519526FFE00EBB973 /* WikiFont-Glyphs-iOS.ttf */; }; + 0493C2CC1952373100EBB973 /* CoreDataHousekeeping.m in Sources */ = {isa = PBXBuildFile; fileRef = 0493C2CB1952373100EBB973 /* CoreDataHousekeeping.m */; }; 049566C218F5F4CB0058EA12 /* ZeroConfigState.m in Sources */ = {isa = PBXBuildFile; fileRef = 049566C118F5F4CB0058EA12 /* ZeroConfigState.m */; }; 04992BC018B687AF00A6C22B /* SearchOp.m in Sources */ = {isa = PBXBuildFile; fileRef = 04992BBF18B687AF00A6C22B /* SearchOp.m */; }; 04992BC418B6971F00A6C22B /* SearchThumbUrlsOp.m in Sources */ = {isa = PBXBuildFile; fileRef = 04992BC318B6971F00A6C22B /* SearchThumbUrlsOp.m */; }; @@ -319,6 +320,8 @@ 0493C2D219525F8E00EBB973 /* WikiGlyph_Chars_iOS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WikiGlyph_Chars_iOS.h; sourceTree = "<group>"; }; 0493C2D319526A0100EBB973 /* WikiFont-Glyphs.ttf */ = {isa = PBXFileReference; lastKnownFileType = file; path = "WikiFont-Glyphs.ttf"; sourceTree = "<group>"; }; 0493C2D519526FFE00EBB973 /* WikiFont-Glyphs-iOS.ttf */ = {isa = PBXFileReference; lastKnownFileType = file; path = "WikiFont-Glyphs-iOS.ttf"; sourceTree = "<group>"; }; + 0493C2CA1952373100EBB973 /* CoreDataHousekeeping.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CoreDataHousekeeping.h; sourceTree = "<group>"; }; + 0493C2CB1952373100EBB973 /* CoreDataHousekeeping.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CoreDataHousekeeping.m; sourceTree = "<group>"; }; 049566C018F5F4CB0058EA12 /* ZeroConfigState.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ZeroConfigState.h; sourceTree = "<group>"; }; 049566C118F5F4CB0058EA12 /* ZeroConfigState.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ZeroConfigState.m; sourceTree = "<group>"; }; 04992BBE18B687AF00A6C22B /* SearchOp.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SearchOp.h; sourceTree = "<group>"; }; @@ -920,6 +923,15 @@ path = PaddedLabel; sourceTree = "<group>"; }; + 0493C2C91952373100EBB973 /* Housekeeping */ = { + isa = PBXGroup; + children = ( + 0493C2CA1952373100EBB973 /* CoreDataHousekeeping.h */, + 0493C2CB1952373100EBB973 /* CoreDataHousekeeping.m */, + ); + path = Housekeeping; + sourceTree = "<group>"; + }; 049566BF18F5F4CB0058EA12 /* Zero */ = { isa = PBXGroup; children = ( @@ -1372,6 +1384,7 @@ D4B0ADFF19365F4600F0AC90 /* EventLogging */, 0442F57C1900718600F55DF9 /* Fonts */, 04D34DA31863D2D600610A87 /* HTML Parsing */, + 0493C2C91952373100EBB973 /* Housekeeping */, 0466F44C183A30CC00EA1FD7 /* Images */, 0463639518A844380049EE4F /* Keychain */, 04B60509193522650007185A /* MenuButton */, @@ -1775,6 +1788,7 @@ 04B78A5318A580AF0050EBF5 /* LoginOp.m in Sources */, 04530AF51935BF4D00022512 /* ModalMenuAndContentViewController.m in Sources */, 041A3B6218E11ED90079FF1C /* LanguagesTableVC.m in Sources */, + 0493C2CC1952373100EBB973 /* CoreDataHousekeeping.m in Sources */, 04D34DAD1863D2D600610A87 /* TFHppleElement.m in Sources */, 043F18E118D9691D00D8489A /* TopActionSheetLabel.m in Sources */, 0412CC621925366C0010E616 /* RootViewController.m in Sources */, diff --git a/wikipedia/AppDelegate.m b/wikipedia/AppDelegate.m index fc42a8e..d36a165 100644 --- a/wikipedia/AppDelegate.m +++ b/wikipedia/AppDelegate.m @@ -3,6 +3,7 @@ #import "AppDelegate.h" #import "URLCache.h" +#import "NSDate-Utilities.h" @implementation AppDelegate @@ -56,7 +57,8 @@ @"ZeroWarnWhenLeaving" : @YES, @"ZeroOnDialogShownOnce" : @NO, @"ZeroOffDialogShownOnce" : @NO, - @"FakeZeroOn" : @NO + @"FakeZeroOn" : @NO, + @"LastHousekeepingDate" : [NSDate date] //[NSDate dateWithDaysBeforeNow:10] }; [[NSUserDefaults standardUserDefaults] registerDefaults:userDefaultsDefaults]; } diff --git a/wikipedia/Categories/Article+Convenience.h b/wikipedia/Categories/Article+Convenience.h index ed58405..15d4f52 100644 --- a/wikipedia/Categories/Article+Convenience.h +++ b/wikipedia/Categories/Article+Convenience.h @@ -12,6 +12,8 @@ // larger than 99 x 99 px. - (UIImage *)getThumbnailUsingContext:(NSManagedObjectContext *)context; +- (void)ifNoThumbnailUseFirstSectionImageAsThumbnailUsingContext:(NSManagedObjectContext *)context; + - (NSURL *)desktopURL; @end diff --git a/wikipedia/Categories/Article+Convenience.m b/wikipedia/Categories/Article+Convenience.m index 70a22d0..c7427f1 100644 --- a/wikipedia/Categories/Article+Convenience.m +++ b/wikipedia/Categories/Article+Convenience.m @@ -102,4 +102,25 @@ return encodedUrlString ? [NSURL URLWithString:encodedUrlString] : nil; } +- (void)ifNoThumbnailUseFirstSectionImageAsThumbnailUsingContext:(NSManagedObjectContext *)context; +{ + // If article has no thumbnailImage, use the first section image instead. + if (!self.thumbnailImage) { + [context performBlockAndWait:^(){ + NSArray *firstSectionImage = [self getFirstSectionImageLargerThanSize:THUMBNAIL_MINIMUM_SIZE_TO_CACHE usingContext:context]; + if (firstSectionImage.count == 1) { + SectionImage *sectionImage = (SectionImage *)firstSectionImage[0]; + self.thumbnailImage = sectionImage.image; + + NSError *error = nil; + [context save:&error]; + if (error) { + NSLog(@"\n\nerror = %@\n\n", error); + NSLog(@"\n\nerror = %@\n\n", error.localizedDescription); + } + } + }]; + } +} + @end diff --git a/wikipedia/Housekeeping/CoreDataHousekeeping.h b/wikipedia/Housekeeping/CoreDataHousekeeping.h new file mode 100644 index 0000000..e933a4c --- /dev/null +++ b/wikipedia/Housekeeping/CoreDataHousekeeping.h @@ -0,0 +1,10 @@ +// Created by Monte Hurd on 6/18/14. +// Copyright (c) 2014 Wikimedia Foundation. Provided under MIT-style license; please copy and modify! + +#import <Foundation/Foundation.h> + +@interface CoreDataHousekeeping : NSObject + +-(void)performHouseKeeping; + +@end diff --git a/wikipedia/Housekeeping/CoreDataHousekeeping.m b/wikipedia/Housekeeping/CoreDataHousekeeping.m new file mode 100644 index 0000000..2f4f9c7 --- /dev/null +++ b/wikipedia/Housekeeping/CoreDataHousekeeping.m @@ -0,0 +1,119 @@ +// Created by Monte Hurd on 6/18/14. +// Copyright (c) 2014 Wikimedia Foundation. Provided under MIT-style license; please copy and modify! + +#import "CoreDataHousekeeping.h" +#import "NSDate-Utilities.h" +#import "ArticleDataContextSingleton.h" +#import "ArticleCoreDataObjects.h" + +@interface CoreDataHousekeeping (){ + NSManagedObjectContext *context_; +} + +@end + +@implementation CoreDataHousekeeping + +- (instancetype)init +{ + self = [super init]; + if (self) { + context_ = [ArticleDataContextSingleton sharedInstance].workerContext; + } + return self; +} + +-(void)performHouseKeeping +{ + [context_ performBlockAndWait:^(){ + + [self removeUnsavedUnhistoriedArticles]; + [self removeUnsavedArticleSections]; + + NSError *error = nil; + [context_ save:&error]; + if (error){ + NSLog(@"ImageHousekeeping error = %@", error); + }else{ + [self removeUnusedImages]; + + error = nil; + [context_ save:&error]; + if (error) NSLog(@"ImageHousekeeping error = %@", error); + } + + }]; +} + +-(void)removeUnsavedUnhistoriedArticles +{ + // Removes articles which have neither saved nor history records. + // The user can remove items from both saved pages and history. + // If they've removed this article from both, not need to keep its + // data. + + NSEntityDescription *entity = [NSEntityDescription entityForName: @"Article" inManagedObjectContext: context_]; + + NSFetchRequest *request = [[NSFetchRequest alloc] init]; + [request setEntity:entity]; + // To-many query: http://stackoverflow.com/a/1195519 + [request setPredicate:[NSPredicate predicateWithFormat:@"history.@count == 0 AND saved.@count == 0"]]; + + NSError *error = nil; + NSArray *articles = [context_ executeFetchRequest:request error:&error]; + for (Article *article in articles) { + NSLog(@"removing article w/o history or save records = %@", article.title); + if (article) [context_ deleteObject:article]; + } +} + +-(void)removeUnsavedArticleSections +{ + // Removes article sections for articles which are unsaved, but which still have a history record. + + // This way the history items will still work if tapped - they will re-download their section data + // automatically because we also set "needsRefresh" to YES below. Also, by removing these sections, + // the "removeUnusedImages" should be able to clean up more images (unused images, that is, + // unreferenced by any sections, are removed by removeUnusedImages). + + NSEntityDescription *entity = [NSEntityDescription entityForName: @"Article" inManagedObjectContext: context_]; + + NSFetchRequest *request = [[NSFetchRequest alloc] init]; + [request setEntity:entity]; + // To-many query: http://stackoverflow.com/a/1195519 + [request setPredicate:[NSPredicate predicateWithFormat:@"history.@count > 0 AND saved.@count == 0"]]; + + NSError *error = nil; + NSArray *articles = [context_ executeFetchRequest:request error:&error]; + for (Article *article in articles) { + NSLog(@"removing sections from article w history w/o saved record = %@", article.title); + for (Section *section in article.section) { + if (section) [context_ deleteObject:section]; + } + article.needsRefresh = @YES; + } +} + +-(void)removeUnusedImages +{ + // Remove core data Images which are not associated with either a SectionImage record or an Article record's + // "thumbnailImage" property. + NSEntityDescription *entity = [NSEntityDescription entityForName: @"Image" inManagedObjectContext: context_]; + + NSFetchRequest *request = [[NSFetchRequest alloc] init]; + [request setEntity:entity]; + + // To-many query: http://stackoverflow.com/a/1195519 (the "@" part). + // Reminder that "article.@count" tells us if any article record "thumbnailImage" property is referencing + // this image. + [request setPredicate:[NSPredicate predicateWithFormat:@"sectionImage.@count == 0 AND article.@count == 0"]]; + + NSError *error = nil; + NSArray *images = [context_ executeFetchRequest:request error:&error]; + for (Image *image in images) { + NSLog(@"unused image = %@", image.fileName); + if (image) [context_ deleteObject:image]; + } +} + +@end diff --git a/wikipedia/View Controllers/History/HistoryViewController.m b/wikipedia/View Controllers/History/HistoryViewController.m index ef898c7..0875c60 100644 --- a/wikipedia/View Controllers/History/HistoryViewController.m +++ b/wikipedia/View Controllers/History/HistoryViewController.m @@ -18,7 +18,6 @@ #import "TopMenuContainerView.h" #import "TopMenuViewController.h" #import "UIViewController+StatusBarHeight.h" -#import "Image+Convenience.h" #define HISTORY_THUMBNAIL_WIDTH 110 #define HISTORY_RESULT_HEIGHT 66 @@ -255,7 +254,6 @@ //NSLog(@"GARBAGE COUNT = %lu", (unsigned long)garbage.count); //NSLog(@"GARBAGE = %@", garbage); if (garbage.count == 0) return; - NSMutableArray *imagesToCollect = [[NSMutableArray alloc] init]; [articleDataContext_.mainContext performBlockAndWait:^(){ for (NSManagedObjectID *historyID in garbage) { @@ -272,34 +270,14 @@ // Article deletes don't cascade to images (intentionally) so delete these manually. if (thumb) [articleDataContext_.mainContext deleteObject:thumb]; - - if (article) { - // Section images might be used in multiple articles, so list them up - // and kill them only after we've cleaned up the sections and can confirm - // they are no longer used... - for (Section *section in article.section) { - for (SectionImage *sectionImage in section.sectionImage) { - Image *image = sectionImage.image; - [imagesToCollect addObject:image]; - } - } - } // Delete the article if (article) [articleDataContext_.mainContext deleteObject:article]; - } + } NSError *error = nil; [articleDataContext_.mainContext save:&error]; - if (error) NSLog(@"GARBAGE pass 1 error = %@", error); - - // Now clean up the images that aren't used in remaining pages - for (Image *image in imagesToCollect) { - [image deleteIfUnused]; - } - // and save again... - [articleDataContext_.mainContext save:&error]; - if (error) NSLog(@"GARBAGE pass 2 error = %@", error); + if (error) NSLog(@"GARBAGE error = %@", error); }]; } diff --git a/wikipedia/View Controllers/WebView/WebViewController.m b/wikipedia/View Controllers/WebView/WebViewController.m index 0452afd..c857519 100644 --- a/wikipedia/View Controllers/WebView/WebViewController.m +++ b/wikipedia/View Controllers/WebView/WebViewController.m @@ -51,6 +51,10 @@ #import "EditFunnel.h" +#import "CoreDataHousekeeping.h" +#import "Article+Convenience.h" +#import "NSDate-Utilities.h" + #define TOC_TOGGLE_ANIMATION_DURATION @0.3f typedef enum { @@ -213,7 +217,23 @@ // viewDidLoad to fire. [self downloadAssetsFilesIfNecessary]; + [self performHousekeepingIfNecessary]; + //[self.view randomlyColorSubviews]; +} + +-(void)performHousekeepingIfNecessary +{ + NSDate *lastHousekeepingDate = [[NSUserDefaults standardUserDefaults] objectForKey:@"LastHousekeepingDate"]; + NSInteger daysSinceLastHouseKeeping = [[NSDate date] daysAfterDate:lastHousekeepingDate]; + NSLog(@"daysSinceLastHouseKeeping = %d", daysSinceLastHouseKeeping); + if (daysSinceLastHouseKeeping > 1) { + NSLog(@"Performing housekeeping..."); + CoreDataHousekeeping *imageHousekeeping = [[CoreDataHousekeeping alloc] init]; + [imageHousekeeping performHouseKeeping]; + [[NSUserDefaults standardUserDefaults] setObject:[NSDate date] forKey:@"LastHousekeepingDate"]; + [[NSUserDefaults standardUserDefaults] synchronize]; + } } -(void)viewWillAppear:(BOOL)animated @@ -1021,6 +1041,9 @@ [super didReceiveMemoryWarning]; // Dispose of any resources that can be recreated. + //CoreDataHousekeeping *imageHousekeeping = [[CoreDataHousekeeping alloc] init]; + //[imageHousekeeping performHouseKeeping]; + // Do not remove the following commented toggle. It's for testing W0 stuff. //[[SessionSingleton sharedInstance].zeroConfigState toggleFakeZeroOn]; } @@ -1403,6 +1426,13 @@ } if (mode == DISPLAY_LEAD_SECTION) break; } + + // If article has no thumbnailImage, use the first section image instead. + // Actually sets article.thumbnailImage to point to the image record of the first section + // image. That way, if the housekeeping code removes all section images, it won't remove this + // particular one because it checks to see if an article is referencing an image before it + // removes them. + [article ifNoThumbnailUseFirstSectionImageAsThumbnailUsingContext:articleDataContext_.mainContext]; // Pull the scroll offset out so the article object doesn't have to be passed into the block below. CGPoint scrollOffset = CGPointMake(article.lastScrollX.floatValue, article.lastScrollY.floatValue); -- To view, visit https://gerrit.wikimedia.org/r/140836 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I82711860b0017f95d250e86d7a3a2e0b4b0bc5ac Gerrit-PatchSet: 1 Gerrit-Project: apps/ios/wikipedia Gerrit-Branch: master Gerrit-Owner: Mhurd <mh...@wikimedia.org> Gerrit-Reviewer: Brion VIBBER <br...@wikimedia.org> Gerrit-Reviewer: Mhurd <mh...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits