Merge branch 'charlesmchen/corruptDatabaseViews'

pull/1/head
Matthew Chen 7 years ago
parent c5e7b10bd9
commit 9cdf489ace

@ -120,6 +120,15 @@ NS_ASSUME_NONNULL_BEGIN
}]];
#endif
[items
addObject:[OWSTableItem
itemWithTitle:@"Increment Database Extension Versions"
actionBlock:^() {
for (NSString *extensionName in OWSPrimaryStorage.sharedManager.registeredExtensionNames) {
[OWSStorage incrementVersionOfDatabaseExtension:extensionName];
}
}]];
return [OWSTableSection sectionWithTitle:self.name items:items];
}

@ -14,6 +14,8 @@ NS_ASSUME_NONNULL_BEGIN
@interface OWSBatchMessageProcessor : NSObject
+ (instancetype)sharedInstance;
+ (NSString *)databaseExtensionName;
+ (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage;
- (void)enqueueEnvelopeData:(NSData *)envelopeData

@ -458,6 +458,11 @@ NSString *const OWSMessageContentJobFinderExtensionGroup = @"OWSMessageContentJo
#pragma mark - class methods
+ (NSString *)databaseExtensionName
{
return OWSMessageContentJobFinderExtensionName;
}
+ (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage
{
[OWSMessageContentJobFinder asyncRegisterDatabaseExtension:storage];

@ -25,9 +25,8 @@ NS_ASSUME_NONNULL_BEGIN
*/
- (nullable NSNumber *)nextExpirationTimestampWithTransaction:(YapDatabaseReadTransaction *_Nonnull)transaction;
/**
* Database extensions required for class to work.
*/
+ (NSString *)databaseExtensionName;
+ (void)asyncRegisterDatabaseExtensions:(OWSStorage *)storage;
#ifdef DEBUG

@ -185,7 +185,7 @@ static NSString *const OWSDisappearingMessageFinderExpiresAtIndex = @"index_mess
dict[OWSDisappearingMessageFinderThreadIdColumn] = message.uniqueThreadId;
}];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil];
}
#ifdef DEBUG
@ -196,6 +196,11 @@ static NSString *const OWSDisappearingMessageFinderExpiresAtIndex = @"index_mess
}
#endif
+ (NSString *)databaseExtensionName
{
return OWSDisappearingMessageFinderExpiresAtIndex;
}
+ (void)asyncRegisterDatabaseExtensions:(OWSStorage *)storage
{
[storage asyncRegisterExtension:[self indexDatabaseExtension] withName:OWSDisappearingMessageFinderExpiresAtIndex];

@ -14,9 +14,7 @@ NS_ASSUME_NONNULL_BEGIN
- (void)run;
/**
* Database extensions required for class to work.
*/
+ (NSString *)databaseExtensionName;
+ (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage;
#ifdef DEBUG

@ -114,7 +114,7 @@ static NSString *const OWSFailedAttachmentDownloadsJobAttachmentStateIndex = @"i
dict[OWSFailedAttachmentDownloadsJobAttachmentStateColumn] = @(attachment.state);
}];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil];
}
#ifdef DEBUG
@ -126,6 +126,11 @@ static NSString *const OWSFailedAttachmentDownloadsJobAttachmentStateIndex = @"i
}
#endif
+ (NSString *)databaseExtensionName
{
return OWSFailedAttachmentDownloadsJobAttachmentStateIndex;
}
+ (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage
{
[storage asyncRegisterExtension:[self indexDatabaseExtension]

@ -14,9 +14,7 @@ NS_ASSUME_NONNULL_BEGIN
- (void)run;
/**
* Database extensions required for class to work.
*/
+ (NSString *)databaseExtensionName;
+ (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage;
#ifdef DEBUG

@ -124,7 +124,7 @@ static NSString *const OWSFailedMessagesJobMessageStateIndex = @"index_outoing_m
dict[OWSFailedMessagesJobMessageStateColumn] = @(message.messageState);
}];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil];
}
#ifdef DEBUG
@ -136,6 +136,11 @@ static NSString *const OWSFailedMessagesJobMessageStateIndex = @"index_outoing_m
}
#endif
+ (NSString *)databaseExtensionName
{
return OWSFailedMessagesJobMessageStateIndex;
}
+ (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage
{
[storage asyncRegisterExtension:[self indexDatabaseExtension] withName:OWSFailedMessagesJobMessageStateIndex];

@ -14,6 +14,8 @@ NS_ASSUME_NONNULL_BEGIN
@interface OWSMessageReceiver : NSObject
+ (instancetype)sharedInstance;
+ (NSString *)databaseExtensionName;
+ (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage;
- (void)handleReceivedEnvelope:(OWSSignalServiceProtosEnvelope *)envelope;

@ -413,6 +413,11 @@ NSString *const OWSMessageDecryptJobFinderExtensionGroup = @"OWSMessageProcessin
#pragma mark - class methods
+ (NSString *)databaseExtensionName
{
return OWSMessageDecryptJobFinderExtensionName;
}
+ (void)asyncRegisterDatabaseExtension:(OWSStorage *)storage
{
[OWSMessageDecryptJobFinder asyncRegisterDatabaseExtension:storage];

@ -12,9 +12,7 @@ NS_ASSUME_NONNULL_BEGIN
- (instancetype)initWithPrimaryStorage:(OWSPrimaryStorage *)primaryStorage NS_DESIGNATED_INITIALIZER;
/**
* Must be called before using this finder.
*/
+ (NSString *)databaseExtensionName;
+ (void)asyncRegisterExtensionWithPrimaryStorage:(OWSStorage *)storage;
/**

@ -89,7 +89,12 @@ NSString *const OWSIncomingMessageFinderColumnSourceDeviceId = @"OWSIncomingMess
YapDatabaseSecondaryIndexHandler *handler = [YapDatabaseSecondaryIndexHandler withObjectBlock:block];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler];
return [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil];
}
+ (NSString *)databaseExtensionName
{
return OWSIncomingMessageFinderExtensionName;
}
+ (void)asyncRegisterExtensionWithPrimaryStorage:(OWSStorage *)storage

@ -28,6 +28,7 @@ NS_ASSUME_NONNULL_BEGIN
#pragma mark - Extension registration
+ (NSString *)databaseExtensionName;
+ (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage;
@end

@ -111,6 +111,11 @@ static NSString *const OWSMediaGalleryFinderExtensionName = @"OWSMediaGalleryFin
#pragma mark - Extension registration
+ (NSString *)databaseExtensionName
{
return OWSMediaGalleryFinderExtensionName;
}
+ (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage
{
[storage asyncRegisterExtension:[self mediaGalleryDatabaseExtension]
@ -158,7 +163,7 @@ static NSString *const OWSMediaGalleryFinderExtensionName = @"OWSMediaGalleryFin
YapDatabaseViewOptions *options = [YapDatabaseViewOptions new];
options.allowedCollections = [[YapWhitelistBlacklist alloc] initWithWhitelist:[NSSet setWithObject:TSMessage.collection]];
return [[YapDatabaseAutoView alloc] initWithGrouping:grouping sorting:sorting versionTag:@"1" options:options];
}

@ -22,7 +22,7 @@ NS_ASSUME_NONNULL_BEGIN
NSString *const OWSPrimaryStorageExceptionName_CouldNotCreateDatabaseDirectory
= @"TSStorageManagerExceptionName_CouldNotCreateDatabaseDirectory";
void runSyncRegistrationsForStorage(OWSStorage *storage)
void RunSyncRegistrationsForStorage(OWSStorage *storage)
{
OWSCAssert(storage);
@ -30,7 +30,7 @@ void runSyncRegistrationsForStorage(OWSStorage *storage)
[TSDatabaseView registerCrossProcessNotifier:storage];
}
void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t completion)
void RunAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t completion)
{
OWSCAssert(storage);
OWSCAssert(completion);
@ -43,7 +43,8 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl
[TSDatabaseView asyncRegisterThreadInteractionsDatabaseView:storage];
[TSDatabaseView asyncRegisterThreadDatabaseView:storage];
[TSDatabaseView asyncRegisterUnreadDatabaseView:storage];
[storage asyncRegisterExtension:[TSDatabaseSecondaryIndexes registerTimeStampIndex] withName:@"idx"];
[storage asyncRegisterExtension:[TSDatabaseSecondaryIndexes registerTimeStampIndex]
withName:[TSDatabaseSecondaryIndexes registerTimeStampIndexExtensionName]];
[OWSMessageReceiver asyncRegisterDatabaseExtension:storage];
[OWSBatchMessageProcessor asyncRegisterDatabaseExtension:storage];
@ -63,6 +64,24 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl
[TSDatabaseView asyncRegisterLazyRestoreAttachmentsDatabaseView:storage completion:completion];
}
void VerifyRegistrationsForPrimaryStorage(OWSStorage *storage)
{
OWSCAssert(storage);
[[storage newDatabaseConnection] asyncReadWithBlock:^(YapDatabaseReadTransaction *transaction) {
for (NSString *extensionName in storage.registeredExtensionNames) {
DDLogVerbose(@"Verifying database extension: %@", extensionName);
YapDatabaseViewTransaction *_Nullable viewTransaction = [transaction ext:extensionName];
if (!viewTransaction) {
OWSProdLogAndCFail(
@"VerifyRegistrationsForPrimaryStorage missing database extension: %@", extensionName);
[OWSStorage incrementVersionOfDatabaseExtension:extensionName];
}
}
}];
}
#pragma mark -
@interface OWSPrimaryStorage ()
@ -119,7 +138,7 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl
- (void)runSyncRegistrations
{
runSyncRegistrationsForStorage(self);
RunSyncRegistrationsForStorage(self);
// See comments on OWSDatabaseConnection.
//
@ -137,7 +156,7 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl
DDLogVerbose(@"%@ async registrations enqueuing.", self.logTag);
runAsyncRegistrationsForStorage(self, ^{
RunAsyncRegistrationsForStorage(self, ^{
OWSAssertIsOnMainThread();
OWSAssert(!self.areAsyncRegistrationsComplete);
@ -147,9 +166,16 @@ void runAsyncRegistrationsForStorage(OWSStorage *storage, dispatch_block_t compl
self.areAsyncRegistrationsComplete = YES;
completion();
[self verifyDatabaseViews];
});
}
- (void)verifyDatabaseViews
{
VerifyRegistrationsForPrimaryStorage(self);
}
+ (void)protectFiles
{
DDLogInfo(

@ -64,9 +64,11 @@ typedef void (^OWSStorageMigrationBlock)(void);
- (YapDatabaseConnection *)newDatabaseConnection;
#ifdef DEBUG
#pragma mark - Extension Registration
+ (void)incrementVersionOfDatabaseExtension:(NSString *)extensionName;
- (BOOL)registerExtension:(YapDatabaseExtension *)extension withName:(NSString *)extensionName;
#endif
- (void)asyncRegisterExtension:(YapDatabaseExtension *)extension withName:(NSString *)extensionName;
- (void)asyncRegisterExtension:(YapDatabaseExtension *)extension
@ -75,6 +77,10 @@ typedef void (^OWSStorageMigrationBlock)(void);
- (nullable id)registeredExtension:(NSString *)extensionName;
- (NSArray<NSString *> *)registeredExtensionNames;
#pragma mark -
- (unsigned long long)databaseFileSize;
- (unsigned long long)databaseWALFileSize;
- (unsigned long long)databaseSHMFileSize;

@ -6,6 +6,7 @@
#import "AppContext.h"
#import "NSData+Base64.h"
#import "NSNotificationCenter+OWS.h"
#import "NSUserDefaults+OWS.h"
#import "OWSBackgroundTask.h"
#import "OWSFileSystem.h"
#import "OWSPrimaryStorage.h"
@ -14,7 +15,11 @@
#import <Curve25519Kit/Randomness.h>
#import <SAMKeychain/SAMKeychain.h>
#import <YapDatabase/YapDatabase.h>
#import <YapDatabase/YapDatabaseAutoView.h>
#import <YapDatabase/YapDatabaseCrossProcessNotification.h>
#import <YapDatabase/YapDatabaseCryptoUtils.h>
#import <YapDatabase/YapDatabaseSecondaryIndex.h>
#import <YapDatabase/YapDatabaseSecondaryIndexPrivate.h>
NS_ASSUME_NONNULL_BEGIN
@ -36,6 +41,8 @@ const NSUInteger kDatabasePasswordLength = 30;
typedef NSData *_Nullable (^LoadDatabaseMetadataBlock)(NSError **_Nullable);
typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
NSString *const kNSUserDefaults_DatabaseExtensionVersionMap = @"kNSUserDefaults_DatabaseExtensionVersionMap";
#pragma mark -
@interface YapDatabaseConnection ()
@ -234,6 +241,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
@property (atomic, nullable) YapDatabase *database;
@property (nonatomic) NSMutableArray<NSString *> *extensionNames;
@end
#pragma mark -
@ -249,6 +258,8 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
selector:@selector(resetStorage)
name:OWSResetStorageNotification
object:nil];
self.extensionNames = [NSMutableArray new];
}
return self;
@ -461,8 +472,90 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
return dbConnection;
}
#pragma mark - Extension Registration
+ (void)incrementVersionOfDatabaseExtension:(NSString *)extensionName
{
DDLogError(@"%@ %s", self.logTag, __PRETTY_FUNCTION__);
NSUserDefaults *appUserDefaults = [NSUserDefaults appUserDefaults];
OWSAssert(appUserDefaults);
NSMutableDictionary<NSString *, NSNumber *> *_Nullable versionMap =
[[appUserDefaults valueForKey:kNSUserDefaults_DatabaseExtensionVersionMap] mutableCopy];
if (!versionMap) {
versionMap = [NSMutableDictionary new];
}
NSNumber *_Nullable versionSuffix = versionMap[extensionName];
versionMap[extensionName] = @(versionSuffix.intValue + 1);
[appUserDefaults setValue:versionMap forKey:kNSUserDefaults_DatabaseExtensionVersionMap];
[appUserDefaults synchronize];
}
- (nullable NSString *)appendSuffixToDatabaseExtensionVersionIfNecessary:(nullable NSString *)versionTag
extensionName:(NSString *)extensionName
{
OWSAssertIsOnMainThread();
NSUserDefaults *appUserDefaults = [NSUserDefaults appUserDefaults];
OWSAssert(appUserDefaults);
NSDictionary<NSString *, NSNumber *> *_Nullable versionMap =
[appUserDefaults valueForKey:kNSUserDefaults_DatabaseExtensionVersionMap];
NSNumber *_Nullable versionSuffix = versionMap[extensionName];
if (versionSuffix) {
NSString *result =
[NSString stringWithFormat:@"%@.%@", (versionTag.length < 1 ? @"0" : versionTag), versionSuffix];
DDLogWarn(@"%@ database extension version: %@ + %@ -> %@", self.logTag, versionTag, versionSuffix, result);
return result;
}
return versionTag;
}
- (YapDatabaseExtension *)updateExtensionVersion:(YapDatabaseExtension *)extension withName:(NSString *)extensionName
{
OWSAssert(extension);
OWSAssert(extensionName.length > 0);
if ([extension isKindOfClass:[YapDatabaseAutoView class]]) {
YapDatabaseAutoView *databaseView = (YapDatabaseAutoView *)extension;
YapDatabaseAutoView *databaseViewCopy = [[YapDatabaseAutoView alloc]
initWithGrouping:databaseView.grouping
sorting:databaseView.sorting
versionTag:[self appendSuffixToDatabaseExtensionVersionIfNecessary:databaseView.versionTag
extensionName:extensionName]
options:databaseView.options];
return databaseViewCopy;
} else if ([extension isKindOfClass:[YapDatabaseSecondaryIndex class]]) {
YapDatabaseSecondaryIndex *secondaryIndex = (YapDatabaseSecondaryIndex *)extension;
OWSAssert(secondaryIndex->setup);
OWSAssert(secondaryIndex->handler);
YapDatabaseSecondaryIndex *secondaryIndexCopy = [[YapDatabaseSecondaryIndex alloc]
initWithSetup:secondaryIndex->setup
handler:secondaryIndex->handler
versionTag:[self appendSuffixToDatabaseExtensionVersionIfNecessary:secondaryIndex.versionTag
extensionName:extensionName]
options:secondaryIndex->options];
return secondaryIndexCopy;
} else if ([extension isKindOfClass:[YapDatabaseCrossProcessNotification class]]) {
// versionTag doesn't matter for YapDatabaseCrossProcessNotification.
return extension;
} else {
// This method needs to be able to update the versionTag of all extensions.
// If we start using other extension types, we need to modify this method to
// handle them as well.
OWSProdLogAndFail(@"%@ Unknown extension type: %@", self.logTag, [extension class]);
return extension;
}
}
- (BOOL)registerExtension:(YapDatabaseExtension *)extension withName:(NSString *)extensionName
{
extension = [self updateExtensionVersion:extension withName:extensionName];
OWSAssert(![self.extensionNames containsObject:extensionName]);
[self.extensionNames addObject:extensionName];
return [self.database registerExtension:extension withName:extensionName];
}
@ -476,6 +569,11 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
withName:(NSString *)extensionName
completion:(nullable dispatch_block_t)completion
{
extension = [self updateExtensionVersion:extension withName:extensionName];
OWSAssert(![self.extensionNames containsObject:extensionName]);
[self.extensionNames addObject:extensionName];
[self.database asyncRegisterExtension:extension
withName:extensionName
completionBlock:^(BOOL ready) {
@ -498,6 +596,11 @@ typedef NSData *_Nullable (^CreateDatabaseMetadataBlock)(void);
return [self.database registeredExtension:extensionName];
}
- (NSArray<NSString *> *)registeredExtensionNames
{
return [self.extensionNames copy];
}
#pragma mark - Password
+ (void)deleteDatabaseFiles

@ -7,6 +7,8 @@
@interface TSDatabaseSecondaryIndexes : NSObject
+ (NSString *)registerTimeStampIndexExtensionName;
+ (YapDatabaseSecondaryIndex *)registerTimeStampIndex;
+ (void)enumerateMessagesWithTimestamp:(uint64_t)timestamp

@ -3,12 +3,18 @@
//
#import "TSDatabaseSecondaryIndexes.h"
#import "OWSStorage.h"
#import "TSInteraction.h"
#define TSTimeStampSQLiteIndex @"messagesTimeStamp"
@implementation TSDatabaseSecondaryIndexes
+ (NSString *)registerTimeStampIndexExtensionName
{
return @"idx";
}
+ (YapDatabaseSecondaryIndex *)registerTimeStampIndex {
YapDatabaseSecondaryIndexSetup *setup = [[YapDatabaseSecondaryIndexSetup alloc] init];
[setup addColumn:TSTimeStampSQLiteIndex withType:YapDatabaseSecondaryIndexTypeReal];
@ -25,7 +31,8 @@
YapDatabaseSecondaryIndexHandler *handler = [YapDatabaseSecondaryIndexHandler withObjectBlock:block];
YapDatabaseSecondaryIndex *secondaryIndex = [[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler];
YapDatabaseSecondaryIndex *secondaryIndex =
[[YapDatabaseSecondaryIndex alloc] initWithSetup:setup handler:handler versionTag:nil];
return secondaryIndex;
}
@ -37,7 +44,7 @@
{
NSString *formattedString = [NSString stringWithFormat:@"WHERE %@ = %lld", TSTimeStampSQLiteIndex, timestamp];
YapDatabaseQuery *query = [YapDatabaseQuery queryWithFormat:formattedString];
[[transaction ext:@"idx"] enumerateKeysMatchingQuery:query usingBlock:block];
[[transaction ext:[self registerTimeStampIndexExtensionName]] enumerateKeysMatchingQuery:query usingBlock:block];
}
@end

@ -128,6 +128,13 @@ NS_ASSUME_NONNULL_BEGIN
OWSFail(_messageFormat, ##__VA_ARGS__); \
}
#define OWSProdLogAndCFail(_messageFormat, ...) \
{ \
DDLogError(_messageFormat, ##__VA_ARGS__); \
[DDLog flushLog]; \
OWSCFail(_messageFormat, ##__VA_ARGS__); \
}
// This function is intended for use in Swift.
void SwiftAssertIsOnMainThread(NSString *functionName);

Loading…
Cancel
Save