From d48fd158b7dfb075d333f06b4609eb995f500d87 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Sun, 28 Aug 2016 20:42:07 -0400 Subject: [PATCH] Build contact/group exports with data streams Ideally we'd be using file streams so we wouldn't have to load it all into ram but none of the attachment uploading infrastructure takes streams (yet!) // FREEBIE --- src/Devices/OWSChunkedOutputStream.h | 16 +++++ src/Devices/OWSChunkedOutputStream.m | 35 ++++++++++ src/Devices/OWSContactsOutputStream.h | 15 +++++ src/Devices/OWSContactsOutputStream.m | 42 ++++++++++++ src/Devices/OWSGroupsOutputStream.h | 15 +++++ src/Devices/OWSGroupsOutputStream.m | 42 ++++++++++++ .../DeviceSyncing/OWSSyncContactsMessage.m | 65 ++++--------------- .../DeviceSyncing/OWSSyncGroupsMessage.m | 62 ++++-------------- 8 files changed, 190 insertions(+), 102 deletions(-) create mode 100644 src/Devices/OWSChunkedOutputStream.h create mode 100644 src/Devices/OWSChunkedOutputStream.m create mode 100644 src/Devices/OWSContactsOutputStream.h create mode 100644 src/Devices/OWSContactsOutputStream.m create mode 100644 src/Devices/OWSGroupsOutputStream.h create mode 100644 src/Devices/OWSGroupsOutputStream.m diff --git a/src/Devices/OWSChunkedOutputStream.h b/src/Devices/OWSChunkedOutputStream.h new file mode 100644 index 000000000..e545b4f19 --- /dev/null +++ b/src/Devices/OWSChunkedOutputStream.h @@ -0,0 +1,16 @@ +// Copyright © 2016 Open Whisper Systems. All rights reserved. + +NS_ASSUME_NONNULL_BEGIN + +@class PBCodedOutputStream; + +@interface OWSChunkedOutputStream : NSObject + +@property (nonatomic, readonly) PBCodedOutputStream *delegateStream; + ++ (instancetype)streamWithOutputStream:(NSOutputStream *)output; +- (void)flush; + +@end + +NS_ASSUME_NONNULL_END diff --git a/src/Devices/OWSChunkedOutputStream.m b/src/Devices/OWSChunkedOutputStream.m new file mode 100644 index 000000000..b7e3dec6a --- /dev/null +++ b/src/Devices/OWSChunkedOutputStream.m @@ -0,0 +1,35 @@ +// Copyright © 2016 Open Whisper Systems. All rights reserved. + +#import "OWSChunkedOutputStream.h" +#import + +NS_ASSUME_NONNULL_BEGIN + +@implementation OWSChunkedOutputStream + ++ (instancetype)streamWithOutputStream:(NSOutputStream *)output +{ + return [[self alloc] initWithOutputStream:output]; +} + +- (instancetype)initWithOutputStream:(NSOutputStream *)outputStream +{ + self = [super init]; + if (!self) { + return self; + } + + _delegateStream = [PBCodedOutputStream streamWithOutputStream:outputStream]; + + return self; +} + +- (void)flush +{ + [self.delegateStream flush]; +} + + +@end + +NS_ASSUME_NONNULL_END diff --git a/src/Devices/OWSContactsOutputStream.h b/src/Devices/OWSContactsOutputStream.h new file mode 100644 index 000000000..6cdcebaeb --- /dev/null +++ b/src/Devices/OWSContactsOutputStream.h @@ -0,0 +1,15 @@ +// Copyright © 2016 Open Whisper Systems. All rights reserved. + +#import "OWSChunkedOutputStream.h" + +NS_ASSUME_NONNULL_BEGIN + +@class Contact; + +@interface OWSContactsOutputStream : OWSChunkedOutputStream + +- (void)writeContact:(Contact *)contact; + +@end + +NS_ASSUME_NONNULL_END diff --git a/src/Devices/OWSContactsOutputStream.m b/src/Devices/OWSContactsOutputStream.m new file mode 100644 index 000000000..31a19e297 --- /dev/null +++ b/src/Devices/OWSContactsOutputStream.m @@ -0,0 +1,42 @@ +// Copyright © 2016 Open Whisper Systems. All rights reserved. + +#import "OWSContactsOutputStream.h" +#import "Contact.h" +#import "OWSSignalServiceProtos.pb.h" +#import + +NS_ASSUME_NONNULL_BEGIN + +@implementation OWSContactsOutputStream + +- (void)writeContact:(Contact *)contact +{ + OWSSignalServiceProtosContactDetailsBuilder *contactBuilder = [OWSSignalServiceProtosContactDetailsBuilder new]; + [contactBuilder setName:contact.fullName]; + [contactBuilder setNumber:contact.textSecureIdentifiers.firstObject]; + + NSData *avatarPng; + if (contact.image) { + OWSSignalServiceProtosContactDetailsAvatarBuilder *avatarBuilder = + [OWSSignalServiceProtosContactDetailsAvatarBuilder new]; + + [avatarBuilder setContentType:@"image/png"]; + avatarPng = UIImagePNGRepresentation(contact.image); + [avatarBuilder setLength:(uint32_t)avatarPng.length]; + [contactBuilder setAvatarBuilder:avatarBuilder]; + } + + NSData *contactData = [[contactBuilder build] data]; + + uint32_t contactDataLength = (uint32_t)contactData.length; + [self.delegateStream writeRawVarint32:contactDataLength]; + [self.delegateStream writeRawData:contactData]; + + if (contact.image) { + [self.delegateStream writeRawData:avatarPng]; + } +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/src/Devices/OWSGroupsOutputStream.h b/src/Devices/OWSGroupsOutputStream.h new file mode 100644 index 000000000..9d5ce70e8 --- /dev/null +++ b/src/Devices/OWSGroupsOutputStream.h @@ -0,0 +1,15 @@ +// Copyright © 2016 Open Whisper Systems. All rights reserved. + +#import "OWSChunkedOutputStream.h" + +NS_ASSUME_NONNULL_BEGIN + +@class TSGroupModel; + +@interface OWSGroupsOutputStream : OWSChunkedOutputStream + +- (void)writeGroup:(TSGroupModel *)group; + +@end + +NS_ASSUME_NONNULL_END diff --git a/src/Devices/OWSGroupsOutputStream.m b/src/Devices/OWSGroupsOutputStream.m new file mode 100644 index 000000000..93b6e2dc6 --- /dev/null +++ b/src/Devices/OWSGroupsOutputStream.m @@ -0,0 +1,42 @@ +// Copyright © 2016 Open Whisper Systems. All rights reserved. + +#import "OWSGroupsOutputStream.h" +#import "OWSSignalServiceProtos.pb.h" +#import "TSGroupModel.h" +#import + +NS_ASSUME_NONNULL_BEGIN + +@implementation OWSGroupsOutputStream + +- (void)writeGroup:(TSGroupModel *)group +{ + OWSSignalServiceProtosGroupDetailsBuilder *groupBuilder = [OWSSignalServiceProtosGroupDetailsBuilder new]; + [groupBuilder setId:group.groupId]; + [groupBuilder setName:group.groupName]; + [groupBuilder setMembersArray:group.groupMemberIds]; + + NSData *avatarPng; + if (group.groupImage) { + OWSSignalServiceProtosGroupDetailsAvatarBuilder *avatarBuilder = + [OWSSignalServiceProtosGroupDetailsAvatarBuilder new]; + + [avatarBuilder setContentType:@"image/png"]; + avatarPng = UIImagePNGRepresentation(group.groupImage); + [avatarBuilder setLength:(uint32_t)avatarPng.length]; + [groupBuilder setAvatarBuilder:avatarBuilder]; + } + + NSData *groupData = [[groupBuilder build] data]; + uint32_t groupDataLength = (uint32_t)groupData.length; + [self.delegateStream writeRawVarint32:groupDataLength]; + [self.delegateStream writeRawData:groupData]; + + if (avatarPng) { + [self.delegateStream writeRawData:avatarPng]; + } +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/src/Messages/DeviceSyncing/OWSSyncContactsMessage.m b/src/Messages/DeviceSyncing/OWSSyncContactsMessage.m index 3c9fa8cb6..438ea8e9e 100644 --- a/src/Messages/DeviceSyncing/OWSSyncContactsMessage.m +++ b/src/Messages/DeviceSyncing/OWSSyncContactsMessage.m @@ -4,10 +4,10 @@ #import "Contact.h" #import "ContactsManagerProtocol.h" #import "NSDate+millisecondTimeStamp.h" +#import "OWSContactsOutputStream.h" #import "OWSSignalServiceProtos.pb.h" #import "TSAttachment.h" #import "TSAttachmentStream.h" -#import NS_ASSUME_NONNULL_BEGIN @@ -54,60 +54,21 @@ NS_ASSUME_NONNULL_BEGIN - (NSData *)buildPlainTextAttachmentData { - NSString *fileName = - [NSString stringWithFormat:@"%@_%@", [[NSProcessInfo processInfo] globallyUniqueString], @"contacts.dat"]; - NSURL *fileURL = [NSURL fileURLWithPath:[NSTemporaryDirectory() stringByAppendingPathComponent:fileName]]; - NSOutputStream *fileOutputStream = [NSOutputStream outputStreamWithURL:fileURL append:NO]; - [fileOutputStream open]; - - PBCodedOutputStream *outputStream = [PBCodedOutputStream streamWithOutputStream:fileOutputStream]; - DDLogInfo(@"Writing contacts data to %@", fileURL); - for (Contact *contact in self.contactsManager.signalContacts) { - OWSSignalServiceProtosContactDetailsBuilder *contactBuilder = [OWSSignalServiceProtosContactDetailsBuilder new]; - - [contactBuilder setName:contact.fullName]; - [contactBuilder setNumber:contact.textSecureIdentifiers.firstObject]; - - NSData *avatarPng; - if (contact.image) { - OWSSignalServiceProtosContactDetailsAvatarBuilder *avatarBuilder = - [OWSSignalServiceProtosContactDetailsAvatarBuilder new]; - - [avatarBuilder setContentType:@"image/png"]; - avatarPng = UIImagePNGRepresentation(contact.image); - [avatarBuilder setLength:(uint32_t)avatarPng.length]; - [contactBuilder setAvatarBuilder:avatarBuilder]; - } + // TODO use temp file stream to avoid loading everything into memory at once + // First though, we need to re-engineer our attachment process to accept streams (encrypting with stream, + // and uploading with streams). + NSOutputStream *dataOutputStream = [NSOutputStream outputStreamToMemory]; + [dataOutputStream open]; + OWSContactsOutputStream *contactsOutputStream = [OWSContactsOutputStream streamWithOutputStream:dataOutputStream]; - NSData *contactData = [[contactBuilder build] data]; + for (Contact *contact in self.contactsManager.signalContacts) { + [contactsOutputStream writeContact:contact]; + } - uint32_t contactDataLength = (uint32_t)contactData.length; - [outputStream writeRawVarint32:contactDataLength]; - [outputStream writeRawData:contactData]; + [contactsOutputStream flush]; + [dataOutputStream close]; - if (contact.image) { - [outputStream writeRawData:avatarPng]; - } - } - [outputStream flush]; - [fileOutputStream close]; - - // TODO pass stream to builder rather than data as a singular hulk. - [NSInputStream inputStreamWithURL:fileURL]; - NSError *error; - NSData *data = [NSData dataWithContentsOfURL:fileURL options:NSDataReadingMappedIfSafe error:&error]; - if (error) { - DDLogError(@"Failed to read back contact data after writing it to %@ with error:%@", fileURL, error); - } - return data; - - // TODO delete contacts file. - // NSError *error; - // NSFileManager *manager = [NSFileManager defaultManager]; - // [manager removeItemAtURL:fileURL error:&error]; - // if (error) { - // DDLogError(@"Failed removing temp file at url:%@ with error:%@", fileURL, error); - // } + return [dataOutputStream propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; } @end diff --git a/src/Messages/DeviceSyncing/OWSSyncGroupsMessage.m b/src/Messages/DeviceSyncing/OWSSyncGroupsMessage.m index 0c224fd79..1ae57c7fe 100644 --- a/src/Messages/DeviceSyncing/OWSSyncGroupsMessage.m +++ b/src/Messages/DeviceSyncing/OWSSyncGroupsMessage.m @@ -2,6 +2,7 @@ #import "OWSSyncGroupsMessage.h" #import "NSDate+millisecondTimeStamp.h" +#import "OWSGroupsOutputStream.h" #import "OWSSignalServiceProtos.pb.h" #import "TSAttachment.h" #import "TSGroupModel.h" @@ -47,65 +48,26 @@ NS_ASSUME_NONNULL_BEGIN - (NSData *)buildPlainTextAttachmentData { - NSString *fileName = - [NSString stringWithFormat:@"%@_%@", [[NSProcessInfo processInfo] globallyUniqueString], @"contacts.dat"]; - NSURL *fileURL = [NSURL fileURLWithPath:[NSTemporaryDirectory() stringByAppendingPathComponent:fileName]]; - NSOutputStream *fileOutputStream = [NSOutputStream outputStreamWithURL:fileURL append:NO]; - [fileOutputStream open]; - - PBCodedOutputStream *outputStream = [PBCodedOutputStream streamWithOutputStream:fileOutputStream]; - DDLogInfo(@"Writing groups data to %@", fileURL); + // TODO use temp file stream to avoid loading everything into memory at once + // First though, we need to re-engineer our attachment process to accept streams (encrypting with stream, + // and uploading with streams). + NSOutputStream *dataOutputStream = [NSOutputStream outputStreamToMemory]; + [dataOutputStream open]; + OWSGroupsOutputStream *groupsOutputStream = [OWSGroupsOutputStream streamWithOutputStream:dataOutputStream]; + [TSGroupThread enumerateCollectionObjectsUsingBlock:^(id obj, BOOL *stop) { if (![obj isKindOfClass:[TSGroupThread class]]) { DDLogError(@"Unexpected class in group collection: %@", obj); return; } TSGroupModel *group = ((TSGroupThread *)obj).groupModel; - OWSSignalServiceProtosGroupDetailsBuilder *groupBuilder = [OWSSignalServiceProtosGroupDetailsBuilder new]; - [groupBuilder setId:group.groupId]; - [groupBuilder setName:group.groupName]; - [groupBuilder setMembersArray:group.groupMemberIds]; - - NSData *avatarPng; - if (group.groupImage) { - OWSSignalServiceProtosGroupDetailsAvatarBuilder *avatarBuilder = - [OWSSignalServiceProtosGroupDetailsAvatarBuilder new]; - - [avatarBuilder setContentType:@"image/png"]; - avatarPng = UIImagePNGRepresentation(group.groupImage); - [avatarBuilder setLength:(uint32_t)avatarPng.length]; - [groupBuilder setAvatarBuilder:avatarBuilder]; - } - - NSData *groupData = [[groupBuilder build] data]; - uint32_t groupDataLength = (uint32_t)groupData.length; - [outputStream writeRawVarint32:groupDataLength]; - [outputStream writeRawData:groupData]; - - if (avatarPng) { - [outputStream writeRawData:avatarPng]; - } + [groupsOutputStream writeGroup:group]; }]; - [outputStream flush]; - [fileOutputStream close]; + [groupsOutputStream flush]; + [dataOutputStream close]; - // TODO pass stream to builder rather than data as a singular hulk. - [NSInputStream inputStreamWithURL:fileURL]; - NSError *error; - NSData *data = [NSData dataWithContentsOfURL:fileURL options:NSDataReadingMappedIfSafe error:&error]; - if (error) { - DDLogError(@"Failed to read back contact data after writing it to %@ with error:%@", fileURL, error); - } - return data; - - // TODO delete contacts file. - // NSError *error; - // NSFileManager *manager = [NSFileManager defaultManager]; - // [manager removeItemAtURL:fileURL error:&error]; - // if (error) { - // DDLogError(@"Failed removing temp file at url:%@ with error:%@", fileURL, error); - // } + return [dataOutputStream propertyForKey:NSStreamDataWrittenToMemoryStreamKey]; } @end