Build contact/group exports with data streams

Ideally we'd be using file streams so we wouldn't have to load it all
into ram but none of the attachment uploading infrastructure takes
streams (yet!)

// FREEBIE

src/Devices/OWSChunkedOutputStream.h

@@ -0,0 +1,16 @@
+//  Copyright © 2016 Open Whisper Systems. All rights reserved.
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class PBCodedOutputStream;
+
+@interface OWSChunkedOutputStream : NSObject
+
+@property (nonatomic, readonly) PBCodedOutputStream *delegateStream;
+
++ (instancetype)streamWithOutputStream:(NSOutputStream *)output;
+- (void)flush;
+
+@end
+
+NS_ASSUME_NONNULL_END

src/Devices/OWSChunkedOutputStream.m

@@ -0,0 +1,35 @@
+//  Copyright © 2016 Open Whisper Systems. All rights reserved.
+
+#import "OWSChunkedOutputStream.h"
+#import <ProtocolBuffers/CodedOutputStream.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@implementation OWSChunkedOutputStream
+
++ (instancetype)streamWithOutputStream:(NSOutputStream *)output
+{
+    return [[self alloc] initWithOutputStream:output];
+}
+
+- (instancetype)initWithOutputStream:(NSOutputStream *)outputStream
+{
+    self = [super init];
+    if (!self) {
+        return self;
+    }
+
+    _delegateStream = [PBCodedOutputStream streamWithOutputStream:outputStream];
+
+    return self;
+}
+
+- (void)flush
+{
+    [self.delegateStream flush];
+}
+
+
+@end
+
+NS_ASSUME_NONNULL_END

src/Devices/OWSContactsOutputStream.h

@@ -0,0 +1,15 @@
+//  Copyright © 2016 Open Whisper Systems. All rights reserved.
+
+#import "OWSChunkedOutputStream.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class Contact;
+
+@interface OWSContactsOutputStream : OWSChunkedOutputStream
+
+- (void)writeContact:(Contact *)contact;
+
+@end
+
+NS_ASSUME_NONNULL_END

src/Devices/OWSContactsOutputStream.m

@@ -0,0 +1,42 @@
+//  Copyright © 2016 Open Whisper Systems. All rights reserved.
+
+#import "OWSContactsOutputStream.h"
+#import "Contact.h"
+#import "OWSSignalServiceProtos.pb.h"
+#import <ProtocolBuffers/CodedOutputStream.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@implementation OWSContactsOutputStream
+
+- (void)writeContact:(Contact *)contact
+{
+    OWSSignalServiceProtosContactDetailsBuilder *contactBuilder = [OWSSignalServiceProtosContactDetailsBuilder new];
+    [contactBuilder setName:contact.fullName];
+    [contactBuilder setNumber:contact.textSecureIdentifiers.firstObject];
+
+    NSData *avatarPng;
+    if (contact.image) {
+        OWSSignalServiceProtosContactDetailsAvatarBuilder *avatarBuilder =
+            [OWSSignalServiceProtosContactDetailsAvatarBuilder new];
+
+        [avatarBuilder setContentType:@"image/png"];
+        avatarPng = UIImagePNGRepresentation(contact.image);
+        [avatarBuilder setLength:(uint32_t)avatarPng.length];
+        [contactBuilder setAvatarBuilder:avatarBuilder];
+    }
+
+    NSData *contactData = [[contactBuilder build] data];
+
+    uint32_t contactDataLength = (uint32_t)contactData.length;
+    [self.delegateStream writeRawVarint32:contactDataLength];
+    [self.delegateStream writeRawData:contactData];
+
+    if (contact.image) {
+        [self.delegateStream writeRawData:avatarPng];
+    }
+}
+
+@end
+
+NS_ASSUME_NONNULL_END

src/Devices/OWSGroupsOutputStream.h

@@ -0,0 +1,15 @@
+//  Copyright © 2016 Open Whisper Systems. All rights reserved.
+
+#import "OWSChunkedOutputStream.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class TSGroupModel;
+
+@interface OWSGroupsOutputStream : OWSChunkedOutputStream
+
+- (void)writeGroup:(TSGroupModel *)group;
+
+@end
+
+NS_ASSUME_NONNULL_END

src/Devices/OWSGroupsOutputStream.m

@@ -0,0 +1,42 @@
+//  Copyright © 2016 Open Whisper Systems. All rights reserved.
+
+#import "OWSGroupsOutputStream.h"
+#import "OWSSignalServiceProtos.pb.h"
+#import "TSGroupModel.h"
+#import <ProtocolBuffers/CodedOutputStream.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@implementation OWSGroupsOutputStream
+
+- (void)writeGroup:(TSGroupModel *)group
+{
+    OWSSignalServiceProtosGroupDetailsBuilder *groupBuilder = [OWSSignalServiceProtosGroupDetailsBuilder new];
+    [groupBuilder setId:group.groupId];
+    [groupBuilder setName:group.groupName];
+    [groupBuilder setMembersArray:group.groupMemberIds];
+
+    NSData *avatarPng;
+    if (group.groupImage) {
+        OWSSignalServiceProtosGroupDetailsAvatarBuilder *avatarBuilder =
+            [OWSSignalServiceProtosGroupDetailsAvatarBuilder new];
+
+        [avatarBuilder setContentType:@"image/png"];
+        avatarPng = UIImagePNGRepresentation(group.groupImage);
+        [avatarBuilder setLength:(uint32_t)avatarPng.length];
+        [groupBuilder setAvatarBuilder:avatarBuilder];
+    }
+
+    NSData *groupData = [[groupBuilder build] data];
+    uint32_t groupDataLength = (uint32_t)groupData.length;
+    [self.delegateStream writeRawVarint32:groupDataLength];
+    [self.delegateStream writeRawData:groupData];
+
+    if (avatarPng) {
+        [self.delegateStream writeRawData:avatarPng];
+    }
+}
+
+@end
+
+NS_ASSUME_NONNULL_END

src/Messages/DeviceSyncing/OWSSyncContactsMessage.m

@@ -4,10 +4,10 @@
 #import "Contact.h"
 #import "ContactsManagerProtocol.h"
 #import "NSDate+millisecondTimeStamp.h"
+#import "OWSContactsOutputStream.h"
 #import "OWSSignalServiceProtos.pb.h"
 #import "TSAttachment.h"
 #import "TSAttachmentStream.h"
-#import <ProtocolBuffers/CodedOutputStream.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
@@ -54,60 +54,21 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (NSData *)buildPlainTextAttachmentData
 {
-    NSString *fileName =
+    // TODO use temp file stream to avoid loading everything into memory at once
-        [NSString stringWithFormat:@"%@_%@", [[NSProcessInfo processInfo] globallyUniqueString], @"contacts.dat"];
+    // First though, we need to re-engineer our attachment process to accept streams (encrypting with stream,
-    NSURL *fileURL = [NSURL fileURLWithPath:[NSTemporaryDirectory() stringByAppendingPathComponent:fileName]];
+    // and uploading with streams).
-    NSOutputStream *fileOutputStream = [NSOutputStream outputStreamWithURL:fileURL append:NO];
+    NSOutputStream *dataOutputStream = [NSOutputStream outputStreamToMemory];
-    [fileOutputStream open];
+    [dataOutputStream open];
-
+    OWSContactsOutputStream *contactsOutputStream = [OWSContactsOutputStream streamWithOutputStream:dataOutputStream];
-    PBCodedOutputStream *outputStream = [PBCodedOutputStream streamWithOutputStream:fileOutputStream];
-    DDLogInfo(@"Writing contacts data to %@", fileURL);
-    for (Contact *contact in self.contactsManager.signalContacts) {
-        OWSSignalServiceProtosContactDetailsBuilder *contactBuilder = [OWSSignalServiceProtosContactDetailsBuilder new];
-
-        [contactBuilder setName:contact.fullName];
-        [contactBuilder setNumber:contact.textSecureIdentifiers.firstObject];
 
-        NSData *avatarPng;
+    for (Contact *contact in self.contactsManager.signalContacts) {
-        if (contact.image) {
+        [contactsOutputStream writeContact:contact];
-            OWSSignalServiceProtosContactDetailsAvatarBuilder *avatarBuilder =
-                [OWSSignalServiceProtosContactDetailsAvatarBuilder new];
-
-            [avatarBuilder setContentType:@"image/png"];
-            avatarPng = UIImagePNGRepresentation(contact.image);
-            [avatarBuilder setLength:(uint32_t)avatarPng.length];
-            [contactBuilder setAvatarBuilder:avatarBuilder];
     }
 
-        NSData *contactData = [[contactBuilder build] data];
+    [contactsOutputStream flush];
+    [dataOutputStream close];
 
-        uint32_t contactDataLength = (uint32_t)contactData.length;
+    return [dataOutputStream propertyForKey:NSStreamDataWrittenToMemoryStreamKey];
-        [outputStream writeRawVarint32:contactDataLength];
-        [outputStream writeRawData:contactData];
-
-        if (contact.image) {
-            [outputStream writeRawData:avatarPng];
-        }
-    }
-    [outputStream flush];
-    [fileOutputStream close];
-
-    // TODO pass stream to builder rather than data as a singular hulk.
-    [NSInputStream inputStreamWithURL:fileURL];
-    NSError *error;
-    NSData *data = [NSData dataWithContentsOfURL:fileURL options:NSDataReadingMappedIfSafe error:&error];
-    if (error) {
-        DDLogError(@"Failed to read back contact data after writing it to %@ with error:%@", fileURL, error);
-    }
-    return data;
-
-    //    TODO delete contacts file.
-    //    NSError *error;
-    //    NSFileManager *manager = [NSFileManager defaultManager];
-    //    [manager removeItemAtURL:fileURL error:&error];
-    //    if (error) {
-    //        DDLogError(@"Failed removing temp file at url:%@ with error:%@", fileURL, error);
-    //    }
 }
 
 @end

src/Messages/DeviceSyncing/OWSSyncGroupsMessage.m

@@ -2,6 +2,7 @@
 
 #import "OWSSyncGroupsMessage.h"
 #import "NSDate+millisecondTimeStamp.h"
+#import "OWSGroupsOutputStream.h"
 #import "OWSSignalServiceProtos.pb.h"
 #import "TSAttachment.h"
 #import "TSGroupModel.h"
@@ -47,65 +48,26 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (NSData *)buildPlainTextAttachmentData
 {
-    NSString *fileName =
+    // TODO use temp file stream to avoid loading everything into memory at once
-        [NSString stringWithFormat:@"%@_%@", [[NSProcessInfo processInfo] globallyUniqueString], @"contacts.dat"];
+    // First though, we need to re-engineer our attachment process to accept streams (encrypting with stream,
-    NSURL *fileURL = [NSURL fileURLWithPath:[NSTemporaryDirectory() stringByAppendingPathComponent:fileName]];
+    // and uploading with streams).
-    NSOutputStream *fileOutputStream = [NSOutputStream outputStreamWithURL:fileURL append:NO];
+    NSOutputStream *dataOutputStream = [NSOutputStream outputStreamToMemory];
-    [fileOutputStream open];
+    [dataOutputStream open];
-
+    OWSGroupsOutputStream *groupsOutputStream = [OWSGroupsOutputStream streamWithOutputStream:dataOutputStream];
-    PBCodedOutputStream *outputStream = [PBCodedOutputStream streamWithOutputStream:fileOutputStream];
+
-    DDLogInfo(@"Writing groups data to %@", fileURL);
     [TSGroupThread enumerateCollectionObjectsUsingBlock:^(id obj, BOOL *stop) {
         if (![obj isKindOfClass:[TSGroupThread class]]) {
             DDLogError(@"Unexpected class in group collection: %@", obj);
             return;
         }
         TSGroupModel *group = ((TSGroupThread *)obj).groupModel;
-        OWSSignalServiceProtosGroupDetailsBuilder *groupBuilder = [OWSSignalServiceProtosGroupDetailsBuilder new];
+        [groupsOutputStream writeGroup:group];
-        [groupBuilder setId:group.groupId];
-        [groupBuilder setName:group.groupName];
-        [groupBuilder setMembersArray:group.groupMemberIds];
-
-        NSData *avatarPng;
-        if (group.groupImage) {
-            OWSSignalServiceProtosGroupDetailsAvatarBuilder *avatarBuilder =
-                [OWSSignalServiceProtosGroupDetailsAvatarBuilder new];
-
-            [avatarBuilder setContentType:@"image/png"];
-            avatarPng = UIImagePNGRepresentation(group.groupImage);
-            [avatarBuilder setLength:(uint32_t)avatarPng.length];
-            [groupBuilder setAvatarBuilder:avatarBuilder];
-        }
-
-        NSData *groupData = [[groupBuilder build] data];
-        uint32_t groupDataLength = (uint32_t)groupData.length;
-        [outputStream writeRawVarint32:groupDataLength];
-        [outputStream writeRawData:groupData];
-
-        if (avatarPng) {
-            [outputStream writeRawData:avatarPng];
-        }
     }];
 
-    [outputStream flush];
+    [groupsOutputStream flush];
-    [fileOutputStream close];
+    [dataOutputStream close];
 
-    // TODO pass stream to builder rather than data as a singular hulk.
+    return [dataOutputStream propertyForKey:NSStreamDataWrittenToMemoryStreamKey];
-    [NSInputStream inputStreamWithURL:fileURL];
-    NSError *error;
-    NSData *data = [NSData dataWithContentsOfURL:fileURL options:NSDataReadingMappedIfSafe error:&error];
-    if (error) {
-        DDLogError(@"Failed to read back contact data after writing it to %@ with error:%@", fileURL, error);
-    }
-    return data;
-
-    //    TODO delete contacts file.
-    //    NSError *error;
-    //    NSFileManager *manager = [NSFileManager defaultManager];
-    //    [manager removeItemAtURL:fileURL error:&error];
-    //    if (error) {
-    //        DDLogError(@"Failed removing temp file at url:%@ with error:%@", fileURL, error);
-    //    }
 }
 
 @end