Updated the migration to handle quotes and link previews

SessionMessagingKit/Database/Migrations/_001_InitialSetupMigration.swift

@@ -145,9 +145,7 @@ enum _001_InitialSetupMigration: Migration {
             t.column(.receivedAtTimestampMs, .double).notNull()
             t.column(.expiresInSeconds, .double)
             t.column(.expiresStartedAtMs, .double)
-            
-            t.column(.openGroupInvitationName, .text)
-            t.column(.openGroupInvitationUrl, .text)
+            t.column(.linkPreviewUrl, .text)
             
             t.column(.openGroupServerMessageId, .integer)
                 .indexed()                                            // Quicker querying
@@ -203,17 +201,18 @@ enum _001_InitialSetupMigration: Migration {
         try db.create(table: LinkPreview.self) { t in
             t.column(.url, .text)
                 .notNull()
-                .primaryKey()
-            t.column(.interactionId, .integer)
+                .indexed()                                            // Quicker querying
+            t.column(.timestamp, .double)
                 .notNull()
                 .indexed()                                            // Quicker querying
-                .references(Interaction.self, onDelete: .cascade)     // Delete if interaction deleted
+            t.column(.variant, .integer).notNull()
             t.column(.title, .text)
+            
+            t.primaryKey([.url, .timestamp])
         }
         
         try db.create(table: Attachment.self) { t in
             t.column(.interactionId, .integer)
-                .notNull()
                 .indexed()                                            // Quicker querying
                 .references(Interaction.self, onDelete: .cascade)     // Delete if interaction deleted
             t.column(.serverId, .text)
@@ -231,10 +230,6 @@ enum _001_InitialSetupMigration: Migration {
             t.column(.encryptionKey, .blob)
             t.column(.digest, .blob)
             t.column(.caption, .text)
-            t.column(.quoteId, .text)
-                .references(Quote.self, onDelete: .cascade)           // Delete if Quote deleted
-            t.column(.linkPreviewUrl, .text)
-                .references(LinkPreview.self, onDelete: .cascade)     // Delete if LinkPreview deleted
         }
     }
 }

SessionMessagingKit/Database/Migrations/_002_YDBToGRDBMigration.swift

@@ -348,17 +348,16 @@ enum _002_YDBToGRDBMigration: Migration {
                         let body: String?
                         let expiresInSeconds: UInt32?
                         let expiresStartedAtMs: UInt64?
-                        let openGroupInvitationName: String?
-                        let openGroupInvitationUrl: String?
                         let openGroupServerMessageId: UInt64?
                         let recipientStateMap: [String: TSOutgoingMessageRecipientState]?
-                        let attachmentIds: [String]
+                        let quotedMessage: TSQuotedMessage?
+                        let linkPreview: OWSLinkPreview?
+                        let linkPreviewVariant: LinkPreview.Variant
+                        var attachmentIds: [String]
                         
                         // Handle the common 'TSMessage' values first
                         if let legacyMessage: TSMessage = legacyInteraction as? TSMessage {
                             serverHash = legacyMessage.serverHash
-                            openGroupInvitationName = legacyMessage.openGroupInvitationName
-                            openGroupInvitationUrl = legacyMessage.openGroupInvitationURL
                             
                             // The legacy code only considered '!= 0' ids as valid so set those
                             // values to be null to avoid the unique constraint (it's also more
@@ -367,27 +366,52 @@ enum _002_YDBToGRDBMigration: Migration {
                                 nil :
                                 legacyMessage.openGroupServerMessageID
                             )
-                            attachmentIds = try legacyMessage.attachmentIds.map { legacyId in
-                                guard let attachmentId: String = legacyId as? String else {
-                                    SNLog("[Migration Error] Unable to process attachment id")
-                                    throw GRDBStorageError.migrationFailed
-                                }
-                                
-                                return attachmentId
+                            quotedMessage = legacyMessage.quotedMessage
+                            
+                            // Convert the 'OpenGroupInvitation' into a LinkPreview
+                            if let openGroupInvitationName: String = legacyMessage.openGroupInvitationName, let openGroupInvitationUrl: String = legacyMessage.openGroupInvitationURL {
+                                linkPreviewVariant = .openGroupInvitation
+                                linkPreview = OWSLinkPreview(
+                                    urlString: openGroupInvitationUrl,
+                                    title: openGroupInvitationName,
+                                    imageAttachmentId: nil
+                                )
+                            }
+                            else {
+                                linkPreviewVariant = .standard
+                                linkPreview = legacyMessage.linkPreview
                             }
+                            
+                            // Attachments for deleted messages won't exist
+                            attachmentIds = (legacyMessage.isDeleted ?
+                                [] :
+                                try legacyMessage.attachmentIds.map { legacyId in
+                                    guard let attachmentId: String = legacyId as? String else {
+                                        SNLog("[Migration Error] Unable to process attachment id")
+                                        throw GRDBStorageError.migrationFailed
+                                    }
+                                    
+                                    return attachmentId
+                                }
+                            )
                         }
                         else {
                             serverHash = nil
-                            openGroupInvitationName = nil
-                            openGroupInvitationUrl = nil
                             openGroupServerMessageId = nil
+                            quotedMessage = nil
+                            linkPreviewVariant = .standard
+                            linkPreview = nil
                             attachmentIds = []
                         }
                         
                         // Then handle the behaviours for each message type
                         switch legacyInteraction {
                             case let incomingMessage as TSIncomingMessage:
-                                variant = .standardIncoming
+                                // Note: We want to distinguish deleted messages from normal ones
+                                variant = (incomingMessage.isDeleted ?
+                                    .standardIncomingDeleted :
+                                    .standardIncoming
+                                )
                                 authorId = incomingMessage.authorId
                                 body = incomingMessage.body
                                 expiresInSeconds = incomingMessage.expiresInSeconds
@@ -443,8 +467,7 @@ enum _002_YDBToGRDBMigration: Migration {
                             receivedAtTimestampMs: Double(legacyInteraction.receivedAtTimestamp),
                             expiresInSeconds: expiresInSeconds.map { TimeInterval($0) },
                             expiresStartedAtMs: expiresStartedAtMs.map { Double($0) },
-                            openGroupInvitationName: openGroupInvitationName,
-                            openGroupInvitationUrl: openGroupInvitationUrl,
+                            linkPreviewUrl: linkPreview?.urlString, // Only a soft link so save to set
                             openGroupServerMessageId: openGroupServerMessageId.map { Int64($0) },
                             openGroupWhisperMods: false, // TODO: This
                             openGroupWhisperTo: nil // TODO: This
@@ -455,6 +478,8 @@ enum _002_YDBToGRDBMigration: Migration {
                             throw GRDBStorageError.migrationFailed
                         }
                         
+                        // Handle the recipient states
+                        
                         try recipientStateMap?.forEach { recipientId, legacyState in
                             try RecipientState(
                                 interactionId: interactionId,
@@ -471,11 +496,68 @@ enum _002_YDBToGRDBMigration: Migration {
                                 readTimestampMs: legacyState.readTimestamp?.doubleValue
                             ).insert(db)
                         }
+                        
+                        // Handle any quote
+                        
+                        if let quotedMessage: TSQuotedMessage = quotedMessage {
+                            try Quote(
+                                interactionId: interactionId,
+                                authorId: quotedMessage.authorId,
+                                timestampMs: Double(quotedMessage.timestamp),
+                                body: quotedMessage.body
+                            ).insert(db)
+                            
+                            // Ensure the quote thumbnail works properly
+                            
+                            
+                            // Note: Quote attachments are now attached directly to the interaction
+                            attachmentIds = attachmentIds.appending(
+                                contentsOf: quotedMessage.quotedAttachments.compactMap { attachmentInfo in
+                                    if let attachmentId: String = attachmentInfo.attachmentId {
+                                        return attachmentId
+                                    }
+                                    else if let attachmentId: String = attachmentInfo.thumbnailAttachmentPointerId {
+                                        return attachmentId
+                                    }
+                                    // TODO: Looks like some of these might be busted???
+                                    return attachmentInfo.thumbnailAttachmentStreamId
+                                }
+                            )
+                        }
+                        
+                        // Handle any LinkPreview
+                        
+                        if let linkPreview: OWSLinkPreview = linkPreview, let urlString: String = linkPreview.urlString {
+                            // Note: The `legacyInteraction.timestamp` value is in milliseconds
+                            let timestamp: TimeInterval = LinkPreview.timestampFor(sentTimestampMs: Double(legacyInteraction.timestamp))
+                            
+                            // Note: It's possible for there to be duplicate values here so we use 'save'
+                            // instead of insert (ie. upsert)
+                            try LinkPreview(
+                                url: urlString,
+                                timestamp: timestamp,
+                                variant: linkPreviewVariant,
+                                title: linkPreview.title
+                            ).save(db)
+                            
+                            // Note: LinkPreview attachments are now attached directly to the interaction
+                            attachmentIds = attachmentIds.appending(linkPreview.imageAttachmentId)
+                        }
+                        
+                        // Handle any attachments
                         try attachmentIds.forEach { attachmentId in
                             guard let attachment: TSAttachment = attachments[attachmentId] else {
                                 SNLog("[Migration Error] Unsupported interaction type")
                                 throw GRDBStorageError.migrationFailed
                             }
+                            
+                            let size: CGSize = {
+                                switch attachment {
+                                    case let stream as TSAttachmentStream: return stream.calculateImageSize()
+                                    case let pointer as TSAttachmentPointer: return pointer.mediaSize
+                                    default: return CGSize.zero
+                                }
+                            }()
                             try Attachment(
                                 interactionId: interactionId,
                                 serverId: "\(attachment.serverId)",
@@ -483,16 +565,14 @@ enum _002_YDBToGRDBMigration: Migration {
                                 state: .pending, // TODO: This
                                 contentType: attachment.contentType,
                                 byteCount: UInt(attachment.byteCount),
-                                creationTimestamp: 0, // TODO: This
+                                creationTimestamp: (attachment as? TSAttachmentStream)?.creationTimestamp.timeIntervalSince1970,
                                 sourceFilename: attachment.sourceFilename,
                                 downloadUrl: attachment.downloadURL,
-                                width: 0, // TODO: This attachment.mediaSize,
-                                height: 0, // TODO: This attachment.mediaSize,
+                                width: (size == .zero ? nil : UInt(size.width)),
+                                height: (size == .zero ? nil : UInt(size.height)),
                                 encryptionKey: attachment.encryptionKey,
-                                digest: nil, // TODO: This attachment.digest,
-                                caption: attachment.caption,
-                                quoteId: nil,   // TODO: THis
-                                linkPreviewUrl: nil    // TODO: This
+                                digest: (attachment as? TSAttachmentStream)?.digest,
+                                caption: attachment.caption
                             ).insert(db)
                         }
                 }

SessionMessagingKit/Database/Models/Attachment.swift

@@ -7,14 +7,7 @@ import SessionUtilitiesKit
 public struct Attachment: Codable, FetchableRecord, PersistableRecord, TableRecord, ColumnExpressible {
     public static var databaseTableName: String { "attachment" }
     internal static let interactionForeignKey = ForeignKey([Columns.interactionId], to: [Interaction.Columns.id])
-    internal static let quoteForeignKey = ForeignKey([Columns.quoteId], to: [Quote.Columns.interactionId])
-    internal static let linkPreviewForeignKey = ForeignKey(
-        [Columns.linkPreviewUrl],
-        to: [LinkPreview.Columns.url]
-    )
     private static let interaction = belongsTo(Interaction.self, using: interactionForeignKey)
-    private static let quote = belongsTo(Quote.self, using: quoteForeignKey)
-    private static let linkPreview = belongsTo(LinkPreview.self, using: linkPreviewForeignKey)
     
     public typealias Columns = CodingKeys
     public enum CodingKeys: String, CodingKey, ColumnExpression {
@@ -32,8 +25,6 @@ public struct Attachment: Codable, FetchableRecord, PersistableRecord, TableReco
         case encryptionKey
         case digest
         case caption
-        case quoteId
-        case linkPreviewUrl
     }
     
     public enum Variant: Int, Codable, DatabaseValueConvertible {
@@ -50,8 +41,8 @@ public struct Attachment: Codable, FetchableRecord, PersistableRecord, TableReco
         case failed
     }
     
-    /// The id for the interaction this attachment belongs to
-    public let interactionId: Int64
+    /// The id for the Interaction this attachment belongs to
+    public let interactionId: Int64?
     
     /// The id for the attachment returned by the server
     ///
@@ -78,6 +69,7 @@ public struct Attachment: Codable, FetchableRecord, PersistableRecord, TableReco
     ///
     /// **Uploaded:** This will be the timestamp the file finished uploading
     /// **Downloaded:** This will be the timestamp the file finished downloading
+    /// **Other:** This will be null
     public let creationTimestamp: TimeInterval?
     
     /// Represents the "source" filename sent or received in the protos, not the filename on disk
@@ -103,29 +95,9 @@ public struct Attachment: Codable, FetchableRecord, PersistableRecord, TableReco
     /// Caption for the attachment
     public let caption: String?
     
-    /// The id for the QuotedMessage if this attachment belongs to one
-    ///
-    /// **Note:** If this value is present then this attachment shouldn't be returned as a
-    /// standard attachment for the interaction
-    public let quoteId: String?
-    
-    /// The id for the LinkPreview if this attachment belongs to one
-    ///
-    /// **Note:** If this value is present then this attachment shouldn't be returned as a
-    /// standard attachment for the interaction
-    public let linkPreviewUrl: String?
-    
     // MARK: - Relationships
     
     public var interaction: QueryInterfaceRequest<Interaction> {
         request(for: Attachment.interaction)
     }
-    
-    public var quote: QueryInterfaceRequest<Quote> {
-        request(for: Attachment.quote)
-    }
-    
-    public var linkPreview: QueryInterfaceRequest<LinkPreview> {
-        request(for: Attachment.linkPreview)
-    }
 }

SessionMessagingKit/Database/Models/Interaction.swift

@@ -8,6 +8,10 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
     public static var databaseTableName: String { "interaction" }
     internal static let threadForeignKey = ForeignKey([Columns.threadId], to: [SessionThread.Columns.id])
     internal static let profileForeignKey = ForeignKey([Columns.authorId], to: [Profile.Columns.id])
+    internal static let linkPreviewForeignKey = ForeignKey(
+        [Columns.linkPreviewUrl],
+        to: [LinkPreview.Columns.url]
+    )
     private static let thread = belongsTo(SessionThread.self, using: threadForeignKey)
     private static let profile = hasOne(Profile.self, using: profileForeignKey)
     private static let attachments = hasMany(Attachment.self, using: Attachment.interactionForeignKey)
@@ -29,9 +33,7 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
         
         case expiresInSeconds
         case expiresStartedAtMs
-        
-        case openGroupInvitationName
-        case openGroupInvitationUrl
+        case linkPreviewUrl
         
         // Open Group specific properties
         
@@ -43,6 +45,7 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
     public enum Variant: Int, Codable, DatabaseValueConvertible {
         case standardIncoming
         case standardOutgoing
+        case standardIncomingDeleted
         
         // Info Message Types (spacing the values out to make it easier to extend)
         case infoClosedGroupCreated = 1000
@@ -93,11 +96,10 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
     /// message has expired)
     public fileprivate(set) var expiresStartedAtMs: Double? = nil
     
-    /// When sending an Open Group invitation this will be populated with the name of the open group
-    public let openGroupInvitationName: String?
-    
-    /// When sending an Open Group invitation this will be populated with the url of the open group
-    public let openGroupInvitationUrl: String?
+    /// This value is the url for the link preview for this interaction
+    ///
+    /// **Note:** This is also used for open group invitations
+    public let linkPreviewUrl: String?
     
     // Open Group specific properties
     
@@ -122,10 +124,6 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
     
     public var attachments: QueryInterfaceRequest<Attachment> {
         request(for: Interaction.attachments)
-            .filter(
-                Attachment.Columns.quoteId == nil &&
-                Attachment.Columns.linkPreviewUrl == nil
-            )
     }
 
     public var quote: QueryInterfaceRequest<Quote> {
@@ -133,7 +131,20 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
     }
 
     public var linkPreview: QueryInterfaceRequest<LinkPreview> {
-        request(for: Interaction.linkPreview)
+        let linkPreviewAlias: TableAlias = TableAlias()
+        
+        return LinkPreview
+            .aliased(linkPreviewAlias)
+            .joining(
+                required: LinkPreview.interactions
+                    .filter(literal: [
+                        "(ROUND((\(Interaction.Columns.timestampMs) / 1000 / 100000) - 0.5) * 100000)",
+                        "=",
+                        "\(linkPreviewAlias[LinkPreview.Columns.timestamp])"
+                    ].joined(separator: " "))
+                    .limit(1)   // Avoid joining to multiple interactions
+            )
+            .limit(1)   // Avoid joining to multiple interactions
     }
     
     public var recipientStates: QueryInterfaceRequest<RecipientState> {
@@ -153,8 +164,7 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
         receivedAtTimestampMs: Double,
         expiresInSeconds: TimeInterval?,
         expiresStartedAtMs: Double?,
-        openGroupInvitationName: String?,
-        openGroupInvitationUrl: String?,
+        linkPreviewUrl: String?,
         openGroupServerMessageId: Int64?,
         openGroupWhisperMods: Bool,
         openGroupWhisperTo: String?
@@ -168,8 +178,7 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
         self.receivedAtTimestampMs = receivedAtTimestampMs
         self.expiresInSeconds = expiresInSeconds
         self.expiresStartedAtMs = expiresStartedAtMs
-        self.openGroupInvitationName = openGroupInvitationName
-        self.openGroupInvitationUrl = openGroupInvitationUrl
+        self.linkPreviewUrl = linkPreviewUrl
         self.openGroupServerMessageId = openGroupServerMessageId
         self.openGroupWhisperMods = openGroupWhisperMods
         self.openGroupWhisperTo = openGroupWhisperTo
@@ -180,6 +189,32 @@ public struct Interaction: Codable, FetchableRecord, MutablePersistableRecord, T
     public mutating func didInsert(with rowID: Int64, for column: String?) {
         self.id = rowID
     }
+    
+    public func delete(_ db: Database) throws -> Bool {
+        // If we have a LinkPreview then check if this is the only interaction that has it
+        // and delete the LinkPreview if so
+        if linkPreviewUrl != nil {
+            let interactionAlias: TableAlias = TableAlias()
+            let numInteractions: Int? = try? Interaction
+                .aliased(interactionAlias)
+                .joining(
+                    required: Interaction.linkPreview
+                        .filter(literal: [
+                            "(ROUND((\(interactionAlias[Columns.timestampMs]) / 1000 / 100000) - 0.5) * 100000)",
+                            "=",
+                            "\(LinkPreview.Columns.timestamp)"
+                        ].joined(separator: " "))
+                )
+                .fetchCount(db)
+            let tmp = try linkPreview.fetchAll(db)
+            
+            if numInteractions == 1 {
+                try linkPreview.deleteAll(db)
+            }
+        }
+        
+        return try performDelete(db)
+    }
 }
 
 // MARK: - Convenience

SessionMessagingKit/Database/Models/LinkPreview.swift

@@ -6,33 +6,48 @@ import SessionUtilitiesKit
 
 public struct LinkPreview: Codable, FetchableRecord, PersistableRecord, TableRecord, ColumnExpressible {
     public static var databaseTableName: String { "linkPreview" }
-    internal static let interactionForeignKey = ForeignKey([Columns.interactionId], to: [Interaction.Columns.id])
-    private static let interaction = belongsTo(Interaction.self, using: interactionForeignKey)
-    private static let attachment = hasOne(Attachment.self, using: Attachment.linkPreviewForeignKey)
+    internal static let interactionForeignKey = ForeignKey(
+        [Columns.url],
+        to: [Interaction.Columns.linkPreviewUrl]
+    )
+    internal static let interactions = hasMany(Interaction.self, using: Interaction.linkPreviewForeignKey)
+    
+    /// We want to cache url previews to the nearest 100,000 seconds (~28 hours - simpler than 86,400) to ensure the user isn't shown a preview that is too stale
+    internal static let timstampResolution: Double = 100000
     
     public typealias Columns = CodingKeys
     public enum CodingKeys: String, CodingKey, ColumnExpression {
         case url
-        case interactionId
+        case timestamp
+        case variant
         case title
     }
     
+    public enum Variant: Int, Codable, DatabaseValueConvertible {
+        case standard
+        case openGroupInvitation
+    }
+    
     /// The url for the link preview
     public let url: String
     
-    /// The id for the interaction this LinkPreview belongs to
-    public let interactionId: Int64
+    /// The number of seconds since epoch rounded down to the nearest 100,000 seconds (~day) - This
+    /// allows us to optimise against duplicate urls without having “stale” data last too long
+    public let timestamp: TimeInterval
+    
+    /// The type of link preview
+    public let variant: Variant
     
     /// The title for the link
     public let title: String?
-    
-    // MARK: - Relationships
-    
-    public var interaction: QueryInterfaceRequest<Interaction> {
-        request(for: LinkPreview.interaction)
-    }
-    
-    public var attachment: QueryInterfaceRequest<Attachment> {
-        request(for: LinkPreview.attachment)
+}
+
+// MARK: - Convenience
+
+public extension LinkPreview {
+    static func timestampFor(sentTimestampMs: Double) -> TimeInterval {
+        // We want to round the timestamp down to the nearest 100,000 seconds (~28 hours - simpler than 86,400) to optimise
+        // LinkPreview storage without having too stale data
+        return (floor(sentTimestampMs / 1000 / LinkPreview.timstampResolution) * LinkPreview.timstampResolution)
     }
 }

SessionMessagingKit/Database/Models/Quote.swift

@@ -4,7 +4,7 @@ import Foundation
 import GRDB
 import SessionUtilitiesKit
 
-public struct Quote: Codable, FetchableRecord, MutablePersistableRecord, TableRecord, ColumnExpressible {
+public struct Quote: Codable, FetchableRecord, PersistableRecord, TableRecord, ColumnExpressible {
     public static var databaseTableName: String { "quote" }
     internal static let interactionForeignKey = ForeignKey([Columns.interactionId], to: [Interaction.Columns.id])
     internal static let originalInteractionForeignKey = ForeignKey(
@@ -14,7 +14,6 @@ public struct Quote: Codable, FetchableRecord, MutablePersistableRecord, TableRe
     internal static let profileForeignKey = ForeignKey([Columns.authorId], to: [Profile.Columns.id])
     private static let interaction = belongsTo(Interaction.self, using: interactionForeignKey)
     private static let profile = hasOne(Profile.self, using: profileForeignKey)
-    private static let attachment = hasOne(Attachment.self, using: Attachment.quoteForeignKey)
     private static let quotedInteraction = hasOne(Interaction.self, using: originalInteractionForeignKey)
     
     public typealias Columns = CodingKeys
@@ -47,10 +46,6 @@ public struct Quote: Codable, FetchableRecord, MutablePersistableRecord, TableRe
         request(for: Quote.profile)
     }
     
-    public var attachment: QueryInterfaceRequest<Attachment> {
-        request(for: Quote.attachment)
-    }
-    
     public var originalInteraction: QueryInterfaceRequest<Interaction> {
         request(for: Quote.quotedInteraction)
     }

SessionMessagingKit/Sending & Receiving/Attachments/TSAttachmentStream.h

@@ -60,6 +60,7 @@ typedef void (^OWSThumbnailFailure)(void);
 
 - (BOOL)shouldHaveImageSize;
 - (CGSize)imageSize;
+- (CGSize)calculateImageSize;
 
 - (CGFloat)audioDurationSeconds;
 

SessionUtilitiesKit/General/Dictionary+Description.swift

@@ -16,6 +16,15 @@ public extension Dictionary {
     }
 }
 
+public extension Dictionary {
+    func setting(_ key: Key, _ value: Value?) -> [Key: Value] {
+        var updatedDictionary: [Key: Value] = self
+        updatedDictionary[key] = value
+
+        return updatedDictionary
+    }
+}
+
 public extension Dictionary.Values {
     func asArray() -> [Value] {
         return Array(self)