Split up GIF requests.

// FREEBIE

Signal/src/network/GiphyAPI.swift

@@ -428,8 +428,9 @@ extension GiphyError: LocalizedError {
         guard let height = parsePositiveUInt(dict:renditionDict, key:"height", typeName:"rendition") else {
             return nil
         }
-        // Be lenient when parsing file sizes - we don't require them for stills.
+        guard let fileSize = parsePositiveUInt(dict:renditionDict, key:"size", typeName:"rendition") else {
-        let fileSize = parseLenientUInt(dict:renditionDict, key:"size")
+            return nil
+        }
         guard let urlString = renditionDict["url"] as? String else {
             return nil
         }

Signal/src/network/GiphyDownloader.swift

@@ -10,11 +10,61 @@ enum GiphyRequestPriority {
     case low, high
 }
 
+enum GiphyAssetSegmentState: UInt {
+    case waiting
+    case active
+    case complete
+    case failed
+}
+
+class GiphyAssetSegment {
+    public let index: UInt
+    public let segmentStart: UInt
+    public let segmentLength: UInt
+    public var state: GiphyAssetSegmentState = .waiting
+    private var datas = [Data]()
+
+    init(index: UInt,
+         segmentStart: UInt,
+         segmentLength: UInt) {
+        self.index = index
+        self.segmentStart = segmentStart
+        self.segmentLength = segmentLength
+    }
+
+    public func append(data: Data) {
+        datas.append(data)
+    }
+
+    public func totalDataSize() -> UInt {
+        var result: UInt = 0
+        for data in datas {
+            result += UInt(data.count)
+        }
+        return result
+    }
+
+    public func mergeData(assetData: NSMutableData) {
+        for data in datas {
+            // TODO: In some cases we want to only append a subset.
+            assetData.append(data)
+        }
+    }
+}
+
+enum GiphyAssetRequestState: UInt {
+    case waiting
+    case active
+    case complete
+    case failed
+}
+
 // Represents a request to download a GIF.
 //
 // Should be cancelled if no longer necessary.
 @objc class GiphyAssetRequest: NSObject {
     static let TAG = "[GiphyAssetRequest]"
+    let TAG = "[GiphyAssetRequest]"
 
     let rendition: GiphyRendition
     let priority: GiphyRequestPriority
@@ -28,15 +78,119 @@ enum GiphyRequestPriority {
     // This property is an internal implementation detail of the download process.
     var assetFilePath: String?
 
+    private var segments = [GiphyAssetSegment]()
+    private var assetData = NSMutableData()
+    public var state: GiphyAssetRequestState = .waiting
+
     init(rendition: GiphyRendition,
          priority: GiphyRequestPriority,
          success:@escaping ((GiphyAssetRequest?, GiphyAsset) -> Void),
-         failure:@escaping ((GiphyAssetRequest) -> Void)
+         failure:@escaping ((GiphyAssetRequest) -> Void)) {
-        ) {
         self.rendition = rendition
         self.priority = priority
         self.success = success
         self.failure = failure
+
+        super.init()
+
+        createSegments()
+    }
+
+    private func segmentSize() -> UInt {
+        let fileSize = rendition.fileSize
+        guard fileSize > 0 else {
+            owsFail("\(TAG) rendition missing filesize")
+            requestDidFail()
+            return 0
+        }
+
+        let k1MB: UInt = 1024 * 1024
+        let k500KB: UInt = 500 * 1024
+        let k100KB: UInt = 100 * 1024
+        let k50KB: UInt = 50 * 1024
+        let k10KB: UInt = 10 * 1024
+        let k1KB: UInt = 1 * 1024
+        for segmentSize in [k1MB, k500KB, k100KB, k50KB, k10KB, k1KB ] {
+            if fileSize >= segmentSize {
+                return segmentSize
+            }
+        }
+        return fileSize
+    }
+
+    private func createSegments() {
+        let segmentLength = segmentSize()
+        guard segmentLength > 0 else {
+            return
+        }
+        let fileSize = rendition.fileSize
+
+        var nextSegmentStart: UInt = 0
+        var index: UInt = 0
+        while nextSegmentStart < fileSize {
+            var segmentStart: UInt = nextSegmentStart
+            // The last segment may overlap the penultimate segment
+            // in order to keep the segment sizes uniform.
+            if segmentStart + segmentLength > fileSize {
+                segmentStart = fileSize - segmentLength
+            }
+            segments.append(GiphyAssetSegment(index:index,
+                                              segmentStart:segmentStart,
+                                              segmentLength:segmentLength))
+            nextSegmentStart = segmentStart + segmentLength
+            index += 1
+        }
+    }
+
+    private func firstSegmentWithState(state: GiphyAssetSegmentState) -> GiphyAssetSegment? {
+        for segment in segments {
+            guard segment.state != .failed else {
+                owsFail("\(TAG) unexpected failed segment.")
+                continue
+            }
+            if segment.state == state {
+                return segment
+            }
+        }
+        return nil
+    }
+
+    public func firstWaitingSegment() -> GiphyAssetSegment? {
+        return firstSegmentWithState(state:.waiting)
+    }
+
+    public func firstActiveSegment() -> GiphyAssetSegment? {
+        return firstSegmentWithState(state:.active)
+    }
+
+    public func mergeCompleteSegment(segment: GiphyAssetSegment) {
+        guard segment.totalDataSize() > 0 else {
+            owsFail("\(TAG) could not merge empty segment.")
+            return
+        }
+        segment.mergeData(assetData:assetData)
+    }
+
+    public func writeAssetToFile() -> GiphyAsset? {
+        guard assetData.length > 0 else {
+            owsFail("\(TAG) could not write empty asset to disk.")
+            return nil
+        }
+
+        // We write assets to the temporary directory so that iOS can clean them up.
+        // We try to eagerly clean up these assets when they are no longer in use.
+        let dirPath = NSTemporaryDirectory()
+        let fileExtension = rendition.fileExtension
+        let fileName = (NSUUID().uuidString as NSString).appendingPathExtension(fileExtension)!
+        let filePath = (dirPath as NSString).appendingPathComponent(fileName)
+
+        let success = assetData.write(toFile: filePath, atomically: true)
+        guard success else {
+            owsFail("\(TAG) could not write asset to disk.")
+            return nil
+        }
+        let asset = GiphyAsset(rendition: rendition, filePath : filePath)
+        return asset
     }
 
     public func cancel() {
@@ -147,6 +301,7 @@ class LRUCache<KeyType: Hashable & Equatable, ValueType> {
 }
 
 private var URLSessionTaskGiphyAssetRequest: UInt8 = 0
+private var URLSessionTaskGiphyAssetSegment: UInt8 = 0
 
 // This extension is used to punch an asset request onto a download task.
 extension URLSessionTask {
@@ -158,9 +313,17 @@ extension URLSessionTask {
             objc_setAssociatedObject(self, &URLSessionTaskGiphyAssetRequest, newValue, objc_AssociationPolicy.OBJC_ASSOCIATION_RETAIN_NONATOMIC)
         }
     }
+    var assetSegment: GiphyAssetSegment {
+        get {
+            return objc_getAssociatedObject(self, &URLSessionTaskGiphyAssetSegment) as! GiphyAssetSegment
+        }
+        set {
+            objc_setAssociatedObject(self, &URLSessionTaskGiphyAssetSegment, newValue, objc_AssociationPolicy.OBJC_ASSOCIATION_RETAIN_NONATOMIC)
+        }
+    }
 }
 
-@objc class GiphyDownloader: NSObject, URLSessionTaskDelegate, URLSessionDownloadDelegate {
+@objc class GiphyDownloader: NSObject, URLSessionTaskDelegate, URLSessionDataDelegate {
 
     // MARK: - Properties
 
@@ -201,11 +364,11 @@ extension URLSessionTask {
     // list.
     private var assetRequestQueue = [GiphyAssetRequest]()
     private let kMaxAssetRequestCount = 3
-    private var activeAssetRequests = Set<GiphyAssetRequest>()
+//    private var activeAssetRequests = Set<GiphyAssetRequest>()
 
     // The success and failure callbacks are always called on main queue.
     //
-    // The success callbacks may be called synchronously on cache hit, in 
+    // The success callbacks may be called synchronously on cache hit, in
     // which case the GiphyAssetRequest parameter will be nil.
     public func requestAsset(rendition: GiphyRendition,
                              priority: GiphyRequestPriority,
@@ -227,72 +390,143 @@ extension URLSessionTask {
                                              success:success,
                                              failure:failure)
         assetRequestQueue.append(assetRequest)
-        startRequestIfNecessary()
+        processRequestQueue()
         return assetRequest
     }
 
     public func cancelAllRequests() {
+        AssertIsOnMainThread()
+
         self.assetRequestQueue.forEach { $0.cancel() }
+        self.assetRequestQueue = []
+    }
+
+    private func segmentRequestDidSucceed(assetRequest: GiphyAssetRequest, assetSegment: GiphyAssetSegment) {
+        Logger.verbose("\(self.TAG) segment request succeeded \(assetRequest.rendition.url), \(assetSegment.index), \(assetSegment.segmentStart), \(assetSegment.segmentLength)")
+
+        DispatchQueue.main.async {
+            assetSegment.state = .complete
+            // TODO: Should we move this merge off main thread?
+            assetRequest.mergeCompleteSegment(segment : assetSegment)
+
+            // If the asset request has completed all of its segments,
+            // try to write the asset to file.
+            if assetRequest.firstWaitingSegment() == nil {
+                assetRequest.state = .complete
+
+                // Move write off main thread.
+                DispatchQueue.global().async {
+                    guard let asset = assetRequest.writeAssetToFile() else {
+                        self.segmentRequestDidFail(assetRequest:assetRequest, assetSegment:assetSegment)
+                        return
+                    }
+                    self.assetRequestDidSucceed(assetRequest: assetRequest, asset: asset)
+                }
+            } else {
+                self.processRequestQueue()
+            }
+        }
     }
 
     private func assetRequestDidSucceed(assetRequest: GiphyAssetRequest, asset: GiphyAsset) {
+        Logger.verbose("\(self.TAG) asset request succeeded \(assetRequest.rendition.url)")
+
         DispatchQueue.main.async {
             self.assetMap.set(key:assetRequest.rendition.url, value:asset)
-            self.activeAssetRequests.remove(assetRequest)
+            self.removeAssetRequestFromQueue(assetRequest:assetRequest)
             assetRequest.requestDidSucceed(asset:asset)
-            self.startRequestIfNecessary()
+            self.processRequestQueue()
+        }
+    }
+
+    // TODO: If we wanted to implement segment retry, we'll need to add
+    //       a segmentRequestDidFail() method.
+    private func segmentRequestDidFail(assetRequest: GiphyAssetRequest, assetSegment: GiphyAssetSegment) {
+        Logger.verbose("\(self.TAG) segment request failed \(assetRequest.rendition.url), \(assetSegment.index), \(assetSegment.segmentStart), \(assetSegment.segmentLength)")
+
+        DispatchQueue.main.async {
+            assetSegment.state = .failed
+            assetRequest.state = .failed
+            self.assetRequestDidFail(assetRequest:assetRequest)
         }
     }
 
     private func assetRequestDidFail(assetRequest: GiphyAssetRequest) {
+        Logger.verbose("\(self.TAG) asset request failed \(assetRequest.rendition.url)")
+
         DispatchQueue.main.async {
-            self.activeAssetRequests.remove(assetRequest)
+            self.removeAssetRequestFromQueue(assetRequest:assetRequest)
             assetRequest.requestDidFail()
-            self.startRequestIfNecessary()
+            self.processRequestQueue()
+        }
+    }
+
+    private func removeAssetRequestFromQueue(assetRequest: GiphyAssetRequest) {
+        AssertIsOnMainThread()
+
+        guard assetRequestQueue.contains(assetRequest) else {
+            Logger.warn("\(TAG) could not remove asset request from queue: \(assetRequest.rendition.url)")
+            return
         }
+
+        assetRequestQueue = assetRequestQueue.filter { $0 != assetRequest }
     }
 
-    private func startRequestIfNecessary() {
+    // Start a request if necessary, complete asset requests if possible.
+    private func processRequestQueue() {
         AssertIsOnMainThread()
 
         DispatchQueue.main.async {
-            guard self.activeAssetRequests.count < self.kMaxAssetRequestCount else {
-                return
-            }
             guard let assetRequest = self.popNextAssetRequest() else {
                 return
             }
             guard !assetRequest.wasCancelled else {
                 // Discard the cancelled asset request and try again.
-                self.startRequestIfNecessary()
+                self.processRequestQueue()
                 return
             }
             guard UIApplication.shared.applicationState == .active else {
                 // If app is not active, fail the asset request.
+                assetRequest.state = .failed
                 self.assetRequestDidFail(assetRequest:assetRequest)
-                self.startRequestIfNecessary()
+                self.processRequestQueue()
                 return
             }
 
-            self.activeAssetRequests.insert(assetRequest)
-
             if let asset = self.assetMap.get(key:assetRequest.rendition.url) {
                 // Deferred cache hit, avoids re-downloading assets that were
                 // downloaded while this request was queued.
 
+                assetRequest.state = .complete
                 self.assetRequestDidSucceed(assetRequest : assetRequest, asset: asset)
                 return
             }
 
             guard let downloadSession = self.giphyDownloadSession() else {
                 owsFail("\(self.TAG) Couldn't create session manager.")
+                assetRequest.state = .failed
                 self.assetRequestDidFail(assetRequest:assetRequest)
                 return
             }
 
             // Start a download task.
-            let task = downloadSession.downloadTask(with:assetRequest.rendition.url as URL)
+
+            guard let assetSegment = assetRequest.firstWaitingSegment() else {
+                owsFail("\(self.TAG) queued asset request does not have a waiting segment.")
+                return
+            }
+            assetSegment.state = .active
+            assetRequest.state = .active
+
+            Logger.verbose("\(self.TAG) new segment request \(assetRequest.rendition.url), \(assetSegment.index), \(assetSegment.segmentStart), \(assetSegment.segmentLength)")
+
+            var request = URLRequest(url: assetRequest.rendition.url as URL)
+            let rangeHeaderValue = "bytes=\(assetSegment.segmentStart)-\(assetSegment.segmentStart + assetSegment.segmentLength - 1)"
+            Logger.verbose("\(self.TAG) rangeHeaderValue: \(rangeHeaderValue)")
+            request.addValue(rangeHeaderValue, forHTTPHeaderField: "Range")
+            let task = downloadSession.dataTask(with:request)
             task.assetRequest = assetRequest
+            task.assetSegment = assetSegment
             task.resume()
         }
     }
@@ -300,20 +534,24 @@ extension URLSessionTask {
     private func popNextAssetRequest() -> GiphyAssetRequest? {
         AssertIsOnMainThread()
 
-        var activeAssetRequestURLs = Set<NSURL>()
-        for assetRequest in activeAssetRequests {
-            activeAssetRequestURLs.insert(assetRequest.rendition.url)
-        }
-
         // Prefer the first "high" priority request;
         // fall back to the first "low" priority request.
+        var activeAssetRequestsCount = 0
         for priority in [GiphyRequestPriority.high, GiphyRequestPriority.low] {
-            for (assetRequestIndex, assetRequest) in assetRequestQueue.enumerated() where assetRequest.priority == priority {
+            for assetRequest in assetRequestQueue where assetRequest.priority == priority {
-                guard !activeAssetRequestURLs.contains(assetRequest.rendition.url) else {
+                guard assetRequest.state == .waiting || assetRequest.state ==  .active else {
-                    // Defer requests if there is already an active asset request with the same URL.
+                    continue
+                }
+
+                guard assetRequest.firstActiveSegment() == nil else {
+                    activeAssetRequestsCount += 1
+                    // Ensure that only N requests are active at a time.
+                    guard activeAssetRequestsCount < self.kMaxAssetRequestCount else {
+                        return nil
+                    }
+
                     continue
                 }
-                assetRequestQueue.remove(at:assetRequestIndex)
                 return assetRequest
             }
         }
@@ -329,105 +567,95 @@ extension URLSessionTask {
         completionHandler(.allow)
     }
 
+    public func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
+        let assetRequest = dataTask.assetRequest
+        let assetSegment = dataTask.assetSegment
+        Logger.verbose("\(TAG) session dataTask didReceive: \(data.count) \(assetRequest.rendition.url)")
+        guard !assetRequest.wasCancelled else {
+            dataTask.cancel()
+            segmentRequestDidFail(assetRequest:assetRequest, assetSegment:assetSegment)
+            return
+        }
+        assetSegment.append(data:data)
+    }
+
+    public func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, willCacheResponse proposedResponse: CachedURLResponse, completionHandler: @escaping (CachedURLResponse?) -> Swift.Void) {
+        completionHandler(nil)
+    }
+
     // MARK: URLSessionTaskDelegate
 
     public func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
+//        owsFail("\(TAG) session task didCompleteWithError \(error)")
+        Logger.verbose("\(TAG) session task didCompleteWithError \(error)")
+
         let assetRequest = task.assetRequest
+        let assetSegment = task.assetSegment
         guard !assetRequest.wasCancelled else {
             task.cancel()
-            assetRequestDidFail(assetRequest:assetRequest)
+            segmentRequestDidFail(assetRequest:assetRequest, assetSegment:assetSegment)
             return
         }
         if let error = error {
             Logger.error("\(TAG) download failed with error: \(error)")
-            assetRequestDidFail(assetRequest:assetRequest)
+            segmentRequestDidFail(assetRequest:assetRequest, assetSegment:assetSegment)
             return
         }
         guard let httpResponse = task.response as? HTTPURLResponse else {
             Logger.error("\(TAG) missing or unexpected response: \(task.response)")
-            assetRequestDidFail(assetRequest:assetRequest)
+            segmentRequestDidFail(assetRequest:assetRequest, assetSegment:assetSegment)
             return
         }
         let statusCode = httpResponse.statusCode
         guard statusCode >= 200 && statusCode < 400 else {
             Logger.error("\(TAG) response has invalid status code: \(statusCode)")
-            assetRequestDidFail(assetRequest:assetRequest)
+            segmentRequestDidFail(assetRequest:assetRequest, assetSegment:assetSegment)
             return
         }
-        guard let assetFilePath = assetRequest.assetFilePath else {
+        guard assetSegment.totalDataSize() == assetSegment.segmentLength else {
-            Logger.error("\(TAG) task is missing asset file")
+            Logger.error("\(TAG) segment is missing data: \(statusCode)")
-            assetRequestDidFail(assetRequest:assetRequest)
+            segmentRequestDidFail(assetRequest:assetRequest, assetSegment:assetSegment)
-            return
-        }
-        let asset = GiphyAsset(rendition: assetRequest.rendition, filePath : assetFilePath)
-        assetRequestDidSucceed(assetRequest : assetRequest, asset: asset)
-    }
-
-    // MARK: URLSessionDownloadDelegate
-
-    public func urlSession(_ session: URLSession, downloadTask: URLSessionDownloadTask, didFinishDownloadingTo location: URL) {
-        let assetRequest = downloadTask.assetRequest
-        guard !assetRequest.wasCancelled else {
-            downloadTask.cancel()
-            assetRequestDidFail(assetRequest:assetRequest)
             return
         }
 
-        // We write assets to the temporary directory so that iOS can clean them up.
+        segmentRequestDidSucceed(assetRequest : assetRequest, assetSegment: assetSegment)
-        // We try to eagerly clean up these assets when they are no longer in use.
-        let dirPath = NSTemporaryDirectory()
-        let fileExtension = assetRequest.rendition.fileExtension
-        let fileName = (NSUUID().uuidString as NSString).appendingPathExtension(fileExtension)!
-        let filePath = (dirPath as NSString).appendingPathComponent(fileName)
-
-        do {
-            try FileManager.default.moveItem(at: location, to: URL(fileURLWithPath:filePath))
-            assetRequest.assetFilePath = filePath
-        } catch let error as NSError {
-            owsFail("\(GiphyAsset.TAG) file move failed from: \(location), to: \(filePath), \(error)")
-        }
     }
 
-    var animatedDataCount = [URLSessionDownloadTask: Int64]()
+    // MARK: URLSessionDownloadDelegate
-    var stillDataCount = [URLSessionDownloadTask: Int64]()
-    var totalDataCount = [URLSessionDownloadTask: Int64]()
-    public func urlSession(_ session: URLSession, downloadTask: URLSessionDownloadTask, didWriteData bytesWritten: Int64, totalBytesWritten: Int64, totalBytesExpectedToWrite: Int64) {
-
-        // Log accumulated data usage in debug
-        if _isDebugAssertConfiguration() {
-            let assetRequest = downloadTask.assetRequest
-
-            totalDataCount[downloadTask] = totalBytesWritten
-            if assetRequest.rendition.isStill {
-                stillDataCount[downloadTask] = totalBytesWritten
-            } else {
-                animatedDataCount[downloadTask] = totalBytesWritten
-            }
-
-            let megabyteCount = { (dataCountMap: [URLSessionDownloadTask: Int64]) -> String in
-                let sum = dataCountMap.values.reduce(0, +)
-                let megabyteCount = Float(sum) / 1000 / 1000
-                return String(format: "%06.2f MB", megabyteCount)
-            }
-            Logger.info("\(TAG) Still bytes written:    \(megabyteCount(stillDataCount))")
-            Logger.info("\(TAG) Animated bytes written: \(megabyteCount(animatedDataCount))")
-            Logger.info("\(TAG) Total bytes written:    \(megabyteCount(totalDataCount))")
-        }
-
-        let assetRequest = downloadTask.assetRequest
-        guard !assetRequest.wasCancelled else {
-            downloadTask.cancel()
-            assetRequestDidFail(assetRequest:assetRequest)
-            return
-        }
-    }
 
-    public func urlSession(_ session: URLSession, downloadTask: URLSessionDownloadTask, didResumeAtOffset fileOffset: Int64, expectedTotalBytes: Int64) {
+//    var animatedDataCount = [URLSessionDownloadTask: Int64]()
-        let assetRequest = downloadTask.assetRequest
+//    var stillDataCount = [URLSessionDownloadTask: Int64]()
-        guard !assetRequest.wasCancelled else {
+//    var totalDataCount = [URLSessionDownloadTask: Int64]()
-            downloadTask.cancel()
+//    public func urlSession(_ session: URLSession, downloadTask: URLSessionDownloadTask, didWriteData bytesWritten: Int64, totalBytesWritten: Int64, totalBytesExpectedToWrite: Int64) {
-            assetRequestDidFail(assetRequest:assetRequest)
+//
-            return
+//        owsFail("\(TAG) session downloadTask didWriteData")
-        }
+//
-    }
+////        // Log accumulated data usage in debug
+////        if _isDebugAssertConfiguration() {
+////            let assetRequest = downloadTask.assetRequest
+////
+////            totalDataCount[downloadTask] = totalBytesWritten
+////            if assetRequest.rendition.isStill {
+////                stillDataCount[downloadTask] = totalBytesWritten
+////            } else {
+////                animatedDataCount[downloadTask] = totalBytesWritten
+////            }
+////
+////            let megabyteCount = { (dataCountMap: [URLSessionDownloadTask: Int64]) -> String in
+////                let sum = dataCountMap.values.reduce(0, +)
+////                let megabyteCount = Float(sum) / 1000 / 1000
+////                return String(format: "%06.2f MB", megabyteCount)
+////            }
+////            Logger.info("\(TAG) Still bytes written:    \(megabyteCount(stillDataCount))")
+////            Logger.info("\(TAG) Animated bytes written: \(megabyteCount(animatedDataCount))")
+////            Logger.info("\(TAG) Total bytes written:    \(megabyteCount(totalDataCount))")
+////        }
+////
+////        let assetRequest = downloadTask.assetRequest
+////        guard !assetRequest.wasCancelled else {
+////            downloadTask.cancel()
+////            assetRequestDidFail(assetRequest:assetRequest)
+////            return
+////        }
+//    }
 }