Modify new thumbnail system to include video and GIF thumbnails.

pull/1/head
Matthew Chen 7 years ago
parent 206432fdf0
commit 8748dc9b2e

@ -22,15 +22,15 @@ public struct MediaGalleryItem: Equatable, Hashable {
}
var isVideo: Bool {
return attachmentStream.isVideo()
return attachmentStream.isVideo
}
var isAnimated: Bool {
return attachmentStream.isAnimated()
return attachmentStream.isAnimated
}
var isImage: Bool {
return attachmentStream.isImage()
return attachmentStream.isImage
}
public typealias AsyncThumbnailBlock = (UIImage) -> Void
@ -39,7 +39,7 @@ public struct MediaGalleryItem: Equatable, Hashable {
}
var fullSizedImage: UIImage {
guard let image = attachmentStream.originalImage() else {
guard let image = attachmentStream.originalImage else {
owsFail("\(logTag) in \(#function) unexpectedly unable to build attachment image")
return UIImage()
}

@ -651,7 +651,7 @@ class MessageDetailViewController: OWSViewController, MediaGalleryDataSourceDele
func didTapAudioViewItem(_ viewItem: ConversationViewItem, attachmentStream: TSAttachmentStream) {
SwiftAssertIsOnMainThread(#function)
guard let mediaURL = attachmentStream.originalMediaURL() else {
guard let mediaURL = attachmentStream.originalMediaURL else {
owsFail("\(logTag) in \(#function) mediaURL was unexpectedly nil for attachment: \(attachmentStream)")
return
}

@ -0,0 +1,137 @@
//
// Copyright (c) 2018 Open Whisper Systems. All rights reserved.
//
import Foundation
import AVFoundation
public enum OWSMediaError: Error {
case failure(description: String)
}
@objc public class OWSMediaUtils: NSObject {
@available(*, unavailable, message:"do not instantiate this class.")
private override init() {
}
@objc public class func thumbnail(forImageAtPath path: String, maxDimension: CGFloat) throws -> UIImage {
guard FileManager.default.fileExists(atPath: path) else {
throw OWSMediaError.failure(description: "Media file missing.")
}
guard NSData.ows_isValidImage(atPath: path) else {
throw OWSMediaError.failure(description: "Invalid image.")
}
guard let originalImage = UIImage(contentsOfFile: path) else {
throw OWSMediaError.failure(description: "Could not load original image.")
}
let originalSize = originalImage.size
guard originalSize.width > 0 && originalSize.height > 0 else {
throw OWSMediaError.failure(description: "Original image has invalid size.")
}
var thumbnailSize = CGSize.zero
if originalSize.width > originalSize.height {
thumbnailSize.width = CGFloat(maxDimension)
thumbnailSize.height = round(CGFloat(maxDimension) * originalSize.height / originalSize.width)
} else {
thumbnailSize.width = round(CGFloat(maxDimension) * originalSize.width / originalSize.height)
thumbnailSize.height = CGFloat(maxDimension)
}
guard thumbnailSize.width > 0 && thumbnailSize.height > 0 else {
throw OWSMediaError.failure(description: "Thumbnail has invalid size.")
}
guard originalSize.width > thumbnailSize.width &&
originalSize.height > thumbnailSize.height else {
throw OWSMediaError.failure(description: "Thumbnail isn't smaller than the original.")
}
// We use UIGraphicsBeginImageContextWithOptions() to scale.
// Core Image would provide better quality (e.g. Lanczos) but
// at perf cost we don't want to pay. We could also use
// CoreGraphics directly, but I'm not sure there's any benefit.
guard let thumbnailImage = originalImage.resizedImage(to: thumbnailSize) else {
throw OWSMediaError.failure(description: "Could not thumbnail image.")
}
return thumbnailImage
}
// @objc public class func thumbnail(forImageAtPath path: String, maxDimension : CGFloat) throws -> UIImage {
// guard FileManager.default.fileExists(atPath: path) else {
// throw OWSMediaError.failure(description: "Media file missing.")
// }
// guard NSData.ows_isValidImage(atPath: path) else {
// throw OWSMediaError.failure(description: "Invalid image.")
// }
// let url = URL(fileURLWithPath: path)
// guard let imageSource = CGImageSourceCreateWithURL(url as CFURL, nil) else {
// throw OWSMediaError.failure(description: "Could not create image source.")
// }
// let imageOptions : [String :Any] = [
// kCGImageSourceCreateThumbnailFromImageIfAbsent as String: kCFBooleanTrue as NSNumber,
// kCGImageSourceThumbnailMaxPixelSize as String: maxDimension,
// kCGImageSourceCreateThumbnailWithTransform as String: kCFBooleanTrue as NSNumber]
// guard let thumbnail = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, imageOptions as CFDictionary) else {
// throw OWSMediaError.failure(description: "Could not create image thumbnail.")
// }
// let image = UIImage(cgImage: thumbnail)
// return image
// }
private static let kMaxVideoStillSize: CGFloat = 1024
@objc public class func thumbnail(forVideoAtPath path: String) throws -> UIImage {
return try thumbnail(forVideoAtPath: path, maxSize: CGSize(width: kMaxVideoStillSize, height: kMaxVideoStillSize))
}
@objc public class func thumbnail(forVideoAtPath path: String, maxSize: CGSize) throws -> UIImage {
var maxSize = maxSize
maxSize.width = min(maxSize.width, kMaxVideoStillSize)
maxSize.height = min(maxSize.height, kMaxVideoStillSize)
guard FileManager.default.fileExists(atPath: path) else {
throw OWSMediaError.failure(description: "Media file missing.")
}
let url = URL(fileURLWithPath: path)
let asset = AVURLAsset(url: url, options: nil)
guard isValidVideo(asset: asset) else {
throw OWSMediaError.failure(description: "Invalid video.")
}
let generator = AVAssetImageGenerator(asset: asset)
generator.maximumSize = maxSize
generator.appliesPreferredTrackTransform = true
let time: CMTime = CMTimeMake(1, 60)
let cgImage = try generator.copyCGImage(at: time, actualTime: nil)
let image = UIImage(cgImage: cgImage)
return image
}
@objc public class func isValidVideo(path: String) -> Bool {
guard FileManager.default.fileExists(atPath: path) else {
Logger.error("Media file missing.")
return false
}
let url = URL(fileURLWithPath: path)
let asset = AVURLAsset(url: url, options: nil)
return isValidVideo(asset: asset)
}
private class func isValidVideo(asset: AVURLAsset) -> Bool {
var maxTrackSize = CGSize.zero
for track: AVAssetTrack in asset.tracks(withMediaType: .video) {
let trackSize: CGSize = track.naturalSize
maxTrackSize.width = max(maxTrackSize.width, trackSize.width)
maxTrackSize.height = max(maxTrackSize.height, trackSize.height)
}
if maxTrackSize.width < 1.0 || maxTrackSize.height < 1.0 {
Logger.error("Invalid video size: \(maxTrackSize)")
return false
}
let kMaxValidSize: CGFloat = 3 * 1024.0
if maxTrackSize.width > kMaxValidSize || maxTrackSize.height > kMaxValidSize {
Logger.error("Invalid video dimensions: \(maxTrackSize)")
return false
}
return true
}
}

@ -3,6 +3,11 @@
//
import Foundation
import AVFoundation
public enum OWSThumbnailError: Error {
case failure(description: String)
}
@objc public class OWSLoadedThumbnail: NSObject {
public typealias DataSourceBlock = () throws -> Data
@ -76,16 +81,7 @@ private struct OWSThumbnailRequest {
}
private func canThumbnailAttachment(attachment: TSAttachmentStream) -> Bool {
guard attachment.isImage() else {
return false
}
guard !attachment.isAnimated() else {
return false
}
guard attachment.isValidImage() else {
return false
}
return true
return attachment.isImage || attachment.isAnimated || attachment.isVideo
}
// completion will only be called on success.
@ -122,11 +118,14 @@ private struct OWSThumbnailRequest {
}
let thumbnailRequest = thumbnailRequestStack.removeLast()
if let loadedThumbnail = process(thumbnailRequest: thumbnailRequest) {
do {
let loadedThumbnail = try process(thumbnailRequest: thumbnailRequest)
DispatchQueue.main.async {
thumbnailRequest.success(loadedThumbnail)
}
} else {
} catch {
Logger.error("Could not create thumbnail: \(error)")
DispatchQueue.main.async {
thumbnailRequest.failure()
}
@ -134,75 +133,46 @@ private struct OWSThumbnailRequest {
}
// This should only be called on the serialQueue.
private func process(thumbnailRequest: OWSThumbnailRequest) -> OWSLoadedThumbnail? {
private func process(thumbnailRequest: OWSThumbnailRequest) throws -> OWSLoadedThumbnail {
var possibleAttachment: TSAttachmentStream?
self.dbConnection.read({ (transaction) in
possibleAttachment = TSAttachmentStream.fetch(uniqueId: thumbnailRequest.attachmentId, transaction: transaction)
})
guard let attachment = possibleAttachment else {
Logger.warn("Could not load attachment for thumbnailing.")
return nil
throw OWSThumbnailError.failure(description: "Could not load attachment for thumbnailing.")
}
guard canThumbnailAttachment(attachment: attachment) else {
Logger.warn("Cannot thumbnail attachment.")
return nil
throw OWSThumbnailError.failure(description: "Cannot thumbnail attachment.")
}
if let thumbnails = attachment.thumbnails {
for thumbnail in thumbnails {
if thumbnail.thumbnailDimensionPoints == thumbnailRequest.thumbnailDimensionPoints {
guard let filePath = attachment.path(for: thumbnail) else {
owsFail("Could not determine thumbnail path.")
return nil
throw OWSThumbnailError.failure(description: "Could not determine thumbnail path.")
}
guard let image = UIImage(contentsOfFile: filePath) else {
owsFail("Could not load thumbnail.")
return nil
throw OWSThumbnailError.failure(description: "Could not load thumbnail.")
}
return OWSLoadedThumbnail(image: image, filePath: filePath)
}
}
}
guard let originalFilePath = attachment.originalFilePath() else {
owsFail("Could not determine thumbnail path.")
return nil
}
guard let originalImage = UIImage(contentsOfFile: originalFilePath) else {
owsFail("Could not load original image.")
return nil
}
let originalSize = originalImage.size
guard originalSize.width > 0 && originalSize.height > 0 else {
owsFail("Original image has invalid size.")
return nil
guard let originalFilePath = attachment.originalFilePath else {
throw OWSThumbnailError.failure(description: "Missing original file path.")
}
var thumbnailSize = CGSize.zero
if originalSize.width > originalSize.height {
thumbnailSize.width = CGFloat(thumbnailRequest.thumbnailDimensionPoints)
thumbnailSize.height = round(CGFloat(thumbnailRequest.thumbnailDimensionPoints) * originalSize.height / originalSize.width)
let maxDimension = CGFloat(thumbnailRequest.thumbnailDimensionPoints)
let thumbnailImage: UIImage
if attachment.isImage || attachment.isAnimated {
thumbnailImage = try OWSMediaUtils.thumbnail(forImageAtPath: originalFilePath, maxDimension: maxDimension)
} else if attachment.isVideo {
let maxSize = CGSize(width: maxDimension, height: maxDimension)
thumbnailImage = try OWSMediaUtils.thumbnail(forVideoAtPath: originalFilePath, maxSize: maxSize)
} else {
thumbnailSize.width = round(CGFloat(thumbnailRequest.thumbnailDimensionPoints) * originalSize.width / originalSize.height)
thumbnailSize.height = CGFloat(thumbnailRequest.thumbnailDimensionPoints)
}
guard thumbnailSize.width > 0 && thumbnailSize.height > 0 else {
owsFail("Thumbnail has invalid size.")
return nil
}
guard originalSize.width > thumbnailSize.width &&
originalSize.height > thumbnailSize.height else {
owsFail("Thumbnail isn't smaller than the original.")
return nil
}
// We use UIGraphicsBeginImageContextWithOptions() to scale.
// Core Image would provide better quality (e.g. Lanczos) but
// at perf cost we don't want to pay. We could also use
// CoreGraphics directly, but I'm not sure there's any benefit.
guard let thumbnailImage = originalImage.resizedImage(to: thumbnailSize) else {
owsFail("Could not thumbnail image.")
return nil
throw OWSThumbnailError.failure(description: "Invalid attachment type.")
}
let thumbnailSize = thumbnailImage.size
guard let thumbnailData = UIImageJPEGRepresentation(thumbnailImage, 0.85) else {
owsFail("Could not convert thumbnail to JPEG.")
return nil
throw OWSThumbnailError.failure(description: "Could not convert thumbnail to JPEG.")
}
let temporaryDirectory = NSTemporaryDirectory()
let thumbnailFilename = "\(NSUUID().uuidString).jpg"
@ -210,8 +180,7 @@ private struct OWSThumbnailRequest {
do {
try thumbnailData.write(to: NSURL.fileURL(withPath: thumbnailFilePath), options: .atomicWrite)
} catch let error as NSError {
owsFail("File write failed: \(thumbnailFilePath), \(error)")
return nil
throw OWSThumbnailError.failure(description: "File write failed: \(thumbnailFilePath), \(error)")
}
// It should be safe to assume that an attachment will never end up with two thumbnails of
// the same size since:

@ -58,14 +58,14 @@ typedef void (^OWSThumbnailFailure)(void);
- (nullable NSData *)validStillImageData;
#endif
- (BOOL)isAnimated;
- (BOOL)isImage;
- (BOOL)isVideo;
- (BOOL)isAudio;
@property (nonatomic, readonly) BOOL isAnimated;
@property (nonatomic, readonly) BOOL isImage;
@property (nonatomic, readonly) BOOL isVideo;
@property (nonatomic, readonly) BOOL isAudio;
- (nullable UIImage *)originalImage;
- (nullable NSString *)originalFilePath;
- (nullable NSURL *)originalMediaURL;
@property (nonatomic, readonly, nullable) UIImage *originalImage;
@property (nonatomic, readonly, nullable) NSString *originalFilePath;
@property (nonatomic, readonly, nullable) NSURL *originalMediaURL;
- (NSArray<NSString *> *)allThumbnailPaths;
@ -100,7 +100,7 @@ typedef void (^OWSThumbnailFailure)(void);
// On cache miss, nil will be returned and the completion will be invoked async on main if
// thumbnail can be generated.
- (nullable UIImage *)thumbnailImageWithSizeHint:(CGSize)sizeHint
completion:(OWSThumbnailSuccess)success
success:(OWSThumbnailSuccess)success
failure:(OWSThumbnailFailure)failure;
- (nullable UIImage *)thumbnailImageSmallWithSuccess:(OWSThumbnailSuccess)success failure:(OWSThumbnailFailure)failure;
- (nullable UIImage *)thumbnailImageMediumWithSuccess:(OWSThumbnailSuccess)success failure:(OWSThumbnailFailure)failure;

@ -14,8 +14,6 @@
NS_ASSUME_NONNULL_BEGIN
const CGFloat kMaxVideoStillSize = 1 * 1024;
const NSUInteger kThumbnailDimensionPointsSmall = 300;
const NSUInteger kThumbnailDimensionPointsMedium = 800;
// This size is large enough to render full screen.
@ -374,7 +372,7 @@ typedef void (^OWSLoadedThumbnailSuccess)(OWSLoadedThumbnail *loadedThumbnail);
{
OWSAssert(self.isVideo);
return [NSData ows_isValidVideoAtURL:self.originalMediaURL];
return [OWSMediaUtils isValidVideoWithPath:self.originalFilePath];
}
#pragma mark -
@ -422,108 +420,15 @@ typedef void (^OWSLoadedThumbnailSuccess)(OWSLoadedThumbnail *loadedThumbnail);
[MIMETypeUtil isAnimated:contentType]);
}
- (void)ensureLegacyThumbnail
{
NSString *thumbnailPath = self.legacyThumbnailPath;
if (!thumbnailPath) {
return;
}
if ([[NSFileManager defaultManager] fileExistsAtPath:thumbnailPath]) {
// already exists
return;
}
if (![[NSFileManager defaultManager] fileExistsAtPath:self.originalMediaURL.path]) {
DDLogError(@"%@ while generating thumbnail, source file doesn't exist: %@", self.logTag, self.originalMediaURL);
// If we're not lazy-restoring this message, the attachment should exist on disk.
OWSAssert(self.lazyRestoreFragmentId);
return;
}
// TODO proper resolution?
CGFloat thumbnailSize = 200;
UIImage *_Nullable result;
if (self.isImage || self.isAnimated) {
if (![self isValidImage]) {
DDLogWarn(
@"%@ skipping thumbnail generation for invalid image at path: %@", self.logTag, self.originalFilePath);
return;
}
CGImageSourceRef imageSource = CGImageSourceCreateWithURL((__bridge CFURLRef)self.originalMediaURL, NULL);
OWSAssert(imageSource != NULL);
NSDictionary *imageOptions = @{
(NSString const *)kCGImageSourceCreateThumbnailFromImageIfAbsent : (NSNumber const *)kCFBooleanTrue,
(NSString const *)kCGImageSourceThumbnailMaxPixelSize : @(thumbnailSize),
(NSString const *)kCGImageSourceCreateThumbnailWithTransform : (NSNumber const *)kCFBooleanTrue
};
CGImageRef thumbnail
= CGImageSourceCreateThumbnailAtIndex(imageSource, 0, (__bridge CFDictionaryRef)imageOptions);
CFRelease(imageSource);
result = [[UIImage alloc] initWithCGImage:thumbnail];
CGImageRelease(thumbnail);
} else if (self.isVideo) {
if (![self isValidVideo]) {
DDLogWarn(@"%@ skipping thumbnail for invalid video at path: %@", self.logTag, self.originalFilePath);
return;
}
result = [self videoStillImageWithMaxSize:CGSizeMake(thumbnailSize, thumbnailSize)];
} else {
OWSFail(@"%@ trying to generate thumnail for unexpected attachment: %@ of type: %@",
self.logTag,
self.uniqueId,
self.contentType);
}
if (result == nil) {
DDLogError(@"Unable to build thumbnail for attachmentId: %@", self.uniqueId);
return;
}
NSData *thumbnailData = UIImageJPEGRepresentation(result, 0.9);
OWSAssert(thumbnailData.length > 0);
DDLogDebug(@"%@ generated thumbnail with size: %lu", self.logTag, (unsigned long)thumbnailData.length);
[thumbnailData writeToFile:thumbnailPath atomically:YES];
}
- (nullable UIImage *)videoStillImage
{
if (![self isValidVideo]) {
return nil;
}
// Uses the assets intrinsic size by default
return [self videoStillImageWithMaxSize:CGSizeMake(kMaxVideoStillSize, kMaxVideoStillSize)];
}
- (nullable UIImage *)videoStillImageWithMaxSize:(CGSize)maxSize
{
maxSize.width = MIN(maxSize.width, kMaxVideoStillSize);
maxSize.height = MIN(maxSize.height, kMaxVideoStillSize);
NSURL *_Nullable mediaUrl = self.originalMediaURL;
if (!mediaUrl) {
return nil;
}
AVURLAsset *asset = [[AVURLAsset alloc] initWithURL:mediaUrl options:nil];
AVAssetImageGenerator *generator = [[AVAssetImageGenerator alloc] initWithAsset:asset];
generator.maximumSize = maxSize;
generator.appliesPreferredTrackTransform = YES;
NSError *err = NULL;
CMTime time = CMTimeMake(1, 60);
CGImageRef imgRef = [generator copyCGImageAtTime:time actualTime:NULL error:&err];
if (imgRef == NULL) {
DDLogError(@"Could not generate video still: %@", self.originalFilePath.pathExtension);
NSError *error;
UIImage *_Nullable image = [OWSMediaUtils thumbnailForVideoAtPath:self.originalFilePath error:&error];
if (error || !image) {
DDLogError(@"Could not create video still: %@.", error);
return nil;
}
return [[UIImage alloc] initWithCGImage:imgRef];
return image;
}
+ (void)deleteAttachments

@ -7,6 +7,7 @@
#import "NSData+Image.h"
#import "NSString+SSK.h"
#import "OWSFileSystem.h"
#import <SignalServiceKit/SignalServiceKit-Swift.h>
NS_ASSUME_NONNULL_BEGIN
@ -75,7 +76,7 @@ NS_ASSUME_NONNULL_BEGIN
- (BOOL)isValidVideo
{
return [NSData ows_isValidVideoAtURL:self.dataUrl];
return [OWSMediaUtils isValidVideoWithPath:self.dataUrl.path];
}
- (void)setSourceFilename:(nullable NSString *)sourceFilename

@ -11,6 +11,4 @@
- (BOOL)ows_isValidImage;
- (BOOL)ows_isValidImageWithMimeType:(nullable NSString *)mimeType;
+ (BOOL)ows_isValidVideoAtURL:(NSURL *)url;
@end

@ -261,27 +261,4 @@ typedef NS_ENUM(NSInteger, ImageFormat) {
return (width > 0 && width < kMaxValidSize && height > 0 && height < kMaxValidSize);
}
+ (BOOL)ows_isValidVideoAtURL:(NSURL *)url
{
OWSAssert(url);
AVURLAsset *asset = [[AVURLAsset alloc] initWithURL:url options:nil];
CGSize maxSize = CGSizeZero;
for (AVAssetTrack *track in [asset tracksWithMediaType:AVMediaTypeVideo]) {
CGSize trackSize = track.naturalSize;
maxSize.width = MAX(maxSize.width, trackSize.width);
maxSize.height = MAX(maxSize.height, trackSize.height);
}
if (maxSize.width < 1.f || maxSize.height < 1.f) {
DDLogError(@"Invalid video size: %@", NSStringFromCGSize(maxSize));
return NO;
}
const CGFloat kMaxSize = 3 * 1024.f;
if (maxSize.width > kMaxSize || maxSize.height > kMaxSize) {
DDLogError(@"Invalid video dimensions: %@", NSStringFromCGSize(maxSize));
return NO;
}
return YES;
}
@end

Loading…
Cancel
Save