Merge branch 'mkirk/filtering-high-glyph-chars'

Signal/src/Models/TSMessageAdapaters/TSMessageAdapter.m

@@ -121,7 +121,7 @@
     if ([interaction isKindOfClass:[TSIncomingMessage class]] ||
         [interaction isKindOfClass:[TSOutgoingMessage class]]) {
         TSMessage *message  = (TSMessage *)interaction;
-        adapter.messageBody = message.body;
+        adapter.messageBody = [[DisplayableTextFilter new] displayableText:message.body];
 
         if ([message hasAttachments]) {
             for (NSString *attachmentID in message.attachmentIds) {
@@ -167,11 +167,6 @@
                                NSStringFromClass([attachment class]));
                 }
             }
-        } else { // no attachment, plain text message
-            if ([[DisplayableTextFilter new] shouldPreventDisplayOfText:adapter.messageBody]) {
-                adapter.messageType = TSInfoMessageAdapter;
-                adapter.messageBody = NSLocalizedString(@"INFO_MESSAGE_UNABLE_TO_DISPLAY_MESSAGE", @"Generic error text when message contents are undisplayable");
-            }
         }
     } else if ([interaction isKindOfClass:[TSCall class]]) {
         TSCall *callRecord = (TSCall *)interaction;

Signal/src/util/DisplayableTextFilter.swift

@@ -6,36 +6,34 @@ import Foundation
 
 @objc class DisplayableTextFilter: NSObject {
 
-    // don't bother filtering on small text, lest we inadvertently catch legitimate usage of rare code point stacking
+    let TAG = "[DisplayableTextFilter]"
-    let allowAnyTextLessThanByteSize: Int
 
-    convenience override init() {
+    @objc
-        self.init(allowAnyTextLessThanByteSize: 10000)
+    func displayableText(_ text: String?) -> String? {
-    }
-
-    required init(allowAnyTextLessThanByteSize: Int) {
-        self.allowAnyTextLessThanByteSize = allowAnyTextLessThanByteSize
-    }
-
-    @objc(shouldPreventDisplayOfText:)
-    func shouldPreventDisplay(text: String?) -> Bool {
         guard let text = text else {
-            return false
+            return nil
         }
 
-        let byteCount = text.lengthOfBytes(using: .utf8)
+        if (self.hasExcessiveDiacriticals(text: text)) {
-
+            Logger.warn("\(TAG) filtering text for excessive diacriticals.")
-        guard byteCount >= allowAnyTextLessThanByteSize else {
+            let filteredText = text.folding(options: .diacriticInsensitive, locale: .current)
-            return false
+            assert(!self.hasExcessiveDiacriticals(filteredText))
+            return filteredText
         }
 
-        let characterCount = text.characters.count
+        return text
-        // discard any zalgo style text, which we detect by enforcing avg bytes per character ratio.
+    }
-        if byteCount / characterCount > 10 {
+
-            Logger.warn("filtering undisplayable text bytes: \(byteCount), characterCount: \(characterCount)")
+    private func hasExcessiveDiacriticals(text: String) -> Bool {
-            return true
+        // discard any zalgo style text, by detecting maximum number of glyphs per character
-        } else {
+        for char in text.characters.enumerated() {
-            return false
+            let scalarCount = String(char.element).unicodeScalars.count
+            if scalarCount > 4 {
+                Logger.warn("\(TAG) detected excessive diacriticals at \(char.element) scalarCount: \(scalarCount)")
+                return true
+            }
         }
+
+        return false
     }
 }

Signal/src/view controllers/InboxTableViewCell.m

@@ -63,12 +63,8 @@ NS_ASSUME_NONNULL_BEGIN
     }
     UIImage *avatar = [OWSAvatarBuilder buildImageForThread:thread contactsManager:contactsManager];
     self.threadId = thread.uniqueId;
-    NSString *snippetLabel;
+    NSString *snippetLabel = [[DisplayableTextFilter new] displayableText:thread.lastMessageLabel];
-    if ([[DisplayableTextFilter new] shouldPreventDisplayOfText:thread.lastMessageLabel]) {
+
-        snippetLabel = NSLocalizedString(@"INFO_MESSAGE_UNABLE_TO_DISPLAY_MESSAGE", @"Generic error text when message contents are undisplayable");
-    } else {
-        snippetLabel = thread.lastMessageLabel;
-    }
     NSAttributedString *attributedDate = [self dateAttributedString:thread.lastMessageDate];
     NSUInteger unreadCount             = [[TSMessagesManager sharedManager] unreadMessagesInThread:thread];
 

Signal/test/util/DisplayableTextFilterTest.swift

@@ -16,12 +16,25 @@ class DisplayableTextFilterTest: XCTestCase {
         super.tearDown()
     }
 
-    func testFiltering() {
+    func testDisplayableText() {
         // Ignore default byte size limitations to test other filtering behaviors
-        let filter = DisplayableTextFilter(allowAnyTextLessThanByteSize: 0)
+        let filter = DisplayableTextFilter()
 
-        XCTAssertFalse( filter.shouldPreventDisplay(text: "normal text") )
+        // show plain text
-        XCTAssertFalse( filter.shouldPreventDisplay(text: "🇹🇹🌼🇹🇹🌼🇹🇹") )
+        let boringText = "boring text"
-        XCTAssertTrue( filter.shouldPreventDisplay(text: "L̷̳͔̲͝Ģ̵̮̯̤̩̙͍̬̟͉̹̘̹͍͈̮̦̰̣͟͝O̶̴̮̻̮̗͘͡!̴̷̟͓͓") )
+        XCTAssertEqual(boringText, filter.displayableText(boringText))
+
+        // show high byte emojis
+        let emojiText = "🇹🇹🌼🇹🇹🌼🇹🇹"
+        XCTAssertEqual(emojiText, filter.displayableText(emojiText))
+
+        // show normal diacritic usage
+        let diacriticalText = "Příliš žluťoučký kůň úpěl ďábelské ódy."
+        XCTAssertEqual(diacriticalText, filter.displayableText(diacriticalText))
+
+        // filter excessive diacritics
+        XCTAssertEqual("HAVING TROUBLE READING TEXT?", filter.displayableText("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢") )
+
+        XCTAssertEqual("LGO!", filter.displayableText("L̷̳͔̲͝Ģ̵̮̯̤̩̙͍̬̟͉̹̘̹͍͈̮̦̰̣͟͝O̶̴̮̻̮̗͘͡!̴̷̟͓͓"))
     }
 }