From b00e5a4fd962566fb27df470eb02b1f55e9a1761 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Fri, 8 Jun 2018 08:07:13 -0600 Subject: [PATCH] Fuzzier search matching -[] Backend -[] indexes e5.25 -[x] wire up results: Contacts / Conversations / Messages actual: 3hr -[ ] group thread est: actual: -[x] group name actual: e.25 -[ ] group member name: e.25 -[ ] group member number: e.25 -[ ] contact thread e.5 -[ ] name -[ ] number -[ ] messages e1 -[ ] content -[] Frontend e10.75 -[x] wire up VC's a.5 -[x] show search results only when search box has content a.25 -[] show search results: Contact / Conversation / Messages e2 -[x] wire up matchs -[] style contact cell -[] style conversation cell -[] style messages cell -[] tapping thread search result takes you to conversation e1 -[] tapping message search result takes you to message e1 -[] show snippet text for matched message e1 -[] highlight matched text in thread e3 -[] go to next search result in thread e2 -[] No Results page -[] Hide search unless pulled down --- .../HomeView/HomeViewController.m | 2 +- Signal/test/util/SearcherTest.swift | 138 ++++++++++++++---- .../src/Storage/FullTextSearchFinder.swift | 85 ++++++++++- 3 files changed, 191 insertions(+), 34 deletions(-) diff --git a/Signal/src/ViewControllers/HomeView/HomeViewController.m b/Signal/src/ViewControllers/HomeView/HomeViewController.m index 46338fe88..42cf6d2c7 100644 --- a/Signal/src/ViewControllers/HomeView/HomeViewController.m +++ b/Signal/src/ViewControllers/HomeView/HomeViewController.m @@ -311,7 +311,7 @@ NSString *const kArchivedConversationsReuseIdentifier = @"kArchivedConversations searchController.view.frame = self.view.frame; [self.view addSubview:searchController.view]; // TODO - better/more flexible way to pin below search bar? - [searchController.view autoPinEdgesToSuperviewEdgesWithInsets:UIEdgeInsetsMake(58, 0, 0, 0)]; + [searchController.view autoPinEdgesToSuperviewEdgesWithInsets:UIEdgeInsetsMake(60, 0, 0, 0)]; searchBar.delegate = searchController; OWSAssert(self.tableView.tableHeaderView == nil); diff --git a/Signal/test/util/SearcherTest.swift b/Signal/test/util/SearcherTest.swift index 907cb79a9..924bd26cb 100644 --- a/Signal/test/util/SearcherTest.swift +++ b/Signal/test/util/SearcherTest.swift @@ -8,7 +8,7 @@ import XCTest class ConversationSearcherTest: XCTestCase { - // Mark: Dependencies + // MARK: - Dependencies var searcher: ConversationSearcher { return ConversationSearcher.shared } @@ -17,68 +17,152 @@ class ConversationSearcherTest: XCTestCase { return OWSPrimaryStorage.shared().dbReadWriteConnection } - // Mark: Test Life Cycle + // MARK: - Test Life Cycle override func setUp() { super.setUp() FullTextSearchFinder.syncRegisterDatabaseExtension(storage: OWSPrimaryStorage.shared()) - } - - // Mark: Tests - - func testSearchByGroupName() { + TSContactThread.removeAllObjectsInCollection() TSGroupThread.removeAllObjectsInCollection() - var bookClubThread: ThreadViewModel! - var snackClubThread: ThreadViewModel! self.dbConnection.readWrite { transaction in let bookModel = TSGroupModel(title: "Book Club", memberIds: [], image: nil, groupId: Randomness.generateRandomBytes(16)) let bookClubGroupThread = TSGroupThread.getOrCreateThread(with: bookModel, transaction: transaction) - bookClubThread = ThreadViewModel(thread: bookClubGroupThread, transaction: transaction) + self.bookClubThread = ThreadViewModel(thread: bookClubGroupThread, transaction: transaction) let snackModel = TSGroupModel(title: "Snack Club", memberIds: [], image: nil, groupId: Randomness.generateRandomBytes(16)) let snackClubGroupThread = TSGroupThread.getOrCreateThread(with: snackModel, transaction: transaction) - snackClubThread = ThreadViewModel(thread: snackClubGroupThread, transaction: transaction) + self.snackClubThread = ThreadViewModel(thread: snackClubGroupThread, transaction: transaction) + + let aliceContactThread = TSContactThread.getOrCreateThread(withContactId: "+12345678900", transaction: transaction) + self.aliceThread = ThreadViewModel(thread: aliceContactThread, transaction: transaction) + + let bobContactThread = TSContactThread.getOrCreateThread(withContactId: "+49030183000", transaction: transaction) + self.bobThread = ThreadViewModel(thread: bobContactThread, transaction: transaction) } + } + + // MARK: - Fixtures + + var bookClubThread: ThreadViewModel! + var snackClubThread: ThreadViewModel! + + var aliceThread: ThreadViewModel! + var bobThread: ThreadViewModel! + + // MARK: Tests + + func testSearchByGroupName() { + + var resultSet: SearchResultSet = .empty // No Match - let noMatch = resultSet(searchText: "asdasdasd") - XCTAssert(noMatch.conversations.isEmpty) + resultSet = getResultSet(searchText: "asdasdasd") + XCTAssert(resultSet.conversations.isEmpty) // Partial Match - let bookMatch = resultSet(searchText: "Book") - XCTAssert(bookMatch.conversations.count == 1) - if let foundThread: ThreadViewModel = bookMatch.conversations.first?.thread { + resultSet = getResultSet(searchText: "Book") + XCTAssert(resultSet.conversations.count == 1) + if let foundThread: ThreadViewModel = resultSet.conversations.first?.thread { XCTAssertEqual(bookClubThread, foundThread) } else { XCTFail("no thread found") } - let snackMatch = resultSet(searchText: "Snack") - XCTAssert(snackMatch.conversations.count == 1) - if let foundThread: ThreadViewModel = snackMatch.conversations.first?.thread { + resultSet = getResultSet(searchText: "Snack") + XCTAssert(resultSet.conversations.count == 1) + if let foundThread: ThreadViewModel = resultSet.conversations.first?.thread { XCTAssertEqual(snackClubThread, foundThread) } else { XCTFail("no thread found") } // Multiple Partial Matches - let multipleMatch = resultSet(searchText: "Club") - XCTAssert(multipleMatch.conversations.count == 2) - XCTAssert(multipleMatch.conversations.map { $0.thread }.contains(bookClubThread)) - XCTAssert(multipleMatch.conversations.map { $0.thread }.contains(snackClubThread)) + resultSet = getResultSet(searchText: "Club") + XCTAssertEqual(2, resultSet.conversations.count) + XCTAssert(resultSet.conversations.map { $0.thread }.contains(bookClubThread)) + XCTAssert(resultSet.conversations.map { $0.thread }.contains(snackClubThread)) // Match Name Exactly - let exactMatch = resultSet(searchText: "Book Club") - XCTAssert(exactMatch.conversations.count == 1) - XCTAssertEqual(bookClubThread, exactMatch.conversations.first!.thread) + resultSet = getResultSet(searchText: "Book Club") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(bookClubThread, resultSet.conversations.first!.thread) + } + + func testSearchContactByNumber() { + var resultSet: SearchResultSet = .empty + + // No match + resultSet = getResultSet(searchText: "+5551239999") + XCTAssertEqual(0, resultSet.conversations.count) + + // Exact match + resultSet = getResultSet(searchText: "+12345678900") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + + // Partial match + resultSet = getResultSet(searchText: "+123456") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + + // Prefixes + resultSet = getResultSet(searchText: "12345678900") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + + resultSet = getResultSet(searchText: "49") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(bobThread, resultSet.conversations.first?.thread) + + resultSet = getResultSet(searchText: "1-234-56") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + + resultSet = getResultSet(searchText: "123456") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + + resultSet = getResultSet(searchText: "1.234.56") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + } + + // TODO + func pending_testSearchContactByNumber() { + var resultSet: SearchResultSet = .empty + + // Phone Number formatting should be forgiving + resultSet = getResultSet(searchText: "234.56") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + + resultSet = getResultSet(searchText: "234 56") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + } + + func testSearchContactByName() { + var resultSet: SearchResultSet = .empty + + resultSet = getResultSet(searchText: "Alice") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(aliceThread, resultSet.conversations.first?.thread) + + resultSet = getResultSet(searchText: "Bob") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(bobThread, resultSet.conversations.first?.thread) + + resultSet = getResultSet(searchText: "Barker") + XCTAssertEqual(1, resultSet.conversations.count) + XCTAssertEqual(bobThread, resultSet.conversations.first?.thread) } // Mark: Helpers - private func resultSet(searchText: String) -> SearchResultSet { + private func getResultSet(searchText: String) -> SearchResultSet { var results: SearchResultSet! self.dbConnection.read { transaction in results = self.searcher.results(searchText: searchText, transaction: transaction) diff --git a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift index 3cebd2d84..639389ec2 100644 --- a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift +++ b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift @@ -4,6 +4,20 @@ import Foundation +// Create a searchable index for objects of type T +public class SearchIndexer { + + private let indexBlock: (T) -> String + + public init(indexBlock: @escaping (T) -> String) { + self.indexBlock = indexBlock + } + + public func index(_ item: T) -> String { + return indexBlock(item) + } +} + @objc public class FullTextSearchFinder: NSObject { @@ -13,7 +27,13 @@ public class FullTextSearchFinder: NSObject { return } - ext.enumerateKeysAndObjects(matching: searchText) { (_, _, object, _) in + let normalized = FullTextSearchFinder.normalize(text: searchText) + + // We want a forgiving query for phone numbers + // TODO a stricter "whole word" query for body text? + let prefixQuery = "*\(normalized)*" + + ext.enumerateKeysAndObjects(matching: prefixQuery) { (_, _, object, _) in block(object) } } @@ -22,9 +42,60 @@ public class FullTextSearchFinder: NSObject { return transaction.ext(FullTextSearchFinder.dbExtensionName) as? YapDatabaseFullTextSearchTransaction } + // Mark: Index Building + + private class func normalize(text: String) -> String { + var normalized: String = text.trimmingCharacters(in: .whitespacesAndNewlines) + + // Remove any phone number formatting from the search terms + let nonformattingScalars = normalized.unicodeScalars.lazy.filter { + !CharacterSet.punctuationCharacters.contains($0) + } + + normalized = String(String.UnicodeScalarView(nonformattingScalars)) + + return normalized + } + + private static let groupThreadIndexer: SearchIndexer = SearchIndexer { (groupThread: TSGroupThread) in + let searchableContent = groupThread.groupModel.groupName ?? "" + + // TODO member names, member numbers + + return normalize(text: searchableContent) + } + + private static let contactThreadIndexer: SearchIndexer = SearchIndexer { (contactThread: TSContactThread) in + let searchableContent = contactThread.contactIdentifier() + + // TODO contact name + + return normalize(text: searchableContent) + } + + private static let contactIndexer: SearchIndexer = SearchIndexer { (recipientId: String) in + + let searchableContent = "\(recipientId)" + + // TODO contact name + + return normalize(text: searchableContent) + } + + private class func indexContent(object: Any) -> String? { + if let groupThread = object as? TSGroupThread { + return self.groupThreadIndexer.index(groupThread) + } else if let contactThread = object as? TSContactThread { + return self.contactThreadIndexer.index(contactThread) + } else { + return nil + } + } + // MARK: - Extension Registration - private static let dbExtensionName: String = "FullTextSearchFinderExtension" + // MJK - FIXME, remove dynamic name when done developing. + private static let dbExtensionName: String = "FullTextSearchFinderExtension\(Date())" @objc public class func asyncRegisterDatabaseExtension(storage: OWSStorage) { @@ -37,18 +108,20 @@ public class FullTextSearchFinder: NSObject { } private class var dbExtensionConfig: YapDatabaseFullTextSearch { + // TODO is it worth doing faceted search, i.e. Author / Name / Content? + // seems unlikely that mobile users would use the "author: Alice" search syntax. + // so for now, everything searchable is jammed into a single column let contentColumnName = "content" + let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (dict: NSMutableDictionary, _: String, _: String, object: Any) in - if let groupThread = object as? TSGroupThread { - dict[contentColumnName] = groupThread.groupModel.groupName + if let content: String = indexContent(object: object) { + dict[contentColumnName] = content } } // update search index on contact name changes? // update search index on message insertion? - // TODO is it worth doing faceted search, i.e. Author / Name / Content? - // seems unlikely that mobile users would use the "author: Alice" search syntax. return YapDatabaseFullTextSearch(columnNames: ["content"], handler: handler) } }