From 5db254303ae9cf929103e87b7fb289bd7a36621b Mon Sep 17 00:00:00 2001
From: Morgan Pretty <morgan.t.pretty@gmail.com>
Date: Tue, 20 Jun 2023 14:05:04 +1000
Subject: [PATCH] Removed the wip pruning logic (no longer needed)
Fixed broken unit tests
---
.../SessionUtil+Contacts.swift | 115 ------------------
.../Configs/ConfigContactsSpec.swift | 93 --------------
SessionUtilitiesKit/Database/Storage.swift | 2 +-
3 files changed, 1 insertion(+), 209 deletions(-)
diff --git a/SessionMessagingKit/SessionUtil/Config Handling/SessionUtil+Contacts.swift b/SessionMessagingKit/SessionUtil/Config Handling/SessionUtil+Contacts.swift
index 96b185793..f3d07f91d 100644
--- a/SessionMessagingKit/SessionUtil/Config Handling/SessionUtil+Contacts.swift
+++ b/SessionMessagingKit/SessionUtil/Config Handling/SessionUtil+Contacts.swift
@@ -443,121 +443,6 @@ internal extension SessionUtil {
return updated
}
-
- // MARK: - Pruning
-
- static func pruningIfNeeded(
- _ db: Database,
- conf: UnsafeMutablePointer<config_object>?
- ) throws {
- // First make sure we are actually thowing the correct size constraint error (don't want to prune contacts
- // as a result of some other random error
- do {
- try CExceptionHelper.performSafely { config_push(conf).deallocate() }
- return // If we didn't error then no need to prune
- }
- catch {
- guard (error as NSError).userInfo["NSLocalizedDescription"] as? String == "Config data is too large" else {
- throw error
- }
- }
-
- // Extract the contact data from the config
- var allContactData: [String: ContactData] = extractContacts(
- from: conf,
- latestConfigSentTimestampMs: 0
- )
-
- // Remove the current user profile info (shouldn't be in there but just in case)
- let userPublicKey: String = getUserHexEncodedPublicKey(db)
- var cUserPublicKey: [CChar] = userPublicKey.cArray.nullTerminated()
- contacts_erase(conf, &cUserPublicKey)
-
- /// Do the following in stages (we want to prune as few contacts as possible because we are essentially deleting data and removing these
- /// contacts will result in not just contact data but also associated conversation data for the contact being removed from linked devices
- ///
- ///
- /// **Step 1** First of all we want to try to detect spam-attacks (ie. if someone creates a bunch of accounts and messages you, and you
- /// systematically block every one of those accounts - this can quickly add up)
- ///
- /// We will do this by filtering the contact data to only include blocked contacts, grouping those contacts into contacts created within the
- /// same hour and then only including groups that have more than 10 contacts (ie. if you blocked 20 users within an hour we expect those
- /// contacts were spammers)
- let blockSpamBatchingResolution: TimeInterval = (60 * 60)
- // TODO: Do we want to only do this case for contacts that were created over X time ago? (to avoid unintentionally unblocking accounts that were recently blocked
- let likelySpammerContacts: [ContactData] = allContactData
- .values
- .filter { $0.contact.isBlocked }
- .grouped(by: { $0.created / blockSpamBatchingResolution })
- .filter { _, values in values.count > 20 }
- .values
- .flatMap { $0 }
-
- if !likelySpammerContacts.isEmpty {
- likelySpammerContacts.forEach { contact in
- var cSessionId: [CChar] = contact.contact.id.cArray.nullTerminated()
- contacts_erase(conf, &cSessionId)
-
- allContactData.removeValue(forKey: contact.contact.id)
- }
-
- // If we are no longer erroring then we can stop here
- do { return try CExceptionHelper.performSafely { config_push(conf).deallocate() } }
- catch {}
- }
-
- /// We retrieve the `CONVO_INFO_VOLATILE` records and one-to-one conversation message counts as they will be relevant for subsequent checks
- let volatileThreadInfo: [String: VolatileThreadInfo] = SessionUtil
- .config(for: .convoInfoVolatile, publicKey: userPublicKey)
- .wrappedValue
- .map { SessionUtil.extractConvoVolatileInfo(from: $0) }
- .defaulting(to: [])
- .reduce(into: [:]) { result, next in result[next.threadId] = next }
- let conversationMessageCounts: [String: Int] = try SessionThread
- .filter(SessionThread.Columns.variant == SessionThread.Variant.contact)
- .select(.id)
- .annotated(with: SessionThread.interactions.count)
- .asRequest(of: ThreadCount.self)
- .fetchAll(db)
- .reduce(into: [:]) { result, next in result[next.id] = next.interactionCount }
-
- /// **Step 2** Next up we want to remove contact records which are likely to be invalid, this means contacts which:
- /// - Aren't blocked
- /// - Have no `name` value
- /// - Have no `CONVO_INFO_VOLATILE` record
- /// - Have no messages in their one-to-one conversations
- ///
- /// Any contacts that meet the above criteria are likely either invalid contacts or are contacts which the user hasn't seen or interacted
- /// with for 30+ days
- let likelyInvalidContacts: [ContactData] = allContactData
- .values
- .filter { !$0.contact.isBlocked }
- .filter { $0.profile.name.isEmpty }
- .filter { volatileThreadInfo[$0.contact.id] == nil }
- .filter { (conversationMessageCounts[$0.contact.id] ?? 0) == 0 }
-
- if !likelyInvalidContacts.isEmpty {
- likelyInvalidContacts.forEach { contact in
- var cSessionId: [CChar] = contact.contact.id.cArray.nullTerminated()
- contacts_erase(conf, &cSessionId)
-
- allContactData.removeValue(forKey: contact.contact.id)
- }
-
- // If we are no longer erroring then we can stop here
- do { return try CExceptionHelper.performSafely { config_push(conf).deallocate() } }
- catch {}
- }
-
-
- // TODO: Exclude contacts that have no profile info(?)
- // TODO: Exclude contacts that have a CONVO_INFO_VOLATILE record
- // TODO: Exclude contacts that have a conversation with messages in the database (ie. only delete "empty" contacts)
-
- // TODO: Start pruning valid contacts which have really old conversations...
-
- print("RAWR")
- }
}
// MARK: - External Outgoing Changes
diff --git a/SessionMessagingKitTests/LibSessionUtil/Configs/ConfigContactsSpec.swift b/SessionMessagingKitTests/LibSessionUtil/Configs/ConfigContactsSpec.swift
index 666ca512d..a57839b28 100644
--- a/SessionMessagingKitTests/LibSessionUtil/Configs/ConfigContactsSpec.swift
+++ b/SessionMessagingKitTests/LibSessionUtil/Configs/ConfigContactsSpec.swift
@@ -74,35 +74,6 @@ class ConfigContactsSpec {
}
.to(throwError(NSError(domain: "cpp_exception", code: -2, userInfo: ["NSLocalizedDescription": "Config data is too large"])))
}
-
- // MARK: -- can catch size limit errors thrown when dumping
- it("can catch size limit errors thrown when dumping") {
- var randomGenerator: ARC4RandomNumberGenerator = ARC4RandomNumberGenerator(seed: 1000)
-
- try (0..<100000).forEach { index in
- var contact: contacts_contact = try createContact(
- for: index,
- in: conf,
- rand: &randomGenerator,
- maxing: .allProperties
- )
- contacts_set(conf, &contact)
- }
-
- expect(contacts_size(conf)).to(equal(100000))
- expect(config_needs_push(conf)).to(beTrue())
- expect(config_needs_dump(conf)).to(beTrue())
-
- expect {
- try CExceptionHelper.performSafely {
- var dump: UnsafeMutablePointer<UInt8>? = nil
- var dumpLen: Int = 0
- config_dump(conf, &dump, &dumpLen)
- dump?.deallocate()
- }
- }
- .to(throwError(NSError(domain: "cpp_exception", code: -2, userInfo: ["NSLocalizedDescription": "Config data is too large"])))
- }
}
// MARK: - when checking size limits
@@ -225,70 +196,6 @@ class ConfigContactsSpec {
}
}
- // MARK: - when pruning
- context("when pruning") {
- var mockStorage: Storage!
- var seed: Data!
- var identity: (ed25519KeyPair: KeyPair, x25519KeyPair: KeyPair)!
- var edSK: [UInt8]!
- var error: UnsafeMutablePointer<CChar>?
- var conf: UnsafeMutablePointer<config_object>?
-
- beforeEach {
- mockStorage = Storage(
- customWriter: try! DatabaseQueue(),
- customMigrations: [
- SNUtilitiesKit.migrations(),
- SNMessagingKit.migrations()
- ]
- )
- seed = Data(hex: "0123456789abcdef0123456789abcdef")
-
- // FIXME: Would be good to move these into the libSession-util instead of using Sodium separately
- identity = try! Identity.generate(from: seed)
- edSK = identity.ed25519KeyPair.secretKey
-
- // Initialize a brand new, empty config because we have no dump data to deal with.
- error = nil
- conf = nil
- _ = contacts_init(&conf, &edSK, nil, 0, error)
- error?.deallocate()
- }
-
- it("does something") {
- mockStorage.write { db in
- try SessionThread.fetchOrCreate(db, id: "1", variant: .contact, shouldBeVisible: true)
- try SessionThread.fetchOrCreate(db, id: "2", variant: .contact, shouldBeVisible: true)
- try SessionThread.fetchOrCreate(db, id: "3", variant: .contact, shouldBeVisible: true)
- _ = try Interaction(
- threadId: "1",
- authorId: "1",
- variant: .standardIncoming,
- body: "Test1"
- ).inserted(db)
- _ = try Interaction(
- threadId: "1",
- authorId: "2",
- variant: .standardIncoming,
- body: "Test2"
- ).inserted(db)
- _ = try Interaction(
- threadId: "3",
- authorId: "3",
- variant: .standardIncoming,
- body: "Test3"
- ).inserted(db)
-
- try SessionUtil.pruningIfNeeded(
- db,
- conf: conf
- )
-
- expect(contacts_size(conf)).to(equal(0))
- }
- }
- }
-
// MARK: - generates config correctly
it("generates config correctly") {
diff --git a/SessionUtilitiesKit/Database/Storage.swift b/SessionUtilitiesKit/Database/Storage.swift
index 1ea913cb3..40b0380e9 100644
--- a/SessionUtilitiesKit/Database/Storage.swift
+++ b/SessionUtilitiesKit/Database/Storage.swift
@@ -55,9 +55,9 @@ open class Storage {
// If a custom writer was provided then use that (for unit testing)
guard customWriter == nil else {
dbWriter = customWriter
- perform(migrations: (customMigrations ?? []), async: false, onProgressUpdate: nil, onComplete: { _, _ in })
isValid = true
Storage.internalHasCreatedValidInstance.mutate { $0 = true }
+ perform(migrations: (customMigrations ?? []), async: false, onProgressUpdate: nil, onComplete: { _, _ in })
return
}