diff --git a/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift b/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift
index 1ebb22422..e82ea6917 100644
--- a/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift
+++ b/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift
@@ -54,10 +54,14 @@ public final class AttachmentDownloadJob : NSObject, Job, NSCoding { // NSObject
         let temporaryFilePath = URL(fileURLWithPath: OWSTemporaryDirectoryAccessibleAfterFirstAuth() + UUID().uuidString)
         let handleFailure: (Swift.Error) -> Void = { error in // Intentionally capture self
             OWSFileSystem.deleteFile(temporaryFilePath.absoluteString)
-            storage.withAsync({ transaction in
-                storage.setAttachmentState(to: .failed, for: pointer, associatedWith: self.tsIncomingMessageID, using: transaction)
-            }, completion: { })
-            self.handleFailure(error: error)
+            if let error = error as? Error, case .noAttachment = error {
+                storage.withAsync({ transaction in
+                    storage.setAttachmentState(to: .failed, for: pointer, associatedWith: self.tsIncomingMessageID, using: transaction)
+                }, completion: { })
+                self.handlePermanentFailure(error: error)
+            } else {
+                self.handleFailure(error: error)
+            }
         }
         FileServerAPI.downloadAttachment(from: pointer.downloadURL).done(on: DispatchQueue.global(qos: .userInitiated)) { data in
             do {
@@ -93,6 +97,10 @@ public final class AttachmentDownloadJob : NSObject, Job, NSCoding { // NSObject
     private func handleSuccess() {
         delegate?.handleJobSucceeded(self)
     }
+    
+    private func handlePermanentFailure(error: Swift.Error) {
+        delegate?.handleJobFailedPermanently(self, with: error)
+    }
 
     private func handleFailure(error: Swift.Error) {
         delegate?.handleJobFailed(self, with: error)
diff --git a/SessionMessagingKit/Jobs/AttachmentUploadJob.swift b/SessionMessagingKit/Jobs/AttachmentUploadJob.swift
index 4e9fe1be5..b504272e8 100644
--- a/SessionMessagingKit/Jobs/AttachmentUploadJob.swift
+++ b/SessionMessagingKit/Jobs/AttachmentUploadJob.swift
@@ -48,18 +48,29 @@ public final class AttachmentUploadJob : NSObject, Job, NSCoding { // NSObject/N
         guard !stream.isUploaded else { return handleSuccess() } // Should never occur
         let openGroup = Configuration.shared.storage.getOpenGroup(for: threadID)
         let server = openGroup?.server ?? FileServerAPI.server
+        // FIXME: A lot of what's currently happening in FileServerAPI should really be happening here
         FileServerAPI.uploadAttachment(stream, with: attachmentID, to: server).done(on: DispatchQueue.global(qos: .userInitiated)) { // Intentionally capture self
             self.handleSuccess()
         }.catch(on: DispatchQueue.global(qos: .userInitiated)) { error in
-            self.handleFailure(error: error)
+            if let error = error as? Error, case .noAttachment = error {
+                self.handlePermanentFailure(error: error)
+            } else if let error = error as? DotNetAPI.Error, !error.isRetryable {
+                self.handlePermanentFailure(error: error)
+            } else {
+                self.handleFailure(error: error)
+            }
         }
     }
 
     private func handleSuccess() {
         delegate?.handleJobSucceeded(self)
     }
+    
+    private func handlePermanentFailure(error: Swift.Error) {
+        delegate?.handleJobFailedPermanently(self, with: error)
+    }
 
-    private func handleFailure(error: Error) {
+    private func handleFailure(error: Swift.Error) {
         delegate?.handleJobFailed(self, with: error)
     }
 }
diff --git a/SessionMessagingKit/Jobs/MessageSendJob.swift b/SessionMessagingKit/Jobs/MessageSendJob.swift
index 4574a1e33..467c9e00c 100644
--- a/SessionMessagingKit/Jobs/MessageSendJob.swift
+++ b/SessionMessagingKit/Jobs/MessageSendJob.swift
@@ -69,7 +69,7 @@ public final class MessageSendJob : NSObject, Job, NSCoding { // NSObject/NSCodi
                 if storage.getAttachmentUploadJob(for: attachment.uniqueId!) != nil {
                     // Wait for it to finish
                 } else {
-                    let job = AttachmentUploadJob()
+                    let job = AttachmentUploadJob(attachmentID: attachment.uniqueId!, threadID: message.threadID!)
                     storage.withAsync({ transaction in
                         JobQueue.shared.add(job, using: transaction)
                     }, completion: { })
@@ -83,7 +83,11 @@ public final class MessageSendJob : NSObject, Job, NSCoding { // NSObject/NSCodi
                     self.handleSuccess()
                 }.catch(on: Threading.workQueue) { error in
                     SNLog("Couldn't send message due to error: \(error).")
-                    self.handleFailure(error: error)
+                    if let error = error as? MessageSender.Error, !error.isRetryable {
+                        self.handlePermanentFailure(error: error)
+                    } else {
+                        self.handleFailure(error: error)
+                    }
                 }
             }
         }, completion: { })
@@ -92,6 +96,10 @@ public final class MessageSendJob : NSObject, Job, NSCoding { // NSObject/NSCodi
     private func handleSuccess() {
         delegate?.handleJobSucceeded(self)
     }
+    
+    private func handlePermanentFailure(error: Error) {
+        delegate?.handleJobFailedPermanently(self, with: error)
+    }
 
     private func handleFailure(error: Error) {
         delegate?.handleJobFailed(self, with: error)
diff --git a/SessionMessagingKit/Sending & Receiving/MessageSender.swift b/SessionMessagingKit/Sending & Receiving/MessageSender.swift
index c7cf38752..0c7b3e57b 100644
--- a/SessionMessagingKit/Sending & Receiving/MessageSender.swift	
+++ b/SessionMessagingKit/Sending & Receiving/MessageSender.swift	
@@ -11,6 +11,13 @@ public final class MessageSender : NSObject {
         case proofOfWorkCalculationFailed
         case noUserPublicKey
 
+        internal var isRetryable: Bool {
+            switch self {
+            case .invalidMessage, .protoConversionFailed, .proofOfWorkCalculationFailed: return false
+            default: return true
+            }
+        }
+        
         public var errorDescription: String? {
             switch self {
             case .invalidMessage: return "Invalid message."
diff --git a/SessionMessagingKit/Utilities/DotNetAPI.swift b/SessionMessagingKit/Utilities/DotNetAPI.swift
index 36fcdba1e..4631439bd 100644
--- a/SessionMessagingKit/Utilities/DotNetAPI.swift
+++ b/SessionMessagingKit/Utilities/DotNetAPI.swift
@@ -22,6 +22,10 @@ public class DotNetAPI : NSObject {
         case decryptionFailed
         case maxFileSizeExceeded
 
+        internal var isRetryable: Bool {
+            return false
+        }
+        
         public var errorDescription: String? {
             switch self {
             case .generic: return "An error occurred."