Merge tag 'v1.15.1-beta.1'

app/attachments.js

@@ -1,12 +1,22 @@
 const crypto = require('crypto');
 const path = require('path');
 
+const pify = require('pify');
+const glob = require('glob');
 const fse = require('fs-extra');
 const toArrayBuffer = require('to-arraybuffer');
-const { isArrayBuffer, isString } = require('lodash');
+const { map, isArrayBuffer, isString } = require('lodash');
 
 const PATH = 'attachments.noindex';
 
+exports.getAllAttachments = async userDataPath => {
+  const dir = exports.getPath(userDataPath);
+  const pattern = path.join(dir, '**', '*');
+
+  const files = await pify(glob)(pattern, { nodir: true });
+  return map(files, file => path.relative(dir, file));
+};
+
 //      getPath :: AbsolutePath -> AbsolutePath
 exports.getPath = userDataPath => {
   if (!isString(userDataPath)) {
@@ -120,6 +130,18 @@ exports.createDeleter = root => {
   };
 };
 
+exports.deleteAll = async ({ userDataPath, attachments }) => {
+  const deleteFromDisk = exports.createDeleter(exports.getPath(userDataPath));
+
+  for (let index = 0, max = attachments.length; index < max; index += 1) {
+    const file = attachments[index];
+    // eslint-disable-next-line no-await-in-loop
+    await deleteFromDisk(file);
+  }
+
+  console.log(`deleteAll: deleted ${attachments.length} files`);
+};
+
 //      createName :: Unit -> IO String
 exports.createName = () => {
   const buffer = crypto.randomBytes(32);

app/sql.js

@@ -4,7 +4,7 @@ const rimraf = require('rimraf');
 const sql = require('@journeyapps/sqlcipher');
 const pify = require('pify');
 const uuidv4 = require('uuid/v4');
-const { map, isString } = require('lodash');
+const { map, isString, fromPairs, forEach, last } = require('lodash');
 
 // To get long stack traces
 //   https://github.com/mapbox/node-sqlite3/wiki/API#sqlite3verbose
@@ -15,6 +15,7 @@ module.exports = {
   close,
   removeDB,
 
+  getMessageCount,
   saveMessage,
   saveMessages,
   removeMessage,
@@ -39,6 +40,8 @@ module.exports = {
   getMessagesNeedingUpgrade,
   getMessagesWithVisualMediaAttachments,
   getMessagesWithFileAttachments,
+
+  removeKnownAttachments,
 };
 
 function generateUUID() {
@@ -260,6 +263,16 @@ async function removeDB() {
   rimraf.sync(filePath);
 }
 
+async function getMessageCount() {
+  const row = await db.get('SELECT count(*) from messages;');
+
+  if (!row) {
+    throw new Error('getMessageCount: Unable to get count of messages');
+  }
+
+  return row['count(*)'];
+}
+
 async function saveMessage(data, { forceSave } = {}) {
   const {
     conversationId,
@@ -710,3 +723,92 @@ async function getMessagesWithFileAttachments(conversationId, { limit }) {
 
   return map(rows, row => jsonToObject(row.json));
 }
+
+function getExternalFilesForMessage(message) {
+  const { attachments, contact, quote } = message;
+  const files = [];
+
+  forEach(attachments, attachment => {
+    const { path: file, thumbnail, screenshot } = attachment;
+    if (file) {
+      files.push(file);
+    }
+
+    if (thumbnail && thumbnail.path) {
+      files.push(thumbnail.path);
+    }
+
+    if (screenshot && screenshot.path) {
+      files.push(screenshot.path);
+    }
+  });
+
+  if (quote && quote.attachments && quote.attachments.length) {
+    forEach(quote.attachments, attachment => {
+      const { thumbnail } = attachment;
+
+      if (thumbnail && thumbnail.path) {
+        files.push(thumbnail.path);
+      }
+    });
+  }
+
+  if (contact && contact.length) {
+    forEach(contact, item => {
+      const { avatar } = item;
+
+      if (avatar && avatar.avatar && avatar.avatar.path) {
+        files.push(avatar.avatar.path);
+      }
+    });
+  }
+
+  return files;
+}
+
+async function removeKnownAttachments(allAttachments) {
+  const lookup = fromPairs(map(allAttachments, file => [file, true]));
+  const chunkSize = 50;
+
+  const total = await getMessageCount();
+  console.log(
+    `removeKnownAttachments: About to iterate through ${total} messages`
+  );
+
+  let count = 0;
+  let complete = false;
+  let id = '';
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const rows = await db.all(
+      `SELECT json FROM messages
+       WHERE id > $id
+       ORDER BY id ASC
+       LIMIT $chunkSize;`,
+      {
+        $id: id,
+        $chunkSize: chunkSize,
+      }
+    );
+
+    const messages = map(rows, row => jsonToObject(row.json));
+    forEach(messages, message => {
+      const externalFiles = getExternalFilesForMessage(message);
+      forEach(externalFiles, file => {
+        delete lookup[file];
+      });
+    });
+
+    const lastMessage = last(messages);
+    if (lastMessage) {
+      ({ id } = lastMessage);
+    }
+    complete = messages.length < chunkSize;
+    count += messages.length;
+  }
+
+  console.log(`removeKnownAttachments: Done processing ${count} messages`);
+
+  return Object.keys(lookup);
+}

js/background.js

@@ -338,6 +338,7 @@
         db,
         clearStores: Whisper.Database.clearStores,
         handleDOMException: Whisper.Database.handleDOMException,
+        arrayBufferToString: textsecure.MessageReceiver.arrayBufferToString,
         countCallback: count => {
           window.log.info(`Migration: ${count} messages complete`);
           showMigrationStatus(count);

js/models/conversations.js

@@ -139,6 +139,7 @@
       // Listening for out-of-band data updates
       this.on('delivered', this.updateAndMerge);
       this.on('read', this.updateAndMerge);
+      this.on('expiration-change', this.updateAndMerge);
       this.on('expired', this.onExpired);
     },
 

js/models/messages.js

@@ -389,13 +389,12 @@
       };
     },
     getMessagePropStatus() {
-      if (!this.isOutgoing()) {
-        return null;
-      }
-
       if (this.hasErrors()) {
         return 'error';
       }
+      if (!this.isOutgoing()) {
+        return null;
+      }
 
       const readBy = this.get('read_by') || [];
       if (readBy.length > 0) {
@@ -825,7 +824,6 @@
       this.trigger('pending');
       return promise
         .then(async result => {
-          const now = Date.now();
           this.trigger('done');
 
           // This is used by sendSyncMessage, then set to null
@@ -837,7 +835,7 @@
           this.set({
             sent_to: _.union(sentTo, result.successfulNumbers),
             sent: true,
-            expirationStartTimestamp: now,
+            expirationStartTimestamp: Date.now(),
           });
 
           await window.Signal.Data.saveMessage(this.attributes, {
@@ -847,7 +845,6 @@
           this.sendSyncMessage();
         })
         .catch(result => {
-          const now = Date.now();
           this.trigger('done');
 
           if (result.dataMessage) {
@@ -868,10 +865,12 @@
             this.saveErrors(result.errors);
             if (result.successfulNumbers.length > 0) {
               const sentTo = this.get('sent_to') || [];
+
+              // Note: In a partially-successful group send, we do not start
+              //   the expiration timer.
               this.set({
                 sent_to: _.union(sentTo, result.successfulNumbers),
                 sent: true,
-                expirationStartTimestamp: now,
               });
               promises.push(this.sendSyncMessage());
             }
@@ -1290,8 +1289,8 @@
       }
       return msFromNow;
     },
-    async setToExpire() {
-      if (this.isExpiring() && !this.get('expires_at')) {
+    async setToExpire(force = false) {
+      if (this.isExpiring() && (force || !this.get('expires_at'))) {
         const start = this.get('expirationStartTimestamp');
         const delta = this.get('expireTimer') * 1000;
         const expiresAt = start + delta;

js/modules/data.js

@@ -34,6 +34,7 @@ module.exports = {
   close,
   removeDB,
 
+  getMessageCount,
   saveMessage,
   saveLegacyMessage,
   saveMessages,
@@ -178,8 +179,8 @@ function makeChannel(fnName) {
       });
 
       setTimeout(
-        () => resolve(new Error(`Request to ${fnName} timed out`)),
-        5000
+        () => reject(new Error(`Request to ${fnName} timed out`)),
+        10000
       );
     });
   };
@@ -201,6 +202,10 @@ async function removeDB() {
   await channels.removeDB();
 }
 
+async function getMessageCount() {
+  return channels.getMessageCount();
+}
+
 async function saveMessage(data, { forceSave, Message } = {}) {
   const id = await channels.saveMessage(_cleanData(data), { forceSave });
   Message.refreshExpirationTimer();

js/modules/migrate_to_sql.js

@@ -1,6 +1,6 @@
 /* global window, IDBKeyRange */
 
-const { includes, isFunction, isString, last } = require('lodash');
+const { includes, isFunction, isString, last, forEach } = require('lodash');
 const {
   saveMessages,
   _removeMessages,
@@ -25,6 +25,7 @@ async function migrateToSQL({
   clearStores,
   handleDOMException,
   countCallback,
+  arrayBufferToString,
 }) {
   if (!db) {
     throw new Error('Need db for IndexedDB connection!');
@@ -32,6 +33,9 @@ async function migrateToSQL({
   if (!isFunction(clearStores)) {
     throw new Error('Need clearStores function!');
   }
+  if (!isFunction(arrayBufferToString)) {
+    throw new Error('Need arrayBufferToString function!');
+  }
   if (!isFunction(handleDOMException)) {
     throw new Error('Need handleDOMException function!');
   }
@@ -78,7 +82,21 @@ async function migrateToSQL({
     // eslint-disable-next-line no-await-in-loop
     const status = await migrateStoreToSQLite({
       db,
-      save: saveUnprocesseds,
+      save: async array => {
+        forEach(array, item => {
+          // In the new database, we can't store ArrayBuffers, so we turn these two fields
+          //   into strings like MessageReceiver now does before save.
+          if (item.envelope) {
+            // eslint-disable-next-line no-param-reassign
+            item.envelope = arrayBufferToString(item.envelope);
+          }
+          if (item.decrypted) {
+            // eslint-disable-next-line no-param-reassign
+            item.decrypted = arrayBufferToString(item.decrypted);
+          }
+        });
+        await saveUnprocesseds(array);
+      },
       remove: removeUnprocessed,
       storeName: 'unprocessed',
       handleDOMException,

js/read_syncs.js

@@ -1,4 +1,4 @@
-/* global Backbone, Whisper, ConversationController */
+/* global Backbone, Whisper */
 
 /* eslint-disable more/no-then */
 
@@ -32,9 +32,7 @@
 
         const message = messages.find(
           item =>
-            item.isIncoming() &&
-            item.isUnread() &&
-            item.get('source') === receipt.get('sender')
+            item.isIncoming() && item.get('source') === receipt.get('sender')
         );
         const notificationForMessage = message
           ? Whisper.Notifications.findWhere({ messageId: message.id })
@@ -59,11 +57,39 @@
           return;
         }
 
-        await message.markRead(receipt.get('read_at'));
-        // This notification may result in messages older than this one being
-        //   marked read. We want those messages to have the same expire timer
-        //   start time as this one, so we pass the read_at value through.
-        this.notifyConversation(message, receipt.get('read_at'));
+        const readAt = receipt.get('read_at');
+
+        // If message is unread, we mark it read. Otherwise, we update the expiration
+        //   timer to the time specified by the read sync if it's earlier than
+        //   the previous read time.
+        if (message.isUnread()) {
+          await message.markRead(readAt);
+
+          // onReadMessage may result in messages older than this one being
+          //   marked read. We want those messages to have the same expire timer
+          //   start time as this one, so we pass the readAt value through.
+          const conversation = message.getConversation();
+          if (conversation) {
+            conversation.onReadMessage(message, readAt);
+          }
+        } else {
+          const now = Date.now();
+          const existingTimestamp = message.get('expirationStartTimestamp');
+          const expirationStartTimestamp = Math.min(
+            now,
+            Math.min(existingTimestamp || now, readAt || now)
+          );
+          message.set({ expirationStartTimestamp });
+
+          const force = true;
+          await message.setToExpire(force);
+
+          const conversation = message.getConversation();
+          if (conversation) {
+            conversation.trigger('expiration-change', message);
+          }
+        }
+
         this.remove(receipt);
       } catch (error) {
         window.log.error(
@@ -72,14 +98,5 @@
         );
       }
     },
-    notifyConversation(message, readAt) {
-      const conversation = ConversationController.get({
-        id: message.get('conversationId'),
-      });
-
-      if (conversation) {
-        conversation.onReadMessage(message, readAt);
-      }
-    },
   }))();
 })();

libtextsecure/message_receiver.js

@@ -31,6 +31,11 @@ function MessageReceiver(username, password, signalingKey, options = {}) {
   }
 }
 
+MessageReceiver.stringToArrayBuffer = string =>
+  dcodeIO.ByteBuffer.wrap(string, 'binary').toArrayBuffer();
+MessageReceiver.arrayBufferToString = arrayBuffer =>
+  dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('binary');
+
 MessageReceiver.prototype = new textsecure.EventTarget();
 MessageReceiver.prototype.extend({
   constructor: MessageReceiver,
@@ -265,14 +270,14 @@ MessageReceiver.prototype.extend({
       }
     });
   },
-  queueCached(item) {
+  async queueCached(item) {
     try {
       let envelopePlaintext = item.envelope;
 
-      // Up until 0.42.6 we stored envelope and decrypted as strings in IndexedDB,
-      //   so we need to be ready for them.
       if (typeof envelopePlaintext === 'string') {
-        envelopePlaintext = this.stringToArrayBuffer(envelopePlaintext);
+        envelopePlaintext = MessageReceiver.stringToArrayBuffer(
+          envelopePlaintext
+        );
       }
       const envelope = textsecure.protobuf.Envelope.decode(envelopePlaintext);
 
@@ -280,14 +285,33 @@ MessageReceiver.prototype.extend({
       if (decrypted) {
         let payloadPlaintext = decrypted;
         if (typeof payloadPlaintext === 'string') {
-          payloadPlaintext = this.stringToArrayBuffer(payloadPlaintext);
+          payloadPlaintext = MessageReceiver.stringToArrayBuffer(
+            payloadPlaintext
+          );
         }
         this.queueDecryptedEnvelope(envelope, payloadPlaintext);
       } else {
         this.queueEnvelope(envelope);
       }
     } catch (error) {
-      window.log.error('queueCached error handling item', item.id);
+      window.log.error(
+        'queueCached error handling item',
+        item.id,
+        'removing it. Error:',
+        error && error.stack ? error.stack : error
+      );
+
+      try {
+        const { id } = item;
+        await textsecure.storage.unprocessed.remove(id);
+      } catch (deleteError) {
+        window.log.error(
+          'queueCached error deleting item',
+          item.id,
+          'Error:',
+          deleteError && deleteError.stack ? deleteError.stack : deleteError
+        );
+      }
     }
   },
   getEnvelopeId(envelope) {
@@ -295,13 +319,6 @@ MessageReceiver.prototype.extend({
       envelope.sourceDevice
     } ${envelope.timestamp.toNumber()}`;
   },
-  stringToArrayBuffer(string) {
-    // eslint-disable-next-line new-cap
-    return dcodeIO.ByteBuffer.wrap(string, 'binary').toArrayBuffer();
-  },
-  arrayBufferToString(arrayBuffer) {
-    return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('binary');
-  },
   getAllFromCache() {
     window.log.info('getAllFromCache');
     return textsecure.storage.unprocessed.getAll().then(items => {
@@ -339,7 +356,7 @@ MessageReceiver.prototype.extend({
     const id = this.getEnvelopeId(envelope);
     const data = {
       id,
-      envelope: this.arrayBufferToString(plaintext),
+      envelope: MessageReceiver.arrayBufferToString(plaintext),
       timestamp: Date.now(),
       attempts: 1,
     };
@@ -348,7 +365,7 @@ MessageReceiver.prototype.extend({
   updateCache(envelope, plaintext) {
     const id = this.getEnvelopeId(envelope);
     const data = {
-      decrypted: this.arrayBufferToString(plaintext),
+      decrypted: MessageReceiver.arrayBufferToString(plaintext),
     };
     return textsecure.storage.unprocessed.update(id, data);
   },

main.js

@@ -26,6 +26,7 @@ const packageJson = require('./package.json');
 
 const sql = require('./app/sql');
 const sqlChannels = require('./app/sql_channel');
+const attachments = require('./app/attachments');
 const attachmentChannel = require('./app/attachment_channel');
 const autoUpdate = require('./app/auto_update');
 const createTrayIcon = require('./app/tray_icon');
@@ -630,6 +631,13 @@ app.on('ready', async () => {
   await sql.initialize({ configDir: userDataPath, key });
   await sqlChannels.initialize({ userConfig });
 
+  const allAttachments = await attachments.getAllAttachments(userDataPath);
+  const orphanedAttachments = await sql.removeKnownAttachments(allAttachments);
+  await attachments.deleteAll({
+    userDataPath,
+    attachments: orphanedAttachments,
+  });
+
   ready = true;
 
   autoUpdate.initialize(getMainWindow, locale.messages);

package.json

@@ -3,7 +3,7 @@
   "productName": "Signal",
   "description": "Private messaging from your desktop",
   "repository": "https://github.com/signalapp/Signal-Desktop.git",
-  "version": "1.15.0",
+  "version": "1.15.1-beta.1",
   "license": "GPL-3.0",
   "author": {
     "name": "Open Whisper Systems",
@@ -63,6 +63,7 @@
     "firstline": "^1.2.1",
     "form-data": "^2.3.2",
     "fs-extra": "^5.0.0",
+    "glob": "^7.1.2",
     "google-libphonenumber": "^3.0.7",
     "got": "^8.2.0",
     "intl-tel-input": "^12.1.15",
@@ -119,7 +120,6 @@
     "eslint-plugin-mocha": "^4.12.1",
     "eslint-plugin-more": "^0.3.1",
     "extract-zip": "^1.6.6",
-    "glob": "^7.1.2",
     "grunt": "^1.0.1",
     "grunt-cli": "^1.2.0",
     "grunt-contrib-concat": "^1.0.1",

stylesheets/_index.scss

@@ -19,7 +19,7 @@
 }
 
 .gutter {
-  background-color: $color-black-008;
+  background-color: $color-black-008-no-tranparency;
   float: left;
   width: 300px;
   .content {

stylesheets/_modules.scss

@@ -446,6 +446,7 @@
   color: $color-light-90;
   font-size: 14px;
   line-height: 18px;
+  text-align: start;
 
   overflow-wrap: break-word;
   word-wrap: break-word;
@@ -1946,7 +1947,7 @@
   cursor: pointer;
 
   &:hover {
-    background-color: $color-black-008;
+    background-color: $color-black-016-no-tranparency;
   }
 }
 
@@ -1956,7 +1957,7 @@
 }
 
 .module-conversation-list-item--is-selected {
-  background-color: $color-black-008;
+  background-color: $color-black-016-no-tranparency;
 }
 
 .module-conversation-list-item__avatar {

stylesheets/_variables.scss

@@ -50,6 +50,8 @@ $color-dark-70: #414347;
 $color-dark-85: #1a1c20;
 $color-black: #000000;
 $color-black-008: rgba($color-black, 0.08);
+$color-black-008-no-tranparency: #ededed;
+$color-black-016-no-tranparency: #d9d9d9;
 $color-black-012: rgba($color-black, 0.12);
 $color-black-02: rgba($color-black, 0.2);
 $color-black-04: rgba($color-black, 0.4);