session-desktop/js/backup.js

;(function () {
  'use strict';
  window.Whisper = window.Whisper || {};

  var electronRemote = require('electron').remote;
  var dialog = electronRemote.dialog;
  var BrowserWindow = electronRemote.BrowserWindow;

  var fs = require('fs');
  var path = require('path');

  function stringify(object) {
    for (var key in object) {
      var val = object[key];
      if (val instanceof ArrayBuffer) {
        object[key] = {
          type: 'ArrayBuffer',
          encoding: 'base64',
          data: dcodeIO.ByteBuffer.wrap(val).toString('base64')
        };
      } else if (val instanceof Object) {
        object[key] = stringify(val);
      }
    }
    return object;
  }

  function unstringify(object) {
    if (!(object instanceof Object)) {
      throw new Error('unstringify expects an object');
    }
    for (var key in object) {
      var val = object[key];
      if (val &&
          val.type === 'ArrayBuffer' &&
          val.encoding === 'base64' &&
          typeof val.data === 'string' ) {
        object[key] = dcodeIO.ByteBuffer.wrap(val.data, 'base64').toArrayBuffer();
      } else if (val instanceof Object) {
        object[key] = unstringify(object[key]);
      }
    }
    return object;
  }

  function createOutputStream(writer) {
    var wait = Promise.resolve();
    return {
      write: function(string) {
        wait = wait.then(function() {
          return new Promise(function(resolve) {
            if (writer.write(string)) {
              return resolve();
            }

            //  If write() returns true, we don't need to wait for the drain event
            //   https://nodejs.org/dist/latest-v7.x/docs/api/stream.html#stream_class_stream_writable
            writer.once('drain', resolve);

            // We don't register for the 'error' event here, only in close(). Otherwise,
            //   we'll get "Possible EventEmitter memory leak detected" warnings.
          });
        });
        return wait;
      },
      close: function() {
        return wait.then(function() {
          return new Promise(function(resolve, reject) {
            writer.once('finish', resolve);
            writer.once('error', reject);
            writer.end();
          });
        });
      }
    };
  }

  function exportNonMessages(idb_db, parent, options) {
    return createFileAndWriter(parent, 'db.json').then(function(writer) {
      return exportToJsonFile(idb_db, writer, options);
    });
  }

  /**
  * Export all data from an IndexedDB database
  * @param {IDBDatabase} idb_db
  */
  function exportToJsonFile(idb_db, fileWriter, options) {
    options = options || {};
    _.defaults(options, {excludeClientConfig: false});

    return new Promise(function(resolve, reject) {
      var storeNames = idb_db.objectStoreNames;
      storeNames = _.without(storeNames, 'messages');

      if (options.excludeClientConfig) {
        console.log('exportToJsonFile: excluding client config from export');
        storeNames = _.without(
          storeNames,
          'items',
          'signedPreKeys',
          'preKeys',
          'identityKeys',
          'sessions',
          'unprocessed' // since we won't be able to decrypt them anyway
        );
      }

      var exportedStoreNames = [];
      if (storeNames.length === 0) {
        throw new Error('No stores to export');
      }
      console.log('Exporting from these stores:', storeNames.join(', '));

      var stream = createOutputStream(fileWriter);

      stream.write('{');

      _.each(storeNames, function(storeName) {
        var transaction = idb_db.transaction(storeNames, 'readwrite');
        transaction.onerror = function(e) {
          Whisper.Database.handleDOMException(
            'exportToJsonFile transaction error (store: ' + storeName + ')',
            transaction.error,
            reject
          );
        };
        transaction.oncomplete = function() {
          console.log('transaction complete');
        };

        var store = transaction.objectStore(storeName);
        var request = store.openCursor();
        var count = 0;
        request.onerror = function(e) {
          Whisper.Database.handleDOMException(
            'exportToJsonFile request error (store: ' + storeNames + ')',
            request.error,
            reject
          );
        };
        request.onsuccess = function(event) {
          if (count === 0) {
            console.log('cursor opened');
            stream.write('"' + storeName + '": [');
          }

          var cursor = event.target.result;
          if (cursor) {
            if (count > 0) {
              stream.write(',');
            }
            var jsonString = JSON.stringify(stringify(cursor.value));
            stream.write(jsonString);
            cursor.continue();
            count++;
          } else {
            // no more
            stream.write(']');
            console.log('Exported', count, 'items from store', storeName);

            exportedStoreNames.push(storeName);
            if (exportedStoreNames.length < storeNames.length) {
              stream.write(',');
            } else {
              console.log('Exported all stores');
              stream.write('}');

              stream.close().then(function() {
                console.log('Finished writing all stores to disk');
                resolve();
              });
            }
          }
        };
      });
    });
  }

  function importNonMessages(idb_db, parent, options) {
    var file = 'db.json';
    return readFileAsText(parent, file).then(function(string) {
      return importFromJsonString(idb_db, string, path.join(parent, file), options);
    });
  }

  function eliminateClientConfigInBackup(data, path) {
    var cleaned = _.pick(data, 'conversations', 'groups');
    console.log('Writing configuration-free backup file back to disk');
    try {
      fs.writeFileSync(path, JSON.stringify(cleaned));
    } catch (error) {
      console.log('Error writing cleaned-up backup to disk: ', error.stack);
    }
  }

  /**
  * Import data from JSON into an IndexedDB database. This does not delete any existing data
  *  from the database, so keys could clash
  *
  * @param {IDBDatabase} idb_db
  * @param {string} jsonString - data to import, one key per object store
  */
  function importFromJsonString(idb_db, jsonString, path, options) {
    options = options || {};
    _.defaults(options, {
      forceLightImport: false,
      conversationLookup: {},
      groupLookup: {},
    });

    var conversationLookup = options.conversationLookup;
    var groupLookup = options.groupLookup;
    var result = {
      fullImport: true,
    };

    return new Promise(function(resolve, reject) {
      var importObject = JSON.parse(jsonString);
      delete importObject.debug;

      if (!importObject.sessions || options.forceLightImport) {
        result.fullImport = false;

        delete importObject.items;
        delete importObject.signedPreKeys;
        delete importObject.preKeys;
        delete importObject.identityKeys;
        delete importObject.sessions;
        delete importObject.unprocessed;

        console.log('This is a light import; contacts, groups and messages only');
      }

      // We mutate the on-disk backup to prevent the user from importing client
      //   configuration more than once - that causes lots of encryption errors.
      //   This of course preserves the true data: conversations and groups.
      eliminateClientConfigInBackup(importObject, path);

      var storeNames = _.keys(importObject);
      console.log('Importing to these stores:', storeNames.join(', '));

      var finished = false;
      var finish = function(via) {
        console.log('non-messages import done via', via);
        if (finished) {
          resolve(result);
        }
        finished = true;
      };

      var transaction = idb_db.transaction(storeNames, 'readwrite');
      transaction.onerror = function(e) {
        Whisper.Database.handleDOMException(
          'importFromJsonString transaction error',
          transaction.error,
          reject
        );
      };
      transaction.oncomplete = finish.bind(null, 'transaction complete');

      _.each(storeNames, function(storeName) {
          console.log('Importing items for store', storeName);

          if (!importObject[storeName].length) {
            delete importObject[storeName];
            return;
          }

          var count = 0;
          var skipCount = 0;

          var finishStore = function() {
            // added all objects for this store
            delete importObject[storeName];
            console.log(
              'Done importing to store',
              storeName,
              'Total count:',
              count,
              'Skipped:',
              skipCount
            );
            if (_.keys(importObject).length === 0) {
              // added all object stores
              console.log('DB import complete');
              finish('puts scheduled');
            }
          };

          _.each(importObject[storeName], function(toAdd) {
              toAdd = unstringify(toAdd);

              var haveConversationAlready =
                storeName === 'conversations'
                && conversationLookup[getConversationKey(toAdd)];
              var haveGroupAlready =
                storeName === 'groups' && groupLookup[getGroupKey(toAdd)];

              if (haveConversationAlready || haveGroupAlready) {
                skipCount++;
                count++;
                return;
              }

              var request = transaction.objectStore(storeName).put(toAdd, toAdd.id);
              request.onsuccess = function(event) {
                count++;
                if (count == importObject[storeName].length) {
                  finishStore();
                }
              };
              request.onerror = function(e) {
                Whisper.Database.handleDOMException(
                  'importFromJsonString request error (store: ' + storeName + ')',
                  request.error,
                  reject
                );
              };
          });

          // We have to check here, because we may have skipped every item, resulting
          //   in no onsuccess callback at all.
          if (count === importObject[storeName].length) {
            finishStore();
          }
      });
    });
  }

  function createDirectory(parent, name) {
    return new Promise(function(resolve, reject) {
      var sanitized = sanitizeFileName(name);
      var targetDir = path.join(parent, sanitized);
      fs.mkdir(targetDir, function(error) {
        if (error) {
          return reject(error);
        }

        return resolve(targetDir);
      });
    });
  }

  function createFileAndWriter(parent, name) {
    return new Promise(function(resolve) {
      var sanitized = sanitizeFileName(name);
      var targetPath = path.join(parent, sanitized);
      var options = {
        flags: 'wx'
      };
      return resolve(fs.createWriteStream(targetPath, options));
    });
  }

  function readFileAsText(parent, name) {
    return new Promise(function(resolve, reject) {
      var targetPath = path.join(parent, name);
      fs.readFile(targetPath, 'utf8', function(error, string) {
        if (error) {
          return reject(error);
        }

        return resolve(string);
      });
    });
  }

  function readFileAsArrayBuffer(parent, name) {
    return new Promise(function(resolve, reject) {
      var targetPath = path.join(parent, name);
      // omitting the encoding to get a buffer back
      fs.readFile(targetPath, function(error, buffer) {
        if (error) {
          return reject(error);
        }

        // Buffer instances are also Uint8Array instances
        //   https://nodejs.org/docs/latest/api/buffer.html#buffer_buffers_and_typedarray
        return resolve(buffer.buffer);
      });
    });
  }

  function trimFileName(filename) {
    var components = filename.split('.');
    if (components.length <= 1) {
      return filename.slice(0, 30);
    }

    var extension = components[components.length - 1];
    var name = components.slice(0, components.length - 1);
    if (extension.length > 5) {
      return filename.slice(0, 30);
    }

    return name.join('.').slice(0, 24) + '.' + extension;
  }


  function getAttachmentFileName(attachment) {
    if (attachment.fileName) {
      return trimFileName(attachment.fileName);
    }

    var name = attachment.id;

    if (attachment.contentType) {
      var components = attachment.contentType.split('/');
      name += '.' + (components.length > 1 ? components[1] : attachment.contentType);
    }

    return name;
  }

  function readAttachment(parent, message, attachment) {
    return new Promise(function(resolve, reject) {
      var name = getAttachmentFileName(attachment);
      var sanitized = sanitizeFileName(name);
      var attachmentDir = path.join(parent, message.received_at.toString());

      return readFileAsArrayBuffer(attachmentDir, sanitized).then(function(contents) {
        attachment.data = contents;
        return resolve();
      }, reject);
    });
  }

  function writeAttachment(dir, attachment) {
    var filename = getAttachmentFileName(attachment);
    return createFileAndWriter(dir, filename).then(function(writer) {
      var stream = createOutputStream(writer);
      stream.write(new Buffer(attachment.data));
      return stream.close();
    });
  }

  function writeAttachments(parentDir, name, messageId, attachments) {
    return createDirectory(parentDir, messageId).then(function(dir) {
      return Promise.all(_.map(attachments, function(attachment) {
        return writeAttachment(dir, attachment);
      }));
    }).catch(function(error) {
      console.log(
        'writeAttachments: error exporting conversation',
        name,
        ':',
        error && error.stack ? error.stack : error
      );
      return Promise.reject(error);
    });
  }

  function sanitizeFileName(filename) {
    return filename.toString().replace(/[^a-z0-9.,+()'#\- ]/gi, '_');
  }

  function exportConversation(idb_db, name, conversation, dir) {
    console.log('exporting conversation', name);
    return createFileAndWriter(dir, 'messages.json').then(function(writer) {
      return new Promise(function(resolve, reject) {
        var transaction = idb_db.transaction('messages', 'readwrite');
        transaction.onerror = function(e) {
          Whisper.Database.handleDOMException(
            'exportConversation transaction error (conversation: ' + name + ')',
            transaction.error,
            reject
          );
        };
        transaction.oncomplete = function() {
          // this doesn't really mean anything - we may have attachment processing to do
        };

        var store = transaction.objectStore('messages');
        var index = store.index('conversation');
        var range = IDBKeyRange.bound([conversation.id, 0], [conversation.id, Number.MAX_VALUE]);

        var promiseChain = Promise.resolve();
        var count = 0;
        var request = index.openCursor(range);

        var stream = createOutputStream(writer);
        stream.write('{"messages":[');

        request.onerror = function(e) {
          Whisper.Database.handleDOMException(
            'exportConversation request error (conversation: ' + name + ')',
            request.error,
            reject
          );
        };
        request.onsuccess = function(event) {
          var cursor = event.target.result;
          if (cursor) {
            var message = cursor.value;
            var messageId = message.received_at;
            var attachments = message.attachments;

            // skip message if it is disappearing, no matter the amount of time left
            if (message.expireTimer) {
              cursor.continue();
              return;
            }

            if (count !== 0) {
              stream.write(',');
            }

            message.attachments = _.map(attachments, function(attachment) {
              return _.omit(attachment, ['data']);
            });

            var jsonString = JSON.stringify(stringify(message));
            stream.write(jsonString);

            if (attachments && attachments.length) {
              var process = function() {
                return writeAttachments(dir, name, messageId, attachments);
              };
              promiseChain = promiseChain.then(process);
            }

            count += 1;
            cursor.continue();
          } else {
            stream.write(']}');

            var promise = stream.close();

            return promiseChain.then(promise).then(function() {
              console.log('done exporting conversation', name);
              return resolve();
            }, function(error) {
              console.log(
                'exportConversation: error exporting conversation',
                name,
                ':',
                error && error.stack ? error.stack : error
              );
              return reject(error);
            });
          }
        };
      });
    });
  }

  // Goals for directory names:
  //   1. Human-readable, for easy use and verification by user (names not just ids)
  //   2. Sorted just like the list of conversations in the left-pan (active_at)
  //   3. Disambiguated from other directories (active_at, truncated name, id)
  function getConversationDirName(conversation) {
    var name = conversation.active_at || 'never';
    if (conversation.name) {
      return name + ' (' + conversation.name.slice(0, 30) + ' ' + conversation.id + ')';
    } else {
      return name + ' (' + conversation.id + ')';
    }
  }

  // Goals for logging names:
  //   1. Can be associated with files on disk
  //   2. Adequately disambiguated to enable debugging flow of execution
  //   3. Can be shared to the web without privacy concerns (there's no global redaction
  //      logic for group ids, so we do it manually here)
  function getConversationLoggingName(conversation) {
    var name = conversation.active_at || 'never';
    if (conversation.type === 'private') {
      name += ' (' + conversation.id + ')';
    } else {
      name += ' ([REDACTED_GROUP]' + conversation.id.slice(-3) + ')';
    }
    return name;
  }

  function exportConversations(idb_db, parentDir) {
    return new Promise(function(resolve, reject) {
      var transaction = idb_db.transaction('conversations', 'readwrite');
      transaction.onerror = function(e) {
        Whisper.Database.handleDOMException(
          'exportConversations transaction error',
          transaction.error,
          reject
        );
      };
      transaction.oncomplete = function() {
        // not really very useful - fires at unexpected times
      };

      var promiseChain = Promise.resolve();
      var store = transaction.objectStore('conversations');
      var request = store.openCursor();
      request.onerror = function(e) {
        Whisper.Database.handleDOMException(
          'exportConversations request error',
          request.error,
          reject
        );
      };
      request.onsuccess = function(event) {
        var cursor = event.target.result;
        if (cursor && cursor.value) {
          var conversation = cursor.value;
          var dir = getConversationDirName(conversation);
          var name = getConversationLoggingName(conversation);

          var process = function() {
            return createDirectory(parentDir, dir).then(function(dir) {
              return exportConversation(idb_db, name, conversation, dir);
            });
          };

          console.log('scheduling export for conversation', name);
          promiseChain = promiseChain.then(process);
          cursor.continue();
        } else {
          console.log('Done scheduling conversation exports');
          return promiseChain.then(resolve, reject);
        }
      };
    });
  }

  function getDirectory(options) {
    return new Promise(function(resolve, reject) {
      var browserWindow = BrowserWindow.getFocusedWindow();
      var dialogOptions = {
        title: options.title,
        properties: ['openDirectory'],
        buttonLabel: options.buttonLabel
      };

      dialog.showOpenDialog(browserWindow, dialogOptions, function(directory) {
        if (!directory || !directory[0]) {
          var error = new Error('Error choosing directory');
          error.name = 'ChooseError';
          return reject(error);
        }

        return resolve(directory[0]);
      });
    });
  }

  function getDirContents(dir) {
    return new Promise(function(resolve, reject) {
      fs.readdir(dir, function(err, files) {
        if (err) {
          return reject(err);
        }

        files = _.map(files, function(file) {
          return path.join(dir, file);
        });

        resolve(files);
      });
    });
  }

  function loadAttachments(dir, message) {
    return Promise.all(_.map(message.attachments, function(attachment) {
      return readAttachment(dir, message, attachment);
    }));
  }

  function saveMessage(idb_db, message) {
    return saveAllMessages(idb_db, [message]);
  }

  function saveAllMessages(idb_db, messages) {
    if (!messages.length) {
      return Promise.resolve();
    }

    return new Promise(function(resolve, reject) {
      var finished = false;
      var finish = function(via) {
        console.log('messages done saving via', via);
        if (finished) {
          resolve();
        }
        finished = true;
      };

      var transaction = idb_db.transaction('messages', 'readwrite');
      transaction.onerror = function(e) {
        Whisper.Database.handleDOMException(
          'saveAllMessages transaction error',
          transaction.error,
          reject
        );
      };
      transaction.oncomplete = finish.bind(null, 'transaction complete');

      var store = transaction.objectStore('messages');
      var conversationId = messages[0].conversationId;
      var count = 0;

      _.forEach(messages, function(message) {
        var request = store.put(message, message.id);
        request.onsuccess = function(event) {
          count += 1;
          if (count === messages.length) {
            console.log(
              'Saved',
              messages.length,
              'messages for conversation',
              // Don't know if group or private conversation, so we blindly redact
              '[REDACTED]' + conversationId.slice(-3)
            );
            finish('puts scheduled');
          }
        };
        request.onerror = function(e) {
          Whisper.Database.handleDOMException(
            'saveAllMessages request error',
            request.error,
            reject
          );
        };
      });
    });
  }

  // To reduce the memory impact of attachments, we make individual saves to the
  //   database for every message with an attachment. We load the attachment for a
  //   message, save it, and only then do we move on to the next message. Thus, every
  //   message with attachments needs to be removed from our overall message save with the
  //   filter() call.
  function importConversation(idb_db, dir, options) {
    options = options || {};
    _.defaults(options, {messageLookup: {}});

    var messageLookup = options.messageLookup;
    var conversationId = 'unknown';
    var total = 0;
    var skipped = 0;

    return readFileAsText(dir, 'messages.json').then(function(contents) {
      var promiseChain = Promise.resolve();

      var json = JSON.parse(contents);
      if (json.messages && json.messages.length) {
        conversationId = '[REDACTED]' + (json.messages[0].conversationId || '').slice(-3);
      }
      total = json.messages.length;

      var messages = _.filter(json.messages, function(message) {
        message = unstringify(message);

        if (messageLookup[getMessageKey(message)]) {
          skipped++;
          return false;
        }

        if (message.attachments && message.attachments.length) {
          var process = function() {
            return loadAttachments(dir, message).then(function() {
              return saveMessage(idb_db, message);
            });
          };

          promiseChain = promiseChain.then(process);

          return false;
        }

        return true;
      });

      var promise = Promise.resolve();
      if (messages.length > 0) {
        promise = saveAllMessages(idb_db, messages);
      }

      return promise
        .then(function() {
          return promiseChain;
        })
        .then(function() {
          console.log(
            'Finished importing conversation',
            conversationId,
            'Total:',
            total,
            'Skipped:',
            skipped
          );
        });

    }, function() {
      console.log('Warning: could not access messages.json in directory: ' + dir);
    });
  }

  function importConversations(idb_db, dir, options) {
    return getDirContents(dir).then(function(contents) {
      var promiseChain = Promise.resolve();

      _.forEach(contents, function(conversationDir) {
        if (!fs.statSync(conversationDir).isDirectory()) {
          return;
        }

        var process = function() {
          return importConversation(idb_db, conversationDir, options);
        };

        promiseChain = promiseChain.then(process);
      });

      return promiseChain;
    });
  }

  function getMessageKey(message) {
    var ourNumber = textsecure.storage.user.getNumber();
    var source = message.source || ourNumber;
    if (source === ourNumber) {
      return source + ' ' + message.timestamp;
    }

    var sourceDevice = message.sourceDevice || 1;
    return source + '.' + sourceDevice + ' ' + message.timestamp;
  }
  function loadMessagesLookup(idb_db) {
    return assembleLookup(idb_db, 'messages', getMessageKey);
  }

  function getConversationKey(conversation) {
    return conversation.id;
  }
  function loadConversationLookup(idb_db) {
    return assembleLookup(idb_db, 'conversations', getConversationKey);
  }

  function getGroupKey(group) {
    return group.id;
  }
  function loadGroupsLookup(idb_db) {
    return assembleLookup(idb_db, 'groups', getGroupKey);
  }

  function assembleLookup(idb_db, storeName, keyFunction) {
    var lookup = Object.create(null);

    return new Promise(function(resolve, reject) {
      var transaction = idb_db.transaction(storeName, 'readwrite');
      transaction.onerror = function(e) {
        Whisper.Database.handleDOMException(
          'assembleLookup(' + storeName + ') transaction error',
          transaction.error,
          reject
        );
      };
      transaction.oncomplete = function() {
        // not really very useful - fires at unexpected times
      };

      var promiseChain = Promise.resolve();
      var store = transaction.objectStore(storeName);
      var request = store.openCursor();
      request.onerror = function(e) {
        Whisper.Database.handleDOMException(
          'assembleLookup(' + storeName + ') request error',
          request.error,
          reject
        );
      };
      request.onsuccess = function(event) {
        var cursor = event.target.result;
        if (cursor && cursor.value) {
          lookup[keyFunction(cursor.value)] = true;
          cursor.continue();
        } else {
          console.log('Done creating ' + storeName + ' lookup');
          return resolve(lookup);
        }
      };
    });
  }


  function getTimestamp() {
    return moment().format('YYYY MMM Do [at] h.mm.ss a');
  }

  // directories returned and taken by backup/import are all string paths
  Whisper.Backup = {
    getDirectoryForExport: function() {
      var options = {
        title: i18n('exportChooserTitle'),
        buttonLabel: i18n('exportButton'),
      };
      return getDirectory(options);
    },
    exportToDirectory: function(directory, options) {
      var dir;
      var idb;
      return Whisper.Database.open().then(function(idb_db) {
        idb = idb_db;
        var name = 'Signal Export ' + getTimestamp();
        return createDirectory(directory, name);
      }).then(function(created) {
        dir = created;
        return exportNonMessages(idb, dir, options);
      }).then(function() {
        return exportConversations(idb, dir);
      }).then(function() {
        return dir;
      }).then(function(path) {
        console.log('done backing up!');
        return path;
      }, function(error) {
        console.log(
          'the backup went wrong:',
          error && error.stack ? error.stack : error
        );
        return Promise.reject(error);
      });
    },
    getDirectoryForImport: function() {
      var options = {
        title: i18n('importChooserTitle'),
        buttonLabel: i18n('importButton'),
      };
      return getDirectory(options);
    },
    importFromDirectory: function(directory, options) {
      options = options || {};

      var idb, nonMessageResult;
      return Whisper.Database.open().then(function(idb_db) {
        idb = idb_db;

        return Promise.all([
          loadMessagesLookup(idb_db),
          loadConversationLookup(idb_db),
          loadGroupsLookup(idb_db),
        ]);
      }).then(function(lookups) {
        options.messageLookup = lookups[0];
        options.conversationLookup = lookups[1];
        options.groupLookup = lookups[2];
      }).then(function() {
        return importNonMessages(idb, directory, options);
      }).then(function(result) {
        nonMessageResult = result;
        return importConversations(idb, directory, options);
      }).then(function() {
        console.log('done restoring from backup!');
        return nonMessageResult;
      }, function(error) {
        console.log(
          'the import went wrong:',
          error && error.stack ? error.stack : error
        );
        return Promise.reject(error);
      });
    },
    // for testing
    sanitizeFileName,
    trimFileName,
    getAttachmentFileName,
    getConversationDirName,
    getConversationLoggingName,
  };

}());