From be34f30719e8e50ab850d4a989ce064c80fa12c3 Mon Sep 17 00:00:00 2001 From: Moxie Marlinspike Date: Thu, 16 May 2013 13:16:42 -0700 Subject: [PATCH] Lower maximum size of cursor window on DB upgrade. On some systems, the DB upgrade was failing because there were too many rows for the cursor window. This moves some looping operations into single update statements by using the substr() command, and chunks the rest using a series of LIMITs. --- .../securesms/DatabaseUpgradeActivity.java | 3 +- .../securesms/database/DatabaseFactory.java | 403 +++++++++--------- .../securesms/service/KeyCachingService.java | 7 +- 3 files changed, 201 insertions(+), 212 deletions(-) diff --git a/src/org/thoughtcrime/securesms/DatabaseUpgradeActivity.java b/src/org/thoughtcrime/securesms/DatabaseUpgradeActivity.java index c9180823ae..812ec979e0 100644 --- a/src/org/thoughtcrime/securesms/DatabaseUpgradeActivity.java +++ b/src/org/thoughtcrime/securesms/DatabaseUpgradeActivity.java @@ -103,7 +103,8 @@ public class DatabaseUpgradeActivity extends Activity { protected Void doInBackground(Integer... params) { Log.w("DatabaseUpgradeActivity", "Running background upgrade.."); DatabaseFactory.getInstance(DatabaseUpgradeActivity.this) - .onApplicationLevelUpgrade(masterSecret, params[0], this); + .onApplicationLevelUpgrade(DatabaseUpgradeActivity.this.getApplicationContext(), + masterSecret, params[0], this); return null; } diff --git a/src/org/thoughtcrime/securesms/database/DatabaseFactory.java b/src/org/thoughtcrime/securesms/database/DatabaseFactory.java index 5d5d9a7073..77fee15c66 100644 --- a/src/org/thoughtcrime/securesms/database/DatabaseFactory.java +++ b/src/org/thoughtcrime/securesms/database/DatabaseFactory.java @@ -25,8 +25,10 @@ import android.util.Log; import org.thoughtcrime.securesms.DatabaseUpgradeActivity; import org.thoughtcrime.securesms.crypto.DecryptingPartInputStream; +import org.thoughtcrime.securesms.crypto.DecryptingQueue; import org.thoughtcrime.securesms.crypto.MasterCipher; import org.thoughtcrime.securesms.crypto.MasterSecret; +import org.thoughtcrime.securesms.notifications.MessageNotifier; import org.thoughtcrime.securesms.util.InvalidMessageException; import org.thoughtcrime.securesms.util.Util; @@ -146,126 +148,160 @@ public class DatabaseFactory { instance = null; } - public void onApplicationLevelUpgrade(MasterSecret masterSecret, int fromVersion, + public void onApplicationLevelUpgrade(Context context, MasterSecret masterSecret, int fromVersion, DatabaseUpgradeActivity.DatabaseUpgradeListener listener) { + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + db.beginTransaction(); + if (fromVersion < DatabaseUpgradeActivity.NO_MORE_KEY_EXCHANGE_PREFIX_VERSION) { String KEY_EXCHANGE = "?TextSecureKeyExchange"; String PROCESSED_KEY_EXCHANGE = "?TextSecureKeyExchangd"; String STALE_KEY_EXCHANGE = "?TextSecureKeyExchangs"; + int ROW_LIMIT = 500; MasterCipher masterCipher = new MasterCipher(masterSecret); - int count = 0; - SQLiteDatabase db = databaseHelper.getWritableDatabase(); - Cursor smsCursor = db.query("sms", - new String[] {"_id", "type", "body"}, - "type & " + 0x80000000 + " != 0", - null, null, null, null); + int smsCount = 0; + int threadCount = 0; + int skip = 0; - Cursor threadCursor = db.query("thread", - new String[] {"_id", "snippet_type", "snippet"}, - "snippet_type & " + 0x80000000 + " != 0", - null, null, null, null); + Cursor cursor = db.query("sms", new String[] {"COUNT(*)"}, "type & " + 0x80000000 + " != 0", + null, null, null, null); - if (smsCursor != null) - count = smsCursor.getCount(); + if (cursor != null && cursor.moveToFirst()) { + smsCount = cursor.getInt(0); + cursor.close(); + } - if (threadCursor != null) - count += threadCursor.getCount(); + cursor = db.query("thread", new String[] {"COUNT(*)"}, "snippet_type & " + 0x80000000 + " != 0", + null, null, null, null); - db.beginTransaction(); + if (cursor != null && cursor.moveToFirst()) { + threadCount = cursor.getInt(0); + cursor.close(); + } + + Cursor smsCursor = null; + + Log.w("DatabaseFactory", "Upgrade count: " + (smsCount + threadCount)); + + do { + Log.w("DatabaseFactory", "Looping SMS cursor..."); + if (smsCursor != null) + smsCursor.close(); + + smsCursor = db.query("sms", new String[] {"_id", "type", "body"}, + "type & " + 0x80000000 + " != 0", + null, null, null, "_id", skip + "," + ROW_LIMIT); - while (smsCursor != null && smsCursor.moveToNext()) { - listener.setProgress(smsCursor.getPosition(), count); - - try { - String body = masterCipher.decryptBody(smsCursor.getString(smsCursor.getColumnIndexOrThrow("body"))); - long type = smsCursor.getLong(smsCursor.getColumnIndexOrThrow("type")); - long id = smsCursor.getLong(smsCursor.getColumnIndexOrThrow("_id")); - - if (body.startsWith(KEY_EXCHANGE)) { - body = body.substring(KEY_EXCHANGE.length()); - body = masterCipher.encryptBody(body); - type |= 0x8000; - - db.execSQL("UPDATE sms SET body = ?, type = ? WHERE _id = ?", - new String[] {body, type+"", id+""}); - } else if (body.startsWith(PROCESSED_KEY_EXCHANGE)) { - body = body.substring(PROCESSED_KEY_EXCHANGE.length()); - body = masterCipher.encryptBody(body); - type |= 0x2000; - - db.execSQL("UPDATE sms SET body = ?, type = ? WHERE _id = ?", - new String[] {body, type+"", id+""}); - } else if (body.startsWith(STALE_KEY_EXCHANGE)) { - body = body.substring(STALE_KEY_EXCHANGE.length()); - body = masterCipher.encryptBody(body); - type |= 0x4000; - - db.execSQL("UPDATE sms SET body = ?, type = ? WHERE _id = ?", - new String[] {body, type+"", id+""}); + while (smsCursor != null && smsCursor.moveToNext()) { + listener.setProgress(smsCursor.getPosition() + skip, smsCount + threadCount); + + try { + String body = masterCipher.decryptBody(smsCursor.getString(smsCursor.getColumnIndexOrThrow("body"))); + long type = smsCursor.getLong(smsCursor.getColumnIndexOrThrow("type")); + long id = smsCursor.getLong(smsCursor.getColumnIndexOrThrow("_id")); + + if (body.startsWith(KEY_EXCHANGE)) { + body = body.substring(KEY_EXCHANGE.length()); + body = masterCipher.encryptBody(body); + type |= 0x8000; + + db.execSQL("UPDATE sms SET body = ?, type = ? WHERE _id = ?", + new String[] {body, type+"", id+""}); + } else if (body.startsWith(PROCESSED_KEY_EXCHANGE)) { + body = body.substring(PROCESSED_KEY_EXCHANGE.length()); + body = masterCipher.encryptBody(body); + type |= (0x8000 | 0x2000); + + db.execSQL("UPDATE sms SET body = ?, type = ? WHERE _id = ?", + new String[] {body, type+"", id+""}); + } else if (body.startsWith(STALE_KEY_EXCHANGE)) { + body = body.substring(STALE_KEY_EXCHANGE.length()); + body = masterCipher.encryptBody(body); + type |= (0x8000 | 0x4000); + + db.execSQL("UPDATE sms SET body = ?, type = ? WHERE _id = ?", + new String[] {body, type+"", id+""}); + } + } catch (InvalidMessageException e) { + Log.w("DatabaseFactory", e); } - } catch (InvalidMessageException e) { - Log.w("DatabaseFactory", e); } - } - while (threadCursor != null && threadCursor.moveToNext()) { - listener.setProgress(smsCursor.getCount() + threadCursor.getPosition(), count); + skip += ROW_LIMIT; + } while (smsCursor != null && smsCursor.getCount() > 0); - try { - String snippet = threadCursor.getString(threadCursor.getColumnIndexOrThrow("snippet")); - long snippetType = threadCursor.getLong(threadCursor.getColumnIndexOrThrow("snippet_type")); - long id = threadCursor.getLong(threadCursor.getColumnIndexOrThrow("_id")); - if (!Util.isEmpty(snippet)) { - snippet = masterCipher.decryptBody(snippet); - } - if (snippet.startsWith(KEY_EXCHANGE)) { - snippet = snippet.substring(KEY_EXCHANGE.length()); - snippet = masterCipher.encryptBody(snippet); - snippetType |= 0x8000; - - db.execSQL("UPDATE thread SET snippet = ?, snippet_type = ? WHERE _id = ?", - new String[] {snippet, snippetType+"", id+""}); - } else if (snippet.startsWith(PROCESSED_KEY_EXCHANGE)) { - snippet = snippet.substring(PROCESSED_KEY_EXCHANGE.length()); - snippet = masterCipher.encryptBody(snippet); - snippetType |= 0x2000; - - db.execSQL("UPDATE thread SET snippet = ?, snippet_type = ? WHERE _id = ?", - new String[] {snippet, snippetType+"", id+""}); - } else if (snippet.startsWith(STALE_KEY_EXCHANGE)) { - snippet = snippet.substring(STALE_KEY_EXCHANGE.length()); - snippet = masterCipher.encryptBody(snippet); - snippetType |= 0x4000; - - db.execSQL("UPDATE thread SET snippet = ?, snippet_type = ? WHERE _id = ?", - new String[] {snippet, snippetType+"", id+""}); + Cursor threadCursor = null; + skip = 0; + + do { + Log.w("DatabaseFactory", "Looping thread cursor..."); + + if (threadCursor != null) + threadCursor.close(); + + threadCursor = db.query("thread", new String[] {"_id", "snippet_type", "snippet"}, + "snippet_type & " + 0x80000000 + " != 0", + null, null, null, "_id", skip + "," + ROW_LIMIT); + + while (threadCursor != null && threadCursor.moveToNext()) { + listener.setProgress(smsCount + threadCursor.getPosition(), smsCount + threadCount); + + try { + String snippet = threadCursor.getString(threadCursor.getColumnIndexOrThrow("snippet")); + long snippetType = threadCursor.getLong(threadCursor.getColumnIndexOrThrow("snippet_type")); + long id = threadCursor.getLong(threadCursor.getColumnIndexOrThrow("_id")); + + if (!Util.isEmpty(snippet)) { + snippet = masterCipher.decryptBody(snippet); + } + + if (snippet.startsWith(KEY_EXCHANGE)) { + snippet = snippet.substring(KEY_EXCHANGE.length()); + snippet = masterCipher.encryptBody(snippet); + snippetType |= 0x8000; + + db.execSQL("UPDATE thread SET snippet = ?, snippet_type = ? WHERE _id = ?", + new String[] {snippet, snippetType+"", id+""}); + } else if (snippet.startsWith(PROCESSED_KEY_EXCHANGE)) { + snippet = snippet.substring(PROCESSED_KEY_EXCHANGE.length()); + snippet = masterCipher.encryptBody(snippet); + snippetType |= (0x8000 | 0x2000); + + db.execSQL("UPDATE thread SET snippet = ?, snippet_type = ? WHERE _id = ?", + new String[] {snippet, snippetType+"", id+""}); + } else if (snippet.startsWith(STALE_KEY_EXCHANGE)) { + snippet = snippet.substring(STALE_KEY_EXCHANGE.length()); + snippet = masterCipher.encryptBody(snippet); + snippetType |= (0x8000 | 0x4000); + + db.execSQL("UPDATE thread SET snippet = ?, snippet_type = ? WHERE _id = ?", + new String[] {snippet, snippetType+"", id+""}); + } + } catch (InvalidMessageException e) { + Log.w("DatabaseFactory", e); } - } catch (InvalidMessageException e) { - Log.w("DatabaseFactory", e); } - } - db.setTransactionSuccessful(); - db.endTransaction(); + skip += ROW_LIMIT; + } while (threadCursor != null && threadCursor.getCount() > 0); - smsCursor.close(); - threadCursor.close(); + if (smsCursor != null) + smsCursor.close(); + + if (threadCursor != null) + threadCursor.close(); } if (fromVersion < DatabaseUpgradeActivity.MMS_BODY_VERSION) { Log.w("DatabaseFactory", "Update MMS bodies..."); MasterCipher masterCipher = new MasterCipher(masterSecret); - SQLiteDatabase db = databaseHelper.getWritableDatabase(); - - db.beginTransaction(); - - Cursor mmsCursor = db.query("mms", new String[] {"_id"}, - "msg_box & " + 0x80000000L + " != 0", - null, null, null, null); + Cursor mmsCursor = db.query("mms", new String[] {"_id"}, + "msg_box & " + 0x80000000L + " != 0", + null, null, null, null); Log.w("DatabaseFactory", "Got MMS rows: " + (mmsCursor == null ? "null" : mmsCursor.getCount())); @@ -319,10 +355,13 @@ public class DatabaseFactory { Log.w("DatabaseFactory", "Updated body: " + body + " and part_count: " + partCount); } - - db.setTransactionSuccessful(); - db.endTransaction(); } + + db.setTransactionSuccessful(); + db.endTransaction(); + + DecryptingQueue.schedulePendingDecrypts(context, masterSecret); + MessageNotifier.updateNotification(context, masterSecret); } private static class DatabaseHelper extends SQLiteOpenHelper { @@ -415,53 +454,38 @@ public class DatabaseFactory { db.execSQL("UPDATE sms SET type = ? WHERE type = ?", new String[] {(23L | 0x800000L)+"", 44L+""}); db.execSQL("UPDATE sms SET type = ? WHERE type = ?", new String[] {(20L | 0x800000L)+"", 45L+""}); db.execSQL("UPDATE sms SET type = ? WHERE type = ?", new String[] {(20L | 0x800000L | 0x10000000L)+"", 46L+""}); - db.execSQL("UPDATE sms SET type = ? WHERE type = ?", new String[] {(20L | 0x800000L | 0x20000000L)+"", 47L+""}); + db.execSQL("UPDATE sms SET type = ? WHERE type = ?", new String[] {(20L)+"", 47L+""}); db.execSQL("UPDATE sms SET type = ? WHERE type = ?", new String[] {(20L | 0x800000L | 0x08000000L)+"", 48L+""}); - Cursor cursor = db.query("sms", null,"body LIKE ?", new String[] {SYMMETRIC_ENCRYPT + "%"}, - null, null, null); + db.execSQL("UPDATE sms SET body = substr(body, ?), type = type | ? WHERE body LIKE ?", + new String[] {(SYMMETRIC_ENCRYPT.length()+1)+"", + 0x80000000L+"", + SYMMETRIC_ENCRYPT + "%"}); - updateSmsBodyAndType(db, cursor, SYMMETRIC_ENCRYPT, 0x80000000L); + db.execSQL("UPDATE sms SET body = substr(body, ?), type = type | ? WHERE body LIKE ?", + new String[] {(ASYMMETRIC_LOCAL_ENCRYPT.length()+1)+"", + 0x40000000L+"", + ASYMMETRIC_LOCAL_ENCRYPT + "%"}); - if (cursor != null) - cursor.close(); - - cursor = db.query("sms", null,"body LIKE ?", new String[] {ASYMMETRIC_LOCAL_ENCRYPT + "%"}, - null, null, null); + db.execSQL("UPDATE sms SET body = substr(body, ?), type = type | ? WHERE body LIKE ?", + new String[] {(ASYMMETRIC_ENCRYPT.length()+1)+"", + (0x800000L | 0x20000000L)+"", + ASYMMETRIC_ENCRYPT + "%"}); - updateSmsBodyAndType(db, cursor, ASYMMETRIC_LOCAL_ENCRYPT, 0x40000000L); - - if (cursor != null) - cursor.close(); + db.execSQL("UPDATE sms SET body = substr(body, ?), type = type | ? WHERE body LIKE ?", + new String[] {(KEY_EXCHANGE.length()+1)+"", + 0x8000L+"", + KEY_EXCHANGE + "%"}); - cursor = db.query("sms", null,"body LIKE ?", new String[] {ASYMMETRIC_ENCRYPT + "%"}, - null, null, null); + db.execSQL("UPDATE sms SET body = substr(body, ?), type = type | ? WHERE body LIKE ?", + new String[] {(PROCESSED_KEY_EXCHANGE.length()+1)+"", + (0x8000L | 0x2000L)+"", + PROCESSED_KEY_EXCHANGE + "%"}); - updateSmsBodyAndType(db, cursor, ASYMMETRIC_ENCRYPT, 0L); - - if (cursor != null) - cursor.close(); - - cursor = db.query("sms", null,"body LIKE ?", new String[] {KEY_EXCHANGE + "%"}, - null, null, null); - - updateSmsBodyAndType(db, cursor, KEY_EXCHANGE, 0x8000L); - - if (cursor != null) - cursor.close(); - - cursor = db.query("sms", null,"body LIKE ?", new String[] {PROCESSED_KEY_EXCHANGE + "%"}, - null, null, null); - - updateSmsBodyAndType(db, cursor, PROCESSED_KEY_EXCHANGE, 0x8000L | 0x2000L); - - if (cursor != null) - cursor.close(); - - cursor = db.query("sms", null,"body LIKE ?", new String[] {STALE_KEY_EXCHANGE + "%"}, - null, null, null); - - updateSmsBodyAndType(db, cursor, STALE_KEY_EXCHANGE, 0x8000L | 0x4000L); + db.execSQL("UPDATE sms SET body = substr(body, ?), type = type | ? WHERE body LIKE ?", + new String[] {(STALE_KEY_EXCHANGE.length()+1)+"", + (0x8000L | 0x4000L)+"", + STALE_KEY_EXCHANGE + "%"}); // MMS Updates @@ -481,40 +505,41 @@ public class DatabaseFactory { db.execSQL("ALTER TABLE thread ADD COLUMN snippet_type INTEGER;"); - if (cursor != null) - cursor.close(); - - cursor = db.query("thread", null,"snippet LIKE ?", - new String[] {SYMMETRIC_ENCRYPT + "%"}, null, null, null); - - updateThreadSnippetAndType(db, cursor, SYMMETRIC_ENCRYPT, 0x80000000L); - - if (cursor != null) - cursor.close(); - - cursor = db.query("thread", null,"snippet LIKE ?", - new String[] {KEY_EXCHANGE + "%"}, null, null, null); - - updateThreadSnippetAndType(db, cursor, KEY_EXCHANGE, 0x8000L); - - if (cursor != null) - cursor.close(); - - cursor = db.query("thread", null,"snippet LIKE ?", - new String[] {STALE_KEY_EXCHANGE + "%"}, null, null, null); - - updateThreadSnippetAndType(db, cursor, STALE_KEY_EXCHANGE, 0x8000L | 0x4000L); - - if (cursor != null) - cursor.close(); - - cursor = db.query("thread", null,"snippet LIKE ?", - new String[] {PROCESSED_KEY_EXCHANGE + "%"}, null, null, null); - - updateThreadSnippetAndType(db, cursor, KEY_EXCHANGE, 0x8000L | 0x2000L); - - if (cursor != null) - cursor.close(); + db.execSQL("UPDATE thread SET snippet = substr(snippet, ?), " + + "snippet_type = ? WHERE snippet LIKE ?", + new String[] {(SYMMETRIC_ENCRYPT.length()+1)+"", + 0x80000000L+"", + SYMMETRIC_ENCRYPT + "%"}); + + db.execSQL("UPDATE thread SET snippet = substr(snippet, ?), " + + "snippet_type = ? WHERE snippet LIKE ?", + new String[] {(ASYMMETRIC_LOCAL_ENCRYPT.length()+1)+"", + 0x40000000L+"", + ASYMMETRIC_LOCAL_ENCRYPT + "%"}); + + db.execSQL("UPDATE thread SET snippet = substr(snippet, ?), " + + "snippet_type = ? WHERE snippet LIKE ?", + new String[] {(ASYMMETRIC_ENCRYPT.length()+1)+"", + (0x800000L | 0x20000000L)+"", + ASYMMETRIC_ENCRYPT + "%"}); + + db.execSQL("UPDATE thread SET snippet = substr(snippet, ?), " + + "snippet_type = ? WHERE snippet LIKE ?", + new String[] {(KEY_EXCHANGE.length()+1)+"", + 0x8000L+"", + KEY_EXCHANGE + "%"}); + + db.execSQL("UPDATE thread SET snippet = substr(snippet, ?), " + + "snippet_type = ? WHERE snippet LIKE ?", + new String[] {(STALE_KEY_EXCHANGE.length()+1)+"", + (0x8000L | 0x4000L)+"", + STALE_KEY_EXCHANGE + "%"}); + + db.execSQL("UPDATE thread SET snippet = substr(snippet, ?), " + + "snippet_type = ? WHERE snippet LIKE ?", + new String[] {(PROCESSED_KEY_EXCHANGE.length()+1)+"", + (0x8000L | 0x2000L)+"", + PROCESSED_KEY_EXCHANGE + "%"}); } if (oldVersion < INTRODUCED_MMS_BODY_VERSION) { @@ -545,46 +570,6 @@ public class DatabaseFactory { db.endTransaction(); } - private void updateSmsBodyAndType(SQLiteDatabase db, Cursor cursor, String prefix, long typeMask) - { - while (cursor != null && cursor.moveToNext()) { - String body = cursor.getString(cursor.getColumnIndexOrThrow("body")); - long type = cursor.getLong(cursor.getColumnIndexOrThrow("type")); - long id = cursor.getLong(cursor.getColumnIndexOrThrow("_id")); - - if (body.startsWith(prefix)) { - body = body.substring(prefix.length()); - type |= typeMask; - - db.execSQL("UPDATE sms SET type = ?, body = ? WHERE _id = ?", - new String[]{type+"", body, id+""}); - } - } - - if (cursor != null) - cursor.close(); - } - - private void updateThreadSnippetAndType(SQLiteDatabase db, Cursor cursor, String prefix, long typeMask) - { - while (cursor != null && cursor.moveToNext()) { - String snippet = cursor.getString(cursor.getColumnIndexOrThrow("snippet")); - long type = cursor.getLong(cursor.getColumnIndexOrThrow("snippet_type")); - long id = cursor.getLong(cursor.getColumnIndexOrThrow("_id")); - - if (snippet.startsWith(prefix)) { - snippet = snippet.substring(prefix.length()); - type |= typeMask; - - db.execSQL("UPDATE thread SET snippet_type = ?, snippet = ? WHERE _id = ?", - new String[]{type+"", snippet, id+""}); - } - } - - if (cursor != null) - cursor.close(); - } - private void executeStatements(SQLiteDatabase db, String[] statements) { for (String statement : statements) db.execSQL(statement); diff --git a/src/org/thoughtcrime/securesms/service/KeyCachingService.java b/src/org/thoughtcrime/securesms/service/KeyCachingService.java index 9ba25de02a..851187a415 100644 --- a/src/org/thoughtcrime/securesms/service/KeyCachingService.java +++ b/src/org/thoughtcrime/securesms/service/KeyCachingService.java @@ -32,6 +32,7 @@ import android.util.Log; import android.widget.RemoteViews; import org.thoughtcrime.securesms.ApplicationPreferencesActivity; +import org.thoughtcrime.securesms.DatabaseUpgradeActivity; import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.RoutingActivity; import org.thoughtcrime.securesms.crypto.DecryptingQueue; @@ -78,8 +79,10 @@ public class KeyCachingService extends Service { new Thread() { @Override public void run() { - DecryptingQueue.schedulePendingDecrypts(KeyCachingService.this, masterSecret); - MessageNotifier.updateNotification(KeyCachingService.this, masterSecret); + if (!DatabaseUpgradeActivity.isUpdate(KeyCachingService.this)) { + DecryptingQueue.schedulePendingDecrypts(KeyCachingService.this, masterSecret); + MessageNotifier.updateNotification(KeyCachingService.this, masterSecret); + } } }.start(); }