diff --git a/src/org/thoughtcrime/securesms/attachments/AttachmentId.java b/src/org/thoughtcrime/securesms/attachments/AttachmentId.java index 3d43c84195..6f5160d75a 100644 --- a/src/org/thoughtcrime/securesms/attachments/AttachmentId.java +++ b/src/org/thoughtcrime/securesms/attachments/AttachmentId.java @@ -1,12 +1,15 @@ package org.thoughtcrime.securesms.attachments; +import android.os.Parcel; +import android.os.Parcelable; + import androidx.annotation.NonNull; import com.fasterxml.jackson.annotation.JsonProperty; import org.thoughtcrime.securesms.util.Util; -public class AttachmentId { +public class AttachmentId implements Parcelable { @JsonProperty private final long rowId; @@ -54,4 +57,33 @@ public class AttachmentId { public int hashCode() { return Util.hashCode(rowId, uniqueId); } + + //region Parcelable implementation. + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeLong(rowId); + dest.writeLong(uniqueId); + } + + public static final Parcelable.Creator CREATOR = + new Parcelable.Creator() { + + @Override + public AttachmentId createFromParcel(Parcel in) { + long rowId = in.readLong(); + long uniqueId = in.readLong(); + return new AttachmentId(rowId, uniqueId); + } + + @Override + public AttachmentId[] newArray(int size) { + return new AttachmentId[size]; + } + }; + //endregion } diff --git a/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt index 48bf768e3b..21de7925c7 100644 --- a/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt +++ b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt @@ -1,6 +1,15 @@ package org.thoughtcrime.securesms.attachments -data class DatabaseAttachmentAudioExtras(val attachmentId: AttachmentId, val visualSamples: ByteArray, val durationMs: Long) { +data class DatabaseAttachmentAudioExtras( + val attachmentId: AttachmentId, + /** Small amount of normalized audio byte samples to visualise the content (e.g. draw waveform). */ + val visualSamples: ByteArray, + /** Duration of the audio track in milliseconds. May be [DURATION_UNDEFINED] when is not known. */ + val durationMs: Long = DURATION_UNDEFINED) { + + companion object { + const val DURATION_UNDEFINED = -1L + } override fun equals(other: Any?): Boolean { return other != null && diff --git a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java index 8e637e0036..583e92a813 100644 --- a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java +++ b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java @@ -73,6 +73,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import kotlin.jvm.Synchronized; + public class AttachmentDatabase extends Database { private static final String TAG = AttachmentDatabase.class.getSimpleName(); @@ -106,8 +108,8 @@ public class AttachmentDatabase extends Database { static final String CAPTION = "caption"; public static final String URL = "url"; public static final String DIRECTORY = "parts"; - // audio/* mime type only related columns. - static final String AUDIO_VISUAL_SAMPLES = "audio_visual_samples"; // Small amount of audio byte samples to visualise the content (e.g. draw waveform) + // "audio/*" mime type only related columns. + static final String AUDIO_VISUAL_SAMPLES = "audio_visual_samples"; // Small amount of audio byte samples to visualise the content (e.g. draw waveform). static final String AUDIO_DURATION = "audio_duration"; // Duration of the audio track in milliseconds. public static final int TRANSFER_PROGRESS_DONE = 0; @@ -116,7 +118,7 @@ public class AttachmentDatabase extends Database { public static final int TRANSFER_PROGRESS_FAILED = 3; private static final String PART_ID_WHERE = ROW_ID + " = ? AND " + UNIQUE_ID + " = ?"; - private static final String PART_AUDIO_ONLY_WHERE = CONTENT_TYPE + " LIKE audio/%"; + private static final String PART_AUDIO_ONLY_WHERE = CONTENT_TYPE + " LIKE \"audio/%\""; private static final String[] PROJECTION = new String[] {ROW_ID, MMS_ID, CONTENT_TYPE, NAME, CONTENT_DISPOSITION, @@ -834,6 +836,7 @@ public class AttachmentDatabase extends Database { * Retrieves the audio extra values associated with the attachment. Only "audio/*" mime type attachments are accepted. * @return the related audio extras or null in case any of the audio extra columns are empty or the attachment is not an audio. */ + @Synchronized public @Nullable DatabaseAttachmentAudioExtras getAttachmentAudioExtras(@NonNull AttachmentId attachmentId) { try (Cursor cursor = databaseHelper.getReadableDatabase() // We expect all the audio extra values to be present (not null) or reject the whole record. @@ -859,6 +862,7 @@ public class AttachmentDatabase extends Database { * Updates audio extra columns for the "audio/*" mime type attachments only. * @return true if the update operation was successful. */ + @Synchronized public boolean setAttachmentAudioExtras(@NonNull DatabaseAttachmentAudioExtras extras) { ContentValues values = new ContentValues(); values.put(AUDIO_VISUAL_SAMPLES, extras.getVisualSamples()); diff --git a/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java index 86cf0872ad..64123f535f 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java @@ -103,9 +103,7 @@ public class ClassicOpenHelper extends SQLiteOpenHelper { private static final int GROUP_RECEIPT_TRACKING = 45; private static final int UNREAD_COUNT_VERSION = 46; private static final int MORE_RECIPIENT_FIELDS = 47; - private static final int AUDIO_ATTACHMENT_EXTRAS = 48; - - private static final int DATABASE_VERSION = 48; + private static final int DATABASE_VERSION = 47; private static final String TAG = ClassicOpenHelper.class.getSimpleName(); @@ -1291,11 +1289,6 @@ public class ClassicOpenHelper extends SQLiteOpenHelper { */ } - if (oldVersion < AUDIO_ATTACHMENT_EXTRAS) { - db.execSQL("ALTER TABLE part ADD COLUMN audio_visual_samples BLOB"); - db.execSQL("ALTER TABLE part ADD COLUMN audio_duration INTEGER"); - } - db.setTransactionSuccessful(); db.endTransaction(); } diff --git a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index 82d8bf0a06..de227acdc1 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -92,8 +92,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { private static final int lokiV13 = 34; private static final int lokiV14_BACKUP_FILES = 35; private static final int lokiV15 = 36; + private static final int lokiV16_AUDIO_ATTACHMENT_EXTRAS = 37; - private static final int DATABASE_VERSION = lokiV15; + private static final int DATABASE_VERSION = lokiV16_AUDIO_ATTACHMENT_EXTRAS; private static final String DATABASE_NAME = "signal.db"; private final Context context; @@ -632,6 +633,11 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { db.execSQL(SharedSenderKeysDatabase.getCreateOldClosedGroupRatchetTableCommand()); } + if (oldVersion < lokiV16_AUDIO_ATTACHMENT_EXTRAS) { + db.execSQL("ALTER TABLE part ADD COLUMN audio_visual_samples BLOB"); + db.execSQL("ALTER TABLE part ADD COLUMN audio_duration INTEGER"); + } + db.setTransactionSuccessful(); } finally { db.endTransaction(); diff --git a/src/org/thoughtcrime/securesms/jobmanager/Data.java b/src/org/thoughtcrime/securesms/jobmanager/Data.java index 72e8508a5b..0eaa2e9e4e 100644 --- a/src/org/thoughtcrime/securesms/jobmanager/Data.java +++ b/src/org/thoughtcrime/securesms/jobmanager/Data.java @@ -1,13 +1,20 @@ package org.thoughtcrime.securesms.jobmanager; +import android.os.Parcelable; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import com.fasterxml.jackson.annotation.JsonProperty; +import org.thoughtcrime.securesms.attachments.AttachmentId; +import org.thoughtcrime.securesms.util.ParcelableUtil; + import java.util.HashMap; import java.util.Map; +//TODO AC: For now parcelable objects utilize byteArrays field to store their data into. +// Introduce a dedicated Map field specifically for parcelable needs. public class Data { public static final Data EMPTY = new Data.Builder().build(); @@ -213,6 +220,16 @@ public class Data { return byteArrays.get(key); } + public boolean hasParcelable(@NonNull String key) { + return byteArrays.containsKey(key); + } + + public T getParcelable(@NonNull String key, @NonNull Parcelable.Creator creator) { + throwIfAbsent(byteArrays, key); + byte[] bytes = byteArrays.get(key); + return ParcelableUtil.unmarshall(bytes, creator); + } + private void throwIfAbsent(@NonNull Map map, @NonNull String key) { if (!map.containsKey(key)) { throw new IllegalStateException("Tried to retrieve a value with key '" + key + "', but it wasn't present."); @@ -301,6 +318,12 @@ public class Data { return this; } + public Builder putParcelable(@NonNull String key, @NonNull Parcelable value) { + byte[] bytes = ParcelableUtil.marshall(value); + byteArrays.put(key, bytes); + return this; + } + public Data build() { return new Data(strings, stringArrays, diff --git a/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java b/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java index 1449ace743..9b1ec1f890 100644 --- a/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java +++ b/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java @@ -45,6 +45,7 @@ import org.thoughtcrime.securesms.jobs.TrimThreadJob; import org.thoughtcrime.securesms.jobs.TypingSendJob; import org.thoughtcrime.securesms.jobs.UpdateApkJob; import org.thoughtcrime.securesms.loki.api.BackgroundPollJob; +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob; import org.thoughtcrime.securesms.loki.protocol.ClosedGroupUpdateMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.NullMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.SessionRequestMessageSendJob; @@ -102,6 +103,7 @@ public class WorkManagerFactoryMappings { put(TrimThreadJob.class.getName(), TrimThreadJob.KEY); put(TypingSendJob.class.getName(), TypingSendJob.KEY); put(UpdateApkJob.class.getName(), UpdateApkJob.KEY); + put(PrepareAttachmentAudioExtrasJob.class.getName(), PrepareAttachmentAudioExtrasJob.KEY); }}; public static @Nullable String getFactoryKey(@NonNull String workManagerClass) { diff --git a/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 6f3c34e310..8a89f0bcb6 100644 --- a/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -15,6 +15,7 @@ import org.thoughtcrime.securesms.jobmanager.impl.NetworkOrCellServiceConstraint import org.thoughtcrime.securesms.jobmanager.impl.SqlCipherMigrationConstraint; import org.thoughtcrime.securesms.jobmanager.impl.SqlCipherMigrationConstraintObserver; import org.thoughtcrime.securesms.loki.api.BackgroundPollJob; +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob; import org.thoughtcrime.securesms.loki.protocol.ClosedGroupUpdateMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.NullMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.SessionRequestMessageSendJob; @@ -79,6 +80,7 @@ public final class JobManagerFactories { put(TrimThreadJob.KEY, new TrimThreadJob.Factory()); put(TypingSendJob.KEY, new TypingSendJob.Factory()); put(UpdateApkJob.KEY, new UpdateApkJob.Factory()); + put(PrepareAttachmentAudioExtrasJob.KEY, new PrepareAttachmentAudioExtrasJob.Factory()); }}; } diff --git a/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt b/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt index 815e45ecc3..b23313bb0e 100644 --- a/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt +++ b/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt @@ -26,11 +26,13 @@ import kotlinx.android.synthetic.main.activity_home.* import network.loki.messenger.R import org.thoughtcrime.securesms.ApplicationContext import org.thoughtcrime.securesms.PassphraseRequiredActionBarActivity +import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.conversation.ConversationActivity import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.database.ThreadDatabase import org.thoughtcrime.securesms.database.model.ThreadRecord import org.thoughtcrime.securesms.jobs.MultiDeviceBlockedUpdateJob +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob import org.thoughtcrime.securesms.loki.dialogs.ConversationOptionsBottomSheet import org.thoughtcrime.securesms.loki.dialogs.LightThemeFeatureIntroBottomSheet import org.thoughtcrime.securesms.loki.dialogs.MultiDeviceRemovalBottomSheet diff --git a/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt new file mode 100644 index 0000000000..0f4d7cd492 --- /dev/null +++ b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt @@ -0,0 +1,169 @@ +package org.thoughtcrime.securesms.loki.api + +import android.media.MediaDataSource +import android.os.Build +import android.util.Log +import androidx.annotation.RequiresApi +import org.greenrobot.eventbus.EventBus +import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.attachments.AttachmentId +import org.thoughtcrime.securesms.attachments.DatabaseAttachmentAudioExtras +import org.thoughtcrime.securesms.database.DatabaseFactory +import org.thoughtcrime.securesms.jobmanager.Data +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.jobs.BaseJob +import org.thoughtcrime.securesms.loki.utilities.audio.DecodedAudio +import org.thoughtcrime.securesms.mms.PartAuthority +import java.io.InputStream +import java.lang.IllegalStateException +import java.util.* +import java.util.concurrent.TimeUnit + +/** + * Decodes the audio content of the related attachment entry + * and caches the result with [DatabaseAttachmentAudioExtras] data. + * + * It only process attachments with "audio" mime types. + * + * Due to [DecodedAudio] implementation limitations, it only works for API 23+. + * For any lower targets fake data will be generated. + * + * You can subscribe to [AudioExtrasUpdatedEvent] to be notified about the successful result. + */ +//TODO AC: Rewrite to WorkManager API when +// https://github.com/loki-project/session-android/pull/354 is merged. +class PrepareAttachmentAudioExtrasJob : BaseJob { + + companion object { + private const val TAG = "AttachAudioExtrasJob" + + const val KEY = "PrepareAttachmentAudioExtrasJob" + const val DATA_ATTACH_ID = "attachment_id" + + const val VISUAL_RMS_FRAMES = 32 // The amount of values to be computed for the visualization. + } + + private val attachmentId: AttachmentId + + constructor(attachmentId: AttachmentId) : this(Parameters.Builder() + .setQueue(KEY) + .setLifespan(TimeUnit.DAYS.toMillis(1)) + .build(), + attachmentId) + + private constructor(parameters: Parameters, attachmentId: AttachmentId) : super(parameters) { + this.attachmentId = attachmentId + } + + override fun serialize(): Data { + return Data.Builder().putParcelable(DATA_ATTACH_ID, attachmentId).build(); + } + + override fun getFactoryKey(): String { return KEY + } + + override fun onShouldRetry(e: Exception): Boolean { + return false + } + + override fun onCanceled() { } + + override fun onRun() { + Log.v(TAG, "Processing attachment: $attachmentId") + + val attachDb = DatabaseFactory.getAttachmentDatabase(context) + val attachment = attachDb.getAttachment(attachmentId) + + if (attachment == null) { + throw IllegalStateException("Cannot find attachment with the ID $attachmentId") + } + if (!attachment.contentType.startsWith("audio/")) { + throw IllegalStateException("Attachment $attachmentId is not of audio type.") + } + + // Check if the audio extras already exist. + if (attachDb.getAttachmentAudioExtras(attachmentId) != null) return + + fun extractAttachmentRandomSeed(attachment: Attachment): Int { + return when { + attachment.digest != null -> attachment.digest!!.sum() + attachment.fileName != null -> attachment.fileName.hashCode() + else -> attachment.hashCode() + } + } + + fun generateFakeRms(seed: Int, frames: Int = VISUAL_RMS_FRAMES): ByteArray { + return ByteArray(frames).apply { Random(seed.toLong()).nextBytes(this) } + } + + var rmsValues: ByteArray + var totalDurationMs: Long = DatabaseAttachmentAudioExtras.DURATION_UNDEFINED + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + // Due to API version incompatibility, we just display some random waveform for older API. + rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) + } else { + try { + @Suppress("BlockingMethodInNonBlockingContext") + val decodedAudio = PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { + DecodedAudio.create(InputStreamMediaDataSource(it)) + } + rmsValues = decodedAudio.calculateRms(VISUAL_RMS_FRAMES) + totalDurationMs = (decodedAudio.totalDuration / 1000.0).toLong() + } catch (e: Exception) { + Log.w(TAG, "Failed to decode sample values for the audio attachment \"${attachment.fileName}\".", e) + rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) + } + } + + val audioExtras = DatabaseAttachmentAudioExtras( + attachmentId, + rmsValues, + totalDurationMs + ) + + attachDb.setAttachmentAudioExtras(audioExtras) + + EventBus.getDefault().post(AudioExtrasUpdatedEvent(audioExtras)) + } + + class Factory : Job.Factory { + override fun create(parameters: Parameters, data: Data): PrepareAttachmentAudioExtrasJob { + return PrepareAttachmentAudioExtrasJob(parameters, data.getParcelable(DATA_ATTACH_ID, AttachmentId.CREATOR)) + } + } + + /** Dispatched once the audio extras have been updated. */ + data class AudioExtrasUpdatedEvent(val audioExtras: DatabaseAttachmentAudioExtras) + + @RequiresApi(Build.VERSION_CODES.M) + private class InputStreamMediaDataSource: MediaDataSource { + + private val data: ByteArray + + constructor(inputStream: InputStream): super() { + this.data = inputStream.readBytes() + } + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { + val length: Int = data.size + if (position >= length) { + return -1 // -1 indicates EOF + } + var actualSize = size + if (position + size > length) { + actualSize -= (position + size - length).toInt() + } + System.arraycopy(data, position.toInt(), buffer, offset, actualSize) + return actualSize + } + + override fun getSize(): Long { + return data.size.toLong() + } + + override fun close() { + // We don't need to close the wrapped stream. + } + } +} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt index 399406bc51..ddc79d8722 100644 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt @@ -16,6 +16,7 @@ import java.nio.ByteOrder import java.nio.ShortBuffer import kotlin.jvm.Throws import kotlin.math.ceil +import kotlin.math.roundToInt import kotlin.math.sqrt /** @@ -246,7 +247,7 @@ class DecodedAudio { codec.release() } - fun calculateRms(maxFrames: Int): FloatArray { + fun calculateRms(maxFrames: Int): ByteArray { return calculateRms(this.samples, this.numSamples, this.channels, maxFrames) } } @@ -264,9 +265,9 @@ class DecodedAudio { * If number of samples per channel is less than "maxFrames", * the result array will match the source sample size instead. * - * @return RMS values float array where is each value is within [0..1] range. + * @return normalized RMS values as a signed byte array. */ -private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): FloatArray { +private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): ByteArray { val numFrames: Int val frameStep: Float @@ -310,7 +311,8 @@ private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, m // smoothArray(rmsValues, 1.0f) normalizeArray(rmsValues) - return rmsValues + // Convert normalized result to a signed byte array. + return rmsValues.map { value -> normalizedFloatToByte(value) }.toByteArray() } /** @@ -344,4 +346,14 @@ private fun smoothArray(values: FloatArray, neighborWeight: Float = 1f): FloatAr values[i + 1] * neighborWeight) / (1f + neighborWeight * 2f) } return result +} + +/** Turns a signed byte into a [0..1] float. */ +inline fun byteToNormalizedFloat(value: Byte): Float { + return (value + 128f) / 255f +} + +/** Turns a [0..1] float into a signed byte. */ +inline fun normalizedFloatToByte(value: Float): Byte { + return (255f * value - 128f).roundToInt().toByte() } \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index b3f054031c..a74b638de4 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -5,8 +5,6 @@ import android.content.res.ColorStateList import android.graphics.Color import android.graphics.PorterDuff import android.graphics.drawable.AnimatedVectorDrawable -import android.media.MediaDataSource -import android.os.Build import android.util.AttributeSet import android.view.View import android.view.View.OnTouchListener @@ -15,7 +13,6 @@ import android.widget.FrameLayout import android.widget.ImageView import android.widget.TextView import androidx.annotation.ColorInt -import androidx.annotation.RequiresApi import androidx.core.content.ContextCompat import androidx.core.graphics.ColorUtils import com.pnikosis.materialishprogress.ProgressWheel @@ -24,18 +21,18 @@ import network.loki.messenger.R import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode -import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.ApplicationContext +import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.audio.AudioSlidePlayer import org.thoughtcrime.securesms.components.AnimatingToggle import org.thoughtcrime.securesms.database.AttachmentDatabase +import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.events.PartProgressEvent import org.thoughtcrime.securesms.logging.Log -import org.thoughtcrime.securesms.loki.utilities.audio.DecodedAudio +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob import org.thoughtcrime.securesms.mms.AudioSlide -import org.thoughtcrime.securesms.mms.PartAuthority import org.thoughtcrime.securesms.mms.SlideClickListener import java.io.IOException -import java.io.InputStream import java.util.* import java.util.concurrent.TimeUnit @@ -166,7 +163,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { } // Post to make sure it executes only when the view is attached to a window. - post(::updateSeekBarFromAudio) + post(::updateFromAttachmentAudioExtras) } } audioSlidePlayer = AudioSlidePlayer.createFor(context, audio, this) @@ -254,122 +251,73 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { pauseToPlayDrawable.start() } - private fun updateSeekBarFromAudio() { - if (audioSlidePlayer == null) return - + private fun obtainDatabaseAttachment(): DatabaseAttachment? { + audioSlidePlayer ?: return null val attachment = audioSlidePlayer!!.audioSlide.asAttachment() + return if (attachment is DatabaseAttachment) attachment else null + } - // Parse audio and compute RMS values for the WaveformSeekBar in the background. - asyncCoroutineScope!!.launch { - val rmsFrames = 32 // The amount of values to be computed for the visualization. - - fun extractAttachmentRandomSeed(attachment: Attachment): Int { - return when { - attachment.digest != null -> attachment.digest!!.sum() - attachment.fileName != null -> attachment.fileName.hashCode() - else -> attachment.hashCode() - } - } + private fun updateFromAttachmentAudioExtras() { + val attachment = obtainDatabaseAttachment() ?: return - fun generateFakeRms(seed: Int, frames: Int = rmsFrames): FloatArray { - return Random(seed.toLong()).let { (0 until frames).map { i -> it.nextFloat() }.toFloatArray() } - } + val audioExtras = DatabaseFactory.getAttachmentDatabase(context) + .getAttachmentAudioExtras(attachment.attachmentId) - var rmsValues: FloatArray - var totalDurationMs: Long = -1 - - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { - // Due to API version incompatibility, we just display some random waveform for older API. - rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) - } else { - try { - @Suppress("BlockingMethodInNonBlockingContext") - val decodedAudio = PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { - DecodedAudio.create(InputStreamMediaDataSource(it)) - } - rmsValues = decodedAudio.calculateRms(rmsFrames) - totalDurationMs = (decodedAudio.totalDuration / 1000.0).toLong() - } catch (e: Exception) { - android.util.Log.w(TAG, "Failed to decode sample values for the audio attachment \"${attachment.fileName}\".", e) - rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) - } - } - - android.util.Log.d(TAG, "RMS: ${rmsValues.joinToString()}") + // Schedule a job request if no audio extras were generated yet. + if (audioExtras == null) { + ApplicationContext.getInstance(context).jobManager + .add(PrepareAttachmentAudioExtrasJob(attachment.attachmentId)) + return + } - post { - loadingAnimation.stop() - seekBar.sampleData = rmsValues + loadingAnimation.stop() + seekBar.sampleData = audioExtras.visualSamples - if (totalDurationMs > 0) { - totalDuration.visibility = View.VISIBLE - totalDuration.text = String.format("%02d:%02d", - TimeUnit.MILLISECONDS.toMinutes(totalDurationMs), - TimeUnit.MILLISECONDS.toSeconds(totalDurationMs)) - } - } + if (audioExtras.durationMs > 0) { + totalDuration.visibility = View.VISIBLE + totalDuration.text = String.format("%02d:%02d", + TimeUnit.MILLISECONDS.toMinutes(audioExtras.durationMs), + TimeUnit.MILLISECONDS.toSeconds(audioExtras.durationMs)) } } @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) - fun onEventAsync(event: PartProgressEvent) { + fun onEvent(event: PartProgressEvent) { if (audioSlidePlayer != null && event.attachment == audioSlidePlayer!!.audioSlide.asAttachment()) { downloadProgress.setInstantProgress(event.progress.toFloat() / event.total) } } -} - -private class SeekBarLoadingAnimation( - private val hostView: View, - private val seekBar: WaveformSeekBar): Runnable { - - companion object { - private const val UPDATE_PERIOD = 500L // In milliseconds. - private val random = Random() - } - - fun start() { - stop() - run() - } - fun stop() { - hostView.removeCallbacks(this) - } - - override fun run() { - seekBar.sampleData = (0 until 64).map { random.nextFloat() * 0.5f }.toFloatArray() - hostView.postDelayed(this, UPDATE_PERIOD) + @Subscribe(threadMode = ThreadMode.MAIN) + fun onEvent(event: PrepareAttachmentAudioExtrasJob.AudioExtrasUpdatedEvent) { + if (event.audioExtras.attachmentId == obtainDatabaseAttachment()?.attachmentId) { + updateFromAttachmentAudioExtras() + } } -} - -@RequiresApi(Build.VERSION_CODES.M) -private class InputStreamMediaDataSource: MediaDataSource { - private val data: ByteArray + private class SeekBarLoadingAnimation( + private val hostView: View, + private val seekBar: WaveformSeekBar): Runnable { - constructor(inputStream: InputStream): super() { - this.data = inputStream.readBytes() - } - - override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { - val length: Int = data.size - if (position >= length) { - return -1 // -1 indicates EOF + companion object { + private const val UPDATE_PERIOD = 350L // In milliseconds. + private val random = Random() } - var actualSize = size - if (position + size > length) { - actualSize -= (position + size - length).toInt() + + fun start() { + stop() + hostView.postDelayed(this, UPDATE_PERIOD) } - System.arraycopy(data, position.toInt(), buffer, offset, actualSize) - return actualSize - } - override fun getSize(): Long { - return data.size.toLong() - } + fun stop() { + hostView.removeCallbacks(this) + } - override fun close() { - // We don't need to close the wrapped stream. + override fun run() { + // Generate a random samples with values up to the 50% of the maximum value. + seekBar.sampleData = ByteArray(PrepareAttachmentAudioExtrasJob.VISUAL_RMS_FRAMES) + { (random.nextInt(127) - 64).toByte() } + hostView.postDelayed(this, UPDATE_PERIOD) + } } } \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 95ed82f215..3870bad7fa 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -7,7 +7,6 @@ import android.graphics.Color import android.graphics.Paint import android.graphics.RectF import android.util.AttributeSet -import android.util.Log import android.util.TypedValue import android.view.MotionEvent import android.view.View @@ -15,8 +14,11 @@ import android.view.ViewConfiguration import android.view.animation.DecelerateInterpolator import androidx.core.math.MathUtils import network.loki.messenger.R +import org.thoughtcrime.securesms.loki.utilities.audio.byteToNormalizedFloat import java.lang.Math.abs import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt class WaveformSeekBar : View { @@ -32,8 +34,8 @@ class WaveformSeekBar : View { } private val sampleDataHolder = SampleDataHolder(::invalidate) - /** An array if normalized to [0..1] values representing the audio signal. */ - var sampleData: FloatArray? + /** An array of signed byte values representing the audio signal. */ + var sampleData: ByteArray? get() { return sampleDataHolder.getSamples() } @@ -155,7 +157,8 @@ class WaveformSeekBar : View { var lastBarRight = paddingLeft.toFloat() (0 until barAmount).forEach { barIdx -> - val barValue = sampleDataHolder.computeBarValue(barIdx, barAmount) + // Convert a signed byte to a [0..1] float. + val barValue = byteToNormalizedFloat(sampleDataHolder.computeBarValue(barIdx, barAmount)) val barHeight = max(barMinHeight, getAvailableHeight() * barValue) @@ -246,16 +249,17 @@ class WaveformSeekBar : View { private class SampleDataHolder(private val invalidateDelegate: () -> Any) { - private var sampleDataFrom: FloatArray? = null - private var sampleDataTo: FloatArray? = null + private var sampleDataFrom: ByteArray? = null + private var sampleDataTo: ByteArray? = null private var progress = 1f // Mix between from and to values. private var animation: ValueAnimator? = null - fun computeBarValue(barIdx: Int, barAmount: Int): Float { - fun getSampleValue(sampleData: FloatArray?): Float { + fun computeBarValue(barIdx: Int, barAmount: Int): Byte { + /** @return The array's value at the interpolated index. */ + fun getSampleValue(sampleData: ByteArray?): Byte { if (sampleData == null || sampleData.isEmpty()) - return 0f + return Byte.MIN_VALUE else { val sampleIdx = (barIdx * (sampleData.size / barAmount.toFloat())).toInt() return sampleData[sampleIdx] @@ -268,12 +272,21 @@ class WaveformSeekBar : View { val fromValue = getSampleValue(sampleDataFrom) val toValue = getSampleValue(sampleDataTo) - - return fromValue * (1f - progress) + toValue * progress + val rawResultValue = fromValue * (1f - progress) + toValue * progress + return rawResultValue.roundToInt().toByte() } - fun setSamples(sampleData: FloatArray?) { - sampleDataFrom = sampleDataTo + fun setSamples(sampleData: ByteArray?) { + /** @return a mix between [sampleDataFrom] and [sampleDataTo] arrays according to the current [progress] value. */ + fun computeNewDataFromArray(): ByteArray? { + if (sampleDataTo == null) return null + if (sampleDataFrom == null) return sampleDataTo + + val sampleSize = min(sampleDataFrom!!.size, sampleDataTo!!.size) + return ByteArray(sampleSize) { i -> computeBarValue(i, sampleSize) } + } + + sampleDataFrom = computeNewDataFromArray() sampleDataTo = sampleData progress = 0f @@ -281,7 +294,6 @@ class WaveformSeekBar : View { animation = ValueAnimator.ofFloat(0f, 1f).apply { addUpdateListener { animation -> progress = animation.animatedValue as Float - Log.d("MTPHR", "Progress: $progress") invalidateDelegate() } interpolator = DecelerateInterpolator(3f) @@ -290,7 +302,7 @@ class WaveformSeekBar : View { } } - fun getSamples(): FloatArray? { + fun getSamples(): ByteArray? { return sampleDataTo } } diff --git a/src/org/thoughtcrime/securesms/util/ParcelableUtil.kt b/src/org/thoughtcrime/securesms/util/ParcelableUtil.kt new file mode 100644 index 0000000000..2756500b22 --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/ParcelableUtil.kt @@ -0,0 +1,32 @@ +package org.thoughtcrime.securesms.util + +import android.os.Parcel + +import android.os.Parcelable + +object ParcelableUtil { + @JvmStatic + fun marshall(parcelable: Parcelable): ByteArray { + val parcel = Parcel.obtain() + parcelable.writeToParcel(parcel, 0) + val bytes = parcel.marshall() + parcel.recycle() + return bytes + } + + @JvmStatic + fun unmarshall(bytes: ByteArray): Parcel { + val parcel = Parcel.obtain() + parcel.unmarshall(bytes, 0, bytes.size) + parcel.setDataPosition(0) // This is extremely important! + return parcel + } + + @JvmStatic + fun unmarshall(bytes: ByteArray, creator: Parcelable.Creator): T { + val parcel: Parcel = ParcelableUtil.unmarshall(bytes) + val result = creator.createFromParcel(parcel) + parcel.recycle() + return result + } +} \ No newline at end of file