From fdaadcb2b0991e68e55dea479a87b616cbe68f00 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Fri, 2 Oct 2020 13:32:42 +1000 Subject: [PATCH 01/23] Audio view ported to Kotlin. --- .../securesms/components/AudioView.java | 330 ------------------ .../securesms/components/AudioView.kt | 267 ++++++++++++++ 2 files changed, 267 insertions(+), 330 deletions(-) delete mode 100644 src/org/thoughtcrime/securesms/components/AudioView.java create mode 100644 src/org/thoughtcrime/securesms/components/AudioView.kt diff --git a/src/org/thoughtcrime/securesms/components/AudioView.java b/src/org/thoughtcrime/securesms/components/AudioView.java deleted file mode 100644 index 9e4c7c3e9a..0000000000 --- a/src/org/thoughtcrime/securesms/components/AudioView.java +++ /dev/null @@ -1,330 +0,0 @@ -package org.thoughtcrime.securesms.components; - -import android.annotation.TargetApi; -import android.content.Context; -import android.content.res.ColorStateList; -import android.content.res.TypedArray; -import android.graphics.Color; -import android.graphics.PorterDuff; -import android.graphics.drawable.AnimatedVectorDrawable; -import android.os.Build; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import android.util.AttributeSet; -import android.view.MotionEvent; -import android.view.View; -import android.view.ViewGroup; -import android.widget.FrameLayout; -import android.widget.ImageView; -import android.widget.SeekBar; -import android.widget.TextView; - -import com.pnikosis.materialishprogress.ProgressWheel; - -import org.greenrobot.eventbus.EventBus; -import org.greenrobot.eventbus.Subscribe; -import org.greenrobot.eventbus.ThreadMode; -import org.thoughtcrime.securesms.audio.AudioSlidePlayer; -import org.thoughtcrime.securesms.database.AttachmentDatabase; -import org.thoughtcrime.securesms.events.PartProgressEvent; -import org.thoughtcrime.securesms.logging.Log; -import org.thoughtcrime.securesms.mms.AudioSlide; -import org.thoughtcrime.securesms.mms.SlideClickListener; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import network.loki.messenger.R; - - -public class AudioView extends FrameLayout implements AudioSlidePlayer.Listener { - - private static final String TAG = AudioView.class.getSimpleName(); - - private final @NonNull AnimatingToggle controlToggle; - private final @NonNull ViewGroup container; - private final @NonNull ImageView playButton; - private final @NonNull ImageView pauseButton; - private final @NonNull ImageView downloadButton; - private final @NonNull ProgressWheel downloadProgress; - private final @NonNull SeekBar seekBar; - private final @NonNull TextView timestamp; - - private @Nullable SlideClickListener downloadListener; - private @Nullable AudioSlidePlayer audioSlidePlayer; - private int backwardsCounter; - - public AudioView(Context context) { - this(context, null); - } - - public AudioView(Context context, AttributeSet attrs) { - this(context, attrs, 0); - } - - public AudioView(Context context, AttributeSet attrs, int defStyleAttr) { - super(context, attrs, defStyleAttr); - inflate(context, R.layout.audio_view, this); - - this.container = (ViewGroup) findViewById(R.id.audio_widget_container); - this.controlToggle = (AnimatingToggle) findViewById(R.id.control_toggle); - this.playButton = (ImageView) findViewById(R.id.play); - this.pauseButton = (ImageView) findViewById(R.id.pause); - this.downloadButton = (ImageView) findViewById(R.id.download); - this.downloadProgress = (ProgressWheel) findViewById(R.id.download_progress); - this.seekBar = (SeekBar) findViewById(R.id.seek); - this.timestamp = (TextView) findViewById(R.id.timestamp); - - this.playButton.setOnClickListener(new PlayClickedListener()); - this.pauseButton.setOnClickListener(new PauseClickedListener()); - this.seekBar.setOnSeekBarChangeListener(new SeekBarModifiedListener()); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - this.playButton.setImageDrawable(context.getDrawable(R.drawable.play_icon)); - this.pauseButton.setImageDrawable(context.getDrawable(R.drawable.pause_icon)); - this.playButton.setBackground(context.getDrawable(R.drawable.ic_circle_fill_white_48dp)); - this.pauseButton.setBackground(context.getDrawable(R.drawable.ic_circle_fill_white_48dp)); - } - - if (attrs != null) { - TypedArray typedArray = context.getTheme().obtainStyledAttributes(attrs, R.styleable.AudioView, 0, 0); - setTint(typedArray.getColor(R.styleable.AudioView_foregroundTintColor, Color.WHITE), - typedArray.getColor(R.styleable.AudioView_backgroundTintColor, Color.WHITE)); - container.setBackgroundColor(typedArray.getColor(R.styleable.AudioView_widgetBackground, Color.TRANSPARENT)); - typedArray.recycle(); - } - } - - @Override - protected void onAttachedToWindow() { - super.onAttachedToWindow(); - if (!EventBus.getDefault().isRegistered(this)) EventBus.getDefault().register(this); - } - - @Override - protected void onDetachedFromWindow() { - super.onDetachedFromWindow(); - EventBus.getDefault().unregister(this); - } - - public void setAudio(final @NonNull AudioSlide audio, - final boolean showControls) - { - - if (showControls && audio.isPendingDownload()) { - controlToggle.displayQuick(downloadButton); - seekBar.setEnabled(false); - downloadButton.setOnClickListener(new DownloadClickedListener(audio)); - if (downloadProgress.isSpinning()) downloadProgress.stopSpinning(); - } else if (showControls && audio.getTransferState() == AttachmentDatabase.TRANSFER_PROGRESS_STARTED) { - controlToggle.displayQuick(downloadProgress); - seekBar.setEnabled(false); - downloadProgress.spin(); - } else { - controlToggle.displayQuick(playButton); - seekBar.setEnabled(true); - if (downloadProgress.isSpinning()) downloadProgress.stopSpinning(); - } - - this.audioSlidePlayer = AudioSlidePlayer.createFor(getContext(), audio, this); - } - - public void cleanup() { - if (this.audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { - this.audioSlidePlayer.stop(); - } - } - - public void setDownloadClickListener(@Nullable SlideClickListener listener) { - this.downloadListener = listener; - } - - @Override - public void onStart() { - if (this.pauseButton.getVisibility() != View.VISIBLE) { - togglePlayToPause(); - } - } - - @Override - public void onStop() { - if (this.playButton.getVisibility() != View.VISIBLE) { - togglePauseToPlay(); - } - - if (seekBar.getProgress() + 5 >= seekBar.getMax()) { - backwardsCounter = 4; - onProgress(0.0, 0); - } - } - - @Override - public void setFocusable(boolean focusable) { - super.setFocusable(focusable); - this.playButton.setFocusable(focusable); - this.pauseButton.setFocusable(focusable); - this.seekBar.setFocusable(focusable); - this.seekBar.setFocusableInTouchMode(focusable); - this.downloadButton.setFocusable(focusable); - } - - @Override - public void setClickable(boolean clickable) { - super.setClickable(clickable); - this.playButton.setClickable(clickable); - this.pauseButton.setClickable(clickable); - this.seekBar.setClickable(clickable); - this.seekBar.setOnTouchListener(clickable ? null : new TouchIgnoringListener()); - this.downloadButton.setClickable(clickable); - } - - @Override - public void setEnabled(boolean enabled) { - super.setEnabled(enabled); - this.playButton.setEnabled(enabled); - this.pauseButton.setEnabled(enabled); - this.seekBar.setEnabled(enabled); - this.downloadButton.setEnabled(enabled); - } - - @Override - public void onProgress(double progress, long millis) { - int seekProgress = (int)Math.floor(progress * this.seekBar.getMax()); - - if (seekProgress > seekBar.getProgress() || backwardsCounter > 3) { - backwardsCounter = 0; - this.seekBar.setProgress(seekProgress); - this.timestamp.setText(String.format("%02d:%02d", - TimeUnit.MILLISECONDS.toMinutes(millis), - TimeUnit.MILLISECONDS.toSeconds(millis))); - } else { - backwardsCounter++; - } - } - - public void setTint(int foregroundTint, int backgroundTint) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - this.playButton.setBackgroundTintList(ColorStateList.valueOf(foregroundTint)); - this.playButton.setImageTintList(ColorStateList.valueOf(backgroundTint)); - this.pauseButton.setBackgroundTintList(ColorStateList.valueOf(foregroundTint)); - this.pauseButton.setImageTintList(ColorStateList.valueOf(backgroundTint)); - } else { - this.playButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - this.pauseButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - } - - this.downloadButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - this.downloadProgress.setBarColor(foregroundTint); - - this.timestamp.setTextColor(foregroundTint); - this.seekBar.getProgressDrawable().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - this.seekBar.getThumb().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - } - - private double getProgress() { - if (this.seekBar.getProgress() <= 0 || this.seekBar.getMax() <= 0) { - return 0; - } else { - return (double)this.seekBar.getProgress() / (double)this.seekBar.getMax(); - } - } - - private void togglePlayToPause() { - controlToggle.displayQuick(pauseButton); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - AnimatedVectorDrawable playToPauseDrawable = (AnimatedVectorDrawable)getContext().getDrawable(R.drawable.play_to_pause_animation); - pauseButton.setImageDrawable(playToPauseDrawable); - playToPauseDrawable.start(); - } - } - - private void togglePauseToPlay() { - controlToggle.displayQuick(playButton); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - AnimatedVectorDrawable pauseToPlayDrawable = (AnimatedVectorDrawable)getContext().getDrawable(R.drawable.pause_to_play_animation); - playButton.setImageDrawable(pauseToPlayDrawable); - pauseToPlayDrawable.start(); - } - } - - private class PlayClickedListener implements View.OnClickListener { - @TargetApi(Build.VERSION_CODES.LOLLIPOP) - @Override - public void onClick(View v) { - try { - Log.d(TAG, "playbutton onClick"); - if (audioSlidePlayer != null) { - togglePlayToPause(); - audioSlidePlayer.play(getProgress()); - } - } catch (IOException e) { - Log.w(TAG, e); - } - } - } - - private class PauseClickedListener implements View.OnClickListener { - @TargetApi(Build.VERSION_CODES.LOLLIPOP) - @Override - public void onClick(View v) { - Log.d(TAG, "pausebutton onClick"); - if (audioSlidePlayer != null) { - togglePauseToPlay(); - audioSlidePlayer.stop(); - } - } - } - - private class DownloadClickedListener implements View.OnClickListener { - private final @NonNull AudioSlide slide; - - private DownloadClickedListener(@NonNull AudioSlide slide) { - this.slide = slide; - } - - @Override - public void onClick(View v) { - if (downloadListener != null) downloadListener.onClick(v, slide); - } - } - - private class SeekBarModifiedListener implements SeekBar.OnSeekBarChangeListener { - @Override - public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {} - - @Override - public synchronized void onStartTrackingTouch(SeekBar seekBar) { - if (audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { - audioSlidePlayer.stop(); - } - } - - @Override - public synchronized void onStopTrackingTouch(SeekBar seekBar) { - try { - if (audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { - audioSlidePlayer.play(getProgress()); - } - } catch (IOException e) { - Log.w(TAG, e); - } - } - } - - private static class TouchIgnoringListener implements OnTouchListener { - @Override - public boolean onTouch(View v, MotionEvent event) { - return true; - } - } - - @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) - public void onEventAsync(final PartProgressEvent event) { - if (audioSlidePlayer != null && event.attachment.equals(audioSlidePlayer.getAudioSlide().asAttachment())) { - downloadProgress.setInstantProgress(((float) event.progress) / event.total); - } - } - -} diff --git a/src/org/thoughtcrime/securesms/components/AudioView.kt b/src/org/thoughtcrime/securesms/components/AudioView.kt new file mode 100644 index 0000000000..6e640662d8 --- /dev/null +++ b/src/org/thoughtcrime/securesms/components/AudioView.kt @@ -0,0 +1,267 @@ +package org.thoughtcrime.securesms.components + +import android.content.Context +import android.content.res.ColorStateList +import android.graphics.Color +import android.graphics.PorterDuff +import android.graphics.drawable.AnimatedVectorDrawable +import android.os.Build +import android.util.AttributeSet +import android.view.View +import android.view.View.OnTouchListener +import android.view.ViewGroup +import android.widget.FrameLayout +import android.widget.ImageView +import android.widget.SeekBar +import android.widget.SeekBar.OnSeekBarChangeListener +import android.widget.TextView +import androidx.core.content.ContextCompat +import androidx.core.graphics.BlendModeColorFilterCompat.createBlendModeColorFilterCompat +import androidx.core.graphics.BlendModeCompat +import com.pnikosis.materialishprogress.ProgressWheel +import network.loki.messenger.R +import org.greenrobot.eventbus.EventBus +import org.greenrobot.eventbus.Subscribe +import org.greenrobot.eventbus.ThreadMode +import org.thoughtcrime.securesms.audio.AudioSlidePlayer +import org.thoughtcrime.securesms.database.AttachmentDatabase +import org.thoughtcrime.securesms.events.PartProgressEvent +import org.thoughtcrime.securesms.logging.Log +import org.thoughtcrime.securesms.mms.AudioSlide +import org.thoughtcrime.securesms.mms.SlideClickListener +import java.io.IOException +import java.util.concurrent.TimeUnit +import kotlin.math.floor + +class AudioView: FrameLayout, AudioSlidePlayer.Listener { + + companion object { + private const val TAG = "AudioViewKt" + } + + private val controlToggle: AnimatingToggle + private val container: ViewGroup + private val playButton: ImageView + private val pauseButton: ImageView + private val downloadButton: ImageView + private val downloadProgress: ProgressWheel + private val seekBar: SeekBar + private val timestamp: TextView + + private var downloadListener: SlideClickListener? = null + private var audioSlidePlayer: AudioSlidePlayer? = null + private var backwardsCounter = 0 + + constructor(context: Context): this(context, null) + + constructor(context: Context, attrs: AttributeSet?): this(context, attrs, 0) + + constructor(context: Context, attrs: AttributeSet?, defStyleAttr: Int): super(context, attrs, defStyleAttr) { + View.inflate(context, R.layout.audio_view, this) + container = findViewById(R.id.audio_widget_container) as ViewGroup + controlToggle = findViewById(R.id.control_toggle) as AnimatingToggle + playButton = findViewById(R.id.play) as ImageView + pauseButton = findViewById(R.id.pause) as ImageView + downloadButton = findViewById(R.id.download) as ImageView + downloadProgress = findViewById(R.id.download_progress) as ProgressWheel + seekBar = findViewById(R.id.seek) as SeekBar + timestamp = findViewById(R.id.timestamp) as TextView + + playButton.setOnClickListener { + try { + Log.d(TAG, "playbutton onClick") + if (audioSlidePlayer != null) { + togglePlayToPause() + audioSlidePlayer!!.play(getProgress()) + } + } catch (e: IOException) { + Log.w(TAG, e) + } + } + pauseButton.setOnClickListener { + Log.d(TAG, "pausebutton onClick") + if (audioSlidePlayer != null) { + togglePauseToPlay() + audioSlidePlayer!!.stop() + } + } + seekBar.setOnSeekBarChangeListener(SeekBarModifiedListener()) + + playButton.setImageDrawable(ContextCompat.getDrawable(context, R.drawable.play_icon)) + pauseButton.setImageDrawable(ContextCompat.getDrawable(context, R.drawable.pause_icon)) + playButton.background = ContextCompat.getDrawable(context, R.drawable.ic_circle_fill_white_48dp) + pauseButton.background = ContextCompat.getDrawable(context, R.drawable.ic_circle_fill_white_48dp) + + if (attrs != null) { + val typedArray = context.theme.obtainStyledAttributes(attrs, R.styleable.AudioView, 0, 0) + setTint(typedArray.getColor(R.styleable.AudioView_foregroundTintColor, Color.WHITE), + typedArray.getColor(R.styleable.AudioView_backgroundTintColor, Color.WHITE)) + container.setBackgroundColor(typedArray.getColor(R.styleable.AudioView_widgetBackground, Color.TRANSPARENT)) + typedArray.recycle() + } + } + + override fun onAttachedToWindow() { + super.onAttachedToWindow() + if (!EventBus.getDefault().isRegistered(this)) EventBus.getDefault().register(this) + } + + override fun onDetachedFromWindow() { + super.onDetachedFromWindow() + EventBus.getDefault().unregister(this) + } + + fun setAudio(audio: AudioSlide, showControls: Boolean) { + if (showControls && audio.isPendingDownload) { + controlToggle.displayQuick(downloadButton) + seekBar.isEnabled = false + downloadButton.setOnClickListener { v -> downloadListener?.onClick(v, audio) } + if (downloadProgress.isSpinning) { + downloadProgress.stopSpinning() + } + } else if (showControls && audio.transferState == AttachmentDatabase.TRANSFER_PROGRESS_STARTED) { + controlToggle.displayQuick(downloadProgress) + seekBar.isEnabled = false + downloadProgress.spin() + } else { + controlToggle.displayQuick(playButton) + seekBar.isEnabled = true + if (downloadProgress.isSpinning) { + downloadProgress.stopSpinning() + } + } + audioSlidePlayer = AudioSlidePlayer.createFor(context, audio, this) + } + + fun cleanup() { + if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { + audioSlidePlayer!!.stop() + } + } + + fun setDownloadClickListener(listener: SlideClickListener?) { + downloadListener = listener + } + + fun setTint(foregroundTint: Int, backgroundTint: Int) { + playButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) + playButton.imageTintList = ColorStateList.valueOf(backgroundTint) + pauseButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) + pauseButton.imageTintList = ColorStateList.valueOf(backgroundTint) + + downloadButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN) + downloadProgress.barColor = foregroundTint + timestamp.setTextColor(foregroundTint) + + val colorFilter = createBlendModeColorFilterCompat(foregroundTint, BlendModeCompat.SRC_IN) + seekBar.progressDrawable.colorFilter = colorFilter + seekBar.thumb.colorFilter = colorFilter + } + + override fun onStart() { + if (pauseButton.visibility != View.VISIBLE) { + togglePlayToPause() + } + } + + override fun onStop() { + if (playButton.visibility != View.VISIBLE) { + togglePauseToPlay() + } + if (seekBar.progress + 5 >= seekBar.max) { + backwardsCounter = 4 + onProgress(0.0, 0) + } + } + + override fun setFocusable(focusable: Boolean) { + super.setFocusable(focusable) + playButton.isFocusable = focusable + pauseButton.isFocusable = focusable + seekBar.isFocusable = focusable + seekBar.isFocusableInTouchMode = focusable + downloadButton.isFocusable = focusable + } + + override fun setClickable(clickable: Boolean) { + super.setClickable(clickable) + playButton.isClickable = clickable + pauseButton.isClickable = clickable + seekBar.isClickable = clickable + seekBar.setOnTouchListener(if (clickable) null else + OnTouchListener { _, _ -> return@OnTouchListener true }) // Suppress touch events. + downloadButton.isClickable = clickable + } + + override fun setEnabled(enabled: Boolean) { + super.setEnabled(enabled) + playButton.isEnabled = enabled + pauseButton.isEnabled = enabled + seekBar.isEnabled = enabled + downloadButton.isEnabled = enabled + } + + override fun onProgress(progress: Double, millis: Long) { + val seekProgress = floor(progress * seekBar.max).toInt() + if (seekProgress > seekBar.progress || backwardsCounter > 3) { + backwardsCounter = 0 + seekBar.progress = seekProgress + timestamp.text = String.format("%02d:%02d", + TimeUnit.MILLISECONDS.toMinutes(millis), + TimeUnit.MILLISECONDS.toSeconds(millis)) + } else { + backwardsCounter++ + } + } + + private fun getProgress(): Double { + return if (seekBar.progress <= 0 || seekBar.max <= 0) { + 0.0 + } else { + seekBar.progress.toDouble() / seekBar.max.toDouble() + } + } + + private fun togglePlayToPause() { + controlToggle.displayQuick(pauseButton) + val playToPauseDrawable = ContextCompat.getDrawable(context, R.drawable.play_to_pause_animation) as AnimatedVectorDrawable + pauseButton.setImageDrawable(playToPauseDrawable) + playToPauseDrawable.start() + } + + private fun togglePauseToPlay() { + controlToggle.displayQuick(playButton) + val pauseToPlayDrawable = ContextCompat.getDrawable(context, R.drawable.pause_to_play_animation) as AnimatedVectorDrawable + playButton.setImageDrawable(pauseToPlayDrawable) + pauseToPlayDrawable.start() + } + + private inner class SeekBarModifiedListener : OnSeekBarChangeListener { + override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) {} + + @Synchronized + override fun onStartTrackingTouch(seekBar: SeekBar) { + if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { + audioSlidePlayer!!.stop() + } + } + + @Synchronized + override fun onStopTrackingTouch(seekBar: SeekBar) { + try { + if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { + audioSlidePlayer!!.play(getProgress()) + } + } catch (e: IOException) { + Log.w(TAG, e) + } + } + } + + @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) + fun onEventAsync(event: PartProgressEvent) { + if (audioSlidePlayer != null && event.attachment == audioSlidePlayer!!.audioSlide.asAttachment()) { + downloadProgress.setInstantProgress(event.progress.toFloat() / event.total) + } + } +} \ No newline at end of file From e07cb716c005504e5a1396834cf4bf099bc99cd2 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Wed, 7 Oct 2020 17:43:14 +1100 Subject: [PATCH 02/23] Use waveform seek bar for audio message view. --- res/layout/audio_view.xml | 14 +- res/values/attrs.xml | 16 + .../securesms/audio/AudioSlidePlayer.java | 55 +-- .../securesms/components/AudioView.kt | 134 ++++---- .../securesms/components/WaveformSeekBar.kt | 317 ++++++++++++++++++ 5 files changed, 447 insertions(+), 89 deletions(-) create mode 100644 src/org/thoughtcrime/securesms/components/WaveformSeekBar.kt diff --git a/res/layout/audio_view.xml b/res/layout/audio_view.xml index e5a33d9a41..38687742db 100644 --- a/res/layout/audio_view.xml +++ b/res/layout/audio_view.xml @@ -70,10 +70,18 @@ - + + android:layout_height="30dp" + android:layout_gravity="center_vertical" + app:wave_background_color="#bbb" + app:wave_progress_color="?colorPrimary" + app:wave_gravity="center" + app:wave_width="4dp" + app:wave_corner_radius="2dp" + app:wave_gap="1dp"/> diff --git a/res/values/attrs.xml b/res/values/attrs.xml index 63c47b155d..62fadc1379 100644 --- a/res/values/attrs.xml +++ b/res/values/attrs.xml @@ -287,4 +287,20 @@ + + + + + + + + + + + + + + + + diff --git a/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java b/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java index fa65d129e4..4c7878e543 100644 --- a/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java +++ b/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java @@ -32,6 +32,7 @@ import com.google.android.exoplayer2.source.MediaSource; import com.google.android.exoplayer2.trackselection.DefaultTrackSelector; import com.google.android.exoplayer2.upstream.DefaultDataSourceFactory; +import org.jetbrains.annotations.NotNull; import org.thoughtcrime.securesms.attachments.AttachmentServer; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.AudioSlide; @@ -150,7 +151,11 @@ public class AudioSlidePlayer implements SensorEventListener { case Player.STATE_ENDED: Log.i(TAG, "onComplete"); + + long millis = mediaPlayer.getDuration(); + synchronized (AudioSlidePlayer.this) { + mediaPlayer.release(); mediaPlayer = null; if (audioAttachmentServer != null) { @@ -167,6 +172,7 @@ public class AudioSlidePlayer implements SensorEventListener { } } + notifyOnProgress(1.0, millis); notifyOnStop(); progressEventHandler.removeMessages(0); } @@ -233,6 +239,22 @@ public class AudioSlidePlayer implements SensorEventListener { } } + public synchronized boolean isReady() { + if (mediaPlayer == null) return false; + + return mediaPlayer.getPlaybackState() == Player.STATE_READY && mediaPlayer.getPlayWhenReady(); + } + + public synchronized void seekTo(double progress) throws IOException { + if (mediaPlayer == null) return; + + if (isReady()) { + mediaPlayer.seekTo((long) (mediaPlayer.getDuration() * progress)); + } else { + play(progress); + } + } + public void setListener(@NonNull Listener listener) { this.listener = new WeakReference<>(listener); @@ -256,30 +278,15 @@ public class AudioSlidePlayer implements SensorEventListener { } private void notifyOnStart() { - Util.runOnMain(new Runnable() { - @Override - public void run() { - getListener().onStart(); - } - }); + Util.runOnMain(() -> getListener().onPlayerStart(AudioSlidePlayer.this)); } private void notifyOnStop() { - Util.runOnMain(new Runnable() { - @Override - public void run() { - getListener().onStop(); - } - }); + Util.runOnMain(() -> getListener().onPlayerStop(AudioSlidePlayer.this)); } private void notifyOnProgress(final double progress, final long millis) { - Util.runOnMain(new Runnable() { - @Override - public void run() { - getListener().onProgress(progress, millis); - } - }); + Util.runOnMain(() -> getListener().onPlayerProgress(AudioSlidePlayer.this, progress, millis)); } private @NonNull Listener getListener() { @@ -288,11 +295,11 @@ public class AudioSlidePlayer implements SensorEventListener { if (listener != null) return listener; else return new Listener() { @Override - public void onStart() {} + public void onPlayerStart(@NotNull AudioSlidePlayer player) { } @Override - public void onStop() {} + public void onPlayerStop(@NotNull AudioSlidePlayer player) { } @Override - public void onProgress(double progress, long millis) {} + public void onPlayerProgress(@NotNull AudioSlidePlayer player, double progress, long millis) { } }; } @@ -355,9 +362,9 @@ public class AudioSlidePlayer implements SensorEventListener { } public interface Listener { - void onStart(); - void onStop(); - void onProgress(double progress, long millis); + void onPlayerStart(@NonNull AudioSlidePlayer player); + void onPlayerStop(@NonNull AudioSlidePlayer player); + void onPlayerProgress(@NonNull AudioSlidePlayer player, double progress, long millis); } private static class ProgressEventHandler extends Handler { diff --git a/src/org/thoughtcrime/securesms/components/AudioView.kt b/src/org/thoughtcrime/securesms/components/AudioView.kt index 6e640662d8..ee47ac0b8a 100644 --- a/src/org/thoughtcrime/securesms/components/AudioView.kt +++ b/src/org/thoughtcrime/securesms/components/AudioView.kt @@ -30,6 +30,7 @@ import org.thoughtcrime.securesms.logging.Log import org.thoughtcrime.securesms.mms.AudioSlide import org.thoughtcrime.securesms.mms.SlideClickListener import java.io.IOException +import java.util.* import java.util.concurrent.TimeUnit import kotlin.math.floor @@ -45,7 +46,7 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { private val pauseButton: ImageView private val downloadButton: ImageView private val downloadProgress: ProgressWheel - private val seekBar: SeekBar + private val seekBar: WaveformSeekBar private val timestamp: TextView private var downloadListener: SlideClickListener? = null @@ -58,21 +59,25 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { constructor(context: Context, attrs: AttributeSet?, defStyleAttr: Int): super(context, attrs, defStyleAttr) { View.inflate(context, R.layout.audio_view, this) - container = findViewById(R.id.audio_widget_container) as ViewGroup - controlToggle = findViewById(R.id.control_toggle) as AnimatingToggle - playButton = findViewById(R.id.play) as ImageView - pauseButton = findViewById(R.id.pause) as ImageView - downloadButton = findViewById(R.id.download) as ImageView - downloadProgress = findViewById(R.id.download_progress) as ProgressWheel - seekBar = findViewById(R.id.seek) as SeekBar - timestamp = findViewById(R.id.timestamp) as TextView + container = findViewById(R.id.audio_widget_container) + controlToggle = findViewById(R.id.control_toggle) + playButton = findViewById(R.id.play) + pauseButton = findViewById(R.id.pause) + downloadButton = findViewById(R.id.download) + downloadProgress = findViewById(R.id.download_progress) + seekBar = findViewById(R.id.seek) + timestamp = findViewById(R.id.timestamp) playButton.setOnClickListener { try { Log.d(TAG, "playbutton onClick") if (audioSlidePlayer != null) { togglePlayToPause() - audioSlidePlayer!!.play(getProgress()) + + // Restart the playback if progress bar is near at the end. + val progress = if (seekBar.progress < 0.99f) seekBar.progress.toDouble() else 0.0 + + audioSlidePlayer!!.play(progress) } } catch (e: IOException) { Log.w(TAG, e) @@ -85,7 +90,17 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { audioSlidePlayer!!.stop() } } - seekBar.setOnSeekBarChangeListener(SeekBarModifiedListener()) + seekBar.progressChangeListener = object : WaveformSeekBar.ProgressChangeListener { + override fun onProgressChanged(waveformSeekBar: WaveformSeekBar, progress: Float, fromUser: Boolean) { + if (fromUser && audioSlidePlayer != null) { + synchronized(audioSlidePlayer!!) { + audioSlidePlayer!!.seekTo(progress.toDouble()) + } + } + } + } + //TODO Remove this. + seekBar.sample = Random().let { (0 until 64).map { i -> it.nextFloat() }.toFloatArray() } playButton.setImageDrawable(ContextCompat.getDrawable(context, R.drawable.play_icon)) pauseButton.setImageDrawable(ContextCompat.getDrawable(context, R.drawable.pause_icon)) @@ -153,25 +168,41 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { downloadProgress.barColor = foregroundTint timestamp.setTextColor(foregroundTint) - val colorFilter = createBlendModeColorFilterCompat(foregroundTint, BlendModeCompat.SRC_IN) - seekBar.progressDrawable.colorFilter = colorFilter - seekBar.thumb.colorFilter = colorFilter +// val colorFilter = createBlendModeColorFilterCompat(foregroundTint, BlendModeCompat.SRC_IN) +// seekBar.progressDrawable.colorFilter = colorFilter +// seekBar.thumb.colorFilter = colorFilter } - override fun onStart() { + override fun onPlayerStart(player: AudioSlidePlayer) { if (pauseButton.visibility != View.VISIBLE) { togglePlayToPause() } } - override fun onStop() { + override fun onPlayerStop(player: AudioSlidePlayer) { if (playButton.visibility != View.VISIBLE) { togglePauseToPlay() } - if (seekBar.progress + 5 >= seekBar.max) { - backwardsCounter = 4 - onProgress(0.0, 0) - } + +// if (seekBar.progress + 5 >= seekBar.max) { +// backwardsCounter = 4 +// onProgress(0.0, 0) +// } + } + + override fun onPlayerProgress(player: AudioSlidePlayer, progress: Double, millis: Long) { +// val seekProgress = floor(progress * seekBar.max).toInt() + //TODO Update text. + seekBar.progress = progress.toFloat() +// if (/*seekProgress > 1f || */backwardsCounter > 3) { +// backwardsCounter = 0 +// seekBar.progress = 1f +// timestamp.text = String.format("%02d:%02d", +// TimeUnit.MILLISECONDS.toMinutes(millis), +// TimeUnit.MILLISECONDS.toSeconds(millis)) +// } else { +// backwardsCounter++ +// } } override fun setFocusable(focusable: Boolean) { @@ -201,27 +232,6 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { downloadButton.isEnabled = enabled } - override fun onProgress(progress: Double, millis: Long) { - val seekProgress = floor(progress * seekBar.max).toInt() - if (seekProgress > seekBar.progress || backwardsCounter > 3) { - backwardsCounter = 0 - seekBar.progress = seekProgress - timestamp.text = String.format("%02d:%02d", - TimeUnit.MILLISECONDS.toMinutes(millis), - TimeUnit.MILLISECONDS.toSeconds(millis)) - } else { - backwardsCounter++ - } - } - - private fun getProgress(): Double { - return if (seekBar.progress <= 0 || seekBar.max <= 0) { - 0.0 - } else { - seekBar.progress.toDouble() / seekBar.max.toDouble() - } - } - private fun togglePlayToPause() { controlToggle.displayQuick(pauseButton) val playToPauseDrawable = ContextCompat.getDrawable(context, R.drawable.play_to_pause_animation) as AnimatedVectorDrawable @@ -236,27 +246,27 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { pauseToPlayDrawable.start() } - private inner class SeekBarModifiedListener : OnSeekBarChangeListener { - override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) {} - - @Synchronized - override fun onStartTrackingTouch(seekBar: SeekBar) { - if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { - audioSlidePlayer!!.stop() - } - } - - @Synchronized - override fun onStopTrackingTouch(seekBar: SeekBar) { - try { - if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { - audioSlidePlayer!!.play(getProgress()) - } - } catch (e: IOException) { - Log.w(TAG, e) - } - } - } +// private inner class SeekBarModifiedListener : OnSeekBarChangeListener { +// override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) {} +// +// @Synchronized +// override fun onStartTrackingTouch(seekBar: SeekBar) { +// if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { +// audioSlidePlayer!!.stop() +// } +// } +// +// @Synchronized +// override fun onStopTrackingTouch(seekBar: SeekBar) { +// try { +// if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { +// audioSlidePlayer!!.play(getProgress()) +// } +// } catch (e: IOException) { +// Log.w(TAG, e) +// } +// } +// } @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) fun onEventAsync(event: PartProgressEvent) { diff --git a/src/org/thoughtcrime/securesms/components/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/components/WaveformSeekBar.kt new file mode 100644 index 0000000000..19113d9eb7 --- /dev/null +++ b/src/org/thoughtcrime/securesms/components/WaveformSeekBar.kt @@ -0,0 +1,317 @@ +package org.thoughtcrime.securesms.components + +import android.content.Context +import android.graphics.* +import android.os.Handler +import android.os.Looper +import android.util.AttributeSet +import android.util.TypedValue +import android.view.MotionEvent +import android.view.View +import android.view.ViewConfiguration +import network.loki.messenger.R +import java.lang.IllegalArgumentException +import java.lang.Math.abs + +class WaveformSeekBar : View { + + companion object { + @JvmStatic + inline fun dp(context: Context, dp: Float): Float { + return TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, + dp, + context.resources.displayMetrics + ) + } + + @JvmStatic + inline fun smooth(values: FloatArray, neighborWeight: Float = 1f): FloatArray { + if (values.size < 3) return values + + val result = FloatArray(values.size) + result[0] = values[0] + result[values.size - 1] == values[values.size - 1] + for (i in 1 until values.size - 1) { + result[i] = + (values[i] + values[i - 1] * neighborWeight + values[i + 1] * neighborWeight) / (1f + neighborWeight * 2f) + } + return result + } + } + + var sample: FloatArray = floatArrayOf(0f) + set(value) { + if (value.isEmpty()) throw IllegalArgumentException("Sample array cannot be empty") + +// field = smooth(value, 0.25f) + field = value + invalidate() + } + + + /** Indicates whether the user is currently interacting with the view and performing a seeking gesture. */ + private var userSeeking = false + private var _progress: Float = 0f + /** In [0..1] range. */ + var progress: Float + set(value) { + // Do not let to modify the progress value from the outside + // when the user is currently interacting with the view. + if (userSeeking) return + + _progress = value + invalidate() + progressChangeListener?.onProgressChanged(this, _progress, false) + } + get() { + return _progress + } + + var waveBackgroundColor: Int = Color.LTGRAY + set(value) { + field = value + invalidate() + } + + var waveProgressColor: Int = Color.WHITE + set(value) { + field = value + invalidate() + } + + var waveGap: Float = + dp( + context, + 2f + ) + set(value) { + field = value + invalidate() + } + + var waveWidth: Float = + dp( + context, + 5f + ) + set(value) { + field = value + invalidate() + } + + var waveMinHeight: Float = waveWidth + set(value) { + field = value + invalidate() + } + + var waveCornerRadius: Float = + dp( + context, + 2.5f + ) + set(value) { + field = value + invalidate() + } + + var waveGravity: WaveGravity = + WaveGravity.CENTER + set(value) { + field = value + invalidate() + } + + var progressChangeListener: ProgressChangeListener? = null + + private val postponedProgressUpdateHandler = Handler(Looper.getMainLooper()) + private val postponedProgressUpdateRunnable = Runnable { + progressChangeListener?.onProgressChanged(this, progress, true) + } + + private val wavePaint = Paint(Paint.ANTI_ALIAS_FLAG) + private val waveRect = RectF() + private val progressCanvas = Canvas() + + private var canvasWidth = 0 + private var canvasHeight = 0 + private var maxValue = + dp( + context, + 2f + ) + private var touchDownX = 0f + private var scaledTouchSlop = ViewConfiguration.get(context).scaledTouchSlop + + constructor(context: Context) : this(context, null) + + constructor(context: Context, attrs: AttributeSet?) : this(context, attrs, 0) + + constructor(context: Context, attrs: AttributeSet?, defStyleAttr: Int) + : super(context, attrs, defStyleAttr) { + + val typedAttrs = context.obtainStyledAttributes(attrs, + R.styleable.WaveformSeekBar + ) + + waveWidth = typedAttrs.getDimension(R.styleable.WaveformSeekBar_wave_width, waveWidth) + waveGap = typedAttrs.getDimension(R.styleable.WaveformSeekBar_wave_gap, waveGap) + waveCornerRadius = typedAttrs.getDimension( + R.styleable.WaveformSeekBar_wave_corner_radius, + waveCornerRadius + ) + waveMinHeight = + typedAttrs.getDimension(R.styleable.WaveformSeekBar_wave_min_height, waveMinHeight) + waveBackgroundColor = typedAttrs.getColor( + R.styleable.WaveformSeekBar_wave_background_color, + waveBackgroundColor + ) + waveProgressColor = + typedAttrs.getColor(R.styleable.WaveformSeekBar_wave_progress_color, waveProgressColor) + progress = typedAttrs.getFloat(R.styleable.WaveformSeekBar_wave_progress, progress) + waveGravity = + WaveGravity.fromString( + typedAttrs.getString(R.styleable.WaveformSeekBar_wave_gravity) + ) + + typedAttrs.recycle() + } + + override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { + super.onSizeChanged(w, h, oldw, oldh) + canvasWidth = w + canvasHeight = h + } + + override fun onDraw(canvas: Canvas) { + super.onDraw(canvas) + + val totalWidth = getAvailableWith() + + maxValue = sample.max()!! + val step = (totalWidth / (waveGap + waveWidth)) / sample.size + + var lastWaveRight = paddingLeft.toFloat() + + var i = 0f + while (i < sample.size) { + + var waveHeight = if (maxValue != 0f) { + getAvailableHeight() * (sample[i.toInt()] / maxValue) + } else { + waveMinHeight + } + + if (waveHeight < waveMinHeight) { + waveHeight = waveMinHeight + } + + val top: Float = when (waveGravity) { + WaveGravity.TOP -> paddingTop.toFloat() + WaveGravity.CENTER -> paddingTop + getAvailableHeight() / 2f - waveHeight / 2f + WaveGravity.BOTTOM -> canvasHeight - paddingBottom - waveHeight + } + + waveRect.set(lastWaveRight, top, lastWaveRight + waveWidth, top + waveHeight) + + wavePaint.color = if (waveRect.right <= totalWidth * progress) + waveProgressColor else waveBackgroundColor + + canvas.drawRoundRect(waveRect, waveCornerRadius, waveCornerRadius, wavePaint) + + lastWaveRight = waveRect.right + waveGap + + if (lastWaveRight + waveWidth > totalWidth + paddingLeft) + break + + i += 1f / step + } + } + + override fun onTouchEvent(event: MotionEvent): Boolean { + if (!isEnabled) return false + + when (event.action) { + MotionEvent.ACTION_DOWN -> { + userSeeking = true + if (isParentScrolling()) { + touchDownX = event.x + } else { + updateProgress(event, true) + } + } + MotionEvent.ACTION_MOVE -> { + updateProgress(event, true) + } + MotionEvent.ACTION_UP -> { + userSeeking = false + if (abs(event.x - touchDownX) > scaledTouchSlop) { + updateProgress(event, false) + } + + performClick() + } + } + return true + } + + private fun isParentScrolling(): Boolean { + var parent = parent as View + val root = rootView + + while (true) { + when { + parent.canScrollHorizontally(+1) -> return true + parent.canScrollHorizontally(-1) -> return true + parent.canScrollVertically(+1) -> return true + parent.canScrollVertically(-1) -> return true + } + + if (parent == root) return false + + parent = parent.parent as View + } + } + + private fun updateProgress(event: MotionEvent, delayNotification: Boolean) { + _progress = event.x / getAvailableWith() + invalidate() + + postponedProgressUpdateHandler.removeCallbacks(postponedProgressUpdateRunnable) + if (delayNotification) { + // Re-post delayed user update notification to throttle a bit. + postponedProgressUpdateHandler.postDelayed(postponedProgressUpdateRunnable, 150) + } else { + postponedProgressUpdateRunnable.run() + } + } + + override fun performClick(): Boolean { + super.performClick() + return true + } + + private fun getAvailableWith() = canvasWidth - paddingLeft - paddingRight + private fun getAvailableHeight() = canvasHeight - paddingTop - paddingBottom + + enum class WaveGravity { + TOP, + CENTER, + BOTTOM, + ; + + companion object { + @JvmStatic + fun fromString(gravity: String?): WaveGravity = when (gravity) { + "1" -> TOP + "2" -> CENTER + else -> BOTTOM + } + } + } + + interface ProgressChangeListener { + fun onProgressChanged(waveformSeekBar: WaveformSeekBar, progress: Float, fromUser: Boolean) + } +} \ No newline at end of file From 692741f406605e826515ae5d5ae41876178109c5 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 8 Oct 2020 15:32:47 +1100 Subject: [PATCH 03/23] Audio parsing and RMS computation for waveform visualization. --- build.gradle | 1 + ...sation_activity_attachment_editor_stub.xml | 2 +- .../conversation_item_received_audio.xml | 2 +- res/layout/conversation_item_sent_audio.xml | 2 +- ...{audio_view.xml => message_audio_view.xml} | 6 +- res/values/attrs.xml | 2 +- .../securesms/components/AudioViewOld.java | 331 ++++++++++++++++++ .../conversation/ConversationItem.java | 4 +- .../loki/utilities/audio/DecodedAudio.java | 319 +++++++++++++++++ .../loki/utilities/audio/DecodedAudioExt.kt | 90 +++++ .../views/MessageAudioView.kt} | 174 ++++++--- .../views}/WaveformSeekBar.kt | 42 +-- .../securesms/mms/AttachmentManager.java | 4 +- 13 files changed, 894 insertions(+), 85 deletions(-) rename res/layout/{audio_view.xml => message_audio_view.xml} (96%) create mode 100644 src/org/thoughtcrime/securesms/components/AudioViewOld.java create mode 100644 src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java create mode 100644 src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt rename src/org/thoughtcrime/securesms/{components/AudioView.kt => loki/views/MessageAudioView.kt} (62%) rename src/org/thoughtcrime/securesms/{components => loki/views}/WaveformSeekBar.kt (94%) diff --git a/build.gradle b/build.gradle index 27654e1f04..d011ce7157 100644 --- a/build.gradle +++ b/build.gradle @@ -149,6 +149,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.9.8" implementation "com.squareup.okhttp3:okhttp:3.12.1" implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.3.9' implementation "nl.komponents.kovenant:kovenant:$kovenant_version" implementation "nl.komponents.kovenant:kovenant-android:$kovenant_version" implementation "com.github.lelloman:android-identicons:v11" diff --git a/res/layout/conversation_activity_attachment_editor_stub.xml b/res/layout/conversation_activity_attachment_editor_stub.xml index 034400a61a..c5831c03e8 100644 --- a/res/layout/conversation_activity_attachment_editor_stub.xml +++ b/res/layout/conversation_activity_attachment_editor_stub.xml @@ -32,7 +32,7 @@ app:minHeight="100dp" app:maxHeight="300dp"/> - - - + tools:context="org.thoughtcrime.securesms.loki.views.MessageAudioView"> - - + diff --git a/src/org/thoughtcrime/securesms/components/AudioViewOld.java b/src/org/thoughtcrime/securesms/components/AudioViewOld.java new file mode 100644 index 0000000000..f280cc4a73 --- /dev/null +++ b/src/org/thoughtcrime/securesms/components/AudioViewOld.java @@ -0,0 +1,331 @@ +package org.thoughtcrime.securesms.components; + +import android.annotation.TargetApi; +import android.content.Context; +import android.content.res.ColorStateList; +import android.content.res.TypedArray; +import android.graphics.Color; +import android.graphics.PorterDuff; +import android.graphics.drawable.AnimatedVectorDrawable; +import android.os.Build; +import android.util.AttributeSet; +import android.view.MotionEvent; +import android.view.View; +import android.view.ViewGroup; +import android.widget.FrameLayout; +import android.widget.ImageView; +import android.widget.SeekBar; +import android.widget.TextView; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import com.pnikosis.materialishprogress.ProgressWheel; + +import org.greenrobot.eventbus.EventBus; +import org.greenrobot.eventbus.Subscribe; +import org.greenrobot.eventbus.ThreadMode; +import org.thoughtcrime.securesms.audio.AudioSlidePlayer; +import org.thoughtcrime.securesms.database.AttachmentDatabase; +import org.thoughtcrime.securesms.events.PartProgressEvent; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.mms.AudioSlide; +import org.thoughtcrime.securesms.mms.SlideClickListener; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import network.loki.messenger.R; + + +public class AudioViewOld extends FrameLayout implements AudioSlidePlayer.Listener { + + private static final String TAG = AudioViewOld.class.getSimpleName(); + + private final @NonNull AnimatingToggle controlToggle; + private final @NonNull ViewGroup container; + private final @NonNull ImageView playButton; + private final @NonNull ImageView pauseButton; + private final @NonNull ImageView downloadButton; + private final @NonNull ProgressWheel downloadProgress; + private final @NonNull SeekBar seekBar; + private final @NonNull TextView timestamp; + + private @Nullable SlideClickListener downloadListener; + private @Nullable AudioSlidePlayer audioSlidePlayer; + private int backwardsCounter; + + public AudioViewOld(Context context) { + this(context, null); + } + + public AudioViewOld(Context context, AttributeSet attrs) { + this(context, attrs, 0); + } + + public AudioViewOld(Context context, AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + inflate(context, R.layout.message_audio_view, this); + + this.container = (ViewGroup) findViewById(R.id.audio_widget_container); + this.controlToggle = (AnimatingToggle) findViewById(R.id.control_toggle); + this.playButton = (ImageView) findViewById(R.id.play); + this.pauseButton = (ImageView) findViewById(R.id.pause); + this.downloadButton = (ImageView) findViewById(R.id.download); + this.downloadProgress = (ProgressWheel) findViewById(R.id.download_progress); + this.seekBar = (SeekBar) findViewById(R.id.seek); + this.timestamp = (TextView) findViewById(R.id.timestamp); + + this.playButton.setOnClickListener(new PlayClickedListener()); + this.pauseButton.setOnClickListener(new PauseClickedListener()); + this.seekBar.setOnSeekBarChangeListener(new SeekBarModifiedListener()); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + this.playButton.setImageDrawable(context.getDrawable(R.drawable.play_icon)); + this.pauseButton.setImageDrawable(context.getDrawable(R.drawable.pause_icon)); + this.playButton.setBackground(context.getDrawable(R.drawable.ic_circle_fill_white_48dp)); + this.pauseButton.setBackground(context.getDrawable(R.drawable.ic_circle_fill_white_48dp)); + } + + if (attrs != null) { + TypedArray typedArray = context.getTheme().obtainStyledAttributes(attrs, R.styleable.MessageAudioView, 0, 0); + setTint(typedArray.getColor(R.styleable.MessageAudioView_foregroundTintColor, Color.WHITE), + typedArray.getColor(R.styleable.MessageAudioView_backgroundTintColor, Color.WHITE)); + container.setBackgroundColor(typedArray.getColor(R.styleable.MessageAudioView_widgetBackground, Color.TRANSPARENT)); + typedArray.recycle(); + } + } + + @Override + protected void onAttachedToWindow() { + super.onAttachedToWindow(); + if (!EventBus.getDefault().isRegistered(this)) EventBus.getDefault().register(this); + } + + @Override + protected void onDetachedFromWindow() { + super.onDetachedFromWindow(); + EventBus.getDefault().unregister(this); + } + + public void setAudio(final @NonNull AudioSlide audio, + final boolean showControls) + { + + if (showControls && audio.isPendingDownload()) { + controlToggle.displayQuick(downloadButton); + seekBar.setEnabled(false); + downloadButton.setOnClickListener(new DownloadClickedListener(audio)); + if (downloadProgress.isSpinning()) downloadProgress.stopSpinning(); + } else if (showControls && audio.getTransferState() == AttachmentDatabase.TRANSFER_PROGRESS_STARTED) { + controlToggle.displayQuick(downloadProgress); + seekBar.setEnabled(false); + downloadProgress.spin(); + } else { + controlToggle.displayQuick(playButton); + seekBar.setEnabled(true); + if (downloadProgress.isSpinning()) downloadProgress.stopSpinning(); + } + + this.audioSlidePlayer = AudioSlidePlayer.createFor(getContext(), audio, this); + } + + public void cleanup() { + if (this.audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { + this.audioSlidePlayer.stop(); + } + } + + public void setDownloadClickListener(@Nullable SlideClickListener listener) { + this.downloadListener = listener; + } + + @Override + public void onPlayerStart(@NonNull AudioSlidePlayer player) { + if (this.pauseButton.getVisibility() != View.VISIBLE) { + togglePlayToPause(); + } + } + + @Override + public void onPlayerStop(@NonNull AudioSlidePlayer player) { + if (this.playButton.getVisibility() != View.VISIBLE) { + togglePauseToPlay(); + } + + if (seekBar.getProgress() + 5 >= seekBar.getMax()) { + backwardsCounter = 4; + onPlayerProgress(player, 0.0, 0); + } + } + + @Override + public void setFocusable(boolean focusable) { + super.setFocusable(focusable); + this.playButton.setFocusable(focusable); + this.pauseButton.setFocusable(focusable); + this.seekBar.setFocusable(focusable); + this.seekBar.setFocusableInTouchMode(focusable); + this.downloadButton.setFocusable(focusable); + } + + @Override + public void setClickable(boolean clickable) { + super.setClickable(clickable); + this.playButton.setClickable(clickable); + this.pauseButton.setClickable(clickable); + this.seekBar.setClickable(clickable); + this.seekBar.setOnTouchListener(clickable ? null : new TouchIgnoringListener()); + this.downloadButton.setClickable(clickable); + } + + @Override + public void setEnabled(boolean enabled) { + super.setEnabled(enabled); + this.playButton.setEnabled(enabled); + this.pauseButton.setEnabled(enabled); + this.seekBar.setEnabled(enabled); + this.downloadButton.setEnabled(enabled); + } + + @Override + public void onPlayerProgress(@NonNull AudioSlidePlayer player, double progress, long millis) { + int seekProgress = (int)Math.floor(progress * this.seekBar.getMax()); + + if (seekProgress > seekBar.getProgress() || backwardsCounter > 3) { + backwardsCounter = 0; + this.seekBar.setProgress(seekProgress); + this.timestamp.setText(String.format("%02d:%02d", + TimeUnit.MILLISECONDS.toMinutes(millis), + TimeUnit.MILLISECONDS.toSeconds(millis))); + } else { + backwardsCounter++; + } + } + + public void setTint(int foregroundTint, int backgroundTint) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + this.playButton.setBackgroundTintList(ColorStateList.valueOf(foregroundTint)); + this.playButton.setImageTintList(ColorStateList.valueOf(backgroundTint)); + this.pauseButton.setBackgroundTintList(ColorStateList.valueOf(foregroundTint)); + this.pauseButton.setImageTintList(ColorStateList.valueOf(backgroundTint)); + } else { + this.playButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); + this.pauseButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); + } + + this.downloadButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); + this.downloadProgress.setBarColor(foregroundTint); + + this.timestamp.setTextColor(foregroundTint); + this.seekBar.getProgressDrawable().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); + this.seekBar.getThumb().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); + } + + private double getProgress() { + if (this.seekBar.getProgress() <= 0 || this.seekBar.getMax() <= 0) { + return 0; + } else { + return (double)this.seekBar.getProgress() / (double)this.seekBar.getMax(); + } + } + + private void togglePlayToPause() { + controlToggle.displayQuick(pauseButton); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + AnimatedVectorDrawable playToPauseDrawable = (AnimatedVectorDrawable)getContext().getDrawable(R.drawable.play_to_pause_animation); + pauseButton.setImageDrawable(playToPauseDrawable); + playToPauseDrawable.start(); + } + } + + private void togglePauseToPlay() { + controlToggle.displayQuick(playButton); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + AnimatedVectorDrawable pauseToPlayDrawable = (AnimatedVectorDrawable)getContext().getDrawable(R.drawable.pause_to_play_animation); + playButton.setImageDrawable(pauseToPlayDrawable); + pauseToPlayDrawable.start(); + } + } + + private class PlayClickedListener implements OnClickListener { + @TargetApi(Build.VERSION_CODES.LOLLIPOP) + @Override + public void onClick(View v) { + try { + Log.d(TAG, "playbutton onClick"); + if (audioSlidePlayer != null) { + togglePlayToPause(); + audioSlidePlayer.play(getProgress()); + } + } catch (IOException e) { + Log.w(TAG, e); + } + } + } + + private class PauseClickedListener implements OnClickListener { + @TargetApi(Build.VERSION_CODES.LOLLIPOP) + @Override + public void onClick(View v) { + Log.d(TAG, "pausebutton onClick"); + if (audioSlidePlayer != null) { + togglePauseToPlay(); + audioSlidePlayer.stop(); + } + } + } + + private class DownloadClickedListener implements OnClickListener { + private final @NonNull AudioSlide slide; + + private DownloadClickedListener(@NonNull AudioSlide slide) { + this.slide = slide; + } + + @Override + public void onClick(View v) { + if (downloadListener != null) downloadListener.onClick(v, slide); + } + } + + private class SeekBarModifiedListener implements SeekBar.OnSeekBarChangeListener { + @Override + public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {} + + @Override + public synchronized void onStartTrackingTouch(SeekBar seekBar) { + if (audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { + audioSlidePlayer.stop(); + } + } + + @Override + public synchronized void onStopTrackingTouch(SeekBar seekBar) { + try { + if (audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { + audioSlidePlayer.play(getProgress()); + } + } catch (IOException e) { + Log.w(TAG, e); + } + } + } + + private static class TouchIgnoringListener implements OnTouchListener { + @Override + public boolean onTouch(View v, MotionEvent event) { + return true; + } + } + + @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) + public void onEventAsync(final PartProgressEvent event) { + if (audioSlidePlayer != null && event.attachment.equals(audioSlidePlayer.getAudioSlide().asAttachment())) { + downloadProgress.setInstantProgress(((float) event.progress) / event.total); + } + } + +} diff --git a/src/org/thoughtcrime/securesms/conversation/ConversationItem.java b/src/org/thoughtcrime/securesms/conversation/ConversationItem.java index 6bb7970de7..e98ab75a43 100644 --- a/src/org/thoughtcrime/securesms/conversation/ConversationItem.java +++ b/src/org/thoughtcrime/securesms/conversation/ConversationItem.java @@ -60,7 +60,7 @@ import org.thoughtcrime.securesms.MediaPreviewActivity; import org.thoughtcrime.securesms.MessageDetailsActivity; import org.thoughtcrime.securesms.attachments.DatabaseAttachment; import org.thoughtcrime.securesms.components.AlertView; -import org.thoughtcrime.securesms.components.AudioView; +import org.thoughtcrime.securesms.loki.views.MessageAudioView; import org.thoughtcrime.securesms.components.ConversationItemFooter; import org.thoughtcrime.securesms.components.ConversationItemThumbnail; import org.thoughtcrime.securesms.components.DocumentView; @@ -161,7 +161,7 @@ public class ConversationItem extends TapJackingProofLinearLayout private @NonNull Set batchSelected = new HashSet<>(); private Recipient conversationRecipient; private Stub mediaThumbnailStub; - private Stub audioViewStub; + private Stub audioViewStub; private Stub documentViewStub; private Stub sharedContactStub; private Stub linkPreviewStub; diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java new file mode 100644 index 0000000000..72f83dea88 --- /dev/null +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2015 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.thoughtcrime.securesms.loki.utilities.audio; + +import android.media.AudioFormat; +import android.media.MediaCodec; +import android.media.MediaDataSource; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.os.Build; + +import androidx.annotation.RequiresApi; + +import java.io.FileDescriptor; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; + +/** + * Partially exported class from the old Google's Ringdroid project. + * https://github.com/google/ringdroid/blob/master/app/src/main/java/com/ringdroid/soundfile/SoundFile.java + *

+ * We need this one to parse audio files. Specifically extract RMS values for waveform visualization. + *

+ * NOTE: This class instance creation might be pretty slow (depends on the source audio file size). + * It's recommended to instantiate it in the background. + */ +public class DecodedAudio { + + // Member variables representing frame data + private final long mFileSize; + private final int mAvgBitRate; // Average bit rate in kbps. + private final int mSampleRate; + private final int mChannels; + private final int mNumSamples; // total number of samples per channel in audio file + private final ShortBuffer mDecodedSamples; // shared buffer with mDecodedBytes. + // mDecodedSamples has the following format: + // {s1c1, s1c2, ..., s1cM, s2c1, ..., s2cM, ..., sNc1, ..., sNcM} + // where sicj is the ith sample of the jth channel (a sample is a signed short) + // M is the number of channels (e.g. 2 for stereo) and N is the number of samples per channel. + + // TODO(nfaralli): what is the real list of supported extensions? Is it device dependent? + public static String[] getSupportedExtensions() { + return new String[]{"mp3", "wav", "3gpp", "3gp", "amr", "aac", "m4a", "ogg"}; + } + + public static boolean isFilenameSupported(String filename) { + String[] extensions = getSupportedExtensions(); + for (int i = 0; i < extensions.length; i++) { + if (filename.endsWith("." + extensions[i])) { + return true; + } + } + return false; + } + + public DecodedAudio(FileDescriptor fd, long startOffset, long size) throws IOException { + this(createMediaExtractor(fd, startOffset, size), size); + } + + @RequiresApi(api = Build.VERSION_CODES.M) + public DecodedAudio(MediaDataSource dataSource) throws IOException { + this(createMediaExtractor(dataSource), dataSource.getSize()); + } + + public DecodedAudio(MediaExtractor extractor, long size) throws IOException { + mFileSize = size; + + int numTracks = extractor.getTrackCount(); + // find and select the first audio track present in the file. + MediaFormat format = null; + int trackIndex; + for (trackIndex = 0; trackIndex < numTracks; trackIndex++) { + format = extractor.getTrackFormat(trackIndex); + if (format.getString(MediaFormat.KEY_MIME).startsWith("audio/")) { + extractor.selectTrack(trackIndex); + break; + } + } + if (trackIndex == numTracks) { + throw new IOException("No audio track found in the data source."); + } + + mChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + mSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE); + // Expected total number of samples per channel. + int expectedNumSamples = + (int) ((format.getLong(MediaFormat.KEY_DURATION) / 1000000.f) * mSampleRate + 0.5f); + + MediaCodec codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)); + codec.configure(format, null, null, 0); + codec.start(); + + try { + int pcmEncoding = codec.getOutputFormat().getInteger(MediaFormat.KEY_PCM_ENCODING); + if (pcmEncoding != AudioFormat.ENCODING_PCM_16BIT) { + throw new IOException("Unsupported PCM encoding code: " + pcmEncoding); + } + } catch (NullPointerException e) { + // If KEY_PCM_ENCODING is not specified, means it's ENCODING_PCM_16BIT. + } + + int decodedSamplesSize = 0; // size of the output buffer containing decoded samples. + byte[] decodedSamples = null; + int sampleSize; + MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + long presentationTime; + int totalSizeRead = 0; + boolean doneReading = false; + + // Set the size of the decoded samples buffer to 1MB (~6sec of a stereo stream at 44.1kHz). + // For longer streams, the buffer size will be increased later on, calculating a rough + // estimate of the total size needed to store all the samples in order to resize the buffer + // only once. + ByteBuffer decodedBytes = ByteBuffer.allocate(1 << 20); + boolean firstSampleData = true; + while (true) { + // read data from file and feed it to the decoder input buffers. + int inputBufferIndex = codec.dequeueInputBuffer(100); + if (!doneReading && inputBufferIndex >= 0) { + sampleSize = extractor.readSampleData(codec.getInputBuffer(inputBufferIndex), 0); + if (firstSampleData + && format.getString(MediaFormat.KEY_MIME).equals("audio/mp4a-latm") + && sampleSize == 2) { + // For some reasons on some devices (e.g. the Samsung S3) you should not + // provide the first two bytes of an AAC stream, otherwise the MediaCodec will + // crash. These two bytes do not contain music data but basic info on the + // stream (e.g. channel configuration and sampling frequency), and skipping them + // seems OK with other devices (MediaCodec has already been configured and + // already knows these parameters). + extractor.advance(); + totalSizeRead += sampleSize; + } else if (sampleSize < 0) { + // All samples have been read. + codec.queueInputBuffer( + inputBufferIndex, 0, 0, -1, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + doneReading = true; + } else { + presentationTime = extractor.getSampleTime(); + codec.queueInputBuffer(inputBufferIndex, 0, sampleSize, presentationTime, 0); + extractor.advance(); + totalSizeRead += sampleSize; + } + firstSampleData = false; + } + + // Get decoded stream from the decoder output buffers. + int outputBufferIndex = codec.dequeueOutputBuffer(info, 100); + if (outputBufferIndex >= 0 && info.size > 0) { + if (decodedSamplesSize < info.size) { + decodedSamplesSize = info.size; + decodedSamples = new byte[decodedSamplesSize]; + } + ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferIndex); + outputBuffer.get(decodedSamples, 0, info.size); + outputBuffer.clear(); + // Check if buffer is big enough. Resize it if it's too small. + if (decodedBytes.remaining() < info.size) { + // Getting a rough estimate of the total size, allocate 20% more, and + // make sure to allocate at least 5MB more than the initial size. + int position = decodedBytes.position(); + int newSize = (int) ((position * (1.0 * mFileSize / totalSizeRead)) * 1.2); + if (newSize - position < info.size + 5 * (1 << 20)) { + newSize = position + info.size + 5 * (1 << 20); + } + ByteBuffer newDecodedBytes = null; + // Try to allocate memory. If we are OOM, try to run the garbage collector. + int retry = 10; + while (retry > 0) { + try { + newDecodedBytes = ByteBuffer.allocate(newSize); + break; + } catch (OutOfMemoryError oome) { + // setting android:largeHeap="true" in seem to help not + // reaching this section. + retry--; + } + } + if (retry == 0) { + // Failed to allocate memory... Stop reading more data and finalize the + // instance with the data decoded so far. + break; + } + //ByteBuffer newDecodedBytes = ByteBuffer.allocate(newSize); + decodedBytes.rewind(); + newDecodedBytes.put(decodedBytes); + decodedBytes = newDecodedBytes; + decodedBytes.position(position); + } + decodedBytes.put(decodedSamples, 0, info.size); + codec.releaseOutputBuffer(outputBufferIndex, false); + } /*else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + // Subsequent data will conform to new format. + // We could check that codec.getOutputFormat(), which is the new output format, + // is what we expect. + }*/ + + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 + || (decodedBytes.position() / (2 * mChannels)) >= expectedNumSamples) { + // We got all the decoded data from the decoder. Stop here. + // Theoretically dequeueOutputBuffer(info, ...) should have set info.flags to + // MediaCodec.BUFFER_FLAG_END_OF_STREAM. However some phones (e.g. Samsung S3) + // won't do that for some files (e.g. with mono AAC files), in which case subsequent + // calls to dequeueOutputBuffer may result in the application crashing, without + // even an exception being thrown... Hence the second check. + // (for mono AAC files, the S3 will actually double each sample, as if the stream + // was stereo. The resulting stream is half what it's supposed to be and with a much + // lower pitch.) + break; + } + } + mNumSamples = decodedBytes.position() / (mChannels * 2); // One sample = 2 bytes. + decodedBytes.rewind(); + decodedBytes.order(ByteOrder.LITTLE_ENDIAN); + mDecodedSamples = decodedBytes.asShortBuffer(); + mAvgBitRate = (int) ((mFileSize * 8) * ((float) mSampleRate / mNumSamples) / 1000); + + extractor.release(); + codec.stop(); + codec.release(); + +// // Temporary hack to make it work with the old version. +// int numFrames = mNumSamples / getSamplesPerFrame(); +// if (mNumSamples % getSamplesPerFrame() != 0) { +// numFrames++; +// } +// mFrameGains = new int[numFrames]; +// mFrameLens = new int[numFrames]; +// mFrameOffsets = new int[numFrames]; +// int j; +// int gain, value; +// int frameLens = (int) ((1000 * mAvgBitRate / 8) * +// ((float) getSamplesPerFrame() / mSampleRate)); +// for (trackIndex = 0; trackIndex < numFrames; trackIndex++) { +// gain = -1; +// for (j = 0; j < getSamplesPerFrame(); j++) { +// value = 0; +// for (int k = 0; k < mChannels; k++) { +// if (mDecodedSamples.remaining() > 0) { +// value += java.lang.Math.abs(mDecodedSamples.get()); +// } +// } +// value /= mChannels; +// if (gain < value) { +// gain = value; +// } +// } +// mFrameGains[trackIndex] = (int) Math.sqrt(gain); // here gain = sqrt(max value of 1st channel)... +// mFrameLens[trackIndex] = frameLens; // totally not accurate... +// mFrameOffsets[trackIndex] = (int) (trackIndex * (1000 * mAvgBitRate / 8) * // = i * frameLens +// ((float) getSamplesPerFrame() / mSampleRate)); +// } +// mDecodedSamples.rewind(); +// mNumFrames = numFrames; + } + + public long getFileSizeBytes() { + return mFileSize; + } + + public int getAvgBitrateKbps() { + return mAvgBitRate; + } + + public int getSampleRate() { + return mSampleRate; + } + + public int getChannels() { + return mChannels; + } + + public int getNumSamples() { + return mNumSamples; // Number of samples per channel. + } + + public ShortBuffer getSamples() { + if (mDecodedSamples != null) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && + Build.VERSION.SDK_INT <= Build.VERSION_CODES.N_MR1) { + // Hack for Nougat where asReadOnlyBuffer fails to respect byte ordering. + // See https://code.google.com/p/android/issues/detail?id=223824 + return mDecodedSamples; + } else { + return mDecodedSamples.asReadOnlyBuffer(); + } + } else { + return null; + } + } + + private static MediaExtractor createMediaExtractor(FileDescriptor fd, long startOffset, long size) throws IOException { + MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(fd, startOffset, size); + return extractor; + } + + @RequiresApi(api = Build.VERSION_CODES.M) + private static MediaExtractor createMediaExtractor(MediaDataSource dataSource) throws IOException { + MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(dataSource); + return extractor; + } +} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt new file mode 100644 index 0000000000..3df6fffa9e --- /dev/null +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt @@ -0,0 +1,90 @@ +package org.thoughtcrime.securesms.loki.utilities.audio; + +import java.nio.ShortBuffer +import kotlin.math.ceil +import kotlin.math.sqrt + +/** + * Computes audio RMS values for the first channel only. + * + * A typical RMS calculation algorithm is: + * 1. Square each sample + * 2. Sum the squared samples + * 3. Divide the sum of the squared samples by the number of samples + * 4. Take the square root of step 3., the mean of the squared samples + * + * @param maxFrames Defines amount of output RMS frames. + * If number of samples per channel is less than "maxFrames", + * the result array will match the source sample size instead. + * + * @return Normalized RMS values float array. + */ +fun DecodedAudio.calculateRms(maxFrames: Int): FloatArray { + return calculateRms(this.samples, this.numSamples, this.channels, maxFrames) +} + +private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): FloatArray { + val numFrames: Int + val frameStep: Float + + val samplesPerChannel = numSamples / channels + if (samplesPerChannel <= maxFrames) { + frameStep = 1f + numFrames = samplesPerChannel + } else { + frameStep = numSamples / maxFrames.toFloat() + numFrames = maxFrames + } + + val rmsValues = FloatArray(numFrames) + + var squaredFrameSum = 0.0 + var currentFrameIdx = 0 + + fun calculateFrameRms(nextFrameIdx: Int) { + rmsValues[currentFrameIdx] = sqrt(squaredFrameSum.toFloat()) + + // Advance to the next frame. + squaredFrameSum = 0.0 + currentFrameIdx = nextFrameIdx + } + + (0 until numSamples * channels step channels).forEach { sampleIdx -> + val channelSampleIdx = sampleIdx / channels + val frameIdx = (channelSampleIdx / frameStep).toInt() + + if (currentFrameIdx != frameIdx) { + // Calculate RMS value for the previous frame. + calculateFrameRms(frameIdx) + } + + val samplesInCurrentFrame = ceil((currentFrameIdx + 1) * frameStep) - ceil(currentFrameIdx * frameStep) + squaredFrameSum += (samples[sampleIdx] * samples[sampleIdx]) / samplesInCurrentFrame + } + // Calculate RMS value for the last frame. + calculateFrameRms(-1) + + normalizeArray(rmsValues) + + return rmsValues +} + +/** + * Normalizes the array's values to [0..1] range. + */ +private fun normalizeArray(values: FloatArray) { + var maxValue = -Float.MAX_VALUE + var minValue = +Float.MAX_VALUE + values.forEach { value -> + if (value > maxValue) maxValue = value + if (value < minValue) minValue = value + } + val span = maxValue - minValue + + if (span == 0f) { + values.indices.forEach { i -> values[i] = 0f } + return + } + + values.indices.forEach { i -> values[i] = (values[i] - minValue) / span } +} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/components/AudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt similarity index 62% rename from src/org/thoughtcrime/securesms/components/AudioView.kt rename to src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index ee47ac0b8a..1ba800d6fb 100644 --- a/src/org/thoughtcrime/securesms/components/AudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -1,10 +1,11 @@ -package org.thoughtcrime.securesms.components +package org.thoughtcrime.securesms.loki.views import android.content.Context import android.content.res.ColorStateList import android.graphics.Color import android.graphics.PorterDuff import android.graphics.drawable.AnimatedVectorDrawable +import android.media.MediaDataSource import android.os.Build import android.util.AttributeSet import android.view.View @@ -12,29 +13,32 @@ import android.view.View.OnTouchListener import android.view.ViewGroup import android.widget.FrameLayout import android.widget.ImageView -import android.widget.SeekBar -import android.widget.SeekBar.OnSeekBarChangeListener import android.widget.TextView +import androidx.annotation.RequiresApi import androidx.core.content.ContextCompat -import androidx.core.graphics.BlendModeColorFilterCompat.createBlendModeColorFilterCompat -import androidx.core.graphics.BlendModeCompat import com.pnikosis.materialishprogress.ProgressWheel +import kotlinx.coroutines.* import network.loki.messenger.R import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode +import org.thoughtcrime.securesms.attachments.Attachment import org.thoughtcrime.securesms.audio.AudioSlidePlayer +import org.thoughtcrime.securesms.components.AnimatingToggle import org.thoughtcrime.securesms.database.AttachmentDatabase import org.thoughtcrime.securesms.events.PartProgressEvent import org.thoughtcrime.securesms.logging.Log +import org.thoughtcrime.securesms.loki.utilities.audio.DecodedAudio +import org.thoughtcrime.securesms.loki.utilities.audio.calculateRms import org.thoughtcrime.securesms.mms.AudioSlide +import org.thoughtcrime.securesms.mms.PartAuthority import org.thoughtcrime.securesms.mms.SlideClickListener import java.io.IOException +import java.io.InputStream +import java.lang.Exception import java.util.* -import java.util.concurrent.TimeUnit -import kotlin.math.floor -class AudioView: FrameLayout, AudioSlidePlayer.Listener { +class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { companion object { private const val TAG = "AudioViewKt" @@ -51,14 +55,17 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { private var downloadListener: SlideClickListener? = null private var audioSlidePlayer: AudioSlidePlayer? = null - private var backwardsCounter = 0 +// private var backwardsCounter = 0 + + /** Background coroutine scope that is available when the view is attached to a window. */ + private var asyncCoroutineScope: CoroutineScope? = null constructor(context: Context): this(context, null) constructor(context: Context, attrs: AttributeSet?): this(context, attrs, 0) constructor(context: Context, attrs: AttributeSet?, defStyleAttr: Int): super(context, attrs, defStyleAttr) { - View.inflate(context, R.layout.audio_view, this) + View.inflate(context, R.layout.message_audio_view, this) container = findViewById(R.id.audio_widget_container) controlToggle = findViewById(R.id.control_toggle) playButton = findViewById(R.id.play) @@ -74,7 +81,7 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { if (audioSlidePlayer != null) { togglePlayToPause() - // Restart the playback if progress bar is near at the end. + // Restart the playback if progress bar is nearly at the end. val progress = if (seekBar.progress < 0.99f) seekBar.progress.toDouble() else 0.0 audioSlidePlayer!!.play(progress) @@ -99,8 +106,6 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { } } } - //TODO Remove this. - seekBar.sample = Random().let { (0 until 64).map { i -> it.nextFloat() }.toFloatArray() } playButton.setImageDrawable(ContextCompat.getDrawable(context, R.drawable.play_icon)) pauseButton.setImageDrawable(ContextCompat.getDrawable(context, R.drawable.pause_icon)) @@ -108,10 +113,10 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { pauseButton.background = ContextCompat.getDrawable(context, R.drawable.ic_circle_fill_white_48dp) if (attrs != null) { - val typedArray = context.theme.obtainStyledAttributes(attrs, R.styleable.AudioView, 0, 0) - setTint(typedArray.getColor(R.styleable.AudioView_foregroundTintColor, Color.WHITE), - typedArray.getColor(R.styleable.AudioView_backgroundTintColor, Color.WHITE)) - container.setBackgroundColor(typedArray.getColor(R.styleable.AudioView_widgetBackground, Color.TRANSPARENT)) + val typedArray = context.theme.obtainStyledAttributes(attrs, R.styleable.MessageAudioView, 0, 0) + setTint(typedArray.getColor(R.styleable.MessageAudioView_foregroundTintColor, Color.WHITE), + typedArray.getColor(R.styleable.MessageAudioView_backgroundTintColor, Color.WHITE)) + container.setBackgroundColor(typedArray.getColor(R.styleable.MessageAudioView_widgetBackground, Color.TRANSPARENT)) typedArray.recycle() } } @@ -119,30 +124,42 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { override fun onAttachedToWindow() { super.onAttachedToWindow() if (!EventBus.getDefault().isRegistered(this)) EventBus.getDefault().register(this) + + asyncCoroutineScope = CoroutineScope(Job() + Dispatchers.IO) } override fun onDetachedFromWindow() { super.onDetachedFromWindow() EventBus.getDefault().unregister(this) + + // Cancel all the background operations. + asyncCoroutineScope!!.cancel() + asyncCoroutineScope = null } fun setAudio(audio: AudioSlide, showControls: Boolean) { - if (showControls && audio.isPendingDownload) { - controlToggle.displayQuick(downloadButton) - seekBar.isEnabled = false - downloadButton.setOnClickListener { v -> downloadListener?.onClick(v, audio) } - if (downloadProgress.isSpinning) { - downloadProgress.stopSpinning() + when { + showControls && audio.isPendingDownload -> { + controlToggle.displayQuick(downloadButton) + seekBar.isEnabled = false + downloadButton.setOnClickListener { v -> downloadListener?.onClick(v, audio) } + if (downloadProgress.isSpinning) { + downloadProgress.stopSpinning() + } } - } else if (showControls && audio.transferState == AttachmentDatabase.TRANSFER_PROGRESS_STARTED) { - controlToggle.displayQuick(downloadProgress) - seekBar.isEnabled = false - downloadProgress.spin() - } else { - controlToggle.displayQuick(playButton) - seekBar.isEnabled = true - if (downloadProgress.isSpinning) { - downloadProgress.stopSpinning() + (showControls && audio.transferState == AttachmentDatabase.TRANSFER_PROGRESS_STARTED) -> { + controlToggle.displayQuick(downloadProgress) + seekBar.isEnabled = false + downloadProgress.spin() + } + else -> { + controlToggle.displayQuick(playButton) + seekBar.isEnabled = true + if (downloadProgress.isSpinning) { + downloadProgress.stopSpinning() + } + // Post to make sure it executes only when the view is attached to a window. + post(::updateSeekBarFromAudio) } } audioSlidePlayer = AudioSlidePlayer.createFor(context, audio, this) @@ -246,27 +263,47 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { pauseToPlayDrawable.start() } -// private inner class SeekBarModifiedListener : OnSeekBarChangeListener { -// override fun onProgressChanged(seekBar: SeekBar, progress: Int, fromUser: Boolean) {} -// -// @Synchronized -// override fun onStartTrackingTouch(seekBar: SeekBar) { -// if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { -// audioSlidePlayer!!.stop() -// } -// } -// -// @Synchronized -// override fun onStopTrackingTouch(seekBar: SeekBar) { -// try { -// if (audioSlidePlayer != null && pauseButton.visibility == View.VISIBLE) { -// audioSlidePlayer!!.play(getProgress()) -// } -// } catch (e: IOException) { -// Log.w(TAG, e) -// } -// } -// } + private fun updateSeekBarFromAudio() { + if (audioSlidePlayer == null) return + + val attachment = audioSlidePlayer!!.audioSlide.asAttachment() + + // Parse audio and compute RMS values for the WaveformSeekBar in the background. + asyncCoroutineScope!!.launch { + val rmsFrames = 32 // The amount of values to be computed to supply for the visualization. + + fun extractAttachmentRandomSeed(attachment: Attachment): Int { + return when { + attachment.digest != null -> attachment.digest!!.sum() + attachment.fileName != null -> attachment.fileName.hashCode() + else -> attachment.hashCode() + } + } + + fun generateFakeRms(seed: Int, frames: Int = rmsFrames): FloatArray { + return Random(seed.toLong()).let { (0 until frames).map { i -> it.nextFloat() }.toFloatArray() } + } + + val rmsValues: FloatArray + + rmsValues = if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + // Due to API version incompatibility, we just display some random waveform for older API. + generateFakeRms(extractAttachmentRandomSeed(attachment)) + } else { + try { + @Suppress("BlockingMethodInNonBlockingContext") + PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { + DecodedAudio(InputStreamMediaDataSource(it)).calculateRms(rmsFrames) + } + } catch (e: Exception) { + android.util.Log.w(TAG, "Failed to decode sample values for the audio attachment \"${attachment.fileName}\".", e) + generateFakeRms(extractAttachmentRandomSeed(attachment)) + } + } + + post { seekBar.sample = rmsValues } + } + } @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) fun onEventAsync(event: PartProgressEvent) { @@ -274,4 +311,35 @@ class AudioView: FrameLayout, AudioSlidePlayer.Listener { downloadProgress.setInstantProgress(event.progress.toFloat() / event.total) } } +} + +@RequiresApi(Build.VERSION_CODES.M) +private class InputStreamMediaDataSource: MediaDataSource { + + private val data: ByteArray + + constructor(inputStream: InputStream): super() { + this.data = inputStream.readBytes() + } + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { + val length: Int = data.size + if (position >= length) { + return -1 // -1 indicates EOF + } + var actualSize = size + if (position + size > length) { + actualSize -= (position + size - length).toInt() + } + System.arraycopy(data, position.toInt(), buffer, offset, actualSize) + return actualSize + } + + override fun getSize(): Long { + return data.size.toLong() + } + + override fun close() { + // We don't need to close the wrapped stream. + } } \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/components/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt similarity index 94% rename from src/org/thoughtcrime/securesms/components/WaveformSeekBar.kt rename to src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 19113d9eb7..6eabd872fb 100644 --- a/src/org/thoughtcrime/securesms/components/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -1,4 +1,4 @@ -package org.thoughtcrime.securesms.components +package org.thoughtcrime.securesms.loki.views import android.content.Context import android.graphics.* @@ -81,20 +81,20 @@ class WaveformSeekBar : View { } var waveGap: Float = - dp( - context, - 2f - ) + dp( + context, + 2f + ) set(value) { field = value invalidate() } var waveWidth: Float = - dp( - context, - 5f - ) + dp( + context, + 5f + ) set(value) { field = value invalidate() @@ -107,17 +107,17 @@ class WaveformSeekBar : View { } var waveCornerRadius: Float = - dp( - context, - 2.5f - ) + dp( + context, + 2.5f + ) set(value) { field = value invalidate() } var waveGravity: WaveGravity = - WaveGravity.CENTER + WaveGravity.CENTER set(value) { field = value invalidate() @@ -137,10 +137,10 @@ class WaveformSeekBar : View { private var canvasWidth = 0 private var canvasHeight = 0 private var maxValue = - dp( - context, - 2f - ) + dp( + context, + 2f + ) private var touchDownX = 0f private var scaledTouchSlop = ViewConfiguration.get(context).scaledTouchSlop @@ -171,9 +171,9 @@ class WaveformSeekBar : View { typedAttrs.getColor(R.styleable.WaveformSeekBar_wave_progress_color, waveProgressColor) progress = typedAttrs.getFloat(R.styleable.WaveformSeekBar_wave_progress, progress) waveGravity = - WaveGravity.fromString( - typedAttrs.getString(R.styleable.WaveformSeekBar_wave_gravity) - ) + WaveGravity.fromString( + typedAttrs.getString(R.styleable.WaveformSeekBar_wave_gravity) + ) typedAttrs.recycle() } diff --git a/src/org/thoughtcrime/securesms/mms/AttachmentManager.java b/src/org/thoughtcrime/securesms/mms/AttachmentManager.java index 80af560c00..a4fac15f14 100644 --- a/src/org/thoughtcrime/securesms/mms/AttachmentManager.java +++ b/src/org/thoughtcrime/securesms/mms/AttachmentManager.java @@ -41,7 +41,7 @@ import androidx.annotation.Nullable; import org.thoughtcrime.securesms.MediaPreviewActivity; import org.thoughtcrime.securesms.TransportOption; import org.thoughtcrime.securesms.attachments.Attachment; -import org.thoughtcrime.securesms.components.AudioView; +import org.thoughtcrime.securesms.loki.views.MessageAudioView; import org.thoughtcrime.securesms.components.DocumentView; import org.thoughtcrime.securesms.components.RemovableEditableMediaView; import org.thoughtcrime.securesms.components.ThumbnailView; @@ -91,7 +91,7 @@ public class AttachmentManager { private RemovableEditableMediaView removableMediaView; private ThumbnailView thumbnail; - private AudioView audioView; + private MessageAudioView audioView; private DocumentView documentView; private SignalMapView mapView; From 7a9e73fb131b67d9689437568ccb9c2e1dde947b Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 8 Oct 2020 15:42:32 +1100 Subject: [PATCH 04/23] Properly handle seek bar input events. --- .../thoughtcrime/securesms/audio/AudioSlidePlayer.java | 8 +++----- .../thoughtcrime/securesms/loki/views/WaveformSeekBar.kt | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java b/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java index 4c7878e543..a737e3c264 100644 --- a/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java +++ b/src/org/thoughtcrime/securesms/audio/AudioSlidePlayer.java @@ -246,12 +246,10 @@ public class AudioSlidePlayer implements SensorEventListener { } public synchronized void seekTo(double progress) throws IOException { - if (mediaPlayer == null) return; - - if (isReady()) { - mediaPlayer.seekTo((long) (mediaPlayer.getDuration() * progress)); - } else { + if (mediaPlayer == null || !isReady()) { play(progress); + } else { + mediaPlayer.seekTo((long) (mediaPlayer.getDuration() * progress)); } } diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 6eabd872fb..8434186c2d 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -249,9 +249,11 @@ class WaveformSeekBar : View { if (abs(event.x - touchDownX) > scaledTouchSlop) { updateProgress(event, false) } - performClick() } + MotionEvent.ACTION_CANCEL -> { + userSeeking = false + } } return true } From 6df3264692cc409aa7586491acbde966c5d7943d Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 8 Oct 2020 16:51:34 +1100 Subject: [PATCH 05/23] Audio duration label. Non-continues updates from waveform view seeking. --- res/layout/message_audio_view.xml | 39 ++- .../securesms/components/AudioViewOld.java | 331 ------------------ .../loki/utilities/audio/DecodedAudio.java | 26 +- .../securesms/loki/views/MessageAudioView.kt | 40 ++- .../securesms/loki/views/WaveformSeekBar.kt | 53 +-- 5 files changed, 94 insertions(+), 395 deletions(-) delete mode 100644 src/org/thoughtcrime/securesms/components/AudioViewOld.java diff --git a/res/layout/message_audio_view.xml b/res/layout/message_audio_view.xml index 0272a69e70..8ec2d8b64e 100644 --- a/res/layout/message_audio_view.xml +++ b/res/layout/message_audio_view.xml @@ -73,31 +73,32 @@ + app:wave_gap="1dp" + tools:wave_background_color="#bbb" + tools:wave_progress_color="?colorPrimary"/> - + - + diff --git a/src/org/thoughtcrime/securesms/components/AudioViewOld.java b/src/org/thoughtcrime/securesms/components/AudioViewOld.java deleted file mode 100644 index f280cc4a73..0000000000 --- a/src/org/thoughtcrime/securesms/components/AudioViewOld.java +++ /dev/null @@ -1,331 +0,0 @@ -package org.thoughtcrime.securesms.components; - -import android.annotation.TargetApi; -import android.content.Context; -import android.content.res.ColorStateList; -import android.content.res.TypedArray; -import android.graphics.Color; -import android.graphics.PorterDuff; -import android.graphics.drawable.AnimatedVectorDrawable; -import android.os.Build; -import android.util.AttributeSet; -import android.view.MotionEvent; -import android.view.View; -import android.view.ViewGroup; -import android.widget.FrameLayout; -import android.widget.ImageView; -import android.widget.SeekBar; -import android.widget.TextView; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import com.pnikosis.materialishprogress.ProgressWheel; - -import org.greenrobot.eventbus.EventBus; -import org.greenrobot.eventbus.Subscribe; -import org.greenrobot.eventbus.ThreadMode; -import org.thoughtcrime.securesms.audio.AudioSlidePlayer; -import org.thoughtcrime.securesms.database.AttachmentDatabase; -import org.thoughtcrime.securesms.events.PartProgressEvent; -import org.thoughtcrime.securesms.logging.Log; -import org.thoughtcrime.securesms.mms.AudioSlide; -import org.thoughtcrime.securesms.mms.SlideClickListener; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import network.loki.messenger.R; - - -public class AudioViewOld extends FrameLayout implements AudioSlidePlayer.Listener { - - private static final String TAG = AudioViewOld.class.getSimpleName(); - - private final @NonNull AnimatingToggle controlToggle; - private final @NonNull ViewGroup container; - private final @NonNull ImageView playButton; - private final @NonNull ImageView pauseButton; - private final @NonNull ImageView downloadButton; - private final @NonNull ProgressWheel downloadProgress; - private final @NonNull SeekBar seekBar; - private final @NonNull TextView timestamp; - - private @Nullable SlideClickListener downloadListener; - private @Nullable AudioSlidePlayer audioSlidePlayer; - private int backwardsCounter; - - public AudioViewOld(Context context) { - this(context, null); - } - - public AudioViewOld(Context context, AttributeSet attrs) { - this(context, attrs, 0); - } - - public AudioViewOld(Context context, AttributeSet attrs, int defStyleAttr) { - super(context, attrs, defStyleAttr); - inflate(context, R.layout.message_audio_view, this); - - this.container = (ViewGroup) findViewById(R.id.audio_widget_container); - this.controlToggle = (AnimatingToggle) findViewById(R.id.control_toggle); - this.playButton = (ImageView) findViewById(R.id.play); - this.pauseButton = (ImageView) findViewById(R.id.pause); - this.downloadButton = (ImageView) findViewById(R.id.download); - this.downloadProgress = (ProgressWheel) findViewById(R.id.download_progress); - this.seekBar = (SeekBar) findViewById(R.id.seek); - this.timestamp = (TextView) findViewById(R.id.timestamp); - - this.playButton.setOnClickListener(new PlayClickedListener()); - this.pauseButton.setOnClickListener(new PauseClickedListener()); - this.seekBar.setOnSeekBarChangeListener(new SeekBarModifiedListener()); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - this.playButton.setImageDrawable(context.getDrawable(R.drawable.play_icon)); - this.pauseButton.setImageDrawable(context.getDrawable(R.drawable.pause_icon)); - this.playButton.setBackground(context.getDrawable(R.drawable.ic_circle_fill_white_48dp)); - this.pauseButton.setBackground(context.getDrawable(R.drawable.ic_circle_fill_white_48dp)); - } - - if (attrs != null) { - TypedArray typedArray = context.getTheme().obtainStyledAttributes(attrs, R.styleable.MessageAudioView, 0, 0); - setTint(typedArray.getColor(R.styleable.MessageAudioView_foregroundTintColor, Color.WHITE), - typedArray.getColor(R.styleable.MessageAudioView_backgroundTintColor, Color.WHITE)); - container.setBackgroundColor(typedArray.getColor(R.styleable.MessageAudioView_widgetBackground, Color.TRANSPARENT)); - typedArray.recycle(); - } - } - - @Override - protected void onAttachedToWindow() { - super.onAttachedToWindow(); - if (!EventBus.getDefault().isRegistered(this)) EventBus.getDefault().register(this); - } - - @Override - protected void onDetachedFromWindow() { - super.onDetachedFromWindow(); - EventBus.getDefault().unregister(this); - } - - public void setAudio(final @NonNull AudioSlide audio, - final boolean showControls) - { - - if (showControls && audio.isPendingDownload()) { - controlToggle.displayQuick(downloadButton); - seekBar.setEnabled(false); - downloadButton.setOnClickListener(new DownloadClickedListener(audio)); - if (downloadProgress.isSpinning()) downloadProgress.stopSpinning(); - } else if (showControls && audio.getTransferState() == AttachmentDatabase.TRANSFER_PROGRESS_STARTED) { - controlToggle.displayQuick(downloadProgress); - seekBar.setEnabled(false); - downloadProgress.spin(); - } else { - controlToggle.displayQuick(playButton); - seekBar.setEnabled(true); - if (downloadProgress.isSpinning()) downloadProgress.stopSpinning(); - } - - this.audioSlidePlayer = AudioSlidePlayer.createFor(getContext(), audio, this); - } - - public void cleanup() { - if (this.audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { - this.audioSlidePlayer.stop(); - } - } - - public void setDownloadClickListener(@Nullable SlideClickListener listener) { - this.downloadListener = listener; - } - - @Override - public void onPlayerStart(@NonNull AudioSlidePlayer player) { - if (this.pauseButton.getVisibility() != View.VISIBLE) { - togglePlayToPause(); - } - } - - @Override - public void onPlayerStop(@NonNull AudioSlidePlayer player) { - if (this.playButton.getVisibility() != View.VISIBLE) { - togglePauseToPlay(); - } - - if (seekBar.getProgress() + 5 >= seekBar.getMax()) { - backwardsCounter = 4; - onPlayerProgress(player, 0.0, 0); - } - } - - @Override - public void setFocusable(boolean focusable) { - super.setFocusable(focusable); - this.playButton.setFocusable(focusable); - this.pauseButton.setFocusable(focusable); - this.seekBar.setFocusable(focusable); - this.seekBar.setFocusableInTouchMode(focusable); - this.downloadButton.setFocusable(focusable); - } - - @Override - public void setClickable(boolean clickable) { - super.setClickable(clickable); - this.playButton.setClickable(clickable); - this.pauseButton.setClickable(clickable); - this.seekBar.setClickable(clickable); - this.seekBar.setOnTouchListener(clickable ? null : new TouchIgnoringListener()); - this.downloadButton.setClickable(clickable); - } - - @Override - public void setEnabled(boolean enabled) { - super.setEnabled(enabled); - this.playButton.setEnabled(enabled); - this.pauseButton.setEnabled(enabled); - this.seekBar.setEnabled(enabled); - this.downloadButton.setEnabled(enabled); - } - - @Override - public void onPlayerProgress(@NonNull AudioSlidePlayer player, double progress, long millis) { - int seekProgress = (int)Math.floor(progress * this.seekBar.getMax()); - - if (seekProgress > seekBar.getProgress() || backwardsCounter > 3) { - backwardsCounter = 0; - this.seekBar.setProgress(seekProgress); - this.timestamp.setText(String.format("%02d:%02d", - TimeUnit.MILLISECONDS.toMinutes(millis), - TimeUnit.MILLISECONDS.toSeconds(millis))); - } else { - backwardsCounter++; - } - } - - public void setTint(int foregroundTint, int backgroundTint) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - this.playButton.setBackgroundTintList(ColorStateList.valueOf(foregroundTint)); - this.playButton.setImageTintList(ColorStateList.valueOf(backgroundTint)); - this.pauseButton.setBackgroundTintList(ColorStateList.valueOf(foregroundTint)); - this.pauseButton.setImageTintList(ColorStateList.valueOf(backgroundTint)); - } else { - this.playButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - this.pauseButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - } - - this.downloadButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - this.downloadProgress.setBarColor(foregroundTint); - - this.timestamp.setTextColor(foregroundTint); - this.seekBar.getProgressDrawable().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - this.seekBar.getThumb().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); - } - - private double getProgress() { - if (this.seekBar.getProgress() <= 0 || this.seekBar.getMax() <= 0) { - return 0; - } else { - return (double)this.seekBar.getProgress() / (double)this.seekBar.getMax(); - } - } - - private void togglePlayToPause() { - controlToggle.displayQuick(pauseButton); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - AnimatedVectorDrawable playToPauseDrawable = (AnimatedVectorDrawable)getContext().getDrawable(R.drawable.play_to_pause_animation); - pauseButton.setImageDrawable(playToPauseDrawable); - playToPauseDrawable.start(); - } - } - - private void togglePauseToPlay() { - controlToggle.displayQuick(playButton); - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - AnimatedVectorDrawable pauseToPlayDrawable = (AnimatedVectorDrawable)getContext().getDrawable(R.drawable.pause_to_play_animation); - playButton.setImageDrawable(pauseToPlayDrawable); - pauseToPlayDrawable.start(); - } - } - - private class PlayClickedListener implements OnClickListener { - @TargetApi(Build.VERSION_CODES.LOLLIPOP) - @Override - public void onClick(View v) { - try { - Log.d(TAG, "playbutton onClick"); - if (audioSlidePlayer != null) { - togglePlayToPause(); - audioSlidePlayer.play(getProgress()); - } - } catch (IOException e) { - Log.w(TAG, e); - } - } - } - - private class PauseClickedListener implements OnClickListener { - @TargetApi(Build.VERSION_CODES.LOLLIPOP) - @Override - public void onClick(View v) { - Log.d(TAG, "pausebutton onClick"); - if (audioSlidePlayer != null) { - togglePauseToPlay(); - audioSlidePlayer.stop(); - } - } - } - - private class DownloadClickedListener implements OnClickListener { - private final @NonNull AudioSlide slide; - - private DownloadClickedListener(@NonNull AudioSlide slide) { - this.slide = slide; - } - - @Override - public void onClick(View v) { - if (downloadListener != null) downloadListener.onClick(v, slide); - } - } - - private class SeekBarModifiedListener implements SeekBar.OnSeekBarChangeListener { - @Override - public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {} - - @Override - public synchronized void onStartTrackingTouch(SeekBar seekBar) { - if (audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { - audioSlidePlayer.stop(); - } - } - - @Override - public synchronized void onStopTrackingTouch(SeekBar seekBar) { - try { - if (audioSlidePlayer != null && pauseButton.getVisibility() == View.VISIBLE) { - audioSlidePlayer.play(getProgress()); - } - } catch (IOException e) { - Log.w(TAG, e); - } - } - } - - private static class TouchIgnoringListener implements OnTouchListener { - @Override - public boolean onTouch(View v, MotionEvent event) { - return true; - } - } - - @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) - public void onEventAsync(final PartProgressEvent event) { - if (audioSlidePlayer != null && event.attachment.equals(audioSlidePlayer.getAudioSlide().asAttachment())) { - downloadProgress.setInstantProgress(((float) event.progress) / event.total); - } - } - -} diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java index 72f83dea88..ef9067b54b 100644 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java @@ -46,6 +46,7 @@ public class DecodedAudio { private final long mFileSize; private final int mAvgBitRate; // Average bit rate in kbps. private final int mSampleRate; + private final long mDuration; // In microseconds. private final int mChannels; private final int mNumSamples; // total number of samples per channel in audio file private final ShortBuffer mDecodedSamples; // shared buffer with mDecodedBytes. @@ -81,29 +82,31 @@ public class DecodedAudio { public DecodedAudio(MediaExtractor extractor, long size) throws IOException { mFileSize = size; + MediaFormat mediaFormat = null; int numTracks = extractor.getTrackCount(); // find and select the first audio track present in the file. - MediaFormat format = null; int trackIndex; for (trackIndex = 0; trackIndex < numTracks; trackIndex++) { - format = extractor.getTrackFormat(trackIndex); + MediaFormat format = extractor.getTrackFormat(trackIndex); if (format.getString(MediaFormat.KEY_MIME).startsWith("audio/")) { extractor.selectTrack(trackIndex); + mediaFormat = format; break; } } - if (trackIndex == numTracks) { + if (mediaFormat == null) { throw new IOException("No audio track found in the data source."); } - mChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - mSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE); + mChannels = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); + mSampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); + mDuration = mediaFormat.getLong(MediaFormat.KEY_DURATION); // Expected total number of samples per channel. int expectedNumSamples = - (int) ((format.getLong(MediaFormat.KEY_DURATION) / 1000000.f) * mSampleRate + 0.5f); + (int) ((mDuration / 1000000.f) * mSampleRate + 0.5f); - MediaCodec codec = MediaCodec.createDecoderByType(format.getString(MediaFormat.KEY_MIME)); - codec.configure(format, null, null, 0); + MediaCodec codec = MediaCodec.createDecoderByType(mediaFormat.getString(MediaFormat.KEY_MIME)); + codec.configure(mediaFormat, null, null, 0); codec.start(); try { @@ -135,7 +138,7 @@ public class DecodedAudio { if (!doneReading && inputBufferIndex >= 0) { sampleSize = extractor.readSampleData(codec.getInputBuffer(inputBufferIndex), 0); if (firstSampleData - && format.getString(MediaFormat.KEY_MIME).equals("audio/mp4a-latm") + && mediaFormat.getString(MediaFormat.KEY_MIME).equals("audio/mp4a-latm") && sampleSize == 2) { // For some reasons on some devices (e.g. the Samsung S3) you should not // provide the first two bytes of an AAC stream, otherwise the MediaCodec will @@ -285,6 +288,11 @@ public class DecodedAudio { return mChannels; } + /** @return Total duration in milliseconds. */ + public long getDuration() { + return mDuration; + } + public int getNumSamples() { return mNumSamples; // Number of samples per channel. } diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index 1ba800d6fb..d67bd1e236 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -14,8 +14,10 @@ import android.view.ViewGroup import android.widget.FrameLayout import android.widget.ImageView import android.widget.TextView +import androidx.annotation.ColorInt import androidx.annotation.RequiresApi import androidx.core.content.ContextCompat +import androidx.core.graphics.ColorUtils import com.pnikosis.materialishprogress.ProgressWheel import kotlinx.coroutines.* import network.loki.messenger.R @@ -37,6 +39,7 @@ import java.io.IOException import java.io.InputStream import java.lang.Exception import java.util.* +import java.util.concurrent.TimeUnit class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { @@ -51,7 +54,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { private val downloadButton: ImageView private val downloadProgress: ProgressWheel private val seekBar: WaveformSeekBar - private val timestamp: TextView + private val totalDuration: TextView private var downloadListener: SlideClickListener? = null private var audioSlidePlayer: AudioSlidePlayer? = null @@ -73,7 +76,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { downloadButton = findViewById(R.id.download) downloadProgress = findViewById(R.id.download_progress) seekBar = findViewById(R.id.seek) - timestamp = findViewById(R.id.timestamp) + totalDuration = findViewById(R.id.total_duration) playButton.setOnClickListener { try { @@ -158,6 +161,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { if (downloadProgress.isSpinning) { downloadProgress.stopSpinning() } + // Post to make sure it executes only when the view is attached to a window. post(::updateSeekBarFromAudio) } @@ -175,7 +179,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { downloadListener = listener } - fun setTint(foregroundTint: Int, backgroundTint: Int) { + fun setTint(@ColorInt foregroundTint: Int, @ColorInt backgroundTint: Int) { playButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) playButton.imageTintList = ColorStateList.valueOf(backgroundTint) pauseButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) @@ -183,11 +187,13 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { downloadButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN) downloadProgress.barColor = foregroundTint - timestamp.setTextColor(foregroundTint) + totalDuration.setTextColor(foregroundTint) // val colorFilter = createBlendModeColorFilterCompat(foregroundTint, BlendModeCompat.SRC_IN) // seekBar.progressDrawable.colorFilter = colorFilter // seekBar.thumb.colorFilter = colorFilter + seekBar.waveProgressColor = foregroundTint + seekBar.waveBackgroundColor = ColorUtils.blendARGB(foregroundTint, backgroundTint, 0.75f) } override fun onPlayerStart(player: AudioSlidePlayer) { @@ -284,24 +290,36 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { return Random(seed.toLong()).let { (0 until frames).map { i -> it.nextFloat() }.toFloatArray() } } - val rmsValues: FloatArray + var rmsValues: FloatArray = floatArrayOf() + var totalDurationMs: Long = -1 - rmsValues = if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { // Due to API version incompatibility, we just display some random waveform for older API. - generateFakeRms(extractAttachmentRandomSeed(attachment)) + rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) } else { try { @Suppress("BlockingMethodInNonBlockingContext") - PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { - DecodedAudio(InputStreamMediaDataSource(it)).calculateRms(rmsFrames) + val decodedAudio = PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { + DecodedAudio(InputStreamMediaDataSource(it)) } + rmsValues = decodedAudio.calculateRms(rmsFrames) + totalDurationMs = (decodedAudio.duration / 1000.0).toLong() } catch (e: Exception) { android.util.Log.w(TAG, "Failed to decode sample values for the audio attachment \"${attachment.fileName}\".", e) - generateFakeRms(extractAttachmentRandomSeed(attachment)) + rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) } } - post { seekBar.sample = rmsValues } + post { + seekBar.sample = rmsValues + + if (totalDurationMs > 0) { + totalDuration.visibility = View.VISIBLE + totalDuration.text = String.format("%02d:%02d", + TimeUnit.MILLISECONDS.toMinutes(totalDurationMs), + TimeUnit.MILLISECONDS.toSeconds(totalDurationMs)) + } + } } } diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 8434186c2d..6d562fc6ba 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -80,21 +80,13 @@ class WaveformSeekBar : View { invalidate() } - var waveGap: Float = - dp( - context, - 2f - ) + var waveGap: Float = dp(context, 2f) set(value) { field = value invalidate() } - var waveWidth: Float = - dp( - context, - 5f - ) + var waveWidth: Float = dp(context, 5f) set(value) { field = value invalidate() @@ -106,11 +98,7 @@ class WaveformSeekBar : View { invalidate() } - var waveCornerRadius: Float = - dp( - context, - 2.5f - ) + var waveCornerRadius: Float = dp(context, 2.5f) set(value) { field = value invalidate() @@ -235,24 +223,26 @@ class WaveformSeekBar : View { when (event.action) { MotionEvent.ACTION_DOWN -> { userSeeking = true +// preUserSeekingProgress = _progress if (isParentScrolling()) { touchDownX = event.x } else { - updateProgress(event, true) + updateProgress(event, false) } } MotionEvent.ACTION_MOVE -> { - updateProgress(event, true) + updateProgress(event, false) } MotionEvent.ACTION_UP -> { userSeeking = false if (abs(event.x - touchDownX) > scaledTouchSlop) { - updateProgress(event, false) + updateProgress(event, true) } performClick() } MotionEvent.ACTION_CANCEL -> { userSeeking = false +// updateProgress(preUserSeekingProgress, false) } } return true @@ -276,19 +266,32 @@ class WaveformSeekBar : View { } } - private fun updateProgress(event: MotionEvent, delayNotification: Boolean) { - _progress = event.x / getAvailableWith() + private fun updateProgress(event: MotionEvent, notify: Boolean) { + updateProgress(event.x / getAvailableWith(), notify) + } + + private fun updateProgress(progress: Float, notify: Boolean) { + _progress = progress invalidate() - postponedProgressUpdateHandler.removeCallbacks(postponedProgressUpdateRunnable) - if (delayNotification) { - // Re-post delayed user update notification to throttle a bit. - postponedProgressUpdateHandler.postDelayed(postponedProgressUpdateRunnable, 150) - } else { + if (notify) { postponedProgressUpdateRunnable.run() } } +// private fun updateProgress(event: MotionEvent, delayNotification: Boolean) { +// _progress = event.x / getAvailableWith() +// invalidate() +// +// postponedProgressUpdateHandler.removeCallbacks(postponedProgressUpdateRunnable) +// if (delayNotification) { +// // Re-post delayed user update notification to throttle a bit. +// postponedProgressUpdateHandler.postDelayed(postponedProgressUpdateRunnable, 150) +// } else { +// postponedProgressUpdateRunnable.run() +// } +// } + override fun performClick(): Boolean { super.performClick() return true From 8cbb34f1746f794ff1cd4103a55d0f9600e7f066 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 8 Oct 2020 17:26:10 +1100 Subject: [PATCH 06/23] Normalization and smooth functions moved to the extension file. --- .../loki/utilities/audio/DecodedAudio.java | 16 ---------- .../loki/utilities/audio/DecodedAudioExt.kt | 20 +++++++++++-- .../securesms/loki/views/MessageAudioView.kt | 4 ++- .../securesms/loki/views/WaveformSeekBar.kt | 29 +------------------ 4 files changed, 21 insertions(+), 48 deletions(-) diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java index ef9067b54b..6c58f3c57f 100644 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java @@ -1,19 +1,3 @@ -/* - * Copyright (C) 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.thoughtcrime.securesms.loki.utilities.audio; import android.media.AudioFormat; diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt index 3df6fffa9e..3802bb3575 100644 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt @@ -17,13 +17,13 @@ import kotlin.math.sqrt * If number of samples per channel is less than "maxFrames", * the result array will match the source sample size instead. * - * @return Normalized RMS values float array. + * @return RMS values float array where is each value is within [0..1] range. */ fun DecodedAudio.calculateRms(maxFrames: Int): FloatArray { return calculateRms(this.samples, this.numSamples, this.channels, maxFrames) } -private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): FloatArray { +fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): FloatArray { val numFrames: Int val frameStep: Float @@ -65,6 +65,7 @@ private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, m calculateFrameRms(-1) normalizeArray(rmsValues) +// smoothArray(rmsValues, 1.0f) return rmsValues } @@ -72,7 +73,7 @@ private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, m /** * Normalizes the array's values to [0..1] range. */ -private fun normalizeArray(values: FloatArray) { +fun normalizeArray(values: FloatArray) { var maxValue = -Float.MAX_VALUE var minValue = +Float.MAX_VALUE values.forEach { value -> @@ -87,4 +88,17 @@ private fun normalizeArray(values: FloatArray) { } values.indices.forEach { i -> values[i] = (values[i] - minValue) / span } +} + +fun smoothArray(values: FloatArray, neighborWeight: Float = 1f): FloatArray { + if (values.size < 3) return values + + val result = FloatArray(values.size) + result[0] = values[0] + result[values.size - 1] == values[values.size - 1] + for (i in 1 until values.size - 1) { + result[i] = (values[i] + values[i - 1] * neighborWeight + + values[i + 1] * neighborWeight) / (1f + neighborWeight * 2f) + } + return result } \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index d67bd1e236..aaf1b7518e 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -276,7 +276,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { // Parse audio and compute RMS values for the WaveformSeekBar in the background. asyncCoroutineScope!!.launch { - val rmsFrames = 32 // The amount of values to be computed to supply for the visualization. + val rmsFrames = 32 // The amount of values to be computed for the visualization. fun extractAttachmentRandomSeed(attachment: Attachment): Int { return when { @@ -310,6 +310,8 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { } } + android.util.Log.d(TAG, "RMS: ${rmsValues.joinToString()}") + post { seekBar.sample = rmsValues diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 6d562fc6ba..a064bad4f2 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -24,32 +24,15 @@ class WaveformSeekBar : View { context.resources.displayMetrics ) } - - @JvmStatic - inline fun smooth(values: FloatArray, neighborWeight: Float = 1f): FloatArray { - if (values.size < 3) return values - - val result = FloatArray(values.size) - result[0] = values[0] - result[values.size - 1] == values[values.size - 1] - for (i in 1 until values.size - 1) { - result[i] = - (values[i] + values[i - 1] * neighborWeight + values[i + 1] * neighborWeight) / (1f + neighborWeight * 2f) - } - return result - } } var sample: FloatArray = floatArrayOf(0f) set(value) { if (value.isEmpty()) throw IllegalArgumentException("Sample array cannot be empty") - -// field = smooth(value, 0.25f) field = value invalidate() } - /** Indicates whether the user is currently interacting with the view and performing a seeking gesture. */ private var userSeeking = false private var _progress: Float = 0f @@ -124,11 +107,6 @@ class WaveformSeekBar : View { private var canvasWidth = 0 private var canvasHeight = 0 - private var maxValue = - dp( - context, - 2f - ) private var touchDownX = 0f private var scaledTouchSlop = ViewConfiguration.get(context).scaledTouchSlop @@ -177,7 +155,6 @@ class WaveformSeekBar : View { val totalWidth = getAvailableWith() - maxValue = sample.max()!! val step = (totalWidth / (waveGap + waveWidth)) / sample.size var lastWaveRight = paddingLeft.toFloat() @@ -185,11 +162,7 @@ class WaveformSeekBar : View { var i = 0f while (i < sample.size) { - var waveHeight = if (maxValue != 0f) { - getAvailableHeight() * (sample[i.toInt()] / maxValue) - } else { - waveMinHeight - } + var waveHeight = getAvailableHeight() * sample[i.toInt()] if (waveHeight < waveMinHeight) { waveHeight = waveMinHeight From c7d89985a1a00fe099cb3e4b30bb580c45f3b705 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 8 Oct 2020 19:31:20 +1100 Subject: [PATCH 07/23] Waveform change animation. --- res/layout/message_audio_view.xml | 13 +- res/values/attrs.xml | 16 +- .../securesms/loki/views/MessageAudioView.kt | 6 +- .../securesms/loki/views/WaveformSeekBar.kt | 174 +++++++++++------- 4 files changed, 129 insertions(+), 80 deletions(-) diff --git a/res/layout/message_audio_view.xml b/res/layout/message_audio_view.xml index 8ec2d8b64e..6f14afd919 100644 --- a/res/layout/message_audio_view.xml +++ b/res/layout/message_audio_view.xml @@ -79,12 +79,13 @@ android:layout_gravity="center_vertical" android:layout_marginStart="4dp" android:layout_marginEnd="4dp" - app:wave_gravity="center" - app:wave_width="4dp" - app:wave_corner_radius="2dp" - app:wave_gap="1dp" - tools:wave_background_color="#bbb" - tools:wave_progress_color="?colorPrimary"/> + app:bar_gravity="center" + app:bar_width="4dp" + app:bar_corner_radius="2dp" + app:bar_gap="1dp" + tools:progress="0.5" + tools:bar_background_color="#bbb" + tools:bar_progress_color="?colorPrimary"/> - - - - - - - + + + + + + + - + diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index aaf1b7518e..19a1d930ec 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -192,8 +192,8 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { // val colorFilter = createBlendModeColorFilterCompat(foregroundTint, BlendModeCompat.SRC_IN) // seekBar.progressDrawable.colorFilter = colorFilter // seekBar.thumb.colorFilter = colorFilter - seekBar.waveProgressColor = foregroundTint - seekBar.waveBackgroundColor = ColorUtils.blendARGB(foregroundTint, backgroundTint, 0.75f) + seekBar.barProgressColor = foregroundTint + seekBar.barBackgroundColor = ColorUtils.blendARGB(foregroundTint, backgroundTint, 0.75f) } override fun onPlayerStart(player: AudioSlidePlayer) { @@ -313,7 +313,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { android.util.Log.d(TAG, "RMS: ${rmsValues.joinToString()}") post { - seekBar.sample = rmsValues + seekBar.sampleData = rmsValues if (totalDurationMs > 0) { totalDuration.visibility = View.VISIBLE diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index a064bad4f2..5a283890ba 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -1,17 +1,23 @@ package org.thoughtcrime.securesms.loki.views +import android.animation.ValueAnimator import android.content.Context -import android.graphics.* +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.RectF import android.os.Handler import android.os.Looper import android.util.AttributeSet +import android.util.Log import android.util.TypedValue import android.view.MotionEvent import android.view.View import android.view.ViewConfiguration +import android.view.animation.DecelerateInterpolator import network.loki.messenger.R -import java.lang.IllegalArgumentException import java.lang.Math.abs +import kotlin.math.max class WaveformSeekBar : View { @@ -19,17 +25,20 @@ class WaveformSeekBar : View { @JvmStatic inline fun dp(context: Context, dp: Float): Float { return TypedValue.applyDimension( - TypedValue.COMPLEX_UNIT_DIP, - dp, - context.resources.displayMetrics + TypedValue.COMPLEX_UNIT_DIP, + dp, + context.resources.displayMetrics ) } } - var sample: FloatArray = floatArrayOf(0f) + private val sampleDataHolder = SampleDataHolder(::invalidate) + var sampleData: FloatArray? + get() { + return sampleDataHolder.getSamples() + } set(value) { - if (value.isEmpty()) throw IllegalArgumentException("Sample array cannot be empty") - field = value + sampleDataHolder.setSamples(value) invalidate() } @@ -51,44 +60,43 @@ class WaveformSeekBar : View { return _progress } - var waveBackgroundColor: Int = Color.LTGRAY + var barBackgroundColor: Int = Color.LTGRAY set(value) { field = value invalidate() } - var waveProgressColor: Int = Color.WHITE + var barProgressColor: Int = Color.WHITE set(value) { field = value invalidate() } - var waveGap: Float = dp(context, 2f) + var barGap: Float = dp(context, 2f) set(value) { field = value invalidate() } - var waveWidth: Float = dp(context, 5f) + var barWidth: Float = dp(context, 5f) set(value) { field = value invalidate() } - var waveMinHeight: Float = waveWidth + var barMinHeight: Float = barWidth set(value) { field = value invalidate() } - var waveCornerRadius: Float = dp(context, 2.5f) + var barCornerRadius: Float = dp(context, 2.5f) set(value) { field = value invalidate() } - var waveGravity: WaveGravity = - WaveGravity.CENTER + var barGravity: WaveGravity = WaveGravity.CENTER set(value) { field = value invalidate() @@ -101,8 +109,8 @@ class WaveformSeekBar : View { progressChangeListener?.onProgressChanged(this, progress, true) } - private val wavePaint = Paint(Paint.ANTI_ALIAS_FLAG) - private val waveRect = RectF() + private val barPaint = Paint(Paint.ANTI_ALIAS_FLAG) + private val barRect = RectF() private val progressCanvas = Canvas() private var canvasWidth = 0 @@ -117,28 +125,25 @@ class WaveformSeekBar : View { constructor(context: Context, attrs: AttributeSet?, defStyleAttr: Int) : super(context, attrs, defStyleAttr) { - val typedAttrs = context.obtainStyledAttributes(attrs, - R.styleable.WaveformSeekBar - ) - - waveWidth = typedAttrs.getDimension(R.styleable.WaveformSeekBar_wave_width, waveWidth) - waveGap = typedAttrs.getDimension(R.styleable.WaveformSeekBar_wave_gap, waveGap) - waveCornerRadius = typedAttrs.getDimension( - R.styleable.WaveformSeekBar_wave_corner_radius, - waveCornerRadius + val typedAttrs = context.obtainStyledAttributes(attrs, R.styleable.WaveformSeekBar) + barWidth = typedAttrs.getDimension(R.styleable.WaveformSeekBar_bar_width, barWidth) + barGap = typedAttrs.getDimension(R.styleable.WaveformSeekBar_bar_gap, barGap) + barCornerRadius = typedAttrs.getDimension( + R.styleable.WaveformSeekBar_bar_corner_radius, + barCornerRadius ) - waveMinHeight = - typedAttrs.getDimension(R.styleable.WaveformSeekBar_wave_min_height, waveMinHeight) - waveBackgroundColor = typedAttrs.getColor( - R.styleable.WaveformSeekBar_wave_background_color, - waveBackgroundColor + barMinHeight = + typedAttrs.getDimension(R.styleable.WaveformSeekBar_bar_min_height, barMinHeight) + barBackgroundColor = typedAttrs.getColor( + R.styleable.WaveformSeekBar_bar_background_color, + barBackgroundColor ) - waveProgressColor = - typedAttrs.getColor(R.styleable.WaveformSeekBar_wave_progress_color, waveProgressColor) - progress = typedAttrs.getFloat(R.styleable.WaveformSeekBar_wave_progress, progress) - waveGravity = + barProgressColor = + typedAttrs.getColor(R.styleable.WaveformSeekBar_bar_progress_color, barProgressColor) + progress = typedAttrs.getFloat(R.styleable.WaveformSeekBar_progress, progress) + barGravity = WaveGravity.fromString( - typedAttrs.getString(R.styleable.WaveformSeekBar_wave_gravity) + typedAttrs.getString(R.styleable.WaveformSeekBar_bar_gravity) ) typedAttrs.recycle() @@ -146,47 +151,39 @@ class WaveformSeekBar : View { override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { super.onSizeChanged(w, h, oldw, oldh) + canvasWidth = w canvasHeight = h + invalidate() } override fun onDraw(canvas: Canvas) { super.onDraw(canvas) - val totalWidth = getAvailableWith() - - val step = (totalWidth / (waveGap + waveWidth)) / sample.size + val totalWidth = getAvailableWidth() + val barAmount = (totalWidth / (barWidth + barGap)).toInt() - var lastWaveRight = paddingLeft.toFloat() + var lastBarRight = paddingLeft.toFloat() - var i = 0f - while (i < sample.size) { + (0 until barAmount).forEach { barIdx -> + val barValue = sampleDataHolder.computeBarValue(barIdx, barAmount) - var waveHeight = getAvailableHeight() * sample[i.toInt()] + val barHeight = max(barMinHeight, getAvailableHeight() * barValue) - if (waveHeight < waveMinHeight) { - waveHeight = waveMinHeight - } - - val top: Float = when (waveGravity) { + val top: Float = when (barGravity) { WaveGravity.TOP -> paddingTop.toFloat() - WaveGravity.CENTER -> paddingTop + getAvailableHeight() / 2f - waveHeight / 2f - WaveGravity.BOTTOM -> canvasHeight - paddingBottom - waveHeight + WaveGravity.CENTER -> paddingTop + getAvailableHeight() * 0.5f - barHeight * 0.5f + WaveGravity.BOTTOM -> canvasHeight - paddingBottom - barHeight } - waveRect.set(lastWaveRight, top, lastWaveRight + waveWidth, top + waveHeight) - - wavePaint.color = if (waveRect.right <= totalWidth * progress) - waveProgressColor else waveBackgroundColor - - canvas.drawRoundRect(waveRect, waveCornerRadius, waveCornerRadius, wavePaint) + barRect.set(lastBarRight, top, lastBarRight + barWidth, top + barHeight) - lastWaveRight = waveRect.right + waveGap + barPaint.color = if (barRect.right <= totalWidth * progress) + barProgressColor else barBackgroundColor - if (lastWaveRight + waveWidth > totalWidth + paddingLeft) - break + canvas.drawRoundRect(barRect, barCornerRadius, barCornerRadius, barPaint) - i += 1f / step + lastBarRight = barRect.right + barGap } } @@ -240,7 +237,7 @@ class WaveformSeekBar : View { } private fun updateProgress(event: MotionEvent, notify: Boolean) { - updateProgress(event.x / getAvailableWith(), notify) + updateProgress(event.x / getAvailableWidth(), notify) } private fun updateProgress(progress: Float, notify: Boolean) { @@ -270,9 +267,60 @@ class WaveformSeekBar : View { return true } - private fun getAvailableWith() = canvasWidth - paddingLeft - paddingRight + private fun getAvailableWidth() = canvasWidth - paddingLeft - paddingRight private fun getAvailableHeight() = canvasHeight - paddingTop - paddingBottom + private class SampleDataHolder(private val invalidateDelegate: () -> Any) { + + private var sampleDataFrom: FloatArray? = null + private var sampleDataTo: FloatArray? = null + private var progress = 1f // Mix between from and to values. + + private var animation: ValueAnimator? = null + + fun computeBarValue(barIdx: Int, barAmount: Int): Float { + fun getSampleValue(sampleData: FloatArray?): Float { + if (sampleData == null || sampleData.isEmpty()) + return 0f + else { + val sampleIdx = (barIdx * (sampleData.size / barAmount.toFloat())).toInt() + return sampleData[sampleIdx] + } + } + + if (progress == 1f) { + return getSampleValue(sampleDataTo) + } + + val fromValue = getSampleValue(sampleDataFrom) + val toValue = getSampleValue(sampleDataTo) + + return fromValue * (1f - progress) + toValue * progress + } + + fun setSamples(sampleData: FloatArray?) { + //TODO Animate from the current value. + sampleDataFrom = sampleDataTo + sampleDataTo = sampleData + + animation?.cancel() + animation = ValueAnimator.ofFloat(0f, 1f).apply { + addUpdateListener { animation -> + progress = animation.animatedValue as Float + Log.d("MTPHR", "Progress: $progress") + invalidateDelegate() + } + interpolator = DecelerateInterpolator(3f) + duration = 500 + start() + } + } + + fun getSamples(): FloatArray? { + return sampleDataTo + } + } + enum class WaveGravity { TOP, CENTER, From cb67bfa4a51467be3d42f66347940c79c523c45e Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Mon, 12 Oct 2020 16:10:45 +1100 Subject: [PATCH 08/23] Decoded audio ported to Kotlin. --- .../loki/utilities/audio/DecodedAudio.java | 311 ---------------- .../loki/utilities/audio/DecodedAudio.kt | 347 ++++++++++++++++++ .../loki/utilities/audio/DecodedAudioExt.kt | 104 ------ .../securesms/loki/views/MessageAudioView.kt | 5 +- .../securesms/loki/views/WaveformSeekBar.kt | 24 +- 5 files changed, 351 insertions(+), 440 deletions(-) delete mode 100644 src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java create mode 100644 src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt delete mode 100644 src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java deleted file mode 100644 index 6c58f3c57f..0000000000 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.java +++ /dev/null @@ -1,311 +0,0 @@ -package org.thoughtcrime.securesms.loki.utilities.audio; - -import android.media.AudioFormat; -import android.media.MediaCodec; -import android.media.MediaDataSource; -import android.media.MediaExtractor; -import android.media.MediaFormat; -import android.os.Build; - -import androidx.annotation.RequiresApi; - -import java.io.FileDescriptor; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.ShortBuffer; - -/** - * Partially exported class from the old Google's Ringdroid project. - * https://github.com/google/ringdroid/blob/master/app/src/main/java/com/ringdroid/soundfile/SoundFile.java - *

- * We need this one to parse audio files. Specifically extract RMS values for waveform visualization. - *

- * NOTE: This class instance creation might be pretty slow (depends on the source audio file size). - * It's recommended to instantiate it in the background. - */ -public class DecodedAudio { - - // Member variables representing frame data - private final long mFileSize; - private final int mAvgBitRate; // Average bit rate in kbps. - private final int mSampleRate; - private final long mDuration; // In microseconds. - private final int mChannels; - private final int mNumSamples; // total number of samples per channel in audio file - private final ShortBuffer mDecodedSamples; // shared buffer with mDecodedBytes. - // mDecodedSamples has the following format: - // {s1c1, s1c2, ..., s1cM, s2c1, ..., s2cM, ..., sNc1, ..., sNcM} - // where sicj is the ith sample of the jth channel (a sample is a signed short) - // M is the number of channels (e.g. 2 for stereo) and N is the number of samples per channel. - - // TODO(nfaralli): what is the real list of supported extensions? Is it device dependent? - public static String[] getSupportedExtensions() { - return new String[]{"mp3", "wav", "3gpp", "3gp", "amr", "aac", "m4a", "ogg"}; - } - - public static boolean isFilenameSupported(String filename) { - String[] extensions = getSupportedExtensions(); - for (int i = 0; i < extensions.length; i++) { - if (filename.endsWith("." + extensions[i])) { - return true; - } - } - return false; - } - - public DecodedAudio(FileDescriptor fd, long startOffset, long size) throws IOException { - this(createMediaExtractor(fd, startOffset, size), size); - } - - @RequiresApi(api = Build.VERSION_CODES.M) - public DecodedAudio(MediaDataSource dataSource) throws IOException { - this(createMediaExtractor(dataSource), dataSource.getSize()); - } - - public DecodedAudio(MediaExtractor extractor, long size) throws IOException { - mFileSize = size; - - MediaFormat mediaFormat = null; - int numTracks = extractor.getTrackCount(); - // find and select the first audio track present in the file. - int trackIndex; - for (trackIndex = 0; trackIndex < numTracks; trackIndex++) { - MediaFormat format = extractor.getTrackFormat(trackIndex); - if (format.getString(MediaFormat.KEY_MIME).startsWith("audio/")) { - extractor.selectTrack(trackIndex); - mediaFormat = format; - break; - } - } - if (mediaFormat == null) { - throw new IOException("No audio track found in the data source."); - } - - mChannels = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); - mSampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); - mDuration = mediaFormat.getLong(MediaFormat.KEY_DURATION); - // Expected total number of samples per channel. - int expectedNumSamples = - (int) ((mDuration / 1000000.f) * mSampleRate + 0.5f); - - MediaCodec codec = MediaCodec.createDecoderByType(mediaFormat.getString(MediaFormat.KEY_MIME)); - codec.configure(mediaFormat, null, null, 0); - codec.start(); - - try { - int pcmEncoding = codec.getOutputFormat().getInteger(MediaFormat.KEY_PCM_ENCODING); - if (pcmEncoding != AudioFormat.ENCODING_PCM_16BIT) { - throw new IOException("Unsupported PCM encoding code: " + pcmEncoding); - } - } catch (NullPointerException e) { - // If KEY_PCM_ENCODING is not specified, means it's ENCODING_PCM_16BIT. - } - - int decodedSamplesSize = 0; // size of the output buffer containing decoded samples. - byte[] decodedSamples = null; - int sampleSize; - MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); - long presentationTime; - int totalSizeRead = 0; - boolean doneReading = false; - - // Set the size of the decoded samples buffer to 1MB (~6sec of a stereo stream at 44.1kHz). - // For longer streams, the buffer size will be increased later on, calculating a rough - // estimate of the total size needed to store all the samples in order to resize the buffer - // only once. - ByteBuffer decodedBytes = ByteBuffer.allocate(1 << 20); - boolean firstSampleData = true; - while (true) { - // read data from file and feed it to the decoder input buffers. - int inputBufferIndex = codec.dequeueInputBuffer(100); - if (!doneReading && inputBufferIndex >= 0) { - sampleSize = extractor.readSampleData(codec.getInputBuffer(inputBufferIndex), 0); - if (firstSampleData - && mediaFormat.getString(MediaFormat.KEY_MIME).equals("audio/mp4a-latm") - && sampleSize == 2) { - // For some reasons on some devices (e.g. the Samsung S3) you should not - // provide the first two bytes of an AAC stream, otherwise the MediaCodec will - // crash. These two bytes do not contain music data but basic info on the - // stream (e.g. channel configuration and sampling frequency), and skipping them - // seems OK with other devices (MediaCodec has already been configured and - // already knows these parameters). - extractor.advance(); - totalSizeRead += sampleSize; - } else if (sampleSize < 0) { - // All samples have been read. - codec.queueInputBuffer( - inputBufferIndex, 0, 0, -1, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - doneReading = true; - } else { - presentationTime = extractor.getSampleTime(); - codec.queueInputBuffer(inputBufferIndex, 0, sampleSize, presentationTime, 0); - extractor.advance(); - totalSizeRead += sampleSize; - } - firstSampleData = false; - } - - // Get decoded stream from the decoder output buffers. - int outputBufferIndex = codec.dequeueOutputBuffer(info, 100); - if (outputBufferIndex >= 0 && info.size > 0) { - if (decodedSamplesSize < info.size) { - decodedSamplesSize = info.size; - decodedSamples = new byte[decodedSamplesSize]; - } - ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferIndex); - outputBuffer.get(decodedSamples, 0, info.size); - outputBuffer.clear(); - // Check if buffer is big enough. Resize it if it's too small. - if (decodedBytes.remaining() < info.size) { - // Getting a rough estimate of the total size, allocate 20% more, and - // make sure to allocate at least 5MB more than the initial size. - int position = decodedBytes.position(); - int newSize = (int) ((position * (1.0 * mFileSize / totalSizeRead)) * 1.2); - if (newSize - position < info.size + 5 * (1 << 20)) { - newSize = position + info.size + 5 * (1 << 20); - } - ByteBuffer newDecodedBytes = null; - // Try to allocate memory. If we are OOM, try to run the garbage collector. - int retry = 10; - while (retry > 0) { - try { - newDecodedBytes = ByteBuffer.allocate(newSize); - break; - } catch (OutOfMemoryError oome) { - // setting android:largeHeap="true" in seem to help not - // reaching this section. - retry--; - } - } - if (retry == 0) { - // Failed to allocate memory... Stop reading more data and finalize the - // instance with the data decoded so far. - break; - } - //ByteBuffer newDecodedBytes = ByteBuffer.allocate(newSize); - decodedBytes.rewind(); - newDecodedBytes.put(decodedBytes); - decodedBytes = newDecodedBytes; - decodedBytes.position(position); - } - decodedBytes.put(decodedSamples, 0, info.size); - codec.releaseOutputBuffer(outputBufferIndex, false); - } /*else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - // Subsequent data will conform to new format. - // We could check that codec.getOutputFormat(), which is the new output format, - // is what we expect. - }*/ - - if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 - || (decodedBytes.position() / (2 * mChannels)) >= expectedNumSamples) { - // We got all the decoded data from the decoder. Stop here. - // Theoretically dequeueOutputBuffer(info, ...) should have set info.flags to - // MediaCodec.BUFFER_FLAG_END_OF_STREAM. However some phones (e.g. Samsung S3) - // won't do that for some files (e.g. with mono AAC files), in which case subsequent - // calls to dequeueOutputBuffer may result in the application crashing, without - // even an exception being thrown... Hence the second check. - // (for mono AAC files, the S3 will actually double each sample, as if the stream - // was stereo. The resulting stream is half what it's supposed to be and with a much - // lower pitch.) - break; - } - } - mNumSamples = decodedBytes.position() / (mChannels * 2); // One sample = 2 bytes. - decodedBytes.rewind(); - decodedBytes.order(ByteOrder.LITTLE_ENDIAN); - mDecodedSamples = decodedBytes.asShortBuffer(); - mAvgBitRate = (int) ((mFileSize * 8) * ((float) mSampleRate / mNumSamples) / 1000); - - extractor.release(); - codec.stop(); - codec.release(); - -// // Temporary hack to make it work with the old version. -// int numFrames = mNumSamples / getSamplesPerFrame(); -// if (mNumSamples % getSamplesPerFrame() != 0) { -// numFrames++; -// } -// mFrameGains = new int[numFrames]; -// mFrameLens = new int[numFrames]; -// mFrameOffsets = new int[numFrames]; -// int j; -// int gain, value; -// int frameLens = (int) ((1000 * mAvgBitRate / 8) * -// ((float) getSamplesPerFrame() / mSampleRate)); -// for (trackIndex = 0; trackIndex < numFrames; trackIndex++) { -// gain = -1; -// for (j = 0; j < getSamplesPerFrame(); j++) { -// value = 0; -// for (int k = 0; k < mChannels; k++) { -// if (mDecodedSamples.remaining() > 0) { -// value += java.lang.Math.abs(mDecodedSamples.get()); -// } -// } -// value /= mChannels; -// if (gain < value) { -// gain = value; -// } -// } -// mFrameGains[trackIndex] = (int) Math.sqrt(gain); // here gain = sqrt(max value of 1st channel)... -// mFrameLens[trackIndex] = frameLens; // totally not accurate... -// mFrameOffsets[trackIndex] = (int) (trackIndex * (1000 * mAvgBitRate / 8) * // = i * frameLens -// ((float) getSamplesPerFrame() / mSampleRate)); -// } -// mDecodedSamples.rewind(); -// mNumFrames = numFrames; - } - - public long getFileSizeBytes() { - return mFileSize; - } - - public int getAvgBitrateKbps() { - return mAvgBitRate; - } - - public int getSampleRate() { - return mSampleRate; - } - - public int getChannels() { - return mChannels; - } - - /** @return Total duration in milliseconds. */ - public long getDuration() { - return mDuration; - } - - public int getNumSamples() { - return mNumSamples; // Number of samples per channel. - } - - public ShortBuffer getSamples() { - if (mDecodedSamples != null) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && - Build.VERSION.SDK_INT <= Build.VERSION_CODES.N_MR1) { - // Hack for Nougat where asReadOnlyBuffer fails to respect byte ordering. - // See https://code.google.com/p/android/issues/detail?id=223824 - return mDecodedSamples; - } else { - return mDecodedSamples.asReadOnlyBuffer(); - } - } else { - return null; - } - } - - private static MediaExtractor createMediaExtractor(FileDescriptor fd, long startOffset, long size) throws IOException { - MediaExtractor extractor = new MediaExtractor(); - extractor.setDataSource(fd, startOffset, size); - return extractor; - } - - @RequiresApi(api = Build.VERSION_CODES.M) - private static MediaExtractor createMediaExtractor(MediaDataSource dataSource) throws IOException { - MediaExtractor extractor = new MediaExtractor(); - extractor.setDataSource(dataSource); - return extractor; - } -} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt new file mode 100644 index 0000000000..399406bc51 --- /dev/null +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt @@ -0,0 +1,347 @@ +package org.thoughtcrime.securesms.loki.utilities.audio + +import android.media.AudioFormat +import android.media.MediaCodec +import android.media.MediaDataSource +import android.media.MediaExtractor +import android.media.MediaFormat +import android.os.Build + +import androidx.annotation.RequiresApi + +import java.io.FileDescriptor +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.ShortBuffer +import kotlin.jvm.Throws +import kotlin.math.ceil +import kotlin.math.sqrt + +/** + * Decodes the audio data and provides access to its sample data. + * We need this to extract RMS values for waveform visualization. + * + * Use static [DecodedAudio.create] methods to instantiate a [DecodedAudio]. + * + * Partially based on the old [Google's Ringdroid project] + * (https://github.com/google/ringdroid/blob/master/app/src/main/java/com/ringdroid/soundfile/SoundFile.java). + * + * *NOTE:* This class instance creation might be pretty slow (depends on the source audio file size). + * It's recommended to instantiate it in the background. + */ +@Suppress("MemberVisibilityCanBePrivate") +class DecodedAudio { + + companion object { + @JvmStatic + @Throws(IOException::class) + fun create(fd: FileDescriptor, startOffset: Long, size: Long): DecodedAudio { + val mediaExtractor = MediaExtractor().apply { setDataSource(fd, startOffset, size) } + return DecodedAudio(mediaExtractor, size) + } + + @JvmStatic + @RequiresApi(api = Build.VERSION_CODES.M) + @Throws(IOException::class) + fun create(dataSource: MediaDataSource): DecodedAudio { + val mediaExtractor = MediaExtractor().apply { setDataSource(dataSource) } + return DecodedAudio(mediaExtractor, dataSource.size) + } + } + + val dataSize: Long + + /** Average bit rate in kbps. */ + val avgBitRate: Int + + val sampleRate: Int + + /** In microseconds. */ + val totalDuration: Long + + val channels: Int + + /** Total number of samples per channel in audio file. */ + val numSamples: Int + + val samples: ShortBuffer + get() { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && + Build.VERSION.SDK_INT <= Build.VERSION_CODES.N_MR1 + ) { + // Hack for Nougat where asReadOnlyBuffer fails to respect byte ordering. + // See https://code.google.com/p/android/issues/detail?id=223824 + decodedSamples + } else { + decodedSamples.asReadOnlyBuffer() + } + } + + /** + * Shared buffer with mDecodedBytes. + * Has the following format: + * {s1c1, s1c2, ..., s1cM, s2c1, ..., s2cM, ..., sNc1, ..., sNcM} + * where sicj is the ith sample of the jth channel (a sample is a signed short) + * M is the number of channels (e.g. 2 for stereo) and N is the number of samples per channel. + */ + private val decodedSamples: ShortBuffer + + @Throws(IOException::class) + private constructor(extractor: MediaExtractor, size: Long) { + dataSize = size + + var mediaFormat: MediaFormat? = null + // Find and select the first audio track present in the file. + for (trackIndex in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(trackIndex) + if (format.getString(MediaFormat.KEY_MIME)!!.startsWith("audio/")) { + extractor.selectTrack(trackIndex) + mediaFormat = format + break + } + } + if (mediaFormat == null) { + throw IOException("No audio track found in the data source.") + } + + channels = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT) + sampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) + totalDuration = mediaFormat.getLong(MediaFormat.KEY_DURATION) + + // Expected total number of samples per channel. + val expectedNumSamples = ((totalDuration / 1000000f) * sampleRate + 0.5f).toInt() + + val codec = MediaCodec.createDecoderByType(mediaFormat.getString(MediaFormat.KEY_MIME)!!) + codec.configure(mediaFormat, null, null, 0) + codec.start() + + // Check if the track is in PCM 16 bit encoding. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + try { + val pcmEncoding = codec.outputFormat.getInteger(MediaFormat.KEY_PCM_ENCODING) + if (pcmEncoding != AudioFormat.ENCODING_PCM_16BIT) { + throw IOException("Unsupported PCM encoding code: $pcmEncoding") + } + } catch (e: NullPointerException) { + // If KEY_PCM_ENCODING is not specified, means it's ENCODING_PCM_16BIT. + } + } + + var decodedSamplesSize: Int = 0 // size of the output buffer containing decoded samples. + var decodedSamples: ByteArray? = null + var sampleSize: Int + val info = MediaCodec.BufferInfo() + var presentationTime: Long + var totalSizeRead: Int = 0 + var doneReading = false + + // Set the size of the decoded samples buffer to 1MB (~6sec of a stereo stream at 44.1kHz). + // For longer streams, the buffer size will be increased later on, calculating a rough + // estimate of the total size needed to store all the samples in order to resize the buffer + // only once. + var decodedBytes: ByteBuffer = ByteBuffer.allocate(1 shl 20) + var firstSampleData = true + while (true) { + // read data from file and feed it to the decoder input buffers. + val inputBufferIndex: Int = codec.dequeueInputBuffer(100) + if (!doneReading && inputBufferIndex >= 0) { + sampleSize = extractor.readSampleData(codec.getInputBuffer(inputBufferIndex)!!, 0) + if (firstSampleData + && mediaFormat.getString(MediaFormat.KEY_MIME)!! == "audio/mp4a-latm" + && sampleSize == 2 + ) { + // For some reasons on some devices (e.g. the Samsung S3) you should not + // provide the first two bytes of an AAC stream, otherwise the MediaCodec will + // crash. These two bytes do not contain music data but basic info on the + // stream (e.g. channel configuration and sampling frequency), and skipping them + // seems OK with other devices (MediaCodec has already been configured and + // already knows these parameters). + extractor.advance() + totalSizeRead += sampleSize + } else if (sampleSize < 0) { + // All samples have been read. + codec.queueInputBuffer( + inputBufferIndex, 0, 0, -1, MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + doneReading = true + } else { + presentationTime = extractor.sampleTime + codec.queueInputBuffer(inputBufferIndex, 0, sampleSize, presentationTime, 0) + extractor.advance() + totalSizeRead += sampleSize + } + firstSampleData = false + } + + // Get decoded stream from the decoder output buffers. + val outputBufferIndex: Int = codec.dequeueOutputBuffer(info, 100) + if (outputBufferIndex >= 0 && info.size > 0) { + if (decodedSamplesSize < info.size) { + decodedSamplesSize = info.size + decodedSamples = ByteArray(decodedSamplesSize) + } + val outputBuffer: ByteBuffer = codec.getOutputBuffer(outputBufferIndex)!! + outputBuffer.get(decodedSamples!!, 0, info.size) + outputBuffer.clear() + // Check if buffer is big enough. Resize it if it's too small. + if (decodedBytes.remaining() < info.size) { + // Getting a rough estimate of the total size, allocate 20% more, and + // make sure to allocate at least 5MB more than the initial size. + val position = decodedBytes.position() + var newSize = ((position * (1.0 * dataSize / totalSizeRead)) * 1.2).toInt() + if (newSize - position < info.size + 5 * (1 shl 20)) { + newSize = position + info.size + 5 * (1 shl 20) + } + var newDecodedBytes: ByteBuffer? = null + // Try to allocate memory. If we are OOM, try to run the garbage collector. + var retry = 10 + while (retry > 0) { + try { + newDecodedBytes = ByteBuffer.allocate(newSize) + break + } catch (e: OutOfMemoryError) { + // setting android:largeHeap="true" in seem to help not + // reaching this section. + retry-- + } + } + if (retry == 0) { + // Failed to allocate memory... Stop reading more data and finalize the + // instance with the data decoded so far. + break + } + decodedBytes.rewind() + newDecodedBytes!!.put(decodedBytes) + decodedBytes = newDecodedBytes + decodedBytes.position(position) + } + decodedBytes.put(decodedSamples, 0, info.size) + codec.releaseOutputBuffer(outputBufferIndex, false) + } + + if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 + || (decodedBytes.position() / (2 * channels)) >= expectedNumSamples + ) { + // We got all the decoded data from the decoder. Stop here. + // Theoretically dequeueOutputBuffer(info, ...) should have set info.flags to + // MediaCodec.BUFFER_FLAG_END_OF_STREAM. However some phones (e.g. Samsung S3) + // won't do that for some files (e.g. with mono AAC files), in which case subsequent + // calls to dequeueOutputBuffer may result in the application crashing, without + // even an exception being thrown... Hence the second check. + // (for mono AAC files, the S3 will actually double each sample, as if the stream + // was stereo. The resulting stream is half what it's supposed to be and with a much + // lower pitch.) + break + } + } + numSamples = decodedBytes.position() / (channels * 2) // One sample = 2 bytes. + decodedBytes.rewind() + decodedBytes.order(ByteOrder.LITTLE_ENDIAN) + this.decodedSamples = decodedBytes.asShortBuffer() + avgBitRate = ((dataSize * 8) * (sampleRate.toFloat() / numSamples) / 1000).toInt() + + extractor.release() + codec.stop() + codec.release() + } + + fun calculateRms(maxFrames: Int): FloatArray { + return calculateRms(this.samples, this.numSamples, this.channels, maxFrames) + } +} + +/** + * Computes audio RMS values for the first channel only. + * + * A typical RMS calculation algorithm is: + * 1. Square each sample + * 2. Sum the squared samples + * 3. Divide the sum of the squared samples by the number of samples + * 4. Take the square root of step 3., the mean of the squared samples + * + * @param maxFrames Defines amount of output RMS frames. + * If number of samples per channel is less than "maxFrames", + * the result array will match the source sample size instead. + * + * @return RMS values float array where is each value is within [0..1] range. + */ +private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): FloatArray { + val numFrames: Int + val frameStep: Float + + val samplesPerChannel = numSamples / channels + if (samplesPerChannel <= maxFrames) { + frameStep = 1f + numFrames = samplesPerChannel + } else { + frameStep = numSamples / maxFrames.toFloat() + numFrames = maxFrames + } + + val rmsValues = FloatArray(numFrames) + + var squaredFrameSum = 0.0 + var currentFrameIdx = 0 + + fun calculateFrameRms(nextFrameIdx: Int) { + rmsValues[currentFrameIdx] = sqrt(squaredFrameSum.toFloat()) + + // Advance to the next frame. + squaredFrameSum = 0.0 + currentFrameIdx = nextFrameIdx + } + + (0 until numSamples * channels step channels).forEach { sampleIdx -> + val channelSampleIdx = sampleIdx / channels + val frameIdx = (channelSampleIdx / frameStep).toInt() + + if (currentFrameIdx != frameIdx) { + // Calculate RMS value for the previous frame. + calculateFrameRms(frameIdx) + } + + val samplesInCurrentFrame = ceil((currentFrameIdx + 1) * frameStep) - ceil(currentFrameIdx * frameStep) + squaredFrameSum += (samples[sampleIdx] * samples[sampleIdx]) / samplesInCurrentFrame + } + // Calculate RMS value for the last frame. + calculateFrameRms(-1) + +// smoothArray(rmsValues, 1.0f) + normalizeArray(rmsValues) + + return rmsValues +} + +/** + * Normalizes the array's values to [0..1] range. + */ +private fun normalizeArray(values: FloatArray) { + var maxValue = -Float.MAX_VALUE + var minValue = +Float.MAX_VALUE + values.forEach { value -> + if (value > maxValue) maxValue = value + if (value < minValue) minValue = value + } + val span = maxValue - minValue + + if (span == 0f) { + values.indices.forEach { i -> values[i] = 0f } + return + } + + values.indices.forEach { i -> values[i] = (values[i] - minValue) / span } +} + +private fun smoothArray(values: FloatArray, neighborWeight: Float = 1f): FloatArray { + if (values.size < 3) return values + + val result = FloatArray(values.size) + result[0] = values[0] + result[values.size - 1] == values[values.size - 1] + for (i in 1 until values.size - 1) { + result[i] = (values[i] + values[i - 1] * neighborWeight + + values[i + 1] * neighborWeight) / (1f + neighborWeight * 2f) + } + return result +} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt deleted file mode 100644 index 3802bb3575..0000000000 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudioExt.kt +++ /dev/null @@ -1,104 +0,0 @@ -package org.thoughtcrime.securesms.loki.utilities.audio; - -import java.nio.ShortBuffer -import kotlin.math.ceil -import kotlin.math.sqrt - -/** - * Computes audio RMS values for the first channel only. - * - * A typical RMS calculation algorithm is: - * 1. Square each sample - * 2. Sum the squared samples - * 3. Divide the sum of the squared samples by the number of samples - * 4. Take the square root of step 3., the mean of the squared samples - * - * @param maxFrames Defines amount of output RMS frames. - * If number of samples per channel is less than "maxFrames", - * the result array will match the source sample size instead. - * - * @return RMS values float array where is each value is within [0..1] range. - */ -fun DecodedAudio.calculateRms(maxFrames: Int): FloatArray { - return calculateRms(this.samples, this.numSamples, this.channels, maxFrames) -} - -fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): FloatArray { - val numFrames: Int - val frameStep: Float - - val samplesPerChannel = numSamples / channels - if (samplesPerChannel <= maxFrames) { - frameStep = 1f - numFrames = samplesPerChannel - } else { - frameStep = numSamples / maxFrames.toFloat() - numFrames = maxFrames - } - - val rmsValues = FloatArray(numFrames) - - var squaredFrameSum = 0.0 - var currentFrameIdx = 0 - - fun calculateFrameRms(nextFrameIdx: Int) { - rmsValues[currentFrameIdx] = sqrt(squaredFrameSum.toFloat()) - - // Advance to the next frame. - squaredFrameSum = 0.0 - currentFrameIdx = nextFrameIdx - } - - (0 until numSamples * channels step channels).forEach { sampleIdx -> - val channelSampleIdx = sampleIdx / channels - val frameIdx = (channelSampleIdx / frameStep).toInt() - - if (currentFrameIdx != frameIdx) { - // Calculate RMS value for the previous frame. - calculateFrameRms(frameIdx) - } - - val samplesInCurrentFrame = ceil((currentFrameIdx + 1) * frameStep) - ceil(currentFrameIdx * frameStep) - squaredFrameSum += (samples[sampleIdx] * samples[sampleIdx]) / samplesInCurrentFrame - } - // Calculate RMS value for the last frame. - calculateFrameRms(-1) - - normalizeArray(rmsValues) -// smoothArray(rmsValues, 1.0f) - - return rmsValues -} - -/** - * Normalizes the array's values to [0..1] range. - */ -fun normalizeArray(values: FloatArray) { - var maxValue = -Float.MAX_VALUE - var minValue = +Float.MAX_VALUE - values.forEach { value -> - if (value > maxValue) maxValue = value - if (value < minValue) minValue = value - } - val span = maxValue - minValue - - if (span == 0f) { - values.indices.forEach { i -> values[i] = 0f } - return - } - - values.indices.forEach { i -> values[i] = (values[i] - minValue) / span } -} - -fun smoothArray(values: FloatArray, neighborWeight: Float = 1f): FloatArray { - if (values.size < 3) return values - - val result = FloatArray(values.size) - result[0] = values[0] - result[values.size - 1] == values[values.size - 1] - for (i in 1 until values.size - 1) { - result[i] = (values[i] + values[i - 1] * neighborWeight + - values[i + 1] * neighborWeight) / (1f + neighborWeight * 2f) - } - return result -} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index 19a1d930ec..4a750842cf 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -31,7 +31,6 @@ import org.thoughtcrime.securesms.database.AttachmentDatabase import org.thoughtcrime.securesms.events.PartProgressEvent import org.thoughtcrime.securesms.logging.Log import org.thoughtcrime.securesms.loki.utilities.audio.DecodedAudio -import org.thoughtcrime.securesms.loki.utilities.audio.calculateRms import org.thoughtcrime.securesms.mms.AudioSlide import org.thoughtcrime.securesms.mms.PartAuthority import org.thoughtcrime.securesms.mms.SlideClickListener @@ -300,10 +299,10 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { try { @Suppress("BlockingMethodInNonBlockingContext") val decodedAudio = PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { - DecodedAudio(InputStreamMediaDataSource(it)) + DecodedAudio.create(InputStreamMediaDataSource(it)) } rmsValues = decodedAudio.calculateRms(rmsFrames) - totalDurationMs = (decodedAudio.duration / 1000.0).toLong() + totalDurationMs = (decodedAudio.totalDuration / 1000.0).toLong() } catch (e: Exception) { android.util.Log.w(TAG, "Failed to decode sample values for the audio attachment \"${attachment.fileName}\".", e) rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 5a283890ba..470224bb68 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -23,7 +23,7 @@ class WaveformSeekBar : View { companion object { @JvmStatic - inline fun dp(context: Context, dp: Float): Float { + fun dp(context: Context, dp: Float): Float { return TypedValue.applyDimension( TypedValue.COMPLEX_UNIT_DIP, dp, @@ -104,14 +104,8 @@ class WaveformSeekBar : View { var progressChangeListener: ProgressChangeListener? = null - private val postponedProgressUpdateHandler = Handler(Looper.getMainLooper()) - private val postponedProgressUpdateRunnable = Runnable { - progressChangeListener?.onProgressChanged(this, progress, true) - } - private val barPaint = Paint(Paint.ANTI_ALIAS_FLAG) private val barRect = RectF() - private val progressCanvas = Canvas() private var canvasWidth = 0 private var canvasHeight = 0 @@ -245,23 +239,10 @@ class WaveformSeekBar : View { invalidate() if (notify) { - postponedProgressUpdateRunnable.run() + progressChangeListener?.onProgressChanged(this, progress, true) } } -// private fun updateProgress(event: MotionEvent, delayNotification: Boolean) { -// _progress = event.x / getAvailableWith() -// invalidate() -// -// postponedProgressUpdateHandler.removeCallbacks(postponedProgressUpdateRunnable) -// if (delayNotification) { -// // Re-post delayed user update notification to throttle a bit. -// postponedProgressUpdateHandler.postDelayed(postponedProgressUpdateRunnable, 150) -// } else { -// postponedProgressUpdateRunnable.run() -// } -// } - override fun performClick(): Boolean { super.performClick() return true @@ -299,7 +280,6 @@ class WaveformSeekBar : View { } fun setSamples(sampleData: FloatArray?) { - //TODO Animate from the current value. sampleDataFrom = sampleDataTo sampleDataTo = sampleData From 066234a30a1e3c72a5bd47334c21126f37d6de6d Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Mon, 12 Oct 2020 17:36:58 +1100 Subject: [PATCH 09/23] Waveform seek bar loading animation. --- .../securesms/loki/views/MessageAudioView.kt | 32 ++++++++++++++++++- .../securesms/loki/views/WaveformSeekBar.kt | 2 ++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index 4a750842cf..c1b00da652 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -7,6 +7,7 @@ import android.graphics.PorterDuff import android.graphics.drawable.AnimatedVectorDrawable import android.media.MediaDataSource import android.os.Build +import android.os.Handler import android.util.AttributeSet import android.view.View import android.view.View.OnTouchListener @@ -57,11 +58,12 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { private var downloadListener: SlideClickListener? = null private var audioSlidePlayer: AudioSlidePlayer? = null -// private var backwardsCounter = 0 /** Background coroutine scope that is available when the view is attached to a window. */ private var asyncCoroutineScope: CoroutineScope? = null + private val loadingAnimation: SeekBarLoadingAnimation + constructor(context: Context): this(context, null) constructor(context: Context, attrs: AttributeSet?): this(context, attrs, 0) @@ -121,6 +123,9 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { container.setBackgroundColor(typedArray.getColor(R.styleable.MessageAudioView_widgetBackground, Color.TRANSPARENT)) typedArray.recycle() } + + loadingAnimation = SeekBarLoadingAnimation(this, seekBar) + loadingAnimation.start() } override fun onAttachedToWindow() { @@ -312,6 +317,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { android.util.Log.d(TAG, "RMS: ${rmsValues.joinToString()}") post { + loadingAnimation.stop() seekBar.sampleData = rmsValues if (totalDurationMs > 0) { @@ -332,6 +338,30 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { } } +private class SeekBarLoadingAnimation( + private val hostView: View, + private val seekBar: WaveformSeekBar): Runnable { + + companion object { + private const val UPDATE_PERIOD = 500L // In milliseconds. + private val random = Random() + } + + fun start() { + stop() + run() + } + + fun stop() { + hostView.removeCallbacks(this) + } + + override fun run() { + seekBar.sampleData = (0 until 64).map { random.nextFloat() * 0.5f }.toFloatArray() + hostView.postDelayed(this, UPDATE_PERIOD) + } +} + @RequiresApi(Build.VERSION_CODES.M) private class InputStreamMediaDataSource: MediaDataSource { diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 470224bb68..411e1793b0 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -33,6 +33,7 @@ class WaveformSeekBar : View { } private val sampleDataHolder = SampleDataHolder(::invalidate) + /** An array if normalized to [0..1] values representing the audio signal. */ var sampleData: FloatArray? get() { return sampleDataHolder.getSamples() @@ -282,6 +283,7 @@ class WaveformSeekBar : View { fun setSamples(sampleData: FloatArray?) { sampleDataFrom = sampleDataTo sampleDataTo = sampleData + progress = 0f animation?.cancel() animation = ValueAnimator.ofFloat(0f, 1f).apply { From efbcd0b2076c5b9e7934c07027af794bbedbef42 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Mon, 12 Oct 2020 18:43:35 +1100 Subject: [PATCH 10/23] Themed colors for waveform seek bar and general cleanup. --- res/layout/message_audio_view.xml | 2 +- res/values-notnight-v21/themes.xml | 2 ++ res/values/attrs.xml | 1 + res/values/themes.xml | 2 ++ .../securesms/loki/views/MessageAudioView.kt | 25 ++----------------- .../securesms/loki/views/WaveformSeekBar.kt | 17 +++---------- 6 files changed, 12 insertions(+), 37 deletions(-) diff --git a/res/layout/message_audio_view.xml b/res/layout/message_audio_view.xml index 6f14afd919..c2d2d31700 100644 --- a/res/layout/message_audio_view.xml +++ b/res/layout/message_audio_view.xml @@ -70,7 +70,6 @@ - @color/core_grey_60 @drawable/ic_outline_info_24 + + ?colorControlNormal diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index c1b00da652..29b160ccba 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -7,7 +7,6 @@ import android.graphics.PorterDuff import android.graphics.drawable.AnimatedVectorDrawable import android.media.MediaDataSource import android.os.Build -import android.os.Handler import android.util.AttributeSet import android.view.View import android.view.View.OnTouchListener @@ -37,7 +36,6 @@ import org.thoughtcrime.securesms.mms.PartAuthority import org.thoughtcrime.securesms.mms.SlideClickListener import java.io.IOException import java.io.InputStream -import java.lang.Exception import java.util.* import java.util.concurrent.TimeUnit @@ -193,10 +191,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { downloadProgress.barColor = foregroundTint totalDuration.setTextColor(foregroundTint) -// val colorFilter = createBlendModeColorFilterCompat(foregroundTint, BlendModeCompat.SRC_IN) -// seekBar.progressDrawable.colorFilter = colorFilter -// seekBar.thumb.colorFilter = colorFilter - seekBar.barProgressColor = foregroundTint + // Seek bar's progress color is set from the XML template. Whereas the background is computed. seekBar.barBackgroundColor = ColorUtils.blendARGB(foregroundTint, backgroundTint, 0.75f) } @@ -210,26 +205,10 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { if (playButton.visibility != View.VISIBLE) { togglePauseToPlay() } - -// if (seekBar.progress + 5 >= seekBar.max) { -// backwardsCounter = 4 -// onProgress(0.0, 0) -// } } override fun onPlayerProgress(player: AudioSlidePlayer, progress: Double, millis: Long) { -// val seekProgress = floor(progress * seekBar.max).toInt() - //TODO Update text. seekBar.progress = progress.toFloat() -// if (/*seekProgress > 1f || */backwardsCounter > 3) { -// backwardsCounter = 0 -// seekBar.progress = 1f -// timestamp.text = String.format("%02d:%02d", -// TimeUnit.MILLISECONDS.toMinutes(millis), -// TimeUnit.MILLISECONDS.toSeconds(millis)) -// } else { -// backwardsCounter++ -// } } override fun setFocusable(focusable: Boolean) { @@ -294,7 +273,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { return Random(seed.toLong()).let { (0 until frames).map { i -> it.nextFloat() }.toFloatArray() } } - var rmsValues: FloatArray = floatArrayOf() + var rmsValues: FloatArray var totalDurationMs: Long = -1 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 411e1793b0..dc7c83eb46 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -6,8 +6,6 @@ import android.graphics.Canvas import android.graphics.Color import android.graphics.Paint import android.graphics.RectF -import android.os.Handler -import android.os.Looper import android.util.AttributeSet import android.util.Log import android.util.TypedValue @@ -125,28 +123,23 @@ class WaveformSeekBar : View { barGap = typedAttrs.getDimension(R.styleable.WaveformSeekBar_bar_gap, barGap) barCornerRadius = typedAttrs.getDimension( R.styleable.WaveformSeekBar_bar_corner_radius, - barCornerRadius - ) + barCornerRadius) barMinHeight = typedAttrs.getDimension(R.styleable.WaveformSeekBar_bar_min_height, barMinHeight) barBackgroundColor = typedAttrs.getColor( R.styleable.WaveformSeekBar_bar_background_color, - barBackgroundColor - ) + barBackgroundColor) barProgressColor = typedAttrs.getColor(R.styleable.WaveformSeekBar_bar_progress_color, barProgressColor) progress = typedAttrs.getFloat(R.styleable.WaveformSeekBar_progress, progress) - barGravity = - WaveGravity.fromString( - typedAttrs.getString(R.styleable.WaveformSeekBar_bar_gravity) - ) + barGravity = WaveGravity.fromString( + typedAttrs.getString(R.styleable.WaveformSeekBar_bar_gravity)) typedAttrs.recycle() } override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { super.onSizeChanged(w, h, oldw, oldh) - canvasWidth = w canvasHeight = h invalidate() @@ -188,7 +181,6 @@ class WaveformSeekBar : View { when (event.action) { MotionEvent.ACTION_DOWN -> { userSeeking = true -// preUserSeekingProgress = _progress if (isParentScrolling()) { touchDownX = event.x } else { @@ -207,7 +199,6 @@ class WaveformSeekBar : View { } MotionEvent.ACTION_CANCEL -> { userSeeking = false -// updateProgress(preUserSeekingProgress, false) } } return true From 793e6bf10f853ebc5ecc161f2a26c8fa72cd0cc6 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Mon, 12 Oct 2020 19:01:49 +1100 Subject: [PATCH 11/23] Waveform fill color attribute for message audio view. --- res/layout/conversation_activity_attachment_editor_stub.xml | 3 ++- res/layout/conversation_item_received_audio.xml | 1 + res/layout/conversation_item_sent_audio.xml | 1 + res/layout/message_audio_view.xml | 1 - res/values/attrs.xml | 1 + .../thoughtcrime/securesms/loki/views/MessageAudioView.kt | 6 ++++-- 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/res/layout/conversation_activity_attachment_editor_stub.xml b/res/layout/conversation_activity_attachment_editor_stub.xml index c5831c03e8..12da06a249 100644 --- a/res/layout/conversation_activity_attachment_editor_stub.xml +++ b/res/layout/conversation_activity_attachment_editor_stub.xml @@ -41,7 +41,8 @@ android:paddingBottom="15dp" app:widgetBackground="?conversation_item_bubble_background" app:foregroundTintColor="?android:colorControlNormal" - app:backgroundTintColor="?conversation_item_bubble_background"/> + app:backgroundTintColor="?conversation_item_bubble_background" + app:waveformFillColor="?conversation_item_audio_seek_bar_color"/> diff --git a/res/layout/conversation_item_sent_audio.xml b/res/layout/conversation_item_sent_audio.xml index 22cae85604..7a28a82dad 100644 --- a/res/layout/conversation_item_sent_audio.xml +++ b/res/layout/conversation_item_sent_audio.xml @@ -7,4 +7,5 @@ android:layout_height="wrap_content" app:foregroundTintColor="?android:colorControlNormal" app:backgroundTintColor="?message_sent_background_color" + app:waveformFillColor="?conversation_item_audio_seek_bar_color" android:visibility="gone"/> diff --git a/res/layout/message_audio_view.xml b/res/layout/message_audio_view.xml index c2d2d31700..09e5c9549d 100644 --- a/res/layout/message_audio_view.xml +++ b/res/layout/message_audio_view.xml @@ -78,7 +78,6 @@ android:layout_gravity="center_vertical" android:layout_marginStart="4dp" android:layout_marginEnd="4dp" - app:bar_progress_color="?conversation_item_audio_seek_bar_color" app:bar_gravity="center" app:bar_width="4dp" app:bar_corner_radius="2dp" diff --git a/res/values/attrs.xml b/res/values/attrs.xml index 362f3b9807..207ec30a95 100644 --- a/res/values/attrs.xml +++ b/res/values/attrs.xml @@ -174,6 +174,7 @@ + diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index 29b160ccba..b3f054031c 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -117,7 +117,8 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { if (attrs != null) { val typedArray = context.theme.obtainStyledAttributes(attrs, R.styleable.MessageAudioView, 0, 0) setTint(typedArray.getColor(R.styleable.MessageAudioView_foregroundTintColor, Color.WHITE), - typedArray.getColor(R.styleable.MessageAudioView_backgroundTintColor, Color.WHITE)) + typedArray.getColor(R.styleable.MessageAudioView_backgroundTintColor, Color.WHITE), + typedArray.getColor(R.styleable.MessageAudioView_waveformFillColor, Color.WHITE)) container.setBackgroundColor(typedArray.getColor(R.styleable.MessageAudioView_widgetBackground, Color.TRANSPARENT)) typedArray.recycle() } @@ -181,7 +182,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { downloadListener = listener } - fun setTint(@ColorInt foregroundTint: Int, @ColorInt backgroundTint: Int) { + fun setTint(@ColorInt foregroundTint: Int, @ColorInt backgroundTint: Int, @ColorInt waveformFill: Int) { playButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) playButton.imageTintList = ColorStateList.valueOf(backgroundTint) pauseButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) @@ -192,6 +193,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { totalDuration.setTextColor(foregroundTint) // Seek bar's progress color is set from the XML template. Whereas the background is computed. + seekBar.barProgressColor = waveformFill seekBar.barBackgroundColor = ColorUtils.blendARGB(foregroundTint, backgroundTint, 0.75f) } From 091847b1311e358a134c55ca7c012475d48860c5 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Tue, 13 Oct 2020 20:09:24 +1100 Subject: [PATCH 12/23] Restrict the waveform seek bar progress value to a 0..1 range. --- src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index dc7c83eb46..95ed82f215 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -13,6 +13,7 @@ import android.view.MotionEvent import android.view.View import android.view.ViewConfiguration import android.view.animation.DecelerateInterpolator +import androidx.core.math.MathUtils import network.loki.messenger.R import java.lang.Math.abs import kotlin.math.max @@ -227,11 +228,11 @@ class WaveformSeekBar : View { } private fun updateProgress(progress: Float, notify: Boolean) { - _progress = progress + _progress = MathUtils.clamp(progress, 0f, 1f) invalidate() if (notify) { - progressChangeListener?.onProgressChanged(this, progress, true) + progressChangeListener?.onProgressChanged(this, _progress, true) } } From 674da8827455a2023a117185817f48225b5848b2 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Tue, 20 Oct 2020 18:58:51 +1100 Subject: [PATCH 13/23] Audio extra columns for attachment table. --- .../DatabaseAttachmentAudioExtras.kt | 14 +++++ .../database/AttachmentDatabase.java | 51 ++++++++++++++++++- .../database/helpers/ClassicOpenHelper.java | 9 +++- 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt diff --git a/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt new file mode 100644 index 0000000000..48bf768e3b --- /dev/null +++ b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt @@ -0,0 +1,14 @@ +package org.thoughtcrime.securesms.attachments + +data class DatabaseAttachmentAudioExtras(val attachmentId: AttachmentId, val visualSamples: ByteArray, val durationMs: Long) { + + override fun equals(other: Any?): Boolean { + return other != null && + other is DatabaseAttachmentAudioExtras && + other.attachmentId == attachmentId + } + + override fun hashCode(): Int { + return attachmentId.hashCode() + } +} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java index 63bdd4ca70..8e637e0036 100644 --- a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java +++ b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java @@ -39,6 +39,7 @@ import org.json.JSONException; import org.thoughtcrime.securesms.attachments.Attachment; import org.thoughtcrime.securesms.attachments.AttachmentId; import org.thoughtcrime.securesms.attachments.DatabaseAttachment; +import org.thoughtcrime.securesms.attachments.DatabaseAttachmentAudioExtras; import org.thoughtcrime.securesms.crypto.AttachmentSecret; import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream; import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream; @@ -105,6 +106,9 @@ public class AttachmentDatabase extends Database { static final String CAPTION = "caption"; public static final String URL = "url"; public static final String DIRECTORY = "parts"; + // audio/* mime type only related columns. + static final String AUDIO_VISUAL_SAMPLES = "audio_visual_samples"; // Small amount of audio byte samples to visualise the content (e.g. draw waveform) + static final String AUDIO_DURATION = "audio_duration"; // Duration of the audio track in milliseconds. public static final int TRANSFER_PROGRESS_DONE = 0; public static final int TRANSFER_PROGRESS_STARTED = 1; @@ -112,6 +116,7 @@ public class AttachmentDatabase extends Database { public static final int TRANSFER_PROGRESS_FAILED = 3; private static final String PART_ID_WHERE = ROW_ID + " = ? AND " + UNIQUE_ID + " = ?"; + private static final String PART_AUDIO_ONLY_WHERE = CONTENT_TYPE + " LIKE audio/%"; private static final String[] PROJECTION = new String[] {ROW_ID, MMS_ID, CONTENT_TYPE, NAME, CONTENT_DISPOSITION, @@ -121,6 +126,8 @@ public class AttachmentDatabase extends Database { QUOTE, DATA_RANDOM, THUMBNAIL_RANDOM, WIDTH, HEIGHT, CAPTION, STICKER_PACK_ID, STICKER_PACK_KEY, STICKER_ID, URL}; + private static final String[] PROJECTION_AUDIO_EXTRAS = new String[] {AUDIO_VISUAL_SAMPLES, AUDIO_DURATION}; + public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ROW_ID + " INTEGER PRIMARY KEY, " + MMS_ID + " INTEGER, " + "seq" + " INTEGER DEFAULT 0, " + CONTENT_TYPE + " TEXT, " + NAME + " TEXT, " + "chset" + " INTEGER, " + @@ -133,7 +140,8 @@ public class AttachmentDatabase extends Database { VOICE_NOTE + " INTEGER DEFAULT 0, " + DATA_RANDOM + " BLOB, " + THUMBNAIL_RANDOM + " BLOB, " + QUOTE + " INTEGER DEFAULT 0, " + WIDTH + " INTEGER DEFAULT 0, " + HEIGHT + " INTEGER DEFAULT 0, " + CAPTION + " TEXT DEFAULT NULL, " + URL + " TEXT, " + STICKER_PACK_ID + " TEXT DEFAULT NULL, " + - STICKER_PACK_KEY + " DEFAULT NULL, " + STICKER_ID + " INTEGER DEFAULT -1);"; + STICKER_PACK_KEY + " DEFAULT NULL, " + STICKER_ID + " INTEGER DEFAULT -1," + + AUDIO_VISUAL_SAMPLES + " BLOB, " + AUDIO_DURATION + " INTEGER);"; public static final String[] CREATE_INDEXS = { "CREATE INDEX IF NOT EXISTS part_mms_id_index ON " + TABLE_NAME + " (" + MMS_ID + ");", @@ -822,6 +830,47 @@ public class AttachmentDatabase extends Database { } } + /** + * Retrieves the audio extra values associated with the attachment. Only "audio/*" mime type attachments are accepted. + * @return the related audio extras or null in case any of the audio extra columns are empty or the attachment is not an audio. + */ + public @Nullable DatabaseAttachmentAudioExtras getAttachmentAudioExtras(@NonNull AttachmentId attachmentId) { + try (Cursor cursor = databaseHelper.getReadableDatabase() + // We expect all the audio extra values to be present (not null) or reject the whole record. + .query(TABLE_NAME, + PROJECTION_AUDIO_EXTRAS, + PART_ID_WHERE + + " AND " + AUDIO_VISUAL_SAMPLES + " IS NOT NULL" + + " AND " + AUDIO_DURATION + " IS NOT NULL" + + " AND " + PART_AUDIO_ONLY_WHERE, + attachmentId.toStrings(), + null, null, null, "1")) { + + if (cursor == null || !cursor.moveToFirst()) return null; + + byte[] audioSamples = cursor.getBlob(cursor.getColumnIndexOrThrow(AUDIO_VISUAL_SAMPLES)); + long duration = cursor.getLong(cursor.getColumnIndexOrThrow(AUDIO_DURATION)); + + return new DatabaseAttachmentAudioExtras(attachmentId, audioSamples, duration); + } + } + + /** + * Updates audio extra columns for the "audio/*" mime type attachments only. + * @return true if the update operation was successful. + */ + public boolean setAttachmentAudioExtras(@NonNull DatabaseAttachmentAudioExtras extras) { + ContentValues values = new ContentValues(); + values.put(AUDIO_VISUAL_SAMPLES, extras.getVisualSamples()); + values.put(AUDIO_DURATION, extras.getDurationMs()); + + int alteredRows = databaseHelper.getWritableDatabase().update(TABLE_NAME, + values, + PART_ID_WHERE + " AND " + PART_AUDIO_ONLY_WHERE, + extras.getAttachmentId().toStrings()); + + return alteredRows > 0; + } @VisibleForTesting class ThumbnailFetchCallable implements Callable { diff --git a/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java index 64123f535f..86cf0872ad 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java @@ -103,7 +103,9 @@ public class ClassicOpenHelper extends SQLiteOpenHelper { private static final int GROUP_RECEIPT_TRACKING = 45; private static final int UNREAD_COUNT_VERSION = 46; private static final int MORE_RECIPIENT_FIELDS = 47; - private static final int DATABASE_VERSION = 47; + private static final int AUDIO_ATTACHMENT_EXTRAS = 48; + + private static final int DATABASE_VERSION = 48; private static final String TAG = ClassicOpenHelper.class.getSimpleName(); @@ -1289,6 +1291,11 @@ public class ClassicOpenHelper extends SQLiteOpenHelper { */ } + if (oldVersion < AUDIO_ATTACHMENT_EXTRAS) { + db.execSQL("ALTER TABLE part ADD COLUMN audio_visual_samples BLOB"); + db.execSQL("ALTER TABLE part ADD COLUMN audio_duration INTEGER"); + } + db.setTransactionSuccessful(); db.endTransaction(); } From b6d8898ff922ef2ec038e62710aa30679e97be3f Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 22 Oct 2020 13:52:53 +1100 Subject: [PATCH 14/23] Fixed attachment table queries. Attachment audio extras job. Job manager supports parcelable types now. --- .../securesms/attachments/AttachmentId.java | 34 +++- .../DatabaseAttachmentAudioExtras.kt | 11 +- .../database/AttachmentDatabase.java | 10 +- .../database/helpers/ClassicOpenHelper.java | 9 +- .../database/helpers/SQLCipherOpenHelper.java | 8 +- .../securesms/jobmanager/Data.java | 23 +++ .../migration/WorkManagerFactoryMappings.java | 2 + .../securesms/jobs/JobManagerFactories.java | 2 + .../securesms/loki/activities/HomeActivity.kt | 2 + .../api/PrepareAttachmentAudioExtrasJob.kt | 169 ++++++++++++++++++ .../loki/utilities/audio/DecodedAudio.kt | 20 ++- .../securesms/loki/views/MessageAudioView.kt | 154 ++++++---------- .../securesms/loki/views/WaveformSeekBar.kt | 42 +++-- .../securesms/util/ParcelableUtil.kt | 32 ++++ 14 files changed, 382 insertions(+), 136 deletions(-) create mode 100644 src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt create mode 100644 src/org/thoughtcrime/securesms/util/ParcelableUtil.kt diff --git a/src/org/thoughtcrime/securesms/attachments/AttachmentId.java b/src/org/thoughtcrime/securesms/attachments/AttachmentId.java index 3d43c84195..6f5160d75a 100644 --- a/src/org/thoughtcrime/securesms/attachments/AttachmentId.java +++ b/src/org/thoughtcrime/securesms/attachments/AttachmentId.java @@ -1,12 +1,15 @@ package org.thoughtcrime.securesms.attachments; +import android.os.Parcel; +import android.os.Parcelable; + import androidx.annotation.NonNull; import com.fasterxml.jackson.annotation.JsonProperty; import org.thoughtcrime.securesms.util.Util; -public class AttachmentId { +public class AttachmentId implements Parcelable { @JsonProperty private final long rowId; @@ -54,4 +57,33 @@ public class AttachmentId { public int hashCode() { return Util.hashCode(rowId, uniqueId); } + + //region Parcelable implementation. + @Override + public int describeContents() { + return 0; + } + + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeLong(rowId); + dest.writeLong(uniqueId); + } + + public static final Parcelable.Creator CREATOR = + new Parcelable.Creator() { + + @Override + public AttachmentId createFromParcel(Parcel in) { + long rowId = in.readLong(); + long uniqueId = in.readLong(); + return new AttachmentId(rowId, uniqueId); + } + + @Override + public AttachmentId[] newArray(int size) { + return new AttachmentId[size]; + } + }; + //endregion } diff --git a/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt index 48bf768e3b..21de7925c7 100644 --- a/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt +++ b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt @@ -1,6 +1,15 @@ package org.thoughtcrime.securesms.attachments -data class DatabaseAttachmentAudioExtras(val attachmentId: AttachmentId, val visualSamples: ByteArray, val durationMs: Long) { +data class DatabaseAttachmentAudioExtras( + val attachmentId: AttachmentId, + /** Small amount of normalized audio byte samples to visualise the content (e.g. draw waveform). */ + val visualSamples: ByteArray, + /** Duration of the audio track in milliseconds. May be [DURATION_UNDEFINED] when is not known. */ + val durationMs: Long = DURATION_UNDEFINED) { + + companion object { + const val DURATION_UNDEFINED = -1L + } override fun equals(other: Any?): Boolean { return other != null && diff --git a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java index 8e637e0036..583e92a813 100644 --- a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java +++ b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java @@ -73,6 +73,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import kotlin.jvm.Synchronized; + public class AttachmentDatabase extends Database { private static final String TAG = AttachmentDatabase.class.getSimpleName(); @@ -106,8 +108,8 @@ public class AttachmentDatabase extends Database { static final String CAPTION = "caption"; public static final String URL = "url"; public static final String DIRECTORY = "parts"; - // audio/* mime type only related columns. - static final String AUDIO_VISUAL_SAMPLES = "audio_visual_samples"; // Small amount of audio byte samples to visualise the content (e.g. draw waveform) + // "audio/*" mime type only related columns. + static final String AUDIO_VISUAL_SAMPLES = "audio_visual_samples"; // Small amount of audio byte samples to visualise the content (e.g. draw waveform). static final String AUDIO_DURATION = "audio_duration"; // Duration of the audio track in milliseconds. public static final int TRANSFER_PROGRESS_DONE = 0; @@ -116,7 +118,7 @@ public class AttachmentDatabase extends Database { public static final int TRANSFER_PROGRESS_FAILED = 3; private static final String PART_ID_WHERE = ROW_ID + " = ? AND " + UNIQUE_ID + " = ?"; - private static final String PART_AUDIO_ONLY_WHERE = CONTENT_TYPE + " LIKE audio/%"; + private static final String PART_AUDIO_ONLY_WHERE = CONTENT_TYPE + " LIKE \"audio/%\""; private static final String[] PROJECTION = new String[] {ROW_ID, MMS_ID, CONTENT_TYPE, NAME, CONTENT_DISPOSITION, @@ -834,6 +836,7 @@ public class AttachmentDatabase extends Database { * Retrieves the audio extra values associated with the attachment. Only "audio/*" mime type attachments are accepted. * @return the related audio extras or null in case any of the audio extra columns are empty or the attachment is not an audio. */ + @Synchronized public @Nullable DatabaseAttachmentAudioExtras getAttachmentAudioExtras(@NonNull AttachmentId attachmentId) { try (Cursor cursor = databaseHelper.getReadableDatabase() // We expect all the audio extra values to be present (not null) or reject the whole record. @@ -859,6 +862,7 @@ public class AttachmentDatabase extends Database { * Updates audio extra columns for the "audio/*" mime type attachments only. * @return true if the update operation was successful. */ + @Synchronized public boolean setAttachmentAudioExtras(@NonNull DatabaseAttachmentAudioExtras extras) { ContentValues values = new ContentValues(); values.put(AUDIO_VISUAL_SAMPLES, extras.getVisualSamples()); diff --git a/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java index 86cf0872ad..64123f535f 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/ClassicOpenHelper.java @@ -103,9 +103,7 @@ public class ClassicOpenHelper extends SQLiteOpenHelper { private static final int GROUP_RECEIPT_TRACKING = 45; private static final int UNREAD_COUNT_VERSION = 46; private static final int MORE_RECIPIENT_FIELDS = 47; - private static final int AUDIO_ATTACHMENT_EXTRAS = 48; - - private static final int DATABASE_VERSION = 48; + private static final int DATABASE_VERSION = 47; private static final String TAG = ClassicOpenHelper.class.getSimpleName(); @@ -1291,11 +1289,6 @@ public class ClassicOpenHelper extends SQLiteOpenHelper { */ } - if (oldVersion < AUDIO_ATTACHMENT_EXTRAS) { - db.execSQL("ALTER TABLE part ADD COLUMN audio_visual_samples BLOB"); - db.execSQL("ALTER TABLE part ADD COLUMN audio_duration INTEGER"); - } - db.setTransactionSuccessful(); db.endTransaction(); } diff --git a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index 82d8bf0a06..de227acdc1 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -92,8 +92,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { private static final int lokiV13 = 34; private static final int lokiV14_BACKUP_FILES = 35; private static final int lokiV15 = 36; + private static final int lokiV16_AUDIO_ATTACHMENT_EXTRAS = 37; - private static final int DATABASE_VERSION = lokiV15; + private static final int DATABASE_VERSION = lokiV16_AUDIO_ATTACHMENT_EXTRAS; private static final String DATABASE_NAME = "signal.db"; private final Context context; @@ -632,6 +633,11 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { db.execSQL(SharedSenderKeysDatabase.getCreateOldClosedGroupRatchetTableCommand()); } + if (oldVersion < lokiV16_AUDIO_ATTACHMENT_EXTRAS) { + db.execSQL("ALTER TABLE part ADD COLUMN audio_visual_samples BLOB"); + db.execSQL("ALTER TABLE part ADD COLUMN audio_duration INTEGER"); + } + db.setTransactionSuccessful(); } finally { db.endTransaction(); diff --git a/src/org/thoughtcrime/securesms/jobmanager/Data.java b/src/org/thoughtcrime/securesms/jobmanager/Data.java index 72e8508a5b..0eaa2e9e4e 100644 --- a/src/org/thoughtcrime/securesms/jobmanager/Data.java +++ b/src/org/thoughtcrime/securesms/jobmanager/Data.java @@ -1,13 +1,20 @@ package org.thoughtcrime.securesms.jobmanager; +import android.os.Parcelable; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import com.fasterxml.jackson.annotation.JsonProperty; +import org.thoughtcrime.securesms.attachments.AttachmentId; +import org.thoughtcrime.securesms.util.ParcelableUtil; + import java.util.HashMap; import java.util.Map; +//TODO AC: For now parcelable objects utilize byteArrays field to store their data into. +// Introduce a dedicated Map field specifically for parcelable needs. public class Data { public static final Data EMPTY = new Data.Builder().build(); @@ -213,6 +220,16 @@ public class Data { return byteArrays.get(key); } + public boolean hasParcelable(@NonNull String key) { + return byteArrays.containsKey(key); + } + + public T getParcelable(@NonNull String key, @NonNull Parcelable.Creator creator) { + throwIfAbsent(byteArrays, key); + byte[] bytes = byteArrays.get(key); + return ParcelableUtil.unmarshall(bytes, creator); + } + private void throwIfAbsent(@NonNull Map map, @NonNull String key) { if (!map.containsKey(key)) { throw new IllegalStateException("Tried to retrieve a value with key '" + key + "', but it wasn't present."); @@ -301,6 +318,12 @@ public class Data { return this; } + public Builder putParcelable(@NonNull String key, @NonNull Parcelable value) { + byte[] bytes = ParcelableUtil.marshall(value); + byteArrays.put(key, bytes); + return this; + } + public Data build() { return new Data(strings, stringArrays, diff --git a/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java b/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java index 1449ace743..9b1ec1f890 100644 --- a/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java +++ b/src/org/thoughtcrime/securesms/jobmanager/migration/WorkManagerFactoryMappings.java @@ -45,6 +45,7 @@ import org.thoughtcrime.securesms.jobs.TrimThreadJob; import org.thoughtcrime.securesms.jobs.TypingSendJob; import org.thoughtcrime.securesms.jobs.UpdateApkJob; import org.thoughtcrime.securesms.loki.api.BackgroundPollJob; +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob; import org.thoughtcrime.securesms.loki.protocol.ClosedGroupUpdateMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.NullMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.SessionRequestMessageSendJob; @@ -102,6 +103,7 @@ public class WorkManagerFactoryMappings { put(TrimThreadJob.class.getName(), TrimThreadJob.KEY); put(TypingSendJob.class.getName(), TypingSendJob.KEY); put(UpdateApkJob.class.getName(), UpdateApkJob.KEY); + put(PrepareAttachmentAudioExtrasJob.class.getName(), PrepareAttachmentAudioExtrasJob.KEY); }}; public static @Nullable String getFactoryKey(@NonNull String workManagerClass) { diff --git a/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index 6f3c34e310..8a89f0bcb6 100644 --- a/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/src/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -15,6 +15,7 @@ import org.thoughtcrime.securesms.jobmanager.impl.NetworkOrCellServiceConstraint import org.thoughtcrime.securesms.jobmanager.impl.SqlCipherMigrationConstraint; import org.thoughtcrime.securesms.jobmanager.impl.SqlCipherMigrationConstraintObserver; import org.thoughtcrime.securesms.loki.api.BackgroundPollJob; +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob; import org.thoughtcrime.securesms.loki.protocol.ClosedGroupUpdateMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.NullMessageSendJob; import org.thoughtcrime.securesms.loki.protocol.SessionRequestMessageSendJob; @@ -79,6 +80,7 @@ public final class JobManagerFactories { put(TrimThreadJob.KEY, new TrimThreadJob.Factory()); put(TypingSendJob.KEY, new TypingSendJob.Factory()); put(UpdateApkJob.KEY, new UpdateApkJob.Factory()); + put(PrepareAttachmentAudioExtrasJob.KEY, new PrepareAttachmentAudioExtrasJob.Factory()); }}; } diff --git a/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt b/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt index 815e45ecc3..b23313bb0e 100644 --- a/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt +++ b/src/org/thoughtcrime/securesms/loki/activities/HomeActivity.kt @@ -26,11 +26,13 @@ import kotlinx.android.synthetic.main.activity_home.* import network.loki.messenger.R import org.thoughtcrime.securesms.ApplicationContext import org.thoughtcrime.securesms.PassphraseRequiredActionBarActivity +import org.thoughtcrime.securesms.attachments.AttachmentId import org.thoughtcrime.securesms.conversation.ConversationActivity import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.database.ThreadDatabase import org.thoughtcrime.securesms.database.model.ThreadRecord import org.thoughtcrime.securesms.jobs.MultiDeviceBlockedUpdateJob +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob import org.thoughtcrime.securesms.loki.dialogs.ConversationOptionsBottomSheet import org.thoughtcrime.securesms.loki.dialogs.LightThemeFeatureIntroBottomSheet import org.thoughtcrime.securesms.loki.dialogs.MultiDeviceRemovalBottomSheet diff --git a/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt new file mode 100644 index 0000000000..0f4d7cd492 --- /dev/null +++ b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt @@ -0,0 +1,169 @@ +package org.thoughtcrime.securesms.loki.api + +import android.media.MediaDataSource +import android.os.Build +import android.util.Log +import androidx.annotation.RequiresApi +import org.greenrobot.eventbus.EventBus +import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.attachments.AttachmentId +import org.thoughtcrime.securesms.attachments.DatabaseAttachmentAudioExtras +import org.thoughtcrime.securesms.database.DatabaseFactory +import org.thoughtcrime.securesms.jobmanager.Data +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.jobs.BaseJob +import org.thoughtcrime.securesms.loki.utilities.audio.DecodedAudio +import org.thoughtcrime.securesms.mms.PartAuthority +import java.io.InputStream +import java.lang.IllegalStateException +import java.util.* +import java.util.concurrent.TimeUnit + +/** + * Decodes the audio content of the related attachment entry + * and caches the result with [DatabaseAttachmentAudioExtras] data. + * + * It only process attachments with "audio" mime types. + * + * Due to [DecodedAudio] implementation limitations, it only works for API 23+. + * For any lower targets fake data will be generated. + * + * You can subscribe to [AudioExtrasUpdatedEvent] to be notified about the successful result. + */ +//TODO AC: Rewrite to WorkManager API when +// https://github.com/loki-project/session-android/pull/354 is merged. +class PrepareAttachmentAudioExtrasJob : BaseJob { + + companion object { + private const val TAG = "AttachAudioExtrasJob" + + const val KEY = "PrepareAttachmentAudioExtrasJob" + const val DATA_ATTACH_ID = "attachment_id" + + const val VISUAL_RMS_FRAMES = 32 // The amount of values to be computed for the visualization. + } + + private val attachmentId: AttachmentId + + constructor(attachmentId: AttachmentId) : this(Parameters.Builder() + .setQueue(KEY) + .setLifespan(TimeUnit.DAYS.toMillis(1)) + .build(), + attachmentId) + + private constructor(parameters: Parameters, attachmentId: AttachmentId) : super(parameters) { + this.attachmentId = attachmentId + } + + override fun serialize(): Data { + return Data.Builder().putParcelable(DATA_ATTACH_ID, attachmentId).build(); + } + + override fun getFactoryKey(): String { return KEY + } + + override fun onShouldRetry(e: Exception): Boolean { + return false + } + + override fun onCanceled() { } + + override fun onRun() { + Log.v(TAG, "Processing attachment: $attachmentId") + + val attachDb = DatabaseFactory.getAttachmentDatabase(context) + val attachment = attachDb.getAttachment(attachmentId) + + if (attachment == null) { + throw IllegalStateException("Cannot find attachment with the ID $attachmentId") + } + if (!attachment.contentType.startsWith("audio/")) { + throw IllegalStateException("Attachment $attachmentId is not of audio type.") + } + + // Check if the audio extras already exist. + if (attachDb.getAttachmentAudioExtras(attachmentId) != null) return + + fun extractAttachmentRandomSeed(attachment: Attachment): Int { + return when { + attachment.digest != null -> attachment.digest!!.sum() + attachment.fileName != null -> attachment.fileName.hashCode() + else -> attachment.hashCode() + } + } + + fun generateFakeRms(seed: Int, frames: Int = VISUAL_RMS_FRAMES): ByteArray { + return ByteArray(frames).apply { Random(seed.toLong()).nextBytes(this) } + } + + var rmsValues: ByteArray + var totalDurationMs: Long = DatabaseAttachmentAudioExtras.DURATION_UNDEFINED + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { + // Due to API version incompatibility, we just display some random waveform for older API. + rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) + } else { + try { + @Suppress("BlockingMethodInNonBlockingContext") + val decodedAudio = PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { + DecodedAudio.create(InputStreamMediaDataSource(it)) + } + rmsValues = decodedAudio.calculateRms(VISUAL_RMS_FRAMES) + totalDurationMs = (decodedAudio.totalDuration / 1000.0).toLong() + } catch (e: Exception) { + Log.w(TAG, "Failed to decode sample values for the audio attachment \"${attachment.fileName}\".", e) + rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) + } + } + + val audioExtras = DatabaseAttachmentAudioExtras( + attachmentId, + rmsValues, + totalDurationMs + ) + + attachDb.setAttachmentAudioExtras(audioExtras) + + EventBus.getDefault().post(AudioExtrasUpdatedEvent(audioExtras)) + } + + class Factory : Job.Factory { + override fun create(parameters: Parameters, data: Data): PrepareAttachmentAudioExtrasJob { + return PrepareAttachmentAudioExtrasJob(parameters, data.getParcelable(DATA_ATTACH_ID, AttachmentId.CREATOR)) + } + } + + /** Dispatched once the audio extras have been updated. */ + data class AudioExtrasUpdatedEvent(val audioExtras: DatabaseAttachmentAudioExtras) + + @RequiresApi(Build.VERSION_CODES.M) + private class InputStreamMediaDataSource: MediaDataSource { + + private val data: ByteArray + + constructor(inputStream: InputStream): super() { + this.data = inputStream.readBytes() + } + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { + val length: Int = data.size + if (position >= length) { + return -1 // -1 indicates EOF + } + var actualSize = size + if (position + size > length) { + actualSize -= (position + size - length).toInt() + } + System.arraycopy(data, position.toInt(), buffer, offset, actualSize) + return actualSize + } + + override fun getSize(): Long { + return data.size.toLong() + } + + override fun close() { + // We don't need to close the wrapped stream. + } + } +} \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt index 399406bc51..ddc79d8722 100644 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt @@ -16,6 +16,7 @@ import java.nio.ByteOrder import java.nio.ShortBuffer import kotlin.jvm.Throws import kotlin.math.ceil +import kotlin.math.roundToInt import kotlin.math.sqrt /** @@ -246,7 +247,7 @@ class DecodedAudio { codec.release() } - fun calculateRms(maxFrames: Int): FloatArray { + fun calculateRms(maxFrames: Int): ByteArray { return calculateRms(this.samples, this.numSamples, this.channels, maxFrames) } } @@ -264,9 +265,9 @@ class DecodedAudio { * If number of samples per channel is less than "maxFrames", * the result array will match the source sample size instead. * - * @return RMS values float array where is each value is within [0..1] range. + * @return normalized RMS values as a signed byte array. */ -private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): FloatArray { +private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, maxFrames: Int): ByteArray { val numFrames: Int val frameStep: Float @@ -310,7 +311,8 @@ private fun calculateRms(samples: ShortBuffer, numSamples: Int, channels: Int, m // smoothArray(rmsValues, 1.0f) normalizeArray(rmsValues) - return rmsValues + // Convert normalized result to a signed byte array. + return rmsValues.map { value -> normalizedFloatToByte(value) }.toByteArray() } /** @@ -344,4 +346,14 @@ private fun smoothArray(values: FloatArray, neighborWeight: Float = 1f): FloatAr values[i + 1] * neighborWeight) / (1f + neighborWeight * 2f) } return result +} + +/** Turns a signed byte into a [0..1] float. */ +inline fun byteToNormalizedFloat(value: Byte): Float { + return (value + 128f) / 255f +} + +/** Turns a [0..1] float into a signed byte. */ +inline fun normalizedFloatToByte(value: Float): Byte { + return (255f * value - 128f).roundToInt().toByte() } \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index b3f054031c..a74b638de4 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -5,8 +5,6 @@ import android.content.res.ColorStateList import android.graphics.Color import android.graphics.PorterDuff import android.graphics.drawable.AnimatedVectorDrawable -import android.media.MediaDataSource -import android.os.Build import android.util.AttributeSet import android.view.View import android.view.View.OnTouchListener @@ -15,7 +13,6 @@ import android.widget.FrameLayout import android.widget.ImageView import android.widget.TextView import androidx.annotation.ColorInt -import androidx.annotation.RequiresApi import androidx.core.content.ContextCompat import androidx.core.graphics.ColorUtils import com.pnikosis.materialishprogress.ProgressWheel @@ -24,18 +21,18 @@ import network.loki.messenger.R import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode -import org.thoughtcrime.securesms.attachments.Attachment +import org.thoughtcrime.securesms.ApplicationContext +import org.thoughtcrime.securesms.attachments.DatabaseAttachment import org.thoughtcrime.securesms.audio.AudioSlidePlayer import org.thoughtcrime.securesms.components.AnimatingToggle import org.thoughtcrime.securesms.database.AttachmentDatabase +import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.events.PartProgressEvent import org.thoughtcrime.securesms.logging.Log -import org.thoughtcrime.securesms.loki.utilities.audio.DecodedAudio +import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob import org.thoughtcrime.securesms.mms.AudioSlide -import org.thoughtcrime.securesms.mms.PartAuthority import org.thoughtcrime.securesms.mms.SlideClickListener import java.io.IOException -import java.io.InputStream import java.util.* import java.util.concurrent.TimeUnit @@ -166,7 +163,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { } // Post to make sure it executes only when the view is attached to a window. - post(::updateSeekBarFromAudio) + post(::updateFromAttachmentAudioExtras) } } audioSlidePlayer = AudioSlidePlayer.createFor(context, audio, this) @@ -254,122 +251,73 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { pauseToPlayDrawable.start() } - private fun updateSeekBarFromAudio() { - if (audioSlidePlayer == null) return - + private fun obtainDatabaseAttachment(): DatabaseAttachment? { + audioSlidePlayer ?: return null val attachment = audioSlidePlayer!!.audioSlide.asAttachment() + return if (attachment is DatabaseAttachment) attachment else null + } - // Parse audio and compute RMS values for the WaveformSeekBar in the background. - asyncCoroutineScope!!.launch { - val rmsFrames = 32 // The amount of values to be computed for the visualization. - - fun extractAttachmentRandomSeed(attachment: Attachment): Int { - return when { - attachment.digest != null -> attachment.digest!!.sum() - attachment.fileName != null -> attachment.fileName.hashCode() - else -> attachment.hashCode() - } - } + private fun updateFromAttachmentAudioExtras() { + val attachment = obtainDatabaseAttachment() ?: return - fun generateFakeRms(seed: Int, frames: Int = rmsFrames): FloatArray { - return Random(seed.toLong()).let { (0 until frames).map { i -> it.nextFloat() }.toFloatArray() } - } + val audioExtras = DatabaseFactory.getAttachmentDatabase(context) + .getAttachmentAudioExtras(attachment.attachmentId) - var rmsValues: FloatArray - var totalDurationMs: Long = -1 - - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { - // Due to API version incompatibility, we just display some random waveform for older API. - rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) - } else { - try { - @Suppress("BlockingMethodInNonBlockingContext") - val decodedAudio = PartAuthority.getAttachmentStream(context, attachment.dataUri!!).use { - DecodedAudio.create(InputStreamMediaDataSource(it)) - } - rmsValues = decodedAudio.calculateRms(rmsFrames) - totalDurationMs = (decodedAudio.totalDuration / 1000.0).toLong() - } catch (e: Exception) { - android.util.Log.w(TAG, "Failed to decode sample values for the audio attachment \"${attachment.fileName}\".", e) - rmsValues = generateFakeRms(extractAttachmentRandomSeed(attachment)) - } - } - - android.util.Log.d(TAG, "RMS: ${rmsValues.joinToString()}") + // Schedule a job request if no audio extras were generated yet. + if (audioExtras == null) { + ApplicationContext.getInstance(context).jobManager + .add(PrepareAttachmentAudioExtrasJob(attachment.attachmentId)) + return + } - post { - loadingAnimation.stop() - seekBar.sampleData = rmsValues + loadingAnimation.stop() + seekBar.sampleData = audioExtras.visualSamples - if (totalDurationMs > 0) { - totalDuration.visibility = View.VISIBLE - totalDuration.text = String.format("%02d:%02d", - TimeUnit.MILLISECONDS.toMinutes(totalDurationMs), - TimeUnit.MILLISECONDS.toSeconds(totalDurationMs)) - } - } + if (audioExtras.durationMs > 0) { + totalDuration.visibility = View.VISIBLE + totalDuration.text = String.format("%02d:%02d", + TimeUnit.MILLISECONDS.toMinutes(audioExtras.durationMs), + TimeUnit.MILLISECONDS.toSeconds(audioExtras.durationMs)) } } @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) - fun onEventAsync(event: PartProgressEvent) { + fun onEvent(event: PartProgressEvent) { if (audioSlidePlayer != null && event.attachment == audioSlidePlayer!!.audioSlide.asAttachment()) { downloadProgress.setInstantProgress(event.progress.toFloat() / event.total) } } -} - -private class SeekBarLoadingAnimation( - private val hostView: View, - private val seekBar: WaveformSeekBar): Runnable { - - companion object { - private const val UPDATE_PERIOD = 500L // In milliseconds. - private val random = Random() - } - - fun start() { - stop() - run() - } - fun stop() { - hostView.removeCallbacks(this) - } - - override fun run() { - seekBar.sampleData = (0 until 64).map { random.nextFloat() * 0.5f }.toFloatArray() - hostView.postDelayed(this, UPDATE_PERIOD) + @Subscribe(threadMode = ThreadMode.MAIN) + fun onEvent(event: PrepareAttachmentAudioExtrasJob.AudioExtrasUpdatedEvent) { + if (event.audioExtras.attachmentId == obtainDatabaseAttachment()?.attachmentId) { + updateFromAttachmentAudioExtras() + } } -} - -@RequiresApi(Build.VERSION_CODES.M) -private class InputStreamMediaDataSource: MediaDataSource { - private val data: ByteArray + private class SeekBarLoadingAnimation( + private val hostView: View, + private val seekBar: WaveformSeekBar): Runnable { - constructor(inputStream: InputStream): super() { - this.data = inputStream.readBytes() - } - - override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { - val length: Int = data.size - if (position >= length) { - return -1 // -1 indicates EOF + companion object { + private const val UPDATE_PERIOD = 350L // In milliseconds. + private val random = Random() } - var actualSize = size - if (position + size > length) { - actualSize -= (position + size - length).toInt() + + fun start() { + stop() + hostView.postDelayed(this, UPDATE_PERIOD) } - System.arraycopy(data, position.toInt(), buffer, offset, actualSize) - return actualSize - } - override fun getSize(): Long { - return data.size.toLong() - } + fun stop() { + hostView.removeCallbacks(this) + } - override fun close() { - // We don't need to close the wrapped stream. + override fun run() { + // Generate a random samples with values up to the 50% of the maximum value. + seekBar.sampleData = ByteArray(PrepareAttachmentAudioExtrasJob.VISUAL_RMS_FRAMES) + { (random.nextInt(127) - 64).toByte() } + hostView.postDelayed(this, UPDATE_PERIOD) + } } } \ No newline at end of file diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 95ed82f215..3870bad7fa 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -7,7 +7,6 @@ import android.graphics.Color import android.graphics.Paint import android.graphics.RectF import android.util.AttributeSet -import android.util.Log import android.util.TypedValue import android.view.MotionEvent import android.view.View @@ -15,8 +14,11 @@ import android.view.ViewConfiguration import android.view.animation.DecelerateInterpolator import androidx.core.math.MathUtils import network.loki.messenger.R +import org.thoughtcrime.securesms.loki.utilities.audio.byteToNormalizedFloat import java.lang.Math.abs import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt class WaveformSeekBar : View { @@ -32,8 +34,8 @@ class WaveformSeekBar : View { } private val sampleDataHolder = SampleDataHolder(::invalidate) - /** An array if normalized to [0..1] values representing the audio signal. */ - var sampleData: FloatArray? + /** An array of signed byte values representing the audio signal. */ + var sampleData: ByteArray? get() { return sampleDataHolder.getSamples() } @@ -155,7 +157,8 @@ class WaveformSeekBar : View { var lastBarRight = paddingLeft.toFloat() (0 until barAmount).forEach { barIdx -> - val barValue = sampleDataHolder.computeBarValue(barIdx, barAmount) + // Convert a signed byte to a [0..1] float. + val barValue = byteToNormalizedFloat(sampleDataHolder.computeBarValue(barIdx, barAmount)) val barHeight = max(barMinHeight, getAvailableHeight() * barValue) @@ -246,16 +249,17 @@ class WaveformSeekBar : View { private class SampleDataHolder(private val invalidateDelegate: () -> Any) { - private var sampleDataFrom: FloatArray? = null - private var sampleDataTo: FloatArray? = null + private var sampleDataFrom: ByteArray? = null + private var sampleDataTo: ByteArray? = null private var progress = 1f // Mix between from and to values. private var animation: ValueAnimator? = null - fun computeBarValue(barIdx: Int, barAmount: Int): Float { - fun getSampleValue(sampleData: FloatArray?): Float { + fun computeBarValue(barIdx: Int, barAmount: Int): Byte { + /** @return The array's value at the interpolated index. */ + fun getSampleValue(sampleData: ByteArray?): Byte { if (sampleData == null || sampleData.isEmpty()) - return 0f + return Byte.MIN_VALUE else { val sampleIdx = (barIdx * (sampleData.size / barAmount.toFloat())).toInt() return sampleData[sampleIdx] @@ -268,12 +272,21 @@ class WaveformSeekBar : View { val fromValue = getSampleValue(sampleDataFrom) val toValue = getSampleValue(sampleDataTo) - - return fromValue * (1f - progress) + toValue * progress + val rawResultValue = fromValue * (1f - progress) + toValue * progress + return rawResultValue.roundToInt().toByte() } - fun setSamples(sampleData: FloatArray?) { - sampleDataFrom = sampleDataTo + fun setSamples(sampleData: ByteArray?) { + /** @return a mix between [sampleDataFrom] and [sampleDataTo] arrays according to the current [progress] value. */ + fun computeNewDataFromArray(): ByteArray? { + if (sampleDataTo == null) return null + if (sampleDataFrom == null) return sampleDataTo + + val sampleSize = min(sampleDataFrom!!.size, sampleDataTo!!.size) + return ByteArray(sampleSize) { i -> computeBarValue(i, sampleSize) } + } + + sampleDataFrom = computeNewDataFromArray() sampleDataTo = sampleData progress = 0f @@ -281,7 +294,6 @@ class WaveformSeekBar : View { animation = ValueAnimator.ofFloat(0f, 1f).apply { addUpdateListener { animation -> progress = animation.animatedValue as Float - Log.d("MTPHR", "Progress: $progress") invalidateDelegate() } interpolator = DecelerateInterpolator(3f) @@ -290,7 +302,7 @@ class WaveformSeekBar : View { } } - fun getSamples(): FloatArray? { + fun getSamples(): ByteArray? { return sampleDataTo } } diff --git a/src/org/thoughtcrime/securesms/util/ParcelableUtil.kt b/src/org/thoughtcrime/securesms/util/ParcelableUtil.kt new file mode 100644 index 0000000000..2756500b22 --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/ParcelableUtil.kt @@ -0,0 +1,32 @@ +package org.thoughtcrime.securesms.util + +import android.os.Parcel + +import android.os.Parcelable + +object ParcelableUtil { + @JvmStatic + fun marshall(parcelable: Parcelable): ByteArray { + val parcel = Parcel.obtain() + parcelable.writeToParcel(parcel, 0) + val bytes = parcel.marshall() + parcel.recycle() + return bytes + } + + @JvmStatic + fun unmarshall(bytes: ByteArray): Parcel { + val parcel = Parcel.obtain() + parcel.unmarshall(bytes, 0, bytes.size) + parcel.setDataPosition(0) // This is extremely important! + return parcel + } + + @JvmStatic + fun unmarshall(bytes: ByteArray, creator: Parcelable.Creator): T { + val parcel: Parcel = ParcelableUtil.unmarshall(bytes) + val result = creator.createFromParcel(parcel) + parcel.recycle() + return result + } +} \ No newline at end of file From 04c132853a356f5fe616b9c28543cf31fb3747fc Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 22 Oct 2020 13:55:00 +1100 Subject: [PATCH 15/23] Minor cleanup. --- .../loki/api/PrepareAttachmentAudioExtrasJob.kt | 12 +++++------- .../securesms/loki/views/MessageAudioView.kt | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt index 0f4d7cd492..e0f7a880fc 100644 --- a/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt +++ b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt @@ -116,15 +116,13 @@ class PrepareAttachmentAudioExtrasJob : BaseJob { } } - val audioExtras = DatabaseAttachmentAudioExtras( + attachDb.setAttachmentAudioExtras(DatabaseAttachmentAudioExtras( attachmentId, rmsValues, totalDurationMs - ) + )) - attachDb.setAttachmentAudioExtras(audioExtras) - - EventBus.getDefault().post(AudioExtrasUpdatedEvent(audioExtras)) + EventBus.getDefault().post(AudioExtrasUpdatedEvent(attachmentIdattachmentId)) } class Factory : Job.Factory { @@ -133,8 +131,8 @@ class PrepareAttachmentAudioExtrasJob : BaseJob { } } - /** Dispatched once the audio extras have been updated. */ - data class AudioExtrasUpdatedEvent(val audioExtras: DatabaseAttachmentAudioExtras) + /** Gets dispatched once the audio extras have been updated. */ + data class AudioExtrasUpdatedEvent(val attachmentId: AttachmentId) @RequiresApi(Build.VERSION_CODES.M) private class InputStreamMediaDataSource: MediaDataSource { diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index a74b638de4..460accc0a9 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -290,7 +290,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { @Subscribe(threadMode = ThreadMode.MAIN) fun onEvent(event: PrepareAttachmentAudioExtrasJob.AudioExtrasUpdatedEvent) { - if (event.audioExtras.attachmentId == obtainDatabaseAttachment()?.attachmentId) { + if (event.attachmentId == obtainDatabaseAttachment()?.attachmentId) { updateFromAttachmentAudioExtras() } } From aacbe02327fa89e433a6ad3901809d468939093d Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 22 Oct 2020 13:57:48 +1100 Subject: [PATCH 16/23] A quick fix. --- .../securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt index e0f7a880fc..07da933adf 100644 --- a/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt +++ b/src/org/thoughtcrime/securesms/loki/api/PrepareAttachmentAudioExtrasJob.kt @@ -122,7 +122,7 @@ class PrepareAttachmentAudioExtrasJob : BaseJob { totalDurationMs )) - EventBus.getDefault().post(AudioExtrasUpdatedEvent(attachmentIdattachmentId)) + EventBus.getDefault().post(AudioExtrasUpdatedEvent(attachmentId)) } class Factory : Job.Factory { From 018e3288a4647d1094a5b78382700e50b12ac151 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 22 Oct 2020 16:48:41 +1100 Subject: [PATCH 17/23] Better input event handling for waveform seek bar. --- .../securesms/loki/views/WaveformSeekBar.kt | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt index 3870bad7fa..56ddb0f3c5 100644 --- a/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt +++ b/src/org/thoughtcrime/securesms/loki/views/WaveformSeekBar.kt @@ -15,7 +15,7 @@ import android.view.animation.DecelerateInterpolator import androidx.core.math.MathUtils import network.loki.messenger.R import org.thoughtcrime.securesms.loki.utilities.audio.byteToNormalizedFloat -import java.lang.Math.abs +import kotlin.math.abs import kotlin.math.max import kotlin.math.min import kotlin.math.roundToInt @@ -111,7 +111,9 @@ class WaveformSeekBar : View { private var canvasWidth = 0 private var canvasHeight = 0 + private var touchDownX = 0f + private var touchDownProgress: Float = 0f private var scaledTouchSlop = ViewConfiguration.get(context).scaledTouchSlop constructor(context: Context) : this(context, null) @@ -185,47 +187,31 @@ class WaveformSeekBar : View { when (event.action) { MotionEvent.ACTION_DOWN -> { userSeeking = true - if (isParentScrolling()) { - touchDownX = event.x - } else { - updateProgress(event, false) - } + touchDownX = event.x + touchDownProgress = progress + updateProgress(event, false) } MotionEvent.ACTION_MOVE -> { + // Prevent any parent scrolling if the user scrolled more + // than scaledTouchSlop on horizontal axis. + if (abs(event.x - touchDownX) > scaledTouchSlop) { + parent.requestDisallowInterceptTouchEvent(true) + } updateProgress(event, false) } MotionEvent.ACTION_UP -> { userSeeking = false - if (abs(event.x - touchDownX) > scaledTouchSlop) { - updateProgress(event, true) - } + updateProgress(event, true) performClick() } MotionEvent.ACTION_CANCEL -> { + updateProgress(touchDownProgress, false) userSeeking = false } } return true } - private fun isParentScrolling(): Boolean { - var parent = parent as View - val root = rootView - - while (true) { - when { - parent.canScrollHorizontally(+1) -> return true - parent.canScrollHorizontally(-1) -> return true - parent.canScrollVertically(+1) -> return true - parent.canScrollVertically(-1) -> return true - } - - if (parent == root) return false - - parent = parent.parent as View - } - } - private fun updateProgress(event: MotionEvent, notify: Boolean) { updateProgress(event.x / getAvailableWidth(), notify) } From 6a9365308033120d186b92651482cef6cb25da2f Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 22 Oct 2020 17:30:04 +1100 Subject: [PATCH 18/23] Message audio view is no longer seekable when it's uploading. --- .../thoughtcrime/securesms/loki/views/MessageAudioView.kt | 8 +++++++- src/org/thoughtcrime/securesms/mms/Slide.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index 460accc0a9..5404a003e9 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -96,6 +96,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { audioSlidePlayer!!.stop() } } + seekBar.isEnabled = false seekBar.progressChangeListener = object : WaveformSeekBar.ProgressChangeListener { override fun onProgressChanged(waveformSeekBar: WaveformSeekBar, progress: Float, fromUser: Boolean) { if (fromUser && audioSlidePlayer != null) { @@ -233,7 +234,6 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { super.setEnabled(enabled) playButton.isEnabled = enabled pauseButton.isEnabled = enabled - seekBar.isEnabled = enabled downloadButton.isEnabled = enabled } @@ -299,6 +299,8 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { private val hostView: View, private val seekBar: WaveformSeekBar): Runnable { + private var active = false + companion object { private const val UPDATE_PERIOD = 350L // In milliseconds. private val random = Random() @@ -306,14 +308,18 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { fun start() { stop() + active = true hostView.postDelayed(this, UPDATE_PERIOD) } fun stop() { + active = false hostView.removeCallbacks(this) } override fun run() { + if (!active) return + // Generate a random samples with values up to the 50% of the maximum value. seekBar.sampleData = ByteArray(PrepareAttachmentAudioExtrasJob.VISUAL_RMS_FRAMES) { (random.nextInt(127) - 64).toByte() } diff --git a/src/org/thoughtcrime/securesms/mms/Slide.java b/src/org/thoughtcrime/securesms/mms/Slide.java index cdf5ce5456..19c00664af 100644 --- a/src/org/thoughtcrime/securesms/mms/Slide.java +++ b/src/org/thoughtcrime/securesms/mms/Slide.java @@ -131,7 +131,7 @@ public abstract class Slide { public @NonNull String getContentDescription() { return ""; } - public Attachment asAttachment() { + public @NonNull Attachment asAttachment() { return attachment; } From 5974abee34c76f976e3a9bfa442ebdabc6aa9eb4 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Thu, 22 Oct 2020 17:33:30 +1100 Subject: [PATCH 19/23] Use the build tools version matching compile SDK. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 13c4e23e37..765123923d 100644 --- a/build.gradle +++ b/build.gradle @@ -195,7 +195,7 @@ def abiPostFix = ['armeabi-v7a' : 1, android { flavorDimensions "none" compileSdkVersion 29 - buildToolsVersion '28.0.3' + buildToolsVersion '29.0.3' useLibrary 'org.apache.http.legacy' dexOptions { From 7d9f5a4fd19bd3019a0e65f8d5f216e16b93a07f Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Mon, 26 Oct 2020 12:03:20 +1100 Subject: [PATCH 20/23] Gracefully handle the missing duration field for decoded audio. --- .../securesms/loki/utilities/audio/DecodedAudio.kt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt index ddc79d8722..12c965428b 100644 --- a/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt +++ b/src/org/thoughtcrime/securesms/loki/utilities/audio/DecodedAudio.kt @@ -108,10 +108,19 @@ class DecodedAudio { channels = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT) sampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) - totalDuration = mediaFormat.getLong(MediaFormat.KEY_DURATION) + // On some old APIs (23) this field might be missing. + totalDuration = if (mediaFormat.containsKey(MediaFormat.KEY_DURATION)) { + mediaFormat.getLong(MediaFormat.KEY_DURATION) + } else { + -1L + } // Expected total number of samples per channel. - val expectedNumSamples = ((totalDuration / 1000000f) * sampleRate + 0.5f).toInt() + val expectedNumSamples = if (totalDuration >= 0) { + ((totalDuration / 1000000f) * sampleRate + 0.5f).toInt() + } else { + Int.MAX_VALUE + } val codec = MediaCodec.createDecoderByType(mediaFormat.getString(MediaFormat.KEY_MIME)!!) codec.configure(mediaFormat, null, null, 0) From 42b9468c63005e58116852f8e335771d7bc2f5b3 Mon Sep 17 00:00:00 2001 From: Anton Chekulaev Date: Mon, 26 Oct 2020 23:14:26 +1100 Subject: [PATCH 21/23] New design for audio view progress bar. --- res/drawable/circle_tintable_4dp_inset.xml | 4 +++ res/layout/message_audio_view.xml | 35 ++++++++++--------- .../securesms/loki/views/MessageAudioView.kt | 25 ++++++++----- 3 files changed, 39 insertions(+), 25 deletions(-) create mode 100644 res/drawable/circle_tintable_4dp_inset.xml diff --git a/res/drawable/circle_tintable_4dp_inset.xml b/res/drawable/circle_tintable_4dp_inset.xml new file mode 100644 index 0000000000..92b0e0830c --- /dev/null +++ b/res/drawable/circle_tintable_4dp_inset.xml @@ -0,0 +1,4 @@ + + \ No newline at end of file diff --git a/res/layout/message_audio_view.xml b/res/layout/message_audio_view.xml index 09e5c9549d..0bb4abf2a5 100644 --- a/res/layout/message_audio_view.xml +++ b/res/layout/message_audio_view.xml @@ -15,25 +15,28 @@ - + android:layout_gravity="center_vertical" + android:min="0" + android:max="100" + tools:visibility="gone" + tools:backgroundTint="@android:color/black" + tools:indeterminateTint="@android:color/white"/> downloadListener?.onClick(v, audio) } - if (downloadProgress.isSpinning) { - downloadProgress.stopSpinning() + if (downloadProgress.isIndeterminate) { + downloadProgress.isIndeterminate = false + downloadProgress.progress = 0 } } (showControls && audio.transferState == AttachmentDatabase.TRANSFER_PROGRESS_STARTED) -> { controlToggle.displayQuick(downloadProgress) seekBar.isEnabled = false - downloadProgress.spin() + downloadProgress.isIndeterminate = true } else -> { controlToggle.displayQuick(playButton) seekBar.isEnabled = true - if (downloadProgress.isSpinning) { - downloadProgress.stopSpinning() + if (downloadProgress.isIndeterminate) { + downloadProgress.isIndeterminate = false + downloadProgress.progress = 100 } // Post to make sure it executes only when the view is attached to a window. @@ -187,7 +189,11 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { pauseButton.imageTintList = ColorStateList.valueOf(backgroundTint) downloadButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN) - downloadProgress.barColor = foregroundTint + + downloadProgress.backgroundTintList = ColorStateList.valueOf(foregroundTint) + downloadProgress.progressTintList = ColorStateList.valueOf(backgroundTint) + downloadProgress.indeterminateTintList = ColorStateList.valueOf(backgroundTint) + totalDuration.setTextColor(foregroundTint) // Seek bar's progress color is set from the XML template. Whereas the background is computed. @@ -284,7 +290,8 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { @Subscribe(sticky = true, threadMode = ThreadMode.MAIN) fun onEvent(event: PartProgressEvent) { if (audioSlidePlayer != null && event.attachment == audioSlidePlayer!!.audioSlide.asAttachment()) { - downloadProgress.setInstantProgress(event.progress.toFloat() / event.total) + val progress = ((event.progress.toFloat() / event.total) * 100f).toInt() + downloadProgress.progress = progress } } From 7319e310eb00762a7ceff5d126d31258f5492c35 Mon Sep 17 00:00:00 2001 From: nielsandriesse Date: Wed, 28 Oct 2020 16:06:48 +1100 Subject: [PATCH 22/23] Clean up indentation --- .../DatabaseAttachmentAudioExtras.kt | 14 ++++----- .../database/AttachmentDatabase.java | 31 ++++++++++--------- .../database/helpers/SQLCipherOpenHelper.java | 8 +++-- .../securesms/jobmanager/Data.java | 3 +- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt index 21de7925c7..f10a02272e 100644 --- a/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt +++ b/src/org/thoughtcrime/securesms/attachments/DatabaseAttachmentAudioExtras.kt @@ -1,11 +1,11 @@ package org.thoughtcrime.securesms.attachments data class DatabaseAttachmentAudioExtras( - val attachmentId: AttachmentId, - /** Small amount of normalized audio byte samples to visualise the content (e.g. draw waveform). */ - val visualSamples: ByteArray, - /** Duration of the audio track in milliseconds. May be [DURATION_UNDEFINED] when is not known. */ - val durationMs: Long = DURATION_UNDEFINED) { + val attachmentId: AttachmentId, + /** Small amount of normalized audio byte samples to visualise the content (e.g. draw waveform). */ + val visualSamples: ByteArray, + /** Duration of the audio track in milliseconds. May be [DURATION_UNDEFINED] when it is not known. */ + val durationMs: Long = DURATION_UNDEFINED) { companion object { const val DURATION_UNDEFINED = -1L @@ -13,8 +13,8 @@ data class DatabaseAttachmentAudioExtras( override fun equals(other: Any?): Boolean { return other != null && - other is DatabaseAttachmentAudioExtras && - other.attachmentId == attachmentId + other is DatabaseAttachmentAudioExtras && + other.attachmentId == attachmentId } override fun hashCode(): Int { diff --git a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java index 583e92a813..2e124f1579 100644 --- a/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java +++ b/src/org/thoughtcrime/securesms/database/AttachmentDatabase.java @@ -24,11 +24,12 @@ import android.graphics.Bitmap; import android.media.MediaMetadataRetriever; import android.net.Uri; import android.os.Build; +import android.text.TextUtils; +import android.util.Pair; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; -import android.text.TextUtils; -import android.util.Pair; import com.bumptech.glide.Glide; @@ -52,10 +53,10 @@ import org.thoughtcrime.securesms.mms.PartAuthority; import org.thoughtcrime.securesms.stickers.StickerLocator; import org.thoughtcrime.securesms.util.BitmapDecodingException; import org.thoughtcrime.securesms.util.BitmapUtil; +import org.thoughtcrime.securesms.util.ExternalStorageUtil; import org.thoughtcrime.securesms.util.JsonUtils; import org.thoughtcrime.securesms.util.MediaUtil; import org.thoughtcrime.securesms.util.MediaUtil.ThumbnailData; -import org.thoughtcrime.securesms.util.ExternalStorageUtil; import org.thoughtcrime.securesms.util.Util; import org.thoughtcrime.securesms.video.EncryptedMediaDataSource; @@ -839,15 +840,15 @@ public class AttachmentDatabase extends Database { @Synchronized public @Nullable DatabaseAttachmentAudioExtras getAttachmentAudioExtras(@NonNull AttachmentId attachmentId) { try (Cursor cursor = databaseHelper.getReadableDatabase() - // We expect all the audio extra values to be present (not null) or reject the whole record. - .query(TABLE_NAME, - PROJECTION_AUDIO_EXTRAS, - PART_ID_WHERE + - " AND " + AUDIO_VISUAL_SAMPLES + " IS NOT NULL" + - " AND " + AUDIO_DURATION + " IS NOT NULL" + - " AND " + PART_AUDIO_ONLY_WHERE, - attachmentId.toStrings(), - null, null, null, "1")) { + // We expect all the audio extra values to be present (not null) or reject the whole record. + .query(TABLE_NAME, + PROJECTION_AUDIO_EXTRAS, + PART_ID_WHERE + + " AND " + AUDIO_VISUAL_SAMPLES + " IS NOT NULL" + + " AND " + AUDIO_DURATION + " IS NOT NULL" + + " AND " + PART_AUDIO_ONLY_WHERE, + attachmentId.toStrings(), + null, null, null, "1")) { if (cursor == null || !cursor.moveToFirst()) return null; @@ -869,9 +870,9 @@ public class AttachmentDatabase extends Database { values.put(AUDIO_DURATION, extras.getDurationMs()); int alteredRows = databaseHelper.getWritableDatabase().update(TABLE_NAME, - values, - PART_ID_WHERE + " AND " + PART_AUDIO_ONLY_WHERE, - extras.getAttachmentId().toStrings()); + values, + PART_ID_WHERE + " AND " + PART_AUDIO_ONLY_WHERE, + extras.getAttachmentId().toStrings()); return alteredRows > 0; } diff --git a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index f1e7ff8c0d..92109dde40 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -93,8 +93,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { private static final int lokiV14_BACKUP_FILES = 35; private static final int lokiV15 = 36; private static final int lokiV16 = 37; + private static final int lokiV17 = 38; - private static final int DATABASE_VERSION = lokiV16; + private static final int DATABASE_VERSION = lokiV17; private static final String DATABASE_NAME = "signal.db"; private final Context context; @@ -635,9 +636,12 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { } if (oldVersion < lokiV16) { + db.execSQL(LokiAPIDatabase.getCreateOpenGroupProfilePictureTableCommand()); + } + + if (oldVersion < lokiV17) { db.execSQL("ALTER TABLE part ADD COLUMN audio_visual_samples BLOB"); db.execSQL("ALTER TABLE part ADD COLUMN audio_duration INTEGER"); - db.execSQL(LokiAPIDatabase.getCreateOpenGroupProfilePictureTableCommand()); } db.setTransactionSuccessful(); diff --git a/src/org/thoughtcrime/securesms/jobmanager/Data.java b/src/org/thoughtcrime/securesms/jobmanager/Data.java index 0eaa2e9e4e..eff9fa1478 100644 --- a/src/org/thoughtcrime/securesms/jobmanager/Data.java +++ b/src/org/thoughtcrime/securesms/jobmanager/Data.java @@ -7,13 +7,12 @@ import androidx.annotation.Nullable; import com.fasterxml.jackson.annotation.JsonProperty; -import org.thoughtcrime.securesms.attachments.AttachmentId; import org.thoughtcrime.securesms.util.ParcelableUtil; import java.util.HashMap; import java.util.Map; -//TODO AC: For now parcelable objects utilize byteArrays field to store their data into. +// TODO AC: For now parcelable objects utilize byteArrays field to store their data into. // Introduce a dedicated Map field specifically for parcelable needs. public class Data { From e667c6ee7ed46e2d31d75ace03b47ecbcbaa8f34 Mon Sep 17 00:00:00 2001 From: nielsandriesse Date: Thu, 29 Oct 2020 10:16:49 +1100 Subject: [PATCH 23/23] Make style consistent across platforms --- ...sation_activity_attachment_editor_stub.xml | 4 +-- .../conversation_item_received_audio.xml | 4 +-- res/layout/conversation_item_sent_audio.xml | 4 +-- res/values-notnight-v21/themes.xml | 3 ++- res/values/attrs.xml | 6 +++-- res/values/themes.xml | 4 ++- .../securesms/loki/views/MessageAudioView.kt | 26 +++++++++---------- 7 files changed, 28 insertions(+), 23 deletions(-) diff --git a/res/layout/conversation_activity_attachment_editor_stub.xml b/res/layout/conversation_activity_attachment_editor_stub.xml index 12da06a249..3ee19aac0b 100644 --- a/res/layout/conversation_activity_attachment_editor_stub.xml +++ b/res/layout/conversation_activity_attachment_editor_stub.xml @@ -41,8 +41,8 @@ android:paddingBottom="15dp" app:widgetBackground="?conversation_item_bubble_background" app:foregroundTintColor="?android:colorControlNormal" - app:backgroundTintColor="?conversation_item_bubble_background" - app:waveformFillColor="?conversation_item_audio_seek_bar_color"/> + app:waveformFillColor="?conversation_item_audio_seek_bar_color_outgoing" + app:waveformBackgroundColor="?conversation_item_audio_seek_bar_background_color"/> diff --git a/res/layout/conversation_item_sent_audio.xml b/res/layout/conversation_item_sent_audio.xml index 7a28a82dad..cb7889fa1f 100644 --- a/res/layout/conversation_item_sent_audio.xml +++ b/res/layout/conversation_item_sent_audio.xml @@ -6,6 +6,6 @@ android:layout_width="210dp" android:layout_height="wrap_content" app:foregroundTintColor="?android:colorControlNormal" - app:backgroundTintColor="?message_sent_background_color" - app:waveformFillColor="?conversation_item_audio_seek_bar_color" + app:waveformFillColor="?conversation_item_audio_seek_bar_color_outgoing" + app:waveformBackgroundColor="?conversation_item_audio_seek_bar_background_color" android:visibility="gone"/> diff --git a/res/values-notnight-v21/themes.xml b/res/values-notnight-v21/themes.xml index b49a5cc706..2807ff2ed7 100644 --- a/res/values-notnight-v21/themes.xml +++ b/res/values-notnight-v21/themes.xml @@ -28,7 +28,8 @@ @drawable/ic_outline_info_24 - ?colorControlNormal + @color/accent + @color/white diff --git a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt index 2262578ead..3cccb6f4b8 100644 --- a/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt +++ b/src/org/thoughtcrime/securesms/loki/views/MessageAudioView.kt @@ -30,6 +30,7 @@ import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.events.PartProgressEvent import org.thoughtcrime.securesms.logging.Log import org.thoughtcrime.securesms.loki.api.PrepareAttachmentAudioExtrasJob +import org.thoughtcrime.securesms.loki.utilities.getColorWithID import org.thoughtcrime.securesms.mms.AudioSlide import org.thoughtcrime.securesms.mms.SlideClickListener import java.io.IOException @@ -115,8 +116,8 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { if (attrs != null) { val typedArray = context.theme.obtainStyledAttributes(attrs, R.styleable.MessageAudioView, 0, 0) setTint(typedArray.getColor(R.styleable.MessageAudioView_foregroundTintColor, Color.WHITE), - typedArray.getColor(R.styleable.MessageAudioView_backgroundTintColor, Color.WHITE), - typedArray.getColor(R.styleable.MessageAudioView_waveformFillColor, Color.WHITE)) + typedArray.getColor(R.styleable.MessageAudioView_waveformFillColor, Color.WHITE), + typedArray.getColor(R.styleable.MessageAudioView_waveformBackgroundColor, Color.WHITE)) container.setBackgroundColor(typedArray.getColor(R.styleable.MessageAudioView_widgetBackground, Color.TRANSPARENT)) typedArray.recycle() } @@ -182,23 +183,22 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { downloadListener = listener } - fun setTint(@ColorInt foregroundTint: Int, @ColorInt backgroundTint: Int, @ColorInt waveformFill: Int) { - playButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) - playButton.imageTintList = ColorStateList.valueOf(backgroundTint) - pauseButton.backgroundTintList = ColorStateList.valueOf(foregroundTint) - pauseButton.imageTintList = ColorStateList.valueOf(backgroundTint) + fun setTint(@ColorInt foregroundTint: Int, @ColorInt waveformFill: Int, @ColorInt waveformBackground: Int) { + playButton.backgroundTintList = ColorStateList.valueOf(resources.getColorWithID(R.color.white, context.theme)) + playButton.imageTintList = ColorStateList.valueOf(resources.getColorWithID(R.color.black, context.theme)) + pauseButton.backgroundTintList = ColorStateList.valueOf(resources.getColorWithID(R.color.white, context.theme)) + pauseButton.imageTintList = ColorStateList.valueOf(resources.getColorWithID(R.color.black, context.theme)) downloadButton.setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN) - downloadProgress.backgroundTintList = ColorStateList.valueOf(foregroundTint) - downloadProgress.progressTintList = ColorStateList.valueOf(backgroundTint) - downloadProgress.indeterminateTintList = ColorStateList.valueOf(backgroundTint) + downloadProgress.backgroundTintList = ColorStateList.valueOf(resources.getColorWithID(R.color.white, context.theme)) + downloadProgress.progressTintList = ColorStateList.valueOf(resources.getColorWithID(R.color.black, context.theme)) + downloadProgress.indeterminateTintList = ColorStateList.valueOf(resources.getColorWithID(R.color.black, context.theme)) totalDuration.setTextColor(foregroundTint) - // Seek bar's progress color is set from the XML template. Whereas the background is computed. seekBar.barProgressColor = waveformFill - seekBar.barBackgroundColor = ColorUtils.blendARGB(foregroundTint, backgroundTint, 0.75f) + seekBar.barBackgroundColor = waveformBackground } override fun onPlayerStart(player: AudioSlidePlayer) { @@ -309,7 +309,7 @@ class MessageAudioView: FrameLayout, AudioSlidePlayer.Listener { private var active = false companion object { - private const val UPDATE_PERIOD = 350L // In milliseconds. + private const val UPDATE_PERIOD = 250L // In milliseconds. private val random = Random() }