diff --git a/src/org/thoughtcrime/securesms/ConversationActivity.java b/src/org/thoughtcrime/securesms/ConversationActivity.java index 3eaf71320d..8c1f9cbf6b 100644 --- a/src/org/thoughtcrime/securesms/ConversationActivity.java +++ b/src/org/thoughtcrime/securesms/ConversationActivity.java @@ -59,6 +59,7 @@ import com.google.protobuf.ByteString; import org.thoughtcrime.redphone.RedPhone; import org.thoughtcrime.redphone.RedPhoneService; import org.thoughtcrime.securesms.TransportOptions.OnTransportChangedListener; +import org.thoughtcrime.securesms.audio.AudioCodec; import org.thoughtcrime.securesms.audio.AudioRecorder; import org.thoughtcrime.securesms.audio.AudioSlidePlayer; import org.thoughtcrime.securesms.color.MaterialColor; @@ -138,6 +139,8 @@ import java.security.SecureRandom; import java.util.List; import java.util.concurrent.ExecutionException; +import ws.com.google.android.mms.ContentType; + import static org.thoughtcrime.securesms.TransportOption.Type; import static org.thoughtcrime.securesms.database.GroupDatabase.GroupRecord; import static org.whispersystems.textsecure.internal.push.TextSecureProtos.GroupContext; @@ -1372,14 +1375,10 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity @Override public void onRecorderStarted() { - try { - Vibrator vibrator = (Vibrator)getSystemService(Context.VIBRATOR_SERVICE); - vibrator.vibrate(20); + Vibrator vibrator = (Vibrator)getSystemService(Context.VIBRATOR_SERVICE); + vibrator.vibrate(20); - audioRecorder.startRecording(); - } catch (IOException e) { - Log.w(TAG, e); - } + audioRecorder.startRecording(); } @Override @@ -1393,7 +1392,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity public void onSuccess(final @NonNull Pair result) { try { boolean forceSms = sendButton.isManualSelection() && sendButton.getSelectedTransport().isSms(); - AudioSlide audioSlide = new AudioSlide(ConversationActivity.this, result.first, result.second); + AudioSlide audioSlide = new AudioSlide(ConversationActivity.this, result.first, result.second, ContentType.AUDIO_AAC); SlideDeck slideDeck = new SlideDeck(); slideDeck.addSlide(audioSlide); @@ -1409,7 +1408,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity }.execute(); } }); - } catch (IOException | InvalidMessageException e) { + } catch (InvalidMessageException e) { Log.w(TAG, e); Toast.makeText(ConversationActivity.this, R.string.ConversationActivity_error_sending_voice_message, Toast.LENGTH_LONG).show(); } diff --git a/src/org/thoughtcrime/securesms/audio/AudioCodec.java b/src/org/thoughtcrime/securesms/audio/AudioCodec.java new file mode 100644 index 0000000000..bac295854a --- /dev/null +++ b/src/org/thoughtcrime/securesms/audio/AudioCodec.java @@ -0,0 +1,198 @@ +package org.thoughtcrime.securesms.audio; + +import android.annotation.TargetApi; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.media.MediaRecorder; +import android.os.Build; +import android.util.Log; + +import org.thoughtcrime.securesms.util.Util; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +@TargetApi(Build.VERSION_CODES.JELLY_BEAN) +public class AudioCodec { + + private static final String TAG = AudioCodec.class.getSimpleName(); + + private static final int SAMPLE_RATE = 44100; + private static final int SAMPLE_RATE_INDEX = 4; + private static final int CHANNELS = 1; + private static final int BIT_RATE = 32000; + + private final int bufferSize; + private final MediaCodec mediaCodec; + private final AudioRecord audioRecord; + + private boolean running = true; + private boolean finished = false; + + public AudioCodec() throws IOException { + this.bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT); + this.audioRecord = createAudioRecord(this.bufferSize); + this.mediaCodec = createMediaCodec(this.bufferSize); + + this.mediaCodec.start(); + + try { + audioRecord.startRecording(); + } catch (Exception e) { + Log.w(TAG, e); + mediaCodec.release(); + throw new IOException(e); + } + } + + public synchronized void stop() { + running = false; + while (!finished) Util.wait(this, 0); + } + + public void start(final OutputStream outputStream) { + new Thread(new Runnable() { + @Override + public void run() { + MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + byte[] audioRecordData = new byte[bufferSize]; + ByteBuffer[] codecInputBuffers = mediaCodec.getInputBuffers(); + ByteBuffer[] codecOutputBuffers = mediaCodec.getOutputBuffers(); + + try { + while (true) { + boolean running = isRunning(); + + handleCodecInput(audioRecord, audioRecordData, mediaCodec, codecInputBuffers, running); + handleCodecOutput(mediaCodec, codecOutputBuffers, bufferInfo, outputStream); + + if (!running) break; + } + } catch (IOException e) { + Log.w(TAG, e); + } finally { + mediaCodec.stop(); + audioRecord.stop(); + + mediaCodec.release(); + audioRecord.release(); + + Util.close(outputStream); + setFinished(); + } + } + }, AudioCodec.class.getSimpleName()).start(); + } + + private synchronized boolean isRunning() { + return running; + } + + private synchronized void setFinished() { + finished = true; + notifyAll(); + } + + private void handleCodecInput(AudioRecord audioRecord, byte[] audioRecordData, + MediaCodec mediaCodec, ByteBuffer[] codecInputBuffers, + boolean running) + { + int length = audioRecord.read(audioRecordData, 0, audioRecordData.length); + int codecInputBufferIndex = mediaCodec.dequeueInputBuffer(10 * 1000); + + if (codecInputBufferIndex >= 0) { + ByteBuffer codecBuffer = codecInputBuffers[codecInputBufferIndex]; + codecBuffer.clear(); + codecBuffer.put(audioRecordData); + mediaCodec.queueInputBuffer(codecInputBufferIndex, 0, length, 0, running ? 0 : MediaCodec.BUFFER_FLAG_END_OF_STREAM); + } + } + + private void handleCodecOutput(MediaCodec mediaCodec, + ByteBuffer[] codecOutputBuffers, + MediaCodec.BufferInfo bufferInfo, + OutputStream outputStream) + throws IOException + { + int codecOutputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 0); + + while (codecOutputBufferIndex != MediaCodec.INFO_TRY_AGAIN_LATER) { + if (codecOutputBufferIndex >= 0) { + ByteBuffer encoderOutputBuffer = codecOutputBuffers[codecOutputBufferIndex]; + + encoderOutputBuffer.position(bufferInfo.offset); + encoderOutputBuffer.limit(bufferInfo.offset + bufferInfo.size); + + if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { + byte[] header = createAdtsHeader(bufferInfo.size - bufferInfo.offset); + + + outputStream.write(header); + + byte[] data = new byte[encoderOutputBuffer.remaining()]; + encoderOutputBuffer.get(data); + outputStream.write(data); + } + + encoderOutputBuffer.clear(); + + mediaCodec.releaseOutputBuffer(codecOutputBufferIndex, false); + } else if (codecOutputBufferIndex== MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + codecOutputBuffers = mediaCodec.getOutputBuffers(); + } + + codecOutputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 0); + } + + } + + private byte[] createAdtsHeader(int length) { + int frameLength = length + 7; + byte[] adtsHeader = new byte[7]; + + adtsHeader[0] = (byte) 0xFF; // Sync Word + adtsHeader[1] = (byte) 0xF1; // MPEG-4, Layer (0), No CRC + adtsHeader[2] = (byte) ((MediaCodecInfo.CodecProfileLevel.AACObjectLC - 1) << 6); + adtsHeader[2] |= (((byte) SAMPLE_RATE_INDEX) << 2); + adtsHeader[2] |= (((byte) CHANNELS) >> 2); + adtsHeader[3] = (byte) (((CHANNELS & 3) << 6) | ((frameLength >> 11) & 0x03)); + adtsHeader[4] = (byte) ((frameLength >> 3) & 0xFF); + adtsHeader[5] = (byte) (((frameLength & 0x07) << 5) | 0x1f); + adtsHeader[6] = (byte) 0xFC; + + return adtsHeader; + } + + private AudioRecord createAudioRecord(int bufferSize) { + return new AudioRecord(MediaRecorder.AudioSource.MIC, SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, bufferSize * 10); + } + + private MediaCodec createMediaCodec(int bufferSize) throws IOException { + MediaCodec mediaCodec = MediaCodec.createEncoderByType("audio/mp4a-latm"); + MediaFormat mediaFormat = new MediaFormat(); + + mediaFormat.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm"); + mediaFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE); + mediaFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS); + mediaFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, bufferSize); + mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + mediaFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + + try { + mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + } catch (Exception e) { + Log.w(TAG, e); + mediaCodec.release(); + throw new IOException(e); + } + + return mediaCodec; + } + +} diff --git a/src/org/thoughtcrime/securesms/audio/AudioRecorder.java b/src/org/thoughtcrime/securesms/audio/AudioRecorder.java index 3a03b9d3e9..9c1060b546 100644 --- a/src/org/thoughtcrime/securesms/audio/AudioRecorder.java +++ b/src/org/thoughtcrime/securesms/audio/AudioRecorder.java @@ -1,36 +1,38 @@ package org.thoughtcrime.securesms.audio; +import android.annotation.TargetApi; import android.content.Context; -import android.media.MediaRecorder; import android.net.Uri; +import android.os.Build; import android.os.ParcelFileDescriptor; import android.support.annotation.NonNull; import android.util.Log; import android.util.Pair; -import org.thoughtcrime.securesms.ApplicationContext; import org.thoughtcrime.securesms.crypto.MasterSecret; import org.thoughtcrime.securesms.providers.PersistentBlobProvider; import org.thoughtcrime.securesms.util.MediaUtil; +import org.thoughtcrime.securesms.util.ThreadUtil; import org.thoughtcrime.securesms.util.Util; import org.thoughtcrime.securesms.util.concurrent.ListenableFuture; import org.thoughtcrime.securesms.util.concurrent.SettableFuture; -import org.whispersystems.jobqueue.Job; -import org.whispersystems.jobqueue.JobParameters; import java.io.IOException; +import java.util.concurrent.ExecutorService; +@TargetApi(Build.VERSION_CODES.JELLY_BEAN) public class AudioRecorder { private static final String TAG = AudioRecorder.class.getSimpleName(); + private static final ExecutorService executor = ThreadUtil.newDynamicSingleThreadedExecutor(); + private final Context context; private final MasterSecret masterSecret; private final PersistentBlobProvider blobProvider; - private MediaRecorder mediaRecorder; - private Uri captureUri; - private ParcelFileDescriptor fd; + private AudioCodec audioCodec; + private Uri captureUri; public AudioRecorder(@NonNull Context context, @NonNull MasterSecret masterSecret) { this.context = context; @@ -38,164 +40,77 @@ public class AudioRecorder { this.blobProvider = PersistentBlobProvider.getInstance(context.getApplicationContext()); } - public void startRecording() throws IOException { + public void startRecording() { Log.w(TAG, "startRecording()"); - ApplicationContext.getInstance(context) - .getJobManager() - .add(new StartRecordingJob()); - } - - public @NonNull ListenableFuture> stopRecording() { - Log.w(TAG, "stopRecording()"); - - StopRecordingJob stopRecordingJob = new StopRecordingJob(); - - ApplicationContext.getInstance(context) - .getJobManager() - .add(stopRecordingJob); - - return stopRecordingJob.getFuture(); - } - - private class StopRecordingJob extends Job { - - private final SettableFuture> future = new SettableFuture<>(); - - public StopRecordingJob() { - super(JobParameters.newBuilder() - .withGroupId(AudioRecorder.class.getSimpleName()) - .create()); - } - - public ListenableFuture> getFuture() { - return future; - } - - @Override - public void onAdded() {} - - @Override - public void onRun() { - if (mediaRecorder == null) { - sendToFuture(new IOException("MediaRecorder was never initialized successfully!")); - return; - } - - try { - mediaRecorder.stop(); - } catch (Exception e) { - Log.w(TAG, e); - } + executor.execute(new Runnable() { + @Override + public void run() { + Log.w(TAG, "Running startRecording() + " + Thread.currentThread().getId()); + try { + if (audioCodec != null) { + throw new AssertionError("We can only record once at a time."); + } - try { - fd.close(); - } catch (IOException e) { - Log.w("AudioRecorder", e); - } + ParcelFileDescriptor fds[] = ParcelFileDescriptor.createPipe(); - mediaRecorder.release(); - mediaRecorder = null; + captureUri = blobProvider.create(masterSecret, new ParcelFileDescriptor.AutoCloseInputStream(fds[0])); + audioCodec = new AudioCodec(); - try { - long size = MediaUtil.getMediaSize(context, masterSecret, captureUri); - sendToFuture(new Pair<>(captureUri, size)); - } catch (IOException ioe) { - Log.w(TAG, ioe); - sendToFuture(ioe); + audioCodec.start(new ParcelFileDescriptor.AutoCloseOutputStream(fds[1])); + } catch (IOException e) { + Log.w(TAG, e); + } } + }); + } - captureUri = null; - fd = null; - } - - @Override - public boolean onShouldRetry(Exception e) { - return false; - } + public @NonNull ListenableFuture> stopRecording() { + Log.w(TAG, "stopRecording()"); - @Override - public void onCanceled() {} + final SettableFuture> future = new SettableFuture<>(); - private void sendToFuture(final @NonNull Pair result) { - Util.runOnMain(new Runnable() { - @Override - public void run() { - future.set(result); - } - }); - } - - private void sendToFuture(final @NonNull Exception exception) { - Util.runOnMain(new Runnable() { - @Override - public void run() { - future.setException(exception); + executor.execute(new Runnable() { + @Override + public void run() { + if (audioCodec == null) { + sendToFuture(future, new IOException("MediaRecorder was never initialized successfully!")); + return; } - }); - } - } - - private class StartRecordingJob extends Job { - public StartRecordingJob() { - super(JobParameters.newBuilder() - .withGroupId(AudioRecorder.class.getSimpleName()) - .create()); - } + audioCodec.stop(); - @Override - public void onAdded() {} + try { + long size = MediaUtil.getMediaSize(context, masterSecret, captureUri); + sendToFuture(future, new Pair<>(captureUri, size)); + } catch (IOException ioe) { + Log.w(TAG, ioe); + sendToFuture(future, ioe); + } - @Override - public void onRun() throws Exception { - if (mediaRecorder != null) { - throw new AssertionError("We can only record once at a time."); + audioCodec = null; + captureUri = null; } + }); - ParcelFileDescriptor fds[] = ParcelFileDescriptor.createPipe(); - - fd = fds[1]; - captureUri = blobProvider.create(masterSecret, new ParcelFileDescriptor.AutoCloseInputStream(fds[0])); - mediaRecorder = new MediaRecorder(); - mediaRecorder.setAudioSource(MediaRecorder.AudioSource.MIC); - mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.RAW_AMR); - mediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB); - mediaRecorder.setOutputFile(fds[1].getFileDescriptor()); - - mediaRecorder.prepare(); + return future; + } - try { - mediaRecorder.start(); - } catch (Exception e) { - Log.w(TAG, e); - throw new IOException(e); + private void sendToFuture(final SettableFuture future, final Exception exception) { + Util.runOnMain(new Runnable() { + @Override + public void run() { + future.setException(exception); } - } - - @Override - public boolean onShouldRetry(Exception e) { - return false; - } - - @Override - public void onCanceled() { - try { - if (fd != null) { - fd.close(); - } - - if (captureUri != null) { - blobProvider.delete(captureUri); - } + }); + } - fd = null; - mediaRecorder = null; - captureUri = null; - } catch (IOException e) { - Log.w(TAG, e); + private void sendToFuture(final SettableFuture future, final T result) { + Util.runOnMain(new Runnable() { + @Override + public void run() { + future.set(result); } - } + }); } - } diff --git a/src/org/thoughtcrime/securesms/components/InputPanel.java b/src/org/thoughtcrime/securesms/components/InputPanel.java index 0074f5a650..2567aebac1 100644 --- a/src/org/thoughtcrime/securesms/components/InputPanel.java +++ b/src/org/thoughtcrime/securesms/components/InputPanel.java @@ -72,7 +72,7 @@ public class InputPanel extends LinearLayout implements MicrophoneRecorderView.L this.microphoneRecorderView = ViewUtil.findById(this, R.id.recorder_view); this.microphoneRecorderView.setListener(this); - if (Build.VERSION.SDK_INT < 14) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) { this.microphoneRecorderView.setVisibility(View.GONE); this.microphoneRecorderView.setClickable(false); } diff --git a/src/org/thoughtcrime/securesms/mms/AudioSlide.java b/src/org/thoughtcrime/securesms/mms/AudioSlide.java index cf225353d0..ab76fee39f 100644 --- a/src/org/thoughtcrime/securesms/mms/AudioSlide.java +++ b/src/org/thoughtcrime/securesms/mms/AudioSlide.java @@ -25,6 +25,8 @@ import android.support.annotation.Nullable; import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.attachments.Attachment; +import org.thoughtcrime.securesms.attachments.UriAttachment; +import org.thoughtcrime.securesms.database.AttachmentDatabase; import org.thoughtcrime.securesms.util.ResUtil; import java.io.IOException; @@ -38,6 +40,10 @@ public class AudioSlide extends Slide { super(context, constructAttachmentFromUri(context, uri, ContentType.AUDIO_UNSPECIFIED, dataSize)); } + public AudioSlide(Context context, Uri uri, long dataSize, String contentType) { + super(context, new UriAttachment(uri, contentType, AttachmentDatabase.TRANSFER_PROGRESS_STARTED, dataSize)); + } + public AudioSlide(Context context, Attachment attachment) { super(context, attachment); } diff --git a/src/org/thoughtcrime/securesms/util/ThreadUtil.java b/src/org/thoughtcrime/securesms/util/ThreadUtil.java new file mode 100644 index 0000000000..df29c27306 --- /dev/null +++ b/src/org/thoughtcrime/securesms/util/ThreadUtil.java @@ -0,0 +1,19 @@ +package org.thoughtcrime.securesms.util; + +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class ThreadUtil { + + public static ExecutorService newDynamicSingleThreadedExecutor() { + ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS, + new LinkedBlockingQueue()); + executor.allowCoreThreadTimeOut(true); + + return executor; + } + +} diff --git a/src/org/thoughtcrime/securesms/util/Util.java b/src/org/thoughtcrime/securesms/util/Util.java index e1d3bc08e0..77ec363411 100644 --- a/src/org/thoughtcrime/securesms/util/Util.java +++ b/src/org/thoughtcrime/securesms/util/Util.java @@ -35,6 +35,7 @@ import android.text.Spannable; import android.text.SpannableString; import android.text.TextUtils; import android.text.style.StyleSpan; +import android.util.Log; import android.widget.EditText; import org.thoughtcrime.securesms.BuildConfig; @@ -64,6 +65,8 @@ import ws.com.google.android.mms.pdu.CharacterSets; import ws.com.google.android.mms.pdu.EncodedStringValue; public class Util { + private static final String TAG = Util.class.getSimpleName(); + public static Handler handler = new Handler(Looper.getMainLooper()); public static String join(String[] list, String delimiter) { @@ -158,6 +161,14 @@ public class Util { } } + public static void close(OutputStream out) { + try { + out.close(); + } catch (IOException e) { + Log.w(TAG, e); + } + } + public static String canonicalizeNumber(Context context, String number) throws InvalidNumberException {