From 3a702b3ed8944d9c0e1672346e1fb20a3ead6a63 Mon Sep 17 00:00:00 2001 From: Marcelo Fernandez Date: Mon, 8 Oct 2018 15:46:21 -0300 Subject: [PATCH] Implemented audio input support for JavaScript audio driver --- .../javascript/audio_driver_javascript.cpp | 104 ++++++++++++++++-- platform/javascript/audio_driver_javascript.h | 7 ++ platform/javascript/detect.py | 1 + 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/platform/javascript/audio_driver_javascript.cpp b/platform/javascript/audio_driver_javascript.cpp index 7a6613bb325..a5b627b8dc1 100644 --- a/platform/javascript/audio_driver_javascript.cpp +++ b/platform/javascript/audio_driver_javascript.cpp @@ -44,6 +44,11 @@ extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_js_mix() { AudioDriverJavaScript::singleton->mix_to_js(); } +extern "C" EMSCRIPTEN_KEEPALIVE void audio_driver_process_capture(float sample) { + + AudioDriverJavaScript::singleton->process_capture(sample); +} + void AudioDriverJavaScript::mix_to_js() { int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode()); @@ -51,31 +56,39 @@ void AudioDriverJavaScript::mix_to_js() { int32_t *stream_buffer = reinterpret_cast(internal_buffer); audio_server_process(sample_count, stream_buffer); for (int i = 0; i < sample_count * channel_count; i++) { - internal_buffer[i] = float(stream_buffer[i] >> 16) / 32768.0; + internal_buffer[i] = float(stream_buffer[i] >> 16) / 32768.f; } } +void AudioDriverJavaScript::process_capture(float sample) { + + int32_t sample32 = int32_t(sample * 32768.f) * (1U << 16); + input_buffer_write(sample32); +} + Error AudioDriverJavaScript::init() { /* clang-format off */ EM_ASM({ _audioDriver_audioContext = new (window.AudioContext || window.webkitAudioContext); + _audioDriver_audioInput = null; + _audioDriver_inputStream = null; _audioDriver_scriptNode = null; }); /* clang-format on */ int channel_count = get_total_channels_by_speaker_mode(get_speaker_mode()); /* clang-format off */ - int buffer_length = EM_ASM_INT({ + buffer_length = EM_ASM_INT({ var CHANNEL_COUNT = $0; var channelCount = _audioDriver_audioContext.destination.channelCount; try { // Try letting the browser recommend a buffer length. - _audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(0, 0, channelCount); + _audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(0, 2, channelCount); } catch (e) { // ...otherwise, default to 4096. - _audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(4096, 0, channelCount); + _audioDriver_scriptNode = _audioDriver_audioContext.createScriptProcessor(4096, 2, channelCount); } _audioDriver_scriptNode.connect(_audioDriver_audioContext.destination); @@ -91,6 +104,7 @@ Error AudioDriverJavaScript::init() { memdelete_arr(internal_buffer); internal_buffer = memnew_arr(float, buffer_length *channel_count); } + return internal_buffer ? OK : ERR_OUT_OF_MEMORY; } @@ -101,11 +115,13 @@ void AudioDriverJavaScript::start() { var INTERNAL_BUFFER_PTR = $0; var audioDriverMixFunction = cwrap('audio_driver_js_mix'); + var audioDriverProcessCapture = cwrap('audio_driver_process_capture', null, ['number']); _audioDriver_scriptNode.onaudioprocess = function(audioProcessingEvent) { audioDriverMixFunction(); - // The output buffer contains the samples that will be modified and played. + + var input = audioProcessingEvent.inputBuffer; var output = audioProcessingEvent.outputBuffer; - var input = HEAPF32.subarray( + var internalBuffer = HEAPF32.subarray( INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT, INTERNAL_BUFFER_PTR / HEAPF32.BYTES_PER_ELEMENT + output.length * output.numberOfChannels); @@ -113,8 +129,16 @@ void AudioDriverJavaScript::start() { var outputData = output.getChannelData(channel); // Loop through samples. for (var sample = 0; sample < outputData.length; sample++) { - // Set output equal to input. - outputData[sample] = input[sample * output.numberOfChannels + channel]; + outputData[sample] = internalBuffer[sample * output.numberOfChannels + channel]; + } + } + + if (_audioDriver_audioInput) { + var inputDataL = input.getChannelData(0); + var inputDataR = input.getChannelData(1); + for (var i = 0; i < inputDataL.length; i++) { + audioDriverProcessCapture(inputDataL[i]); + audioDriverProcessCapture(inputDataR[i]); } } }; @@ -152,14 +176,74 @@ void AudioDriverJavaScript::finish() { /* clang-format off */ EM_ASM({ _audioDriver_audioContext = null; + _audioDriver_audioInput = null; _audioDriver_scriptNode = null; }); /* clang-format on */ - memdelete_arr(internal_buffer); - internal_buffer = NULL; + + if (internal_buffer) { + memdelete_arr(internal_buffer); + internal_buffer = NULL; + } +} + +Error AudioDriverJavaScript::capture_start() { + + input_buffer_init(buffer_length); + + /* clang-format off */ + EM_ASM({ + function gotMediaInput(stream) { + _audioDriver_inputStream = stream; + _audioDriver_audioInput = _audioDriver_audioContext.createMediaStreamSource(stream); + _audioDriver_audioInput.connect(_audioDriver_scriptNode); + } + + function gotMediaInputError(e) { + console.log(e); + } + + if (navigator.mediaDevices.getUserMedia) { + navigator.mediaDevices.getUserMedia({"audio": true}).then(gotMediaInput, gotMediaInputError); + } else { + if (!navigator.getUserMedia) + navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia; + navigator.getUserMedia({"audio": true}, gotMediaInput, gotMediaInputError); + } + }); + /* clang-format on */ + + return OK; +} + +Error AudioDriverJavaScript::capture_stop() { + + /* clang-format off */ + EM_ASM({ + if (_audioDriver_inputStream) { + const tracks = _audioDriver_inputStream.getTracks(); + for (var i = 0; i < tracks.length; i++) { + tracks[i].stop(); + } + _audioDriver_inputStream = null; + } + + if (_audioDriver_audioInput) { + _audioDriver_audioInput.disconnect(); + _audioDriver_audioInput = null; + } + + }); + /* clang-format on */ + + input_buffer.clear(); + + return OK; } AudioDriverJavaScript::AudioDriverJavaScript() { + internal_buffer = NULL; + singleton = this; } diff --git a/platform/javascript/audio_driver_javascript.h b/platform/javascript/audio_driver_javascript.h index a65a8ec29fe..c8aeb0b446f 100644 --- a/platform/javascript/audio_driver_javascript.h +++ b/platform/javascript/audio_driver_javascript.h @@ -37,8 +37,12 @@ class AudioDriverJavaScript : public AudioDriver { float *internal_buffer; + int buffer_length; + public: void mix_to_js(); + void process_capture(float sample); + static AudioDriverJavaScript *singleton; virtual const char *get_name() const; @@ -51,6 +55,9 @@ public: virtual void unlock(); virtual void finish(); + virtual Error capture_start(); + virtual Error capture_stop(); + AudioDriverJavaScript(); }; diff --git a/platform/javascript/detect.py b/platform/javascript/detect.py index cf85c3df7fb..22b5f1f87a6 100644 --- a/platform/javascript/detect.py +++ b/platform/javascript/detect.py @@ -122,6 +122,7 @@ def configure(env): ## Link flags env.Append(LINKFLAGS=['-s', 'BINARYEN=1']) + env.Append(LINKFLAGS=['-s', 'BINARYEN_TRAP_MODE=\'clamp\'']) # Allow increasing memory buffer size during runtime. This is efficient # when using WebAssembly (in comparison to asm.js) and works well for