I want to stream audio form web client to c# server over Websocket but i received the same bytes every time .
client javascript code
function audioplayer(){
var stream;
var bufferSize = 1024 * 16;
var audioContext = new AudioContext();
var processor = audioContext.createScriptProcessor(bufferSize, 1, 1);
processor.connect(audioContext.destination);
navigator.mediaDevices.getUserMedia({
video: false,
audio: true
}).then( streamObj => {
stream = streamObj;
input = audioContext.createMediaStreamSource(stream);
input.connect(processor);
processor.onaudioprocess = e => {
microphoneProcess(e);
};
});
//var obj = { command:"mic" , audio : btoa(new Blob(recording)) };
//ws.send(JSON.stringify(obj));
}
function microphoneProcess(e) {
const left = e.inputBuffer.getChannelData(0); // get only one audio channel
const left16 = convertFloat32ToInt16(left); // skip if you don't need this
var obj = { command:"mic" , audio : btoa(left16) }
ws.send(JSON.stringify(obj)) // send to server via web socket
}
function convertFloat32ToInt16(buffer) {
let l = buffer.length;
const buf = new Int16Array(l / 3);
while (l--) {
if (l % 3 === 0) {
buf[l / 3] = buffer[l] * 0xFFFF;
}
}
return buf.buffer;
}
c# server
if (data == null) return;
using(MemoryStream ms = new MemoryStream(data)) {
try {
SoundPlayer player = new SoundPlayer(ms);
player.Stream = ms;
player.Play();
} catch {
}
}
DataRecv.Text = data.Length.ToString();
server give me error
System.InvalidOperationException: 'The wave header is corrupt.'
try to reset the position then play it,
if (data == null) return;
using(MemoryStream ms = new MemoryStream(data)) {
try {
ms.Position = 0;
SoundPlayer player = new SoundPlayer(ms);
player.Stream = ms;
player.Play();
} catch {}
}
DataRecv.Text = data.Length.ToString();
Related
How could I merge or combine two audio.wav loaded from url into a new .wav and play it in the web browser and download it?
The HTML and JS code:
NOTE: To avoid cross-origin load wavs from your local or server or allow it in your .htaccess
<!DOCTYPE html>
<html>
<meta content="text/html;" http-equiv="content-type" charset="utf-8">
<head>
<script>
window.onload = function() {
//Original Wav
//https://www.wavsource.com/snds_2020-10-01_3728627494378403/video_games/pacman/pacman_intro.wav
var audioFileUrl1 = 'pacman_intro.wav'; //Atention to cross-origin** !!!
var audioFileUrl2 = 'pacman_intro.wav'; //I love PacMan intro :P
loadWav(audioFileUrl1, audioFileUrl2); //Load wavs from url
function loadWav(url1,url2){
var arrBytesWav1, arrBytesWav2;
fetch(url1)
.then(function(response) { return response.blob(); })
.then(function(blob) {
var reader = new FileReader();
reader.readAsArrayBuffer(blob);
reader.addEventListener("loadend", function() {
arrBytesWav1 = reader.result;
});
return fetch(url2); //Return the second url as a Promise.
})
//Second load
.then(function(response) { return response.blob(); })
.then(function(blob) {
var reader = new FileReader();
reader.readAsArrayBuffer(blob);
reader.addEventListener("loadend", function() {
arrBytesWav2 = reader.result;
combineWavsBuffers( arrBytesWav1, arrBytesWav2 ); //Combine original wav buffer and play
});
})
.catch( function(error) {
alert('Error wav loading: ' + error.message);
});
}
//Combine two audio .wav buffers and assign to audio control and play it.
function combineWavsBuffers(buffer1, buffer2) {
//Combine array bytes of original wavs buffers
var tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength);
tmp.set( new Uint8Array(buffer1), 0 );
tmp.set( new Uint8Array(buffer2), buffer1.byteLength );
//Get buffer1 audio data to create the new combined wav
var audioData = getAudioData.WavHeader.readHeader(new DataView(buffer1));
console.log('Audio Data: ', audioData);
//Send combined buffer and send audio data to create the audio data of combined
var arrBytesFinal = getWavBytes( tmp, {
isFloat: false, // floating point or 16-bit integer
numChannels: audioData.channels,
sampleRate: audioData.sampleRate,
})
//Create a Blob as Base64 Raw data with audio/wav type
var myBlob = new Blob( [arrBytesFinal] , { type : 'audio/wav; codecs=MS_PCM' });
var combineBase64Wav;
var readerBlob = new FileReader();
readerBlob.addEventListener("loadend", function() {
combineBase64Wav = readerBlob.result.toString();
//Assign to audiocontrol to play the new combined wav.
var audioControl = document.getElementById('audio');
audioControl.src = combineBase64Wav;
audioControl.play();
});
readerBlob.readAsDataURL(myBlob);
console.log( "Buffer1 Size: " + buffer1.byteLength );
console.log( "Buffer2 Size: " + buffer1.byteLength );
console.log( "Combined Size: " + arrBytesFinal.byteLength );
return combineBase64Wav;
}
//Other functions //////////////////////////////////////////////////////////////
// Returns Uint8Array of WAV bytes
function getWavBytes(buffer, options) {
const type = options.isFloat ? Float32Array : Uint16Array
const numFrames = buffer.byteLength / type.BYTES_PER_ELEMENT
const headerBytes = getWavHeader(Object.assign({}, options, { numFrames }))
const wavBytes = new Uint8Array(headerBytes.length + buffer.byteLength);
// prepend header, then add pcmBytes
wavBytes.set(headerBytes, 0)
wavBytes.set(new Uint8Array(buffer), headerBytes.length)
return wavBytes
}
// adapted from https://gist.github.com/also/900023
// returns Uint8Array of WAV header bytes
function getWavHeader(options) {
const numFrames = options.numFrames
const numChannels = options.numChannels || 2
const sampleRate = options.sampleRate || 44100
const bytesPerSample = options.isFloat? 4 : 2
const format = options.isFloat? 3 : 1
const blockAlign = numChannels * bytesPerSample
const byteRate = sampleRate * blockAlign
const dataSize = numFrames * blockAlign
const buffer = new ArrayBuffer(44)
const dv = new DataView(buffer)
let p = 0
function writeString(s) {
for (let i = 0; i < s.length; i++) {
dv.setUint8(p + i, s.charCodeAt(i))
}
p += s.length
}
function writeUint32(d) {
dv.setUint32(p, d, true)
p += 4
}
function writeUint16(d) {
dv.setUint16(p, d, true)
p += 2
}
writeString('RIFF') // ChunkID
writeUint32(dataSize + 36) // ChunkSize
writeString('WAVE') // Format
writeString('fmt ') // Subchunk1ID
writeUint32(16) // Subchunk1Size
writeUint16(format) // AudioFormat
writeUint16(numChannels) // NumChannels
writeUint32(sampleRate) // SampleRate
writeUint32(byteRate) // ByteRate
writeUint16(blockAlign) // BlockAlign
writeUint16(bytesPerSample * 8) // BitsPerSample
writeString('data') // Subchunk2ID
writeUint32(dataSize) // Subchunk2Size
return new Uint8Array(buffer)
}
function getAudioData(){
function WavHeader() {
this.dataOffset = 0;
this.dataLen = 0;
this.channels = 0;
this.sampleRate = 0;
}
function fourccToInt(fourcc) {
return fourcc.charCodeAt(0) << 24 | fourcc.charCodeAt(1) << 16 | fourcc.charCodeAt(2) << 8 | fourcc.charCodeAt(3);
}
WavHeader.RIFF = fourccToInt("RIFF");
WavHeader.WAVE = fourccToInt("WAVE");
WavHeader.fmt_ = fourccToInt("fmt ");
WavHeader.data = fourccToInt("data");
WavHeader.readHeader = function (dataView) {
var w = new WavHeader();
var header = dataView.getUint32(0, false);
if (WavHeader.RIFF != header) {
return;
}
var fileLen = dataView.getUint32(4, true);
if (WavHeader.WAVE != dataView.getUint32(8, false)) {
return;
}
if (WavHeader.fmt_ != dataView.getUint32(12, false)) {
return;
}
var fmtLen = dataView.getUint32(16, true);
var pos = 16 + 4;
switch (fmtLen) {
case 16:
case 18:
w.channels = dataView.getUint16(pos + 2, true);
w.sampleRate = dataView.getUint32(pos + 4, true);
break;
default:
throw 'extended fmt chunk not implemented';
}
pos += fmtLen;
var data = WavHeader.data;
var len = 0;
while (data != header) {
header = dataView.getUint32(pos, false);
len = dataView.getUint32(pos + 4, true);
if (data == header) {
break;
}
pos += (len + 8);
}
w.dataLen = len;
w.dataOffset = pos + 8;
return w;
};
getAudioData.WavHeader = WavHeader;
}
getAudioData();
};//Window onLoad
</script>
</head>
<body>
<!--The audio control HTML element -->
<audio id="audio" src="" controls></audio>
</body>
</html>
I have audio data streaming from the server to the client. It starts as a Node.js buffer (which is a Uint8Array) and is then sent to the AudiWorkletProcessor via port.postMessage(), where it is converted into a Float32Array and stored in this.data. I have spent hours trying to set the output to the audio data contained in the Float32Array. Logging the Float32Array pre-processing shows accurate data, but logging it during processing shows that it is not changing when the new message is posted. This is probably a gap in my low-level audio-programming knowledge.
When data arrives in the client, the following function is called:
process = (data) => {
this.node.port.postMessage(data)
}
As an aside, (and you can let me know) maybe I should be using parameter descriptors instead of postMessage? Anyways, here's my AudioWorkletProcessor:
class BypassProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.isPlaying = true;
this.port.onmessage = this.onmessage.bind(this)
}
static get parameterDescriptors() {
return [{ // Maybe we should use parameters. This is not utilized at present.
name: 'stream',
defaultValue: 0.707
}];
}
convertBlock = (incomingData) => { // incoming data is a UInt8Array
let i, l = incomingData.length;
let outputData = new Float32Array(incomingData.length);
for (i = 0; i < l; i++) {
outputData[i] = (incomingData[i] - 128) / 128.0;
}
return outputData;
}
onmessage(event) {
const { data } = event;
let ui8 = new Uint8Array(data);
this.data = this.convertBlock(ui8)
}
process(inputs, outputs) {
const input = inputs[0];
const output = outputs[0];
if (this.data) {
for (let channel = 0; channel < output.length; ++channel) {
const inputChannel = input[channel]
const outputChannel = output[channel]
for (let i = 0; i < inputChannel.length; ++i) {
outputChannel[i] = this.data[i]
}
}
}
return true;
}
}
registerProcessor('bypass-processor', BypassProcessor);
How can I simply set the output of the AudioWorkletProcessor to the data coming through?
The AudioWorkletProcessor process only each 128 bytes, so you need to manage your own buffers to make sure that is the case for an AudioWorklet, probably by adding a FIFO.
I resolved something like this using a RingBuffer(FIFO) implemented in WebAssembly, in my case I was receiving a buffer with 160 bytes.
Look my AudioWorkletProcessor implementation
import Module from './buffer-kernel.wasmodule.js';
import { HeapAudioBuffer, RingBuffer, ALAW_TO_LINEAR } from './audio-helper.js';
class SpeakerWorkletProcessor extends AudioWorkletProcessor {
constructor(options) {
super();
this.payload = null;
this.bufferSize = options.processorOptions.bufferSize; // Getting buffer size from options
this.channelCount = options.processorOptions.channelCount;
this.inputRingBuffer = new RingBuffer(this.bufferSize, this.channelCount);
this.outputRingBuffer = new RingBuffer(this.bufferSize, this.channelCount);
this.heapInputBuffer = new HeapAudioBuffer(Module, this.bufferSize, this.channelCount);
this.heapOutputBuffer = new HeapAudioBuffer(Module, this.bufferSize, this.channelCount);
this.kernel = new Module.VariableBufferKernel(this.bufferSize);
this.port.onmessage = this.onmessage.bind(this);
}
alawToLinear(incomingData) {
const outputData = new Float32Array(incomingData.length);
for (let i = 0; i < incomingData.length; i++) {
outputData[i] = (ALAW_TO_LINEAR[incomingData[i]] * 1.0) / 32768;
}
return outputData;
}
onmessage(event) {
const { data } = event;
if (data) {
this.payload = this.alawToLinear(new Uint8Array(data)); //Receiving data from my Socket listener and in my case converting PCM alaw to linear
} else {
this.payload = null;
}
}
process(inputs, outputs) {
const output = outputs[0];
if (this.payload) {
this.inputRingBuffer.push([this.payload]); // Pushing data from my Socket
if (this.inputRingBuffer.framesAvailable >= this.bufferSize) { // if the input data size hits the buffer size, so I can "outputted"
this.inputRingBuffer.pull(this.heapInputBuffer.getChannelData());
this.kernel.process(
this.heapInputBuffer.getHeapAddress(),
this.heapOutputBuffer.getHeapAddress(),
this.channelCount,
);
this.outputRingBuffer.push(this.heapOutputBuffer.getChannelData());
}
this.outputRingBuffer.pull(output); // Retriving data from FIFO and putting our output
}
return true;
}
}
registerProcessor(`speaker-worklet-processor`, SpeakerWorkletProcessor);
Look the AudioContext and AudioWorklet instances
this.audioContext = new AudioContext({
latencyHint: 'interactive',
sampleRate: this.sampleRate,
sinkId: audioinput || "default"
});
this.audioBuffer = this.audioContext.createBuffer(1, this.audioSize, this.sampleRate);
this.audioSource = this.audioContext.createBufferSource();
this.audioSource.buffer = this.audioBuffer;
this.audioSource.loop = true;
this.audioContext.audioWorklet
.addModule('workers/speaker-worklet-processor.js')
.then(() => {
this.speakerWorklet = new AudioWorkletNode(
this.audioContext,
'speaker-worklet-processor',
{
channelCount: 1,
processorOptions: {
bufferSize: 160, //Here I'm passing the size of my output, I'm just saying to RingBuffer what size I need
channelCount: 1,
},
},
);
this.audioSource.connect(this.speakerWorklet).connect(this.audioContext.destination);
}).catch((err)=>{
console.log("Receiver ", err);
})
Look how I'm receiving and sending the data from Socket to audioWorklet
protected onMessage(e: any): void { //My Socket message listener
const { data:serverData } = e;
const socketId = e.socketId;
if (this.audioWalking && this.ws && !this.ws.isPaused() && this.ws.info.socketId === socketId) {
const buffer = arrayBufferToBuffer(serverData);
const rtp = RTPParser.parseRtpPacket(buffer);
const sharedPayload = new Uint8Array(new SharedArrayBuffer(rtp.payload.length)); //sharing javascript buffer memory between main thread and worklet thread
sharedPayload.set(rtp.payload, 0);
this.speakerWorklet.port.postMessage(sharedPayload); //Sending data to worklet
}
}
For help people I putted on Github the piece important of this solution
audio-worklet-processor-wasm-example
I followed this example, it have the all explanation how the RingBuffer works
wasm-ring-buffer
In my web application, I have a requirement to play part of mp3 file. This is a local web app, so I don't care about downloads etc, everything is stored locally.
My use case is as follows:
determine file to play
determine start and stop of the sound
load the file [I use BufferLoader]
play
Quite simple.
Right now I just grab the mp3 file, decode it in memory for use with WebAudio API, and play it.
Unfortunately, because the mp3 files can get quite long [30minutes of audio for example] the decoded file in memory can take up to 900MB. That's a bit too much to handle.
Is there any option, where I could decode only part of the file? How could I detect where to start and how far to go?
I cannot anticipate the bitrate, it can be constant, but I would expect variable as well.
Here's an example of what I did:
http://tinyurl.com/z9vjy34
The code [I've tried to make it as compact as possible]:
var MediaPlayerAudioContext = window.AudioContext || window.webkitAudioContext;
var MediaPlayer = function () {
this.mediaPlayerAudioContext = new MediaPlayerAudioContext();
this.currentTextItem = 0;
this.playing = false;
this.active = false;
this.currentPage = null;
this.currentAudioTrack = 0;
};
MediaPlayer.prototype.setPageNumber = function (page_number) {
this.pageTotalNumber = page_number
};
MediaPlayer.prototype.generateAudioTracks = function () {
var audioTracks = [];
var currentBegin;
var currentEnd;
var currentPath;
audioTracks[0] = {
begin: 4.300,
end: 10.000,
path: "example.mp3"
};
this.currentPageAudioTracks = audioTracks;
};
MediaPlayer.prototype.show = function () {
this.mediaPlayerAudioContext = new MediaPlayerAudioContext();
};
MediaPlayer.prototype.hide = function () {
if (this.playing) {
this.stop();
}
this.mediaPlayerAudioContext = null;
this.active = false;
};
MediaPlayer.prototype.play = function () {
this.stopped = false;
console.trace();
this.playMediaPlayer();
};
MediaPlayer.prototype.playbackStarted = function() {
this.playing = true;
};
MediaPlayer.prototype.playMediaPlayer = function () {
var instance = this;
var audioTrack = this.currentPageAudioTracks[this.currentAudioTrack];
var newBufferPath = audioTrack.path;
if (this.mediaPlayerBufferPath && this.mediaPlayerBufferPath === newBufferPath) {
this.currentBufferSource = this.mediaPlayerAudioContext.createBufferSource();
this.currentBufferSource.buffer = this.mediaPlayerBuffer;
this.currentBufferSource.connect(this.mediaPlayerAudioContext.destination);
this.currentBufferSource.onended = function () {
instance.currentBufferSource.disconnect(0);
instance.audioTrackFinishedPlaying()
};
this.playing = true;
this.currentBufferSource.start(0, audioTrack.begin, audioTrack.end - audioTrack.begin);
this.currentAudioStartTimeInAudioContext = this.mediaPlayerAudioContext.currentTime;
this.currentAudioStartTimeOffset = audioTrack.begin;
this.currentTrackStartTime = this.mediaPlayerAudioContext.currentTime - (this.currentTrackResumeOffset || 0);
this.currentTrackResumeOffset = null;
}
else {
function finishedLoading(bufferList) {
instance.mediaPlayerBuffer = bufferList[0];
instance.playMediaPlayer();
}
if (this.currentBufferSource){
this.currentBufferSource.disconnect(0);
this.currentBufferSource.stop(0);
this.currentBufferSource = null;
}
this.mediaPlayerBuffer = null;
this.mediaPlayerBufferPath = newBufferPath;
this.bufferLoader = new BufferLoader(this.mediaPlayerAudioContext, [this.mediaPlayerBufferPath], finishedLoading);
this.bufferLoader.load();
}
};
MediaPlayer.prototype.stop = function () {
this.stopped = true;
if (this.currentBufferSource) {
this.currentBufferSource.onended = null;
this.currentBufferSource.disconnect(0);
this.currentBufferSource.stop(0);
this.currentBufferSource = null;
}
this.bufferLoader = null;
this.mediaPlayerBuffer = null;
this.mediaPlayerBufferPath = null;
this.currentTrackStartTime = null;
this.currentTrackResumeOffset = null;
this.currentAudioTrack = 0;
if (this.currentTextTimeout) {
clearTimeout(this.currentTextTimeout);
this.textHighlightFinished();
this.currentTextTimeout = null;
this.currentTextItem = null;
}
this.playing = false;
};
MediaPlayer.prototype.getNumberOfPages = function () {
return this.pageTotalNumber;
};
MediaPlayer.prototype.playbackFinished = function () {
this.currentAudioTrack = 0;
this.playing = false;
};
MediaPlayer.prototype.audioTrackFinishedPlaying = function () {
this.currentAudioTrack++;
if (this.currentAudioTrack >= this.currentPageAudioTracks.length) {
this.playbackFinished();
} else {
this.playMediaPlayer();
}
};
//
//
// Buffered Loader
//
// Class used to get the sound files
//
function BufferLoader(context, urlList, callback) {
this.context = context;
this.urlList = urlList;
this.onload = callback;
this.bufferList = [];
this.loadCount = 0;
}
// this allows us to handle media files with embedded artwork/id3 tags
function syncStream(node) { // should be done by api itself. and hopefully will.
var buf8 = new Uint8Array(node.buf);
buf8.indexOf = Array.prototype.indexOf;
var i = node.sync, b = buf8;
while (1) {
node.retry++;
i = b.indexOf(0xFF, i);
if (i == -1 || (b[i + 1] & 0xE0 == 0xE0 )) break;
i++;
}
if (i != -1) {
var tmp = node.buf.slice(i); //carefull there it returns copy
delete(node.buf);
node.buf = null;
node.buf = tmp;
node.sync = i;
return true;
}
return false;
}
BufferLoader.prototype.loadBuffer = function (url, index) {
// Load buffer asynchronously
var request = new XMLHttpRequest();
request.open("GET", url, true);
request.responseType = "arraybuffer";
var loader = this;
function decode(sound) {
loader.context.decodeAudioData(
sound.buf,
function (buffer) {
if (!buffer) {
alert('error decoding file data');
return
}
loader.bufferList[index] = buffer;
if (++loader.loadCount == loader.urlList.length)
loader.onload(loader.bufferList);
},
function (error) {
if (syncStream(sound)) {
decode(sound);
} else {
console.error('decodeAudioData error', error);
}
}
);
}
request.onload = function () {
// Asynchronously decode the audio file data in request.response
var sound = {};
sound.buf = request.response;
sound.sync = 0;
sound.retry = 0;
decode(sound);
};
request.onerror = function () {
alert('BufferLoader: XHR error');
};
request.send();
};
BufferLoader.prototype.load = function () {
for (var i = 0; i < this.urlList.length; ++i)
this.loadBuffer(this.urlList[i], i);
};
There is no way of streaming with decodeAudioData(), you need to use MediaElement with createMediaStreamSource and run your stuff then. decodeAudioData() cannot stream on a part.#zre00ne And mp3 will be decoded big!!! Verybig!!!
I'm using the $timeout angular function to call tick() each 512 ms in order to play datas which are in my audio queue. I'm using this to perform a live audio stream. Sometimes there are some cuts in the sounds and I really need to maintain a delta of one second between emitting and receiving sound. So I want to delete some audio datas in my queue corresponding to the duration of each cuts.
Do you know if there is a way to listen to those cuts on the audioContext.destination like :
audioContext.destination.oncuts = function(duration) {
audioQueue.read(duration);
});
Here is my tick and audioQueue functions :
var tick = function() {
$scope.soundclock = Date.now();
$timeout(tick, $scope.tickInterval);
if(startStream && isFocused) {
if(isChrome === true || isOpera === true || isIE === true || isFirefox === true) {
if(audioQueue.length()>=size) {
float32 = audioQueue.read(size);
source = audioContext.createBufferSource();
audioBuffer = audioContext.createBuffer(1, size, sampleRate);
data = audioBuffer.getChannelData(0);
for(var i=0; i<size;i++) {
data[i] = float32[i];
}
source.buffer = audioBuffer;
source.connect(audioContext.destination);
source.start(0);
}
}
if(isSafari === true) {
if(audioQueue.length()>=size) {
float32 = audioQueue.read(size);
source = audioContext.createBufferSource();
audioBuffer = audioContext.createBuffer(1, size, sampleRate);
data = audioBuffer.getChannelData(0);
for(var j=0; j<size;j++) {
data[j] = float32[j];
}
source.buffer = audioBuffer;
source.connect(audioContext.destination);
source.noteOn(0);
}
}
}
};
var audioQueue = {
buffer: new Float32Array(0),
write: function(newAudio){
currentQLength = this.buffer.length;
newBuffer = new Float32Array(currentQLength+newAudio.length);
d = Date.now() - date;
console.log('Queued '+newBuffer.length+' samples. ');
date = Date.now();
newBuffer.set(this.buffer, 0);
newBuffer.set(newAudio, currentQLength);
this.buffer = newBuffer;
},
read: function(nSamples){
samplesToPlay = this.buffer.subarray(0, nSamples);
this.buffer = this.buffer.subarray(nSamples, this.buffer.length);
console.log('Queue at '+this.buffer.length+' samples. ');
return samplesToPlay;
},
length: function(){
return this.buffer.length;
}
};
You need to not rely on Javascript timers (which are, for audio purposes, horribly inaccurate) and schedule your ticks ahead of time. Check out http://www.html5rocks.com/en/tutorials/audio/scheduling/, which I wrote a while ago about scheduling timers.
I'm looking for a browser-based way of recording until a silence occurs.
HTML audio recording from the microphone is possible in Firefox and Chrome - using
Recordmp3js see:
http://nusofthq.com/blog/recording-mp3-using-only-html5-and-javascript-recordmp3-js/
and the code on github: http://github.com/nusofthq/Recordmp3js
I can't see a way to change that code to record until silence.
Record until silence can be done (and tuned) using Java for a native Android App - see here:
Android audio capture silence detection
Google Voice Search demonstrates a browser can doit - but how can I using Javascript?
Any ideas?
If you use the Web Audio API, open up a live microphone audio capture by making a call to : navigator.getUserMedia , then create a node using : createScriptProcessor, then you assign to that node a callback for its event : onaudioprocess . Inside your callback function (below I use script_processor_analysis_node) you have access to the live real-time audio buffer to which you can then parse looking for silence (some length of time where amplitude is low [stays close to zero]).
for normal time domain audio curve see : array_time_domain
which is populated fresh upon each call to callback script_processor_analysis_node ... similarly for frequency domain see array_freq_domain
Turn down your speaker volume or use headphones to avoid feedback from mic -> speaker -> mic ...
<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>capture microphone then show time & frequency domain output</title>
<script type="text/javascript">
var webaudio_tooling_obj = function () {
var audioContext = new AudioContext();
console.log("audio is starting up ...");
var BUFF_SIZE_RENDERER = 16384;
var audioInput = null,
microphone_stream = null,
gain_node = null,
script_processor_node = null,
script_processor_analysis_node = null,
analyser_node = null;
if (!navigator.getUserMedia)
navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia || navigator.msGetUserMedia;
if (navigator.getUserMedia){
navigator.getUserMedia({audio:true},
function(stream) {
start_microphone(stream);
},
function(e) {
alert('Error capturing audio.');
}
);
} else { alert('getUserMedia not supported in this browser.'); }
// ---
function show_some_data(given_typed_array, num_row_to_display, label) {
var size_buffer = given_typed_array.length;
var index = 0;
console.log("__________ " + label);
if (label === "time") {
for (; index < num_row_to_display && index < size_buffer; index += 1) {
var curr_value_time = (given_typed_array[index] / 128) - 1.0;
console.log(curr_value_time);
}
} else if (label === "frequency") {
for (; index < num_row_to_display && index < size_buffer; index += 1) {
console.log(given_typed_array[index]);
}
} else {
throw new Error("ERROR - must pass time or frequency");
}
}
function process_microphone_buffer(event) {
var i, N, inp, microphone_output_buffer;
microphone_output_buffer = event.inputBuffer.getChannelData(0); // just mono - 1 channel for now
}
function start_microphone(stream){
gain_node = audioContext.createGain();
gain_node.connect( audioContext.destination );
microphone_stream = audioContext.createMediaStreamSource(stream);
microphone_stream.connect(gain_node);
script_processor_node = audioContext.createScriptProcessor(BUFF_SIZE_RENDERER, 1, 1);
script_processor_node.onaudioprocess = process_microphone_buffer;
microphone_stream.connect(script_processor_node);
// --- enable volume control for output speakers
document.getElementById('volume').addEventListener('change', function() {
var curr_volume = this.value;
gain_node.gain.value = curr_volume;
console.log("curr_volume ", curr_volume);
});
// --- setup FFT
script_processor_analysis_node = audioContext.createScriptProcessor(2048, 1, 1);
script_processor_analysis_node.connect(gain_node);
analyser_node = audioContext.createAnalyser();
analyser_node.smoothingTimeConstant = 0;
analyser_node.fftSize = 2048;
microphone_stream.connect(analyser_node);
analyser_node.connect(script_processor_analysis_node);
var buffer_length = analyser_node.frequencyBinCount;
var array_freq_domain = new Uint8Array(buffer_length);
var array_time_domain = new Uint8Array(buffer_length);
console.log("buffer_length " + buffer_length);
script_processor_analysis_node.onaudioprocess = function() {
// get the average for the first channel
analyser_node.getByteFrequencyData(array_freq_domain);
analyser_node.getByteTimeDomainData(array_time_domain);
// draw the spectrogram
if (microphone_stream.playbackState == microphone_stream.PLAYING_STATE) {
show_some_data(array_freq_domain, 5, "frequency");
show_some_data(array_time_domain, 5, "time"); // store this to record to aggregate buffer/file
// examine array_time_domain for near zero values over some time period
}
};
}
}(); // webaudio_tooling_obj = function()
</script>
</head>
<body>
<p>Volume</p>
<input id="volume" type="range" min="0" max="1" step="0.1" value="0.5"/>
</body>
</html>
This is an old post but I am sure many will have the same problem so I am posting my solution here.
Use hark.js
Below is sample demo code i used for my electron app
hark = require('./node_modules/hark/hark.bundle.js')
navigator.getUserMedia({ audio : true}, onMediaSuccess, function(){});
function onMediaSuccess(blog) {
var options = {};
var speechEvents = hark(blog, options);
speechEvents.on('speaking', function() {
console.log('speaking');
});
speechEvents.on('stopped_speaking', function() {
console.log('stopped_speaking');
});
};
The solution from #Scott Stensland doesn't allow me to parse for silence. I am getting the same value for when I parse the two arrays - that is I'm getting 0 always when parsing arrayFreqDomain and 128 always when parsing arrayTimeDomain
let analyser = context.createAnalyser();
analyser.smoothingTimeConstant = 0;
analyser.fftSize = 2048;
let buffLength = analyser.frequencyBinCount;
let arrayFreqDomain = new Uint8Array(buffLength);
let arrayTimeDomain = new Uint8Array(buffLength);
processor.connect(analyser);
processor.onaudioprocess = (event) => {
/**
*
* Parse live real-time buffer looking for silence
*
**/
analyser.getByteFrequencyData(arrayFreqDomain);
analyser.getByteTimeDomainData(arrayTimeDomain);
if (context.state === "running") {
let sizeBuffer = arrayTimeDomain.length;
let index = 0;
for (; index < 5 && index < sizeBuffer; index += 1) {
console.log(arrayTimeDomain[index]); <----
}
}
}