I wrote a recorder that records microphone from getUsermedia and audio which is from local using Howler JS.
I created mediastream destination, and
connected each sources (mic, audio) to the destination.
audio seems fine, but microphone is delayed about 2seconds.
I can't figure out the problem.
could you help me guys?
var recorder;
const stop = document.getElementsByClassName("stop");
const record = document.getElementsByClassName("record");
let mediaDest = Howler.ctx.createMediaStreamDestination();
Howler.masterGain.connect(mediaDest);
function onRecordingReady(e) {
// 'e' has 'blob event'
//var audio = document.getElementById("audio");
audioBlob = e.data; // e.data has blob.
//audio.src = URL.createObjectURL(e.data);
}
let audioBlob;
let audioURL = "";
navigator.mediaDevices.getUserMedia({ audio: true }).then(function (stream) {
let userMic = Howler.ctx.createMediaStreamSource(stream);
userMic.connect(mediaDest);
Howler.masterGain.connect(mediaDest);
recorder = new MediaRecorder(mediaDest.stream);
recorder.addEventListener("dataavailable", onRecordingReady);
recorder.addEventListener("stop", function () {
W3Module.convertWebmToMP3(audioBlob).then((mp3blob) => {
const downloadLink = document.createElement("a");
downloadLink.href = URL.createObjectURL(mp3blob);
downloadLink.setAttribute("download", "audio");
//downloadLink.click();
var audio = document.getElementById("audio");
audio.src = URL.createObjectURL(mp3blob);
console.log(mp3blob);
});
});
});
record[0].addEventListener("click", function () {
recorder.start();
});
stop[0].addEventListener("click", function () {
recorder.stop();
});
I figured out the solution.
I didn't know I could connect MediaStreamAudioSourceNode to GainNode.
If someone is suffering this issue, just connect one Node to another Node rather than connect each node to the destination.
I connected the sourceNode to the GainNode, and connected GainNode to the destination.
=========================
It was not the solution...
GainNode playback in realtime whenever input is present...so, even if i can remove the latency, annoying playback occurs.
Related
I have successfully created an audio wave visualizer based on the mdn example here. I now want to add visualization for recorded audio as well. I record the audio using MediaRecorder and save the result as a Blob. However I cannot find a way to connect my AudioContext to the Blob.
This is the relevant code part so far:
var audioContext = new (window.AudioContext || window.webkitAudioContext)();
var analyser = audioContext.createAnalyser();
var dataArray = new Uint8Array(analyser.frequencyBinCount);
if (mediaStream instanceof Blob)
// Recorded audio - does not work
var stream = URL.createObjectURL(mediaStream);
else
// Stream from the microphone - works
stream = mediaStream;
var source = audioContext.createMediaStreamSource(stream);
source.connect(analyser);
mediaStream comes from either:
navigator.mediaDevices.getUserMedia ({
audio: this.audioConstraints,
video: this.videoConstraints,
})
.then( stream => {
mediaStream = stream;
}
or as a result of the recorded data:
mediaRecorder.addEventListener('dataavailable', event => {
mediaChunks.push(event.data);
});
...
mediaStream = new Blob(mediaChunks, { 'type' : 'video/webm' });
How do I connect the AudioContext to the recorded audio? Is it possible with a Blob? Do I need something else? What am I missing?
I've created a fiddle. The relevant part starts at line 118.
Thanks for help and suggestions.
EDIT:
Thanks to Johannes Klauß, I've found a solution.
See the updated fiddle.
You can use the Response API to create an ArrayBuffer and decode that with the audio context to create an AudioBuffer which you can connect to the analyser:
mediaRecorder.addEventListener('dataavailable', event => {
mediaChunks.push(event.data);
});
...
const arrayBuffer = await new Response(new Blob(mediaChunks, { 'type' : 'video/webm' })).arrayBuffer();
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(analyser);
I have a solution to stream using MediaRecorder API:
var socket = new WebSocket("ws://127.0.0.1:8765");
socket.binaryType = "blob";
socket.onopen = function (event) {
const video = document.querySelector('audio');
video.onplay = function() {
mediaStream = video.captureStream();
mediaRecorder = new MediaRecorder(mediaStream, {
mimeType: 'audio/webm'
});
mediaRecorder.addEventListener('dataavailable', (e) => {
socket.send(e.data);
});
mediaRecorder.start(1000);
};
};
But it doesn't play on my server (For example I use ffmpeg to record stream to file) because MediaRecorder API puts headers only to the first chunk. How can I put webm headers to every chunk?
I am using following javascript to record audio and send it to a websocket server:
const recordAudio = () =>
new Promise(async resolve => {
const constraints = {
audio: {
sampleSize: 16,
channelCount: 1,
sampleRate: 8000
},
video: false
};
var mediaRecorder;
const stream = await navigator.mediaDevices.getUserMedia(constraints);
var options = {
audioBitsPerSecond: 128000,
mimeType: 'audio/webm;codecs=pcm'
};
mediaRecorder = new MediaRecorder(stream, options);
var track = stream.getAudioTracks()[0];
var constraints2 = track.getConstraints();
var settings = track.getSettings();
const audioChunks = [];
mediaRecorder.addEventListener("dataavailable", event => {
audioChunks.push(event.data);
webSocket.send(event.data);
});
const start = () => mediaRecorder.start(30);
const stop = () =>
new Promise(resolve => {
mediaRecorder.addEventListener("stop", () => {
const audioBlob = new Blob(audioChunks);
const audioUrl = URL.createObjectURL(audioBlob);
const audio = new Audio(audioUrl);
const play = () => audio.play();
resolve({
audioBlob,
audioUrl,
play
});
});
mediaRecorder.stop();
});
resolve({
start,
stop
});
});
This is for realtime STT and the websocket server refused to send any response. I checked by debugging that the sampleRate is not changing to 8Khz.Upon researching, I found out that this is a known bug on both chrome and firefox. I found some other resources like stackoverflow1 and IBM_STT but I have no idea on how to adapt it to my code.
The above helpful resources refers to buffer but all i have is mediaStream(stream) and event.data(blob) in my code.
I am new to both javascript and Audio Api, so please pardon me if i did something wrong.
If this helps, I have an equivalent code of python to send data from mic to websocket server which works. Library used = Pyaudio. Code :
p = pyaudio.PyAudio()
stream = p.open(format="pyaudio.paInt16",
channels=1,
rate= 8000,
input=True,
frames_per_buffer=10)
print("* recording, please speak")
packet_size = int((30/1000)*8000) # normally 240 packets or 480 bytes
frames = []
#while True:
for i in range(0, 1000):
packet = stream.read(packet_size)
ws.send(packet, binary=True)
To do realtime downsampling follow these steps:
First get stream instance using this:
const stream = await navigator.mediaDevices.getUserMedia(constraints);
Create media stream source from this stream.
var input = audioContext.createMediaStreamSource(stream);
Create script Processor so that you can play with buffers. I am going to create a script processor which takes 4096 samples from the stream at a time, continuously, has 1 input channel and 1 output channel.
var scriptNode = audioContext.createScriptProcessor(4096, 1, 1);
Connect your input with scriptNode. You can connect script Node to the destination as per your requirement.
input.connect(scriptNode);
scriptNode.connect(audioContext.destination);
Now there is a function onaudioprocess in scriptProcessor where you can do whatever you want with 4096 samples. var downsample will contain (1/sampling ratio) number of packets. floatTo16BitPCM will convert that to your required format since the original data is in 32 bit float format.
var inputBuffer = audioProcessingEvent.inputBuffer;
// The output buffer contains the samples that will be modified and played
var outputBuffer = audioProcessingEvent.outputBuffer;
// Loop through the output channels (in this case there is only one)
for (var channel = 0; channel < outputBuffer.numberOfChannels; channel++) {
var inputData = inputBuffer.getChannelData(channel);
var outputData = outputBuffer.getChannelData(channel);
var downsampled = downsample(inputData);
var sixteenBitBuffer = floatTo16BitPCM(downsampled);
}
Your sixteenBitBuffer will contain the data you require.
Functions for downsampling and floatTo16BitPCM are explained in this link of Watson API:IBM Watson Speech to Text Api
You won't need MediaRecorder instance. Watson API is opensource and you can look for a better streamline approach on how they implemented it for their use case. You should be able to salvage important functions from their code.
I'm trying to broadcast a video from my webcam in javascript. I'm using MediaStream to get the video from my webcam, MediaRecorder to record such video in chunks (which would be transmitted to the server), and MediaSource to assemble these chunks and play them seamlessly in a video container called watchVideo on the source below.
It all works perfectly when i'm capturing only video, i.e. constraints = { video: true } ; but if I add audio, the watchVideo doesn't display anything, and the console shows me the following error:
Uncaught DOMException: Failed to execute 'appendBuffer' on 'SourceBuffer': This SourceBuffer has been removed from the parent media source.
This is the relevant part of the code:
var mime = 'video/webm; codecs=vp8';
if (navigator.mediaDevices) {
constraints = { video: true, audio: true };
navigator.mediaDevices.getUserMedia(constraints)
.then(handleUserMedia)
.catch( err => {
console.log("ERROR: " + err);
})
}
function handleUserMedia(stream) {
source = new MediaSource();
watchVideo.src = window.URL.createObjectURL(source);
source.onsourceopen = () => {
buffer = source.addSourceBuffer(mime);
};
var options = { mimeType: mime };
mediaRecorder = new MediaRecorder(stream, options);
mediaRecorder.ondataavailable = handleDataAvailable;
}
function handleDataAvailable(evt) {
var filereader = new FileReader();
filereader.onload = () => {
buffer.appendBuffer(filereader.result );
};
filereader.readAsArrayBuffer(evt.data);
}
I came across the question and it actually helped me more than many answers related to this topic. I don't know if you are still interested in the answer but I have tried
mime="video/webm; codecs="vp9,opus"
and it worked fine with audio and video, I hope this answer will help you
What is the correct way to play live stream with use of WebAudio API.
I am trying with following code, however all I see is that MP3 is being downloaded, but not played; probably MediaElementSource expect a file, not continuous stream.
window.AudioContext = window.AudioContext||window.webkitAudioContext;
context = new AudioContext();
var audio = new Audio();
var source = context.createMediaElementSource(audio);
source.connect(context.destination);
audio.src = '<live mp3 stream>';
audio.play();
try
audio.addEventListener('canplaythrough', function() {
audio.play();
}, false);
You maybe miss audio.crossOrigin = "anonymous" for hosted live stream with CORS enabled.
This is my whole working solution, MP3 as well:
window.AudioContext = window.AudioContext || window.webkitAudioContext;
const context = new AudioContext();
const audio = new Audio();
audio.crossOrigin = 'anonymous'; // Useful to play hosted live stream with CORS enabled
const sourceAudio = context.createMediaElementSource(audio);
sourceAudio.connect(context.destination);
const playHandler = () => {
audio.play();
audio.removeEventListener('canplaythrough', playHandler);
};
const errorHandler = e => {
console.error('Error', e);
audio.removeEventListener('error', errorHandler);
};
audio.addEventListener('canplaythrough', playHandler, false);
audio.addEventListener('error', errorHandler);
audio.src = '<live mp3 stream>';