I have a solution to stream using MediaRecorder API:
var socket = new WebSocket("ws://127.0.0.1:8765");
socket.binaryType = "blob";
socket.onopen = function (event) {
const video = document.querySelector('audio');
video.onplay = function() {
mediaStream = video.captureStream();
mediaRecorder = new MediaRecorder(mediaStream, {
mimeType: 'audio/webm'
});
mediaRecorder.addEventListener('dataavailable', (e) => {
socket.send(e.data);
});
mediaRecorder.start(1000);
};
};
But it doesn't play on my server (For example I use ffmpeg to record stream to file) because MediaRecorder API puts headers only to the first chunk. How can I put webm headers to every chunk?
Related
I wrote a recorder that records microphone from getUsermedia and audio which is from local using Howler JS.
I created mediastream destination, and
connected each sources (mic, audio) to the destination.
audio seems fine, but microphone is delayed about 2seconds.
I can't figure out the problem.
could you help me guys?
var recorder;
const stop = document.getElementsByClassName("stop");
const record = document.getElementsByClassName("record");
let mediaDest = Howler.ctx.createMediaStreamDestination();
Howler.masterGain.connect(mediaDest);
function onRecordingReady(e) {
// 'e' has 'blob event'
//var audio = document.getElementById("audio");
audioBlob = e.data; // e.data has blob.
//audio.src = URL.createObjectURL(e.data);
}
let audioBlob;
let audioURL = "";
navigator.mediaDevices.getUserMedia({ audio: true }).then(function (stream) {
let userMic = Howler.ctx.createMediaStreamSource(stream);
userMic.connect(mediaDest);
Howler.masterGain.connect(mediaDest);
recorder = new MediaRecorder(mediaDest.stream);
recorder.addEventListener("dataavailable", onRecordingReady);
recorder.addEventListener("stop", function () {
W3Module.convertWebmToMP3(audioBlob).then((mp3blob) => {
const downloadLink = document.createElement("a");
downloadLink.href = URL.createObjectURL(mp3blob);
downloadLink.setAttribute("download", "audio");
//downloadLink.click();
var audio = document.getElementById("audio");
audio.src = URL.createObjectURL(mp3blob);
console.log(mp3blob);
});
});
});
record[0].addEventListener("click", function () {
recorder.start();
});
stop[0].addEventListener("click", function () {
recorder.stop();
});
I figured out the solution.
I didn't know I could connect MediaStreamAudioSourceNode to GainNode.
If someone is suffering this issue, just connect one Node to another Node rather than connect each node to the destination.
I connected the sourceNode to the GainNode, and connected GainNode to the destination.
=========================
It was not the solution...
GainNode playback in realtime whenever input is present...so, even if i can remove the latency, annoying playback occurs.
I have been trying to record audio in OGG format on Chrome and send it back to the server, but it always gets their in video/ogg format. Here is what I have:
Capturing audio:
let chunks = [];
let recording = null;
let mediaRecorder = new MediaRecorder(stream);
mediaRecorder.start();
mediaRecorder.onstop = function() {
recording = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });
}
mediaRecorder.ondataavailable = function(e){
chunks.push(e.data);
}
Sending it to the server:
let data = new FormData();
data.append('audio', recording);
jQuery.ajax(...);
The blob gets to the backend, but always in video/ogg!
I ended up using kbumsik/opus-media-recorder, solved the issue for me. A drop-in replacement for MediaRecorder.
You need to remove the VideoTrack from your MediaStream:
const input = document.querySelector("video");
const stop_btn = document.querySelector("button");
input.onplaying = (evt) => {
input.onplaying = null;
console.clear();
const stream = input.captureStream ? input.captureStream() : input.mozCaptureStream();
// get all video tracks (usually a single one)
stream.getVideoTracks().forEach( (track) => {
track.stop(); // stop that track, so the browser doesn't feed it for nothing
stream.removeTrack( track ); // remove it from the MediaStream
} );
const data = [];
const recorder = new MediaRecorder( stream, { mimeType: "audio/webm" } );
recorder.ondataavailable = (evt) => data.push( evt.data );
recorder.onstop = (evt) => exportFile( new Blob( data ) );
stop_btn.onclick = (evt) => recorder.stop();
stop_btn.disabled = false;
recorder.start();
};
console.log( "play the video to start recording" );
function exportFile( blob ) {
stop_btn.remove();
input.src = URL.createObjectURL( blob );
console.log( "video element now playing recoded file" );
}
video { max-height: 150px; }
<video src="https://upload.wikimedia.org/wikipedia/commons/2/22/Volcano_Lava_Sample.webm" controls crossorigin></video>
<button disabled>stop recording</button>
And since StackOverflow's null origined iframes don't allow for safe download links, here is a fiddle with a download link.
You need to set the mimeType of the MediaRecorder. Otherwise the browser will pick whatever format it likes best to encode the media.
let mediaRecorder = new MediaRecorder(stream, { mimeType: 'my/mimetype' });
To be sure that the browser can actually encode the format you want you could use isTypeSupported().
console.log(MediaRecorder.isTypeSupported('my/mimetype'));
Chrome for example doesn't support "audio/ogg; codecs=opus" but supports "audio/webm; codecs=opus". Firefox supports both. Safari none of them.
Once you've configured the MediaRecorder you can use its mimeType when creating the blob.
recording = new Blob(chunks, { 'type' : mediaRecorder.mimeType });
I am trying to develop a canva-like Insta story creator using Canvas and MediaRecorder
The app is working perfectly on a desktop browser - I am able to download the file, and play it on desktop. However, when I send that file to my mobile, it doesn't play(even on Insta). I figure this is an issue with codecs - but don't know how to solve the same.
This is the function that handles the mediaRecorderAPI
Is there any mime type that I can use, that is universal and can play for any device?
initRecorder () {
var dl = document.querySelector("#dl")
let videoStream = this.canvas.captureStream(60);
if(this.isAudioPresent) {
videoStream.addTrack(this.audioStream.getAudioTracks()[0])
}
let mediaRecorder = new MediaRecorder(videoStream, {
videoBitsPerSecond : 2500000,
mime: 'video/webm'
});
let chunks = [];
mediaRecorder.onstop = function(e) {
var blob = new Blob(chunks, { 'type' : 'video/webm' });
chunks = [];
var videoURL = URL.createObjectURL(blob);
dl.href = videoURL;
};
mediaRecorder.ondataavailable = function(e) {
e.data.size && chunks.push(e.data);
};
mediaRecorder.start();
setTimeout(function (){ mediaRecorder.stop(); },this.storytime);
}
```
Figured this out: Different browsers use different transcoding. Insta only accepts MP4 transcoding. Hence, you need to use either a transcoder on the frontend(ffmpeg.js or wasm version of ffmpeg) or send your data to backend and handle there(which I ended up doing)
I am using following javascript to record audio and send it to a websocket server:
const recordAudio = () =>
new Promise(async resolve => {
const constraints = {
audio: {
sampleSize: 16,
channelCount: 1,
sampleRate: 8000
},
video: false
};
var mediaRecorder;
const stream = await navigator.mediaDevices.getUserMedia(constraints);
var options = {
audioBitsPerSecond: 128000,
mimeType: 'audio/webm;codecs=pcm'
};
mediaRecorder = new MediaRecorder(stream, options);
var track = stream.getAudioTracks()[0];
var constraints2 = track.getConstraints();
var settings = track.getSettings();
const audioChunks = [];
mediaRecorder.addEventListener("dataavailable", event => {
audioChunks.push(event.data);
webSocket.send(event.data);
});
const start = () => mediaRecorder.start(30);
const stop = () =>
new Promise(resolve => {
mediaRecorder.addEventListener("stop", () => {
const audioBlob = new Blob(audioChunks);
const audioUrl = URL.createObjectURL(audioBlob);
const audio = new Audio(audioUrl);
const play = () => audio.play();
resolve({
audioBlob,
audioUrl,
play
});
});
mediaRecorder.stop();
});
resolve({
start,
stop
});
});
This is for realtime STT and the websocket server refused to send any response. I checked by debugging that the sampleRate is not changing to 8Khz.Upon researching, I found out that this is a known bug on both chrome and firefox. I found some other resources like stackoverflow1 and IBM_STT but I have no idea on how to adapt it to my code.
The above helpful resources refers to buffer but all i have is mediaStream(stream) and event.data(blob) in my code.
I am new to both javascript and Audio Api, so please pardon me if i did something wrong.
If this helps, I have an equivalent code of python to send data from mic to websocket server which works. Library used = Pyaudio. Code :
p = pyaudio.PyAudio()
stream = p.open(format="pyaudio.paInt16",
channels=1,
rate= 8000,
input=True,
frames_per_buffer=10)
print("* recording, please speak")
packet_size = int((30/1000)*8000) # normally 240 packets or 480 bytes
frames = []
#while True:
for i in range(0, 1000):
packet = stream.read(packet_size)
ws.send(packet, binary=True)
To do realtime downsampling follow these steps:
First get stream instance using this:
const stream = await navigator.mediaDevices.getUserMedia(constraints);
Create media stream source from this stream.
var input = audioContext.createMediaStreamSource(stream);
Create script Processor so that you can play with buffers. I am going to create a script processor which takes 4096 samples from the stream at a time, continuously, has 1 input channel and 1 output channel.
var scriptNode = audioContext.createScriptProcessor(4096, 1, 1);
Connect your input with scriptNode. You can connect script Node to the destination as per your requirement.
input.connect(scriptNode);
scriptNode.connect(audioContext.destination);
Now there is a function onaudioprocess in scriptProcessor where you can do whatever you want with 4096 samples. var downsample will contain (1/sampling ratio) number of packets. floatTo16BitPCM will convert that to your required format since the original data is in 32 bit float format.
var inputBuffer = audioProcessingEvent.inputBuffer;
// The output buffer contains the samples that will be modified and played
var outputBuffer = audioProcessingEvent.outputBuffer;
// Loop through the output channels (in this case there is only one)
for (var channel = 0; channel < outputBuffer.numberOfChannels; channel++) {
var inputData = inputBuffer.getChannelData(channel);
var outputData = outputBuffer.getChannelData(channel);
var downsampled = downsample(inputData);
var sixteenBitBuffer = floatTo16BitPCM(downsampled);
}
Your sixteenBitBuffer will contain the data you require.
Functions for downsampling and floatTo16BitPCM are explained in this link of Watson API:IBM Watson Speech to Text Api
You won't need MediaRecorder instance. Watson API is opensource and you can look for a better streamline approach on how they implemented it for their use case. You should be able to salvage important functions from their code.
I'm currently editing my mp3 file with multiple effects like so
var mainVerse = document.getElementById('audio1');
var s = source;
source.disconnect(audioCtx.destination);
for (var i in filters1) {
s.connect(filters1[i]);
s = filters1[i];
}
s.connect(audioCtx.destination);
The mp3 plays accordingly on the web with the filters on it. Is it possible to create and download a new mp3 file with these new effects, using web audio api or any writing to mp3 container javascript library ? If not whats the best to solve this on the web ?
UPDATE - Using OfflineAudioContext
Using the sample code from https://developer.mozilla.org/en-US/docs/Web/API/OfflineAudioContext/oncomplete
I've tried using the offline node like so;
var audioCtx = new AudioContext();
var offlineCtx = new OfflineAudioContext(2,44100*40,44100);
osource = offlineCtx.createBufferSource();
function getData() {
request = new XMLHttpRequest();
request.open('GET', 'Song1.mp3', true);
request.responseType = 'arraybuffer';
request.onload = function() {
var audioData = request.response;
audioCtx.decodeAudioData(audioData, function(buffer) {
myBuffer = buffer;
osource.buffer = myBuffer;
osource.connect(offlineCtx.destination);
osource.start();
//source.loop = true;
offlineCtx.startRendering().then(function(renderedBuffer) {
console.log('Rendering completed successfully');
var audioCtx = new (window.AudioContext || window.webkitAudioContext)();
var song = audioCtx.createBufferSource();
song.buffer = renderedBuffer;
song.connect(audioCtx.destination);
song.start();
rec = new Recorder(song, {
workerPath: 'Recorderjs/recorderWorker.js'
});
rec.exportWAV(function(e){
rec.clear();
Recorder.forceDownload(e, "filename.wav");
});
}).catch(function(err) {
console.log('Rendering failed: ' + err);
// Note: The promise should reject when startRendering is called a second time on an OfflineAudioContext
});
});
}
request.send();
}
// Run getData to start the process off
getData();
Still getting the recorder to download an empty file, I'm using the song source as the source for the recorder. The song plays and everything with his code but recorder doesn't download it
Use https://github.com/mattdiamond/Recorderjs to record a .wav file. Then use https://github.com/akrennmair/libmp3lame-js to encode it to .mp3.
There's a nifty guide here, if you need a hand: http://audior.ec/blog/recording-mp3-using-only-html5-and-javascript-recordmp3-js/
UPDATE
Try moving
rec = new Recorder(song, {
workerPath: 'Recorderjs/recorderWorker.js'
});
so that it is located above the call to start rendering, and connect it to osource instead, like so:
rec = new Recorder(osource, {
workerPath: 'Recorderjs/recorderWorker.js'
});
osource.connect(offlineCtx.destination);
osource.start();
offlineCtx.startRendering().then(function(renderedBuffer) {
.....