Develop Reference

(Javascript) Microphone and audio from mediastream are out of sync

I wrote a recorder that records microphone from getUsermedia and audio which is from local using Howler JS.
I created mediastream destination, and
connected each sources (mic, audio) to the destination.
audio seems fine, but microphone is delayed about 2seconds.
I can't figure out the problem.
could you help me guys?
var recorder;
const stop = document.getElementsByClassName("stop");
const record = document.getElementsByClassName("record");
let mediaDest = Howler.ctx.createMediaStreamDestination();
Howler.masterGain.connect(mediaDest);
function onRecordingReady(e) {
// 'e' has 'blob event'
//var audio = document.getElementById("audio");
audioBlob = e.data; // e.data has blob.
//audio.src = URL.createObjectURL(e.data);
}
let audioBlob;
let audioURL = "";
navigator.mediaDevices.getUserMedia({ audio: true }).then(function (stream) {
let userMic = Howler.ctx.createMediaStreamSource(stream);
userMic.connect(mediaDest);
Howler.masterGain.connect(mediaDest);
recorder = new MediaRecorder(mediaDest.stream);
recorder.addEventListener("dataavailable", onRecordingReady);
recorder.addEventListener("stop", function () {
W3Module.convertWebmToMP3(audioBlob).then((mp3blob) => {
const downloadLink = document.createElement("a");
downloadLink.href = URL.createObjectURL(mp3blob);
downloadLink.setAttribute("download", "audio");
//downloadLink.click();
var audio = document.getElementById("audio");
audio.src = URL.createObjectURL(mp3blob);
console.log(mp3blob);
});
});
});
record[0].addEventListener("click", function () {
recorder.start();
});
stop[0].addEventListener("click", function () {
recorder.stop();
});

I figured out the solution.
I didn't know I could connect MediaStreamAudioSourceNode to GainNode.
If someone is suffering this issue, just connect one Node to another Node rather than connect each node to the destination.
I connected the sourceNode to the GainNode, and connected GainNode to the destination.
=========================
It was not the solution...
GainNode playback in realtime whenever input is present...so, even if i can remove the latency, annoying playback occurs.

Javascript | MediaRecorder API - Recorded video file on Desktop does not play in Mobile

I am trying to develop a canva-like Insta story creator using Canvas and MediaRecorder
The app is working perfectly on a desktop browser - I am able to download the file, and play it on desktop. However, when I send that file to my mobile, it doesn't play(even on Insta). I figure this is an issue with codecs - but don't know how to solve the same.
This is the function that handles the mediaRecorderAPI
Is there any mime type that I can use, that is universal and can play for any device?
initRecorder () {
var dl = document.querySelector("#dl")
let videoStream = this.canvas.captureStream(60);
if(this.isAudioPresent) {
videoStream.addTrack(this.audioStream.getAudioTracks()[0])
}
let mediaRecorder = new MediaRecorder(videoStream, {
videoBitsPerSecond : 2500000,
mime: 'video/webm'
});
let chunks = [];
mediaRecorder.onstop = function(e) {
var blob = new Blob(chunks, { 'type' : 'video/webm' });
chunks = [];
var videoURL = URL.createObjectURL(blob);
dl.href = videoURL;
};
mediaRecorder.ondataavailable = function(e) {
e.data.size && chunks.push(e.data);
};
mediaRecorder.start();
setTimeout(function (){ mediaRecorder.stop(); },this.storytime);
}
```

Figured this out: Different browsers use different transcoding. Insta only accepts MP4 transcoding. Hence, you need to use either a transcoder on the frontend(ffmpeg.js or wasm version of ffmpeg) or send your data to backend and handle there(which I ended up doing)

JavaScript MediaSource Example

I'm trying to create video player using MediaSource , but I can't make it play while buffering new data. I have this code that downloads the full data then plays it.
var vidElement = document.querySelector('video');
if (window.MediaSource) {
var mediaSource = new MediaSource();
vidElement.src = URL.createObjectURL(mediaSource);
mediaSource.addEventListener('sourceopen', sourceOpen);
} else {
console.log("The Media Source Extensions API is not supported.")
}
function sourceOpen(e) {
URL.revokeObjectURL(vidElement.src);
var mime = 'video/webm; codecs="opus, vp09.00.10.08"';
var mediaSource = e.target;
var sourceBuffer = mediaSource.addSourceBuffer(mime);
var videoUrl = 'droid.webm';
fetch(videoUrl)
.then(function(response) {
return response.arrayBuffer();
})
.then(function(arrayBuffer) {
sourceBuffer.addEventListener('updateend', function(e) {
if (!sourceBuffer.updating && mediaSource.readyState === 'open') {
mediaSource.endOfStream();
}
});
sourceBuffer.appendBuffer(arrayBuffer);
});
}
This code is not working on the local host. I get the MediaSource closed and this line never gets called
mediaSource.endOfStream();
Can any one tell me why the state is closed, please?
Any help about creating player like YouTube or any open source.
and by the way I tried a lot of codes and sources for 2 days now, and it's always the MediaSource giving me errors like the source removed, or not linked.

For starters, on your first line you have a collection...
var V=document.querySelector('video');
Shouldn't that be...
var V=document.querySelector('video')[0];
Before you start operating on it?

Audio recording plays back at inconsistent rate

I am using the code below to record audio in the browser, then use socket.io to send the resulting blob to a node.js server. The data is then saved to a mysql database or as an .ogg file.
The playback from either is strangely garbled at times - in different moments the sound will suddenly accelerate then return to normal. I've not been able to find anything online about this issue.
UPDATE: Further testing leads me to believe it is a hardware/software/compatibility with the Asus D1 soundcard I'm using. recording off of other devices does not result in the same behavior.
var mediaRecorder;
var startRecord = function(recordData){
if (!navigator.getUserMedia)
navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia || navigator.msGetUserMedia;
if (navigator.getUserMedia) {
navigator.getUserMedia({audio:true}, success, function(stream) {
alert('Error capturing audio.');
});
} else alert('getUserMedia not supported in this browser.');
function success(stream) {
var chunks = [];
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.start();
mediaRecorder.onstop = function(e) {
stream.getAudioTracks()[0].stop()
var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' })
var audioURL = URL.createObjectURL(blob);
if (recordData.id) {
socket.emit('audio', {
'id':recordData.id,
'audio':blob,
'option':recordData.type
});
}
mediaRecorder.ondataavailable = function(e) {
chunks.push(e.data);
}
}
};
var stopRecord = function(recordData){
mediaRecorder.stop();
mediaRecorder.state = 'recording'
};

AudioContext Live Streaming

What is the correct way to play live stream with use of WebAudio API.
I am trying with following code, however all I see is that MP3 is being downloaded, but not played; probably MediaElementSource expect a file, not continuous stream.
window.AudioContext = window.AudioContext||window.webkitAudioContext;
context = new AudioContext();
var audio = new Audio();
var source = context.createMediaElementSource(audio);
source.connect(context.destination);
audio.src = '<live mp3 stream>';
audio.play();

try
audio.addEventListener('canplaythrough', function() {
audio.play();
}, false);

You maybe miss audio.crossOrigin = "anonymous" for hosted live stream with CORS enabled.
This is my whole working solution, MP3 as well:
window.AudioContext = window.AudioContext || window.webkitAudioContext;
const context = new AudioContext();
const audio = new Audio();
audio.crossOrigin = 'anonymous'; // Useful to play hosted live stream with CORS enabled
const sourceAudio = context.createMediaElementSource(audio);
sourceAudio.connect(context.destination);
const playHandler = () => {
audio.play();
audio.removeEventListener('canplaythrough', playHandler);
};
const errorHandler = e => {
console.error('Error', e);
audio.removeEventListener('error', errorHandler);
};
audio.addEventListener('canplaythrough', playHandler, false);
audio.addEventListener('error', errorHandler);
audio.src = '<live mp3 stream>';