UPDATE
I found an example of high pass filter using webAudio here. Implemented it like this in my code.
function modifyGain(audioTrack, gainValue){
var ctx = new AudioContext();
var src = ctx.createMediaStreamTrackSource(audioTrack);
var dst = ctx.createMediaStreamDestination();
// create a filter node
var filterNode = ctx.createBiquadFilter();
filterNode.type = 'highpass';
filterNode.frequency.value = 0;
// cutoff frequency: for highpass, audio is attenuated below this frequency
var gainNode = ctx.createGain();
gainNode.gain.value = 1;
src.connect(filterNode);
//filterNode.connect(dst);
filterNode.connect(gainNode);
gainNode.connect(dst);
//alert(ctx.dst)
return dst.stream.getTracks()[0];
}
try {
webcamStream = await navigator.mediaDevices.getUserMedia(mediaConstraints);
document.getElementById("local_video").srcObject = webcamStream;
} catch(err) {
handleGetUserMediaError(err);
return;
}
// Add the tracks from the stream to the RTCPeerConnection
try {
webcamStream.getTracks().forEach(
function (track){
if(track.kind === 'audio'){
track = modifyGain(track, 0.5) //process only audio tracks
}
myPeerConnection.addTrack(track, webcamStream);
});
showLocalVideoContainer();
} catch(err) {
handleGetUserMediaError(err);
}
Before I can actually test if low sounds are silenced by highpass filter I am facing an issue. Using modifyGain mutes audio completely after few seconds. I tried 0, 1500 etc. It goes silence after few seconds.
Original POST
I am using the below constraints to try to suppress noise.
var mediaConstraints = {
audio: {advanced: [
{
echoCancellation: {exact: true}
},
{
autoGainControl: {exact: true}
},
{
noiseSuppression: {exact: true}
},
{
highpassFilter: {exact: true}
}
]
},
video: {
facingMode: "user",
width: { min: 160, ideal: 320, max: 640 },
height: { min: 120, ideal: 240, max: 480 },
}
};
But I want to silence some higher frequencies too. Even if I slowly move my phone on some surface the mic catches the noise and sends it to other peer. It catches even my breathing sound and send that to other side when I place it near my check(like phone call). I want some control over the frequencies which can allow me to select a threshold below which sounds will be not picked by mic.
I have tried searching but I am not sure what exactly will work for my case and how should I do it. I think following are my choices, but I maybe wrong.
Change SDP( codec params?).
Use webAudio and process the mic input before passing it on to other peer.
Use webAudio and process the received audio from other peer and direct it to output device(speaker).
Any help will be appreciated.
Thanks
You can process the audio from your microphone by piping it through web audio & using the BiquadFilter:
const stream = await navigator.mediaDevices.getUserMedia({audio: true});
const ctx = new AudioContext();
const src = ctx.createMediaStreamSource(stream);
const dst = ctx.createMediaStreamDestination();
const biquad = ctx.createBiquadFilter();
[src, biquad, dst].reduce((a, b) => a && a.connect(b));
audio.srcObject = dst.stream;
biquad.type = "highpass";
biquad.gain = 25;
biquad.frequency.value = 1000;
rangeButton.oninput = () => biquad.frequency.value = rangeButton.value;
Here's a working fiddle.
It goes silent after few seconds.
Could be a garbage collection issue. If you're experiencing this, try assigning your stream to a global JS var.
Related
I am a student studying programming.
I am not good at English, so I wrote it using a translator.
I'm studying the mediapipe.
https://google.github.io/mediapipe/solutions/face_mesh
Do you know how to use local video instead of webcam?
let videoElement = document.querySelector(".input_video")
//#mediapipe/camera_utils/camera_utils.js"
const camera = new Camera(videoElement, {
onFrame: async () => {
await holistic.send({ image: videoElement });
},
width: 640,
height: 480,
});
camera.start();
This is the code to get the webcam.
I think I need to change this code but I don't know how to work it.
so I tried to find out about '#mediapipe/camera_utils/camera_utils.js', I couldn't find any data.
And I found using the local video in the codepen demo.
https://codepen.io/mediapipe/details/KKgVaPJ
But I don't know which part of the code to use.
Please teach me the way.
Rather than create a new Camera, you need to send the frames using requestAnimationFrame(). However as the send needs to be in an async function
the requestAnimationFrame needs to be within a Promise.
You have the standard mediapipe setup
let videoElement = document.querySelector(".input_video");
const config = {
locateFile: (file) => {
return 'https://cdn.jsdelivr.net/npm/#mediapipe/face_mesh#' +
`${mpFaceMesh.VERSION}/${file}`;
}
};
const solutionOptions = {
selfieMode: false,
enableFaceGeometry: false,
maxNumFaces: 1,
refineLandmarks: true, //false,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5
};
const faceMesh = new mpFaceMesh.FaceMesh(config);
faceMesh.setOptions(solutionOptions);
faceMesh.onResults(onResults);
but rather than the new Camera() or SourcePicker() you need the an animation frame loop
async function onFrame() {
if (!videoElement.paused && !videoElement.ended) {
await faceMesh.send({
image: videoElement
});
// https://stackoverflow.com/questions/65144038/how-to-use-requestanimationframe-with-promise
await new Promise(requestAnimationFrame);
onFrame();
} else
setTimeout(onFrame, 500);
}
load the video
// must be same domain otherwise it will taint the canvas!
videoElement.src = "./mylocalvideo.mp4";
videoElement.onloadeddata = (evt) => {
let video = evt.target;
canvasElement.width = video.videoWidth;
canvasElement.height = video.videoHeight;
videoElement.play();
onFrame();
}
I am writing code to detect the frequency being played using the Web Audio API FFT, targeting high frequencies above 18kHz. It works perfectly in Chrome, Firefox and Edge but in Safari it is very inaccurate and doesn't register frequencies at all over 19kHz. Is there a known issue for Safari struggling to detect the highest frequencies?
My code is as follows:
async connectMicAudio() {
try {
let stream;
const constraints = {
echoCancellation: false,
noiseSuppression: false,
autoGainControl: false,
mozAutoGainControl: false,
mozNoiseSuppression: false,
};
if (navigator.mediaDevices.getUserMedia) {
stream = await navigator.mediaDevices.getUserMedia({
audio: true,
});
} else {
navigator.getUserMedia = // account for different browsers
navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia;
stream = await navigator.getUserMedia({
audio: true,
});
}
stream.getAudioTracks()[0].applyConstraints(constraints);
this.audioStream = stream;
} catch (error) {
console.log('Error: ', error);
}
let stream = this.audioStream;
const AudioContext = window.AudioContext || window.webkitAudioContext;
const audioContext = new AudioContext();
if (!this.sampleRate) {
this.sampleRate = audioContext.sampleRate;
}
const analyser = audioContext.createAnalyser();
analyser.fftSize = audioParams.FFT_SIZE;
analyser.smoothingTimeConstant = 0;
try {
const microphone = audioContext.createMediaStreamSource(stream);
microphone.connect(analyser);
} catch {
alert('Please allow microphone access.');
}
const processor = audioContext.createScriptProcessor(
audioParams.BUFFER_SIZE,
1,
1
); // single channel audio (mono)
processor.connect(audioContext.destination);
this.analyser = analyser;
this.processor = processor;
}
Then for the FFT I use:
this.processor.onaudioprocess = () => {
let fftData = new Float32Array(analyser.frequencyBinCount);
analyser.getFloatFrequencyData(fftData);
let highPassed = highPass(fftData, this.sampleRate);
let loudest = findLoudestFreqBin(highPassed, this.sampleRate);
console.log('loudest: ', loudest);
};
This extracts the frequency data from each buffer, highPass just zeroes anything under 18k and findLoudestFreqBin just returns the centre point of the bin with the highest amplitude. When I play anything above 19k on Safari, the highest amplitude will always be in a lower bin in the 18-19k range.
As I said this code works perfectly with other browsers so I'm assuming it's an issue with Safari's implementation of the Web Audio API. If anyone has had a similar issue and knows a workaround that would be a huge help.
TLDR
Imagine I have one video and one image. I want to create another video that overlays the image (e.g. watermark) at the center for 2 seconds in the beginning of the video and export it as the final video. I need to do this on the client-side only. Is it possible to use MediaRecorder + Canvas or should I resort to using ffmpeg.js?
Context
I am making a browser-based video editor where the user can upload videos and images and combine them. So far, I implemented this by embedding the video and images inside a canvas element appropriately. The data representation looks somewhat like this:
video: {
url: 'https://archive.com/video.mp4',
duration: 34,
},
images: [{
url: 'https://archive.com/img1.jpg',
start_time: 0,
end_time: 2,
top: 30,
left: 20,
width: 50,
height: 50,
}]
Attempts
I play the video and show/hide images in the canvas. Then, I can use the MediaRecorder to capture the canvas' stream and export it as a data blob at the end. The final output is as expected, but the problem with this approach is I need to play the video from the beginning to the end for me to capture the stream from the canvas. If the video is 60 seconds, exporting it also takes 60 seconds.
function record(canvas) {
return new Promise(function (res, rej) {
const stream = canvas.captureStream();
const mediaRecorder = new MediaRecorder(stream);
const recordedData = [];
// Register recorder events
mediaRecorder.ondataavailable = function (event) {
recordedData.push(event.data);
};
mediaRecorder.onstop = function (event) {
var blob = new Blob(recordedData, {
type: "video/webm",
});
var url = URL.createObjectURL(blob);
res(url);
};
// Start the video and start recording
videoRef.current.currentTime = 0;
videoRef.current.addEventListener(
"play",
(e) => {
mediaRecorder.start();
},
{
once: true,
}
);
videoRef.current.addEventListener(
"ended",
(e) => {
mediaRecorder.stop();
},
{
once: true,
}
);
videoRef.current.play();
});
}
I can use ffmpeg.js to encode the video. I haven't tried this method yet as I will have to convert my image representation into ffmpeg args (I wonder how much work that is).
I am currently working on a project and need to be able to make a recording of my screen and save it locally to my computer.
The recording is being saved as a webm, but everyone of them has a really bad framerate of usually around 10-15 fps. Is there a way to increase the framerate for recording?
I am able to increase the quality of the recording by playing around with the MediaRecorder options and codecs, but this doesn't seem to affect the framerate I am getting at all.
Here is the code I am using to make my recording:
const options = {
mimeType: 'video/webm; codecs="vp9.00.41.8.00.01"',
videoBitsPerSecond: 800 * Mbps,
videoMaximizeFrameRate: true,
};
mediaRecorder = new MediaRecorder(stream, options);
mediaRecorder.ondataavailable = handleDataAvailable;
mediaRecorder.onstop = handleStop;
startBtn.onclick = e => {
mediaRecorder.start();
startBtn.innerHTML = 'Recording';
}
stopBtn.onclick = e => {
mediaRecorder.stop();
startBtn.innerHTML = 'Start';
}
function handleDataAvailable(e) {
recordedChunks.push(e.data);
}
async function handleStop() {
const blob = new Blob(recordedChunks, {
mimeType: 'video/webm'
});
const buffer = Buffer.from(await blob.arrayBuffer());
const { filePath } = await dialog.showSaveDialog({
buttonLabel: 'Save video',
defaultPath: `vid-${Date.now()}.webm`
});
console.log(filePath);
if (filePath) {
writeFile(filePath, buffer, () => console.log('video saved successfully'));
}
}
I have looked through the MDN documentation and haven't found anything about it. I also tried using different codecs with different parameters, but the results are always the same.
The framerate you're getting is typical for any standard screen capture.
The only way to go faster is to utilize the GPU's specific capability to capture and encode. This is out of scope for the web APIs.
I have developed a webpage that listens to the microphone, calculates the root means square (the average loudness) and changes the colour of the text areas on the webpage.
I found the audio-rms package, but the example uses an oscillator and I am not sure how to replace it with a microphone stream.
I then found an article on HTML5 Rocks about capturing audio, and I used some of the code to capture the audio for use in realtime.
I already have some code that is supposed to calculate the rms from the stream, but the problem is that the microphone never sends any audio. By using console logs, I have worked out that the code works up line 8, but it does not work by line 11, which happens to be when calling navigator.mediaDevices.getUserMedia
The code I am using is below and you can view the files on GitHub:
+function () {
var errorCallback = function (e) {
console.log('Permission Rejected!', e);
};
var ctx = new AudioContext()
if (navigator.mediaDevices.getUserMedia) {
//works here
navigator.mediaDevices.getUserMedia({audio: true}, function (stream)
{
//Doesn't work here.
// 2048 sample buffer, 1 channel in, 1 channel out
var processor = ctx.createScriptProcessor(2048, 1, 1)
var source
console.log("processor")
source = ctx.createMediaElementSource(stream)
console.log("media element")
source.connect(processor)
source.connect(ctx.destination)
processor.connect(ctx.destination)
stream.play()
console.log("stream play")
// loop through PCM data and calculate average
// volume for a given 2048 sample buffer
processor.onaudioprocess = function (evt) {
var input = evt.inputBuffer.getChannelData(0)
, len = input.length
, total = i = 0
, rms
while (i < len) total += Math.abs(input[i++])
rms = Math.sqrt(total / len)
console.log(rmsLevel)
if (rmsLevel > 65) { document.getElementById("TL").style.backgroundColor = "rgb(255, 0, 0)"; }
else if (rmsLevel > 60 && rmsLevel <= 65) { document.getElementById("L").style.backgroundColor = "rgb(255, 140, 0)"; }
...
}
}, errorCallback);
} else {
alert("Error. :(")
}
}()
function resetColours() {
document.getElementById("TL").style.backgroundColor = "rgb(110, 110, 110)";
document.getElementById("L").style.backgroundColor = "rgb(110, 110, 110)";
...
}
You've got incorrect usage of navigator.MediaDevices.getUserMedia - you wrote it in the old style of navigator.getUserMedia callbacks, not in the Promise-based way of navigator.MediaDevices.gUM. Take a look at https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia.
Instead of
navigator.mediaDevices.getUserMedia({audio: true}, function (stream) {
...
}, errorcallback );
You should say
navigator.mediaDevices.getUserMedia({audio: true}).then( function (stream) {
...
}).catch(function(err) {
/* handle the error */
});