Develop Reference

Cannot record HTMLCanvasElement when tab is blurred even though render is occuring [duplicate]

When using CanvasCaptureMediaStream and MediaRecorder, is there a way to get an event on each frame?
What I need is not unlike requestAnimationFrame(), but I need it for the CanvasCaptureMediaStream (and/or the MediaRecorder) and not the window. The MediaRecorder could be running at a different frame rate than the window (possibly at a not regularly divisible rate, such as 25 FPS vs 60 FPS), so I want to update the canvas at its frame rate rather than the window's.

This example currently only fully works on FireFox, since chrome simply stops the canvas stream when the tab is blurred... (probably related to this bug, but well, my timer seems to be working but not the recording...)
[Edit]: it actually now works only in chrome, since they have fixed this bug, but not anymore in FF because of this one (caused by e10s).
There doesn't seem to be any event on MediaStream letting you know when a frame has been rendered to it, neither on the MediaRecorder.
Even the currentTime property of the MediaStream (currently only available in FF) doesn't seem to be changing accordingly with the fps argument passed in the captureStream() method.
But what you seem to want is a reliable timer, that won't loose its frequency when i.e the current tab is not focused (which happens for rAF).
Fortunately, the WebAudio API does also have an high precision timer, based on hardware clock, rather than on screen refresh rate.
So we can come with an alternative timed loop, able to keep its frequency even when the tab is blurred.
/*
An alternative timing loop, based on AudioContext's clock
#arg callback : a callback function
with the audioContext's currentTime passed as unique argument
#arg frequency : float in ms;
#returns : a stop function
*/
function audioTimerLoop(callback, frequency) {
// AudioContext time parameters are in seconds
var freq = frequency / 1000;
var aCtx = new AudioContext();
// Chrome needs our oscillator node to be attached to the destination
// So we create a silent Gain Node
var silence = aCtx.createGain();
silence.gain.value = 0;
silence.connect(aCtx.destination);
onOSCend();
var stopped = false;
function onOSCend() {
osc = aCtx.createOscillator();
osc.onended = onOSCend;
osc.connect(silence);
osc.start(0);
osc.stop(aCtx.currentTime + freq);
callback(aCtx.currentTime);
if (stopped) {
osc.onended = function() {
return;
};
}
};
// return a function to stop our loop
return function() {
stopped = true;
};
}
function start() {
// start our loop #25fps
var stopAnim = audioTimerLoop(anim, 1000 / 25);
// maximum stream rate set as 25 fps
cStream = canvas.captureStream(25);
let chunks = [];
var recorder = new MediaRecorder(cStream);
recorder.ondataavailable = e => chunks.push(e.data);
recorder.onstop = e => {
// we can stop our loop
stopAnim();
var url = URL.createObjectURL(new Blob(chunks));
var v = document.createElement('video');
v.src = url;
v.controls = true;
document.body.appendChild(v);
}
recorder.start();
// stops the recorder in 20s, try to change tab during this time
setTimeout(function() {
recorder.stop();
}, 20000)
}
// make something move on the canvas
var ctx = canvas.getContext('2d');
var x = 0;
function anim() {
x = (x + 2) % (canvas.width + 100);
ctx.fillStyle = 'ivory';
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = 'red';
ctx.fillRect(x - 50, 20, 50, 50)
};
btn.onclick = start;
<button id="btn">begin</button>
<canvas id="canvas" width="500" height="200"></canvas>
Nota Bene :
In this example, I set the frequency to 25fps, but we can set it to 60fps and it seems to work correctly even on my old notebook, at least with such a simple animation.

HTML5 Audio gets louder after pausing then playing again

I'm making an audio player with JavaScript, everything works fine until I add a sound visualizer. When I pause the song and then play it again, the sound gets more louder every time I do it, until it gets distorsionated.
I'm newbie with the HTML5 Audio API, I've tried to set the volume as a fixed value, but not works.
The code of the visualizer it's:
function visualizer(audio) {
let context = new AudioContext();
const gainNode = context.createGain();
gainNode.gain.value = 1; // setting it to 100%
gainNode.connect(context.destination);
let src = context.createMediaElementSource(audio);
let analyser = context.createAnalyser();
let canvas = document.getElementById("canvas");
canvas.width = window.innerWidth;
canvas.height = window.innerHeight;
let ctx = canvas.getContext("2d");
src.connect(analyser);
analyser.connect(context.destination);
analyser.fftSize = 2048;
let bufferLength = analyser.frequencyBinCount;
let dataArray = new Uint8Array(bufferLength);
let WIDTH = ctx.canvas.width;
let HEIGHT = ctx.canvas.height;
let barWidth = (WIDTH / bufferLength) * 1.5;
let barHeight;
let x = 0;
let color = randomColor();
function renderFrame() {
requestAnimationFrame(renderFrame);
x = 0;
analyser.getByteFrequencyData(dataArray);
ctx.clearRect(0, 0, WIDTH, HEIGHT);
for (let i = 0; i < bufferLength; i++) {
barHeight = dataArray[i];
ctx.fillStyle = color;
ctx.fillRect(x, HEIGHT - barHeight, barWidth, barHeight);
x += barWidth + 1;
}
}
musicPlay();
renderFrame();
}
And:
function musicPlay() {
status = 'playing';
audio.play();
}
So, I don't know if I'm doing something wrong on the audio analyzer, I've tried to make a global context and don't do the new AudioContext(); every time I enter on the function, also I've tried to specify a fixed volume with:
audio.volume = 1;
or with the GainNode as you can see on the function, but it's not working.
Where is my mistake and why the sound gets louder?
Regards!
--- Update 1 ---
The audio it's loaded from an URL:
function loadAudioElement(url) {
return new Promise(function (resolve, reject) {
let audio = new Audio();
audio.addEventListener('canplay', function () {
/* Resolve the promise, passing through the element. */
resolve(audio);
});
/* Reject the promise on an error. */
audio.addEventListener('error', reject);
audio.src = url;
});
}
And on my player I have:
let playButtonFunction = function () {
if (playstatus === 'pause') {
loadAudioElement(audio.src).then(
visualizer(audio)
);
} else if (playstatus === 'playing') {
musicPause();
}
};

I had a similar issue, did you try to set the audio context to a global object?
This is what I found here:
https://developer.mozilla.org/en-US/docs/Web/API/AudioContext
It's recommended to create one AudioContext and reuse it instead of initializing a new one each time
The AudioContext interface represents an audio-processing graph built from audio modules linked together, each represented by an AudioNode.
An audio context controls both the creation of the nodes it contains and the execution of the audio processing, or decoding. You need to create an AudioContext before you do anything else, as everything happens inside a context. It's recommended to create one AudioContext and reuse it instead of initializing a new one each time, and it's OK to use a single AudioContext for several different audio sources and pipeline concurrently.

Well, as Get Off My Lawn pointed, I was adding by mistake multiple audio elements.
The solution was taking the code of load the song outside the playButtonFunction and only do:
let playButtonFunction = function () {
if (playstatus === 'pause') {
musicPlay();
} else if (playstatus === 'playing') {
musicPause();
}
};
But I still had one problem, with the next/previous functions. In these cases I need call the loadAudioElement function because the song is changing (when you press play/pause no, it's the same song) but with this I have the same problem again.
Well, after a bit of digging, I found that if you want to play a playlist and visualize the music all the time, YOU HAVE TO RELEASE THE OLD CONTEXT BEFORE LOAD THE NEW SONG. Not only to avoid the increase of the song volume, the cpu and memory will also get increased after 3 - 4 songs and the browser will start to run slowly depending on the machine. So:
1 - I made a global variable called clearContextAudio = false;
2 - On my next/previous functions I added this code:
if (closeAudioContext) { //MANDATORY RELEASE THE PREVIOUS RESOURCES TO AVOID OBJECT OVERLAPPING AND CPU-MEMORY USE
context.close();
context = new AudioContext();
}
loadAudioElement(audio.src).then(
visualizer(audio)
);
3 - On my visualizer(audio) function I changed:
let context = new AudioContext();
to
closeAudioContext = true; //MANDATORY RELEASE THE PREVIOUS RESOURCES TO AVOID OBJECT OVERLAPPING AND CPU-MEMORY USE
The value it's initialized to false because the first time there is no song playing, and after play a song you will always need to release the old resources, so the variable will always set to true. Now, you can skip all the times you want a song and not concern about the memory and the overlapping issues.
Hope this helps someone else trying to achieve the same thing! Regards!

Web audio api, stop sound gracefully

The web audio api furnish the method .stop() to stop a sound.
I want my sound to decrease in volume before stopping. To do so I used a gain node. However I'm facing weird issues with this where some sounds just don't play and I can't figure out why.
Here is a dumbed down version of what I do:
https://jsfiddle.net/01p1t09n/1/
You'll hear that if you remove the line with setTimeout() that every sound plays. When setTimeout is there not every sound plays. What really confuses me is that I use push and shift accordingly to find the correct source of the sound, however it seems like it's another that stop playing. The only way I can see this happening is if AudioContext.decodeAudioData isn't synchronous. Just try the jsfiddle to have a better understanding and put your headset on obviously.
Here is the code of the jsfiddle:
let url = "https://raw.githubusercontent.com/gleitz/midi-js-soundfonts/gh-pages/MusyngKite/acoustic_guitar_steel-mp3/A4.mp3";
let soundContainer = {};
let notesMap = {"A4": [] };
let _AudioContext_ = AudioContext || webkitAudioContext;
let audioContext = new _AudioContext_();
var oReq = new XMLHttpRequest();
oReq.open("GET", url, true);
oReq.responseType = "arraybuffer";
oReq.onload = function (oEvent) {
var arrayBuffer = oReq.response;
makeLoop(arrayBuffer);
};
oReq.send(null);
function makeLoop(arrayBuffer){
soundContainer["A4"] = arrayBuffer;
let currentTime = audioContext.currentTime;
for(let i = 0; i < 10; i++){
//playing at same intervals
play("A4", currentTime + i * 0.5);
setTimeout( () => stop("A4"), 500 + i * 500); //remove this line you will hear all the sounds.
}
}
function play(notePlayed, start) {
audioContext.decodeAudioData(soundContainer[notePlayed], (buffer) => {
let source;
let gainNode;
source = audioContext.createBufferSource();
gainNode = audioContext.createGain();
// pushing notes in note map
notesMap[notePlayed].push({ source, gainNode });
source.buffer = buffer;
source.connect(gainNode);
gainNode.connect(audioContext.destination);
gainNode.gain.value = 1;
source.start(start);
});
}
function stop(notePlayed){
let note = notesMap[notePlayed].shift();
note.source.stop();
}
This is just to explain why I do it like this, you can skip it, it's just to explain why I don't use stop()
The reason I'm doing all this is because I want to stop the sound gracefully, so if there is a possibility to do so without using setTimeout I'd gladly take it.
Basically I have a map at the top containing my sounds (notes like A1, A#1, B1,...).
soundMap = {"A": [], "lot": [], "of": [], "sounds": []};
and a play() fct where I populate the arrays once I play the sounds:
play(sound) {
// sound is just { soundName, velocity, start}
let source;
let gainNode;
// sound container is just a map from soundname to the sound data.
this.audioContext.decodeAudioData(this.soundContainer[sound.soundName], (buffer) => {
source = this.audioContext.createBufferSource();
gainNode = this.audioContext.createGain();
gainNode.gain.value = sound.velocity;
// pushing sound in sound map
this.soundMap[sound.soundName].push({ source, gainNode });
source.buffer = buffer;
source.connect(gainNode);
gainNode.connect(this.audioContext.destination);
source.start(sound.start);
});
}
And now the part that stops the sounds :
stop(sound){
//remember above, soundMap is a map from "soundName" to {gain, source}
let dasound = this.soundMap[sound.soundName].shift();
let gain = dasound.gainNode.gain.value - 0.1;
// we lower the gain via incremental values to not have the sound stop abruptly
let i = 0;
for(; gain > 0; i++, gain -= 0.1){ // watchout funky syntax
((gain, i) => {
setTimeout(() => dasound.gainNode.gain.value = gain, 50 * i );
})(gain, i)
}
// we stop the source after the gain is set at 0. stop is in sec
setTimeout(() => note.source.stop(), i * 50);
}

Aaah, yes, yes, yes! I finally found a lot of things by eventually bothering to read "everything" in the doc (diagonally). And let me tell you this api is a diamond in the rough. Anyway, they actually have what I wanted with Audio param :
The AudioParam interface represents an audio-related parameter, usually a parameter of an AudioNode (such as GainNode.gain). An
AudioParam can be set to a specific value or a change in value, and
can be scheduled to happen at a specific time and following a specific
pattern.
It has a function linearRampToValueAtTime()
And they even have an example with what I asked !
// create audio context
var AudioContext = window.AudioContext || window.webkitAudioContext;
var audioCtx = new AudioContext();
// set basic variables for example
var myAudio = document.querySelector('audio');
var pre = document.querySelector('pre');
var myScript = document.querySelector('script');
pre.innerHTML = myScript.innerHTML;
var linearRampPlus = document.querySelector('.linear-ramp-plus');
var linearRampMinus = document.querySelector('.linear-ramp-minus');
// Create a MediaElementAudioSourceNode
// Feed the HTMLMediaElement into it
var source = audioCtx.createMediaElementSource(myAudio);
// Create a gain node and set it's gain value to 0.5
var gainNode = audioCtx.createGain();
// connect the AudioBufferSourceNode to the gainNode
// and the gainNode to the destination
gainNode.gain.setValueAtTime(0, audioCtx.currentTime);
source.connect(gainNode);
gainNode.connect(audioCtx.destination);
// set buttons to do something onclick
linearRampPlus.onclick = function() {
gainNode.gain.linearRampToValueAtTime(1.0, audioCtx.currentTime + 2);
}
linearRampMinus.onclick = function() {
gainNode.gain.linearRampToValueAtTime(0, audioCtx.currentTime + 2);
}
Working example here
They also have different type of timings, like exponential instead of linear ramp which I guess would fit this scenario more.

Web Audio Api, setting the gain

I have been looking at the Web Audio API and am not able to get the audio gain to work. I have a fiddle set up here, so you can understand the application of the function: http://jsfiddle.net/mnu70gy3/
I am hoping to dynamically create a tone on a click event, but am not able to have that tone fade out. Below is the relevant code:
var audioCtx = new AudioContext();
var osc = {}; // set up an object for all the oscillators
var gainNodes = {}; // set up an object for all the gains
var now;
function tone(id,freq) {
// create osc / set gain / connect osc
gainNodes.id = audioCtx.createGain();
osc.id = audioCtx.createOscillator();
osc.id.connect(audioCtx.destination);
// set frequency
osc.id.frequency.value = freq;
// set gain at 1 and fade to 0 in one second
gainNodes.id.gain.value = 1.0;
gainNodes.id.gain.setValueAtTime(0, audioCtx.currentTime + 1);
// start and connect
osc.id.start(0);
osc.id.connect(audioCtx.destination);
}
Any thoughts on if this can be done?

In your code you connect oscillator to the destination twice.
Instead of connecting oscillator -> gain -> destination
gainNodes.id = audioCtx.createGain();
osc.id = audioCtx.createOscillator();
osc.id.connect(gainNodes.id);
// set frequency and gain
osc.id.frequency.value = freq;
gainNodes.id.gain.value = 1.0;
gainNodes.id.gain.setValueAtTime(0, audioCtx.currentTime + 1);
// start and connect
osc.id.start(0);
gainNodes.id.connect(audioCtx.destination);

You need to disconnect your audioCtx.destination when you click on a tile again.
https://jsfiddle.net/2dyq2ajw/
function dismissTone(id,freq) {
gainNodes.id.gain.value = 0;
osc.id.disconnect(audioCtx.destination);
}
if($(this).hasClass('xx'))
tone(thisId,thisFreq);
else
dismissTone(thisId,thisFreq);

Web Audio API AudioParam.value not logging

I am trying to figure out how to read the current value of an AudioParam. When an AudioParam is being modified by an AudioNode through the AudioNode.connect(AudioParam), it doesn't seem to effect AudioParam.value.
Here is an example:
I have an oscillator (source) connected to a gainNode (gain). I have another oscillator (mod) routed to a gainNode (modAmp). ModAmp is then connected to gain.gain. I also have a meter for gain.gain, changing the textbox to display gain.gain.value When we play the oscillator, the gain is audibly moving up and down, but the meter stays constant to the original setting. How can I get a real-time reading of the AudioParam?
http://jsfiddle.net/eliotw/3o0d0ovs/4/
(please note that you have to run the script every time you want to run an oscillator)
//create audio context
window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new window.AudioContext();
//create source and gain, connect them
var source = context.createOscillator();
var gain = context.createGain();
source.connect(gain);
//create modulator and gain for it and connect them
var mod = context.createOscillator();
var modAmp = context.createGain();
mod.connect(modAmp);
//connect modulator gain node to audio param
modAmp.connect(gain.gain);
//connect to audio context
gain.connect(context.destination);
//source values
source.frequency.value = 220;
gain.gain.value = 0.5;
//mod values
mod.frequency.value = 6;
modAmp.gain.value = 0.5;
source.start(0);
mod.start(0);
setInterval(function() {
console.log(gain.gain.value);
},
600
);

You can't, really. The only way is to connect it to a script processor node or Analyzer node and look at the output bits. The latter (using getFloatTimeDomainData) is probably how I'd do it.