I am implementing a synthesizer which uses the nodes of the audio-api to generate sound and my goal is to visualize it using p5.
I currently have a script that analyzes audio with fft and visualizes the frequencies with bars. My audio input at the moment is a locally saved song but I need to change it, so it uses the audiocontext as input.
Currently I can get the audiocontext with p5's own method getAudioContext() but then I have no clue how to set it as input for the visualization.
I know the API has a createBuffer()-Method but I haven't found a way to use it as input for p5.
var fft;
var button;
var song;
var slider;
var audiocontext;
var out;
var prue;
var source;
function preload(){
song = loadSound("src/media/Chopin - Nocturne op.9 No.2.mp3");
button = createButton("Play");
button.mousePressed(togglePlaying);
slider = createSlider(0,1,0.5,0.01);
this.audiocontext = getAudioContext();
}
function setup() {
createCanvas(windowWidth,windowHeight);
fft = new p5.FFT(0.8);
source = context.createBufferSource();
widthBand = (width / 128);
source.connect(context.destination);
}
function draw() {
background(61);
var spectrum = fft.analyze();
noStroke();
for (var i = 0; i<spectrum.length; i++) {
var amp = spectrum[i];
var y = map(amp, 0, 256, height, 0);
fill(i, 255, 255);
rect(i*widthBand,y,widthBand-2, height - y );
}
//Set Volume according to slider
audiocontext.setVolume(slider.value());
}
//Play/Pause Button
function togglePlaying(){
if(!song.isPlaying()){
song.play();
button.html("Pause");
}else if(song.isPlaying()){
song.pause();
button.html("Play");
}
}
Any help would be very appreciated!
Audiocontext is not an input himself but contains one or more input nodes (and output and connections and ...). P5 creates own Audiocontext and operates inside of that.
So, option one: build your app using p5 functionality only. It's a powerful library, all the needed tools (e.g. AudioIn(), MonoSynth() etc.) should be available.
Option two: initialize p5 first and then use p5 created audiocontext to add extra nodes, which can later be used by p5.
var cnv, fft, audiocontext, osc;
//p5 setup.
function setup() {
cnv = createCanvas();
fft = new p5.FFT(0.8);
audiocontext = getAudioContext(); //if p5.Audiocontext doesn't exist
// then new is created. Let's make
// it global.
myCustomSetup(); //now we can create our own input nodes, filters...
fft.setInput(osc); //after which we can connect fft to those created
//nodes
}
function myCustomSetup() {
//p5 audiocontext is usable here, allowing to use full WebAudioApi
//and connect all nodes, created here or by some p5 function.
osc = audiocontext.createOscillator();
}
Related
I'm very much a P5.js newbie so apologies if this is obvious.
I'm trying to build a music visualiser type animation where the amplitude of different audio files control various parameters. However I only want the use to hear the audio file containing the full mastered track.
In this simple example I want the amplitude of the 'kick', 'Hats_1' and 'tb303' files to control the size of the 3 circles. However I don't want these sounds to be output to the speakers (essentially they would be muted, but the amplitude still usable) - instead I only want the user to hear the file 'full track clip'.
`
let kick, kick_amp;
let hats_1, hats_1_amp;
let tb303, tb303_amp;
let full_track, full_track_amp;
let smoothing = 0.4;
function preload() {
kick = loadSound('kick.mp3');
hats_1 = loadSound('Hats 1.mp3');
tb303 = loadSound('303.mp3');
full_track = loadSound('full track clip.mp3');
}
function setup() {
createCanvas(600, 600)
kick.play();
kick_amp = new p5.Amplitude();
kick_amp.setInput(kick);
kick_amp.smooth(smoothing);
hats_1.play();
hats_1_amp = new p5.Amplitude();
hats_1_amp.setInput(hats_1);
kick_amp.smooth(smoothing);
tb303.play();
tb303_amp = new p5.Amplitude();
tb303_amp.setInput(tb303);
tb303_amp.smooth(smoothing);
full_track.play();
full_track_amp = new p5.Amplitude();
full_track_amp.setInput(full_track);
full_track_amp.smooth(smoothing);
}
function draw() {
background(0);
let kick_vol = kick_amp.getLevel();
ellipse( width*0.25,height/2, kick_vol*100, kick_vol*100);
let hats_1_vol = hats_1_amp.getLevel();
ellipse(width*0.75, height/2, hats_1_vol*100, hats_1_vol*100);
let tb303_vol = tb303_amp.getLevel();
ellipse(width*0.5, height*0.25, tb303_vol*100, tb303_vol*100);
}
// Click on the web page to start audio
function touchStarted() {
getAudioContext().resume();
}
`
Is this possible, and if so how do I do it?
Thanks in advance,
Joe
I've tried looking through the P5.Amplitude reference and searching SO but no dice.
Alright so I have this block of code here
ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate)
I am trying to use the audio[0].data.cpu().numpy() part which contains the audio array data.
I want to send it to the front-end, which I know how. But the problem is I don't know what to do with the data. I have done some research on converting numpy to other forms of data but still pretty lost on how to go about this.
What can I do in the front using JavaScript to turn it into audio. Or better yet using a flask server to redirect it to a get route that returns a mp3 file.
I would start looking into Audio Buffers.
Here is an example I copied from here.
This creates white noise, since we are pushing random values into the audio buffer. Here you have to use your numeric values. Please make sure how to set up the sample rate (should be defined in your python tool)
<body>
<h1>AudioBuffer example</h1>
<button>Make white noise</button>
<script>
const button = document.querySelector('button');
const myScript = document.querySelector('script');
let AudioContext = window.AudioContext || window.webkitAudioContext;
let audioCtx;
// Stereo
let channels = 2;
function init() {
audioCtx = new AudioContext();
}
button.onclick = function() {
if(!audioCtx) {
init();
}
// Create an empty two second stereo buffer at the
// sample rate of the AudioContext
let frameCount = audioCtx.sampleRate * 2.0;
let myArrayBuffer = audioCtx.createBuffer(channels, frameCount, audioCtx.sampleRate);
// Fill the buffer with white noise;
//just random values between -1.0 and 1.0
for (let channel = 0; channel < channels; channel++) {
// This gives us the actual array that contains the data
let nowBuffering = myArrayBuffer.getChannelData(channel);
for (let i = 0; i < frameCount; i++) {
// Math.random() is in [0; 1.0]
// audio needs to be in [-1.0; 1.0]
nowBuffering[i] = Math.random() * 2 - 1;
}
}
// Get an AudioBufferSourceNode.
// This is the AudioNode to use when we want to play an AudioBuffer
let source = audioCtx.createBufferSource();
// set the buffer in the AudioBufferSourceNode
source.buffer = myArrayBuffer;
// connect the AudioBufferSourceNode to the
// destination so we can hear the sound
source.connect(audioCtx.destination);
// start the source playing
source.start();
source.onended = () => {
console.log('White noise finished');
}
}
</script>
</body>
I'm trying to write a program using Javascript and the p5.js library to trigger a random image from an array whenever a peak in an audio file is detected. p5's sound library can detect the audio peak for me and then trigger a function upon that audio peak. However, I don't have much experience in Javascript so I'm not sure where to go from here. I've created an array of images and am planning on creating a function using math.Random to grab one of these images. Can I then call that function within my triggerBeat function?
Also, I've set the image as the background so that it's not within p5's draw function, so I'm trying to change the bg variable. I've preloaded the background image, and I've also got code within the preload function to allow the user to upload an audio file.
Sorry if this doesn't make a ton of sense. I'm pretty new to Javascript and I've spent most of today trying to wrap my head around it.
EDIT: updated code
var cnv, song, fft, peakDetect, img, bg;
var imageset = new Array("1.png","2.png","3.png");
function preload(){
img = loadImage("1.png");
var loader = document.querySelector(".loader");
document.getElementById("audiofile").onchange = function(event) {
if(event.target.files[0]) {
if(typeof song != "undefined") {
song.disconnect();
song.stop();
}
song = loadSound(URL.createObjectURL(event.target.files[0]));
loader.classList.add("loading");
}
}
}
function setup() {
cnv = createCanvas(900,900);
drawImage(imageset[0]);
fft = new p5.FFT();
peakDetect = new p5.PeakDetect();
setupSound();
peakDetect.onPeak(drawImage(imageset));
}
function draw() {
drawImage();
}
function drawImage(arr) {
var bg = loadImage(random(arr));
background(bg);
fill(0);
text('play', width/2, height/2);
fft.analyze();
peakDetect.update(fft);
}
function setupSound() {
cnv.mouseClicked( function() {
if (song.isPlaying() ) {
song.stop();
} else {
song.play();
}
});
}
p5 has math functions, one of which is random.
If one argument is given and it is an array, returns a random element from that array.
EDIT
As the result was more messy after answering the initial question, I updated the whole code.
var cnv, song, fft, peakDetect, img, bg;
var imageset = new Array("pic1.png","pic2.png","pic3.png", "pic4.png");
var imagesArr = [];
//next line will make p5 global. Otherwise would the p5 functions be
//accessable from p5 struct functions only.
new p5();
/*******************************************************************
* PRELOAD
* we are using for loading images/audios only
********************************************************************/
function preload(){
//load all images from 'imageset' into 'imagesArr'
for(var i=0; i<imageset.length; i++){
loadImage('../img/'+imageset[i], function(img) {
imagesArr.push(img);
});
}
// next lets load soundfile(s).
//song = loadSound("../snd/test.mp3");
// I used testfile, didn't touch nor tested your code here,
// BUT, again:
// you should only (pre)load you sounds here, setting event should go
// to the setup()
var loader = document.querySelector(".loader");
document.getElementById("audiofile").onchange = function(event) {
if(event.target.files[0]) {
if(typeof song != "undefined") {
song.disconnect();
song.stop();
}
song = loadSound(URL.createObjectURL(event.target.files[0]));
loader.classList.add("loading");
}
}
}
/*******************************************************************
* SETUP
* run once, use for initialisation.
********************************************************************/
function setup() {
//create canvas, draw initial background and text
cnv = createCanvas(900,900);
drawBackground();
text('play', width/2, height/2);
//initiate fft, peakdetect. Set event (onpeak)
fft = new p5.FFT();
peakDetect = new p5.PeakDetect();
setupSound();
peakDetect.onPeak(drawBackground);
}
/*******************************************************************
* DRAW
* endless loop. Here happens all the action.
* But you cannot draw your background here, as it is done by event.
********************************************************************/
function draw(){
//fft and peakdetecting are in use.
fft.analyze();
peakDetect.update(fft);
}
function drawBackground() {
background(255);
background(random(imagesArr));
}
function setupSound() {
cnv.mouseClicked( function() {
if (song.isPlaying() ) {
song.stop();
} else {
song.play();
}
});
}
Have yourArray[Math.floor(Math.random() * yourArray.length)] to get a random img by calling it in your triggerBeat function
I'm making an audio player with JavaScript, everything works fine until I add a sound visualizer. When I pause the song and then play it again, the sound gets more louder every time I do it, until it gets distorsionated.
I'm newbie with the HTML5 Audio API, I've tried to set the volume as a fixed value, but not works.
The code of the visualizer it's:
function visualizer(audio) {
let context = new AudioContext();
const gainNode = context.createGain();
gainNode.gain.value = 1; // setting it to 100%
gainNode.connect(context.destination);
let src = context.createMediaElementSource(audio);
let analyser = context.createAnalyser();
let canvas = document.getElementById("canvas");
canvas.width = window.innerWidth;
canvas.height = window.innerHeight;
let ctx = canvas.getContext("2d");
src.connect(analyser);
analyser.connect(context.destination);
analyser.fftSize = 2048;
let bufferLength = analyser.frequencyBinCount;
let dataArray = new Uint8Array(bufferLength);
let WIDTH = ctx.canvas.width;
let HEIGHT = ctx.canvas.height;
let barWidth = (WIDTH / bufferLength) * 1.5;
let barHeight;
let x = 0;
let color = randomColor();
function renderFrame() {
requestAnimationFrame(renderFrame);
x = 0;
analyser.getByteFrequencyData(dataArray);
ctx.clearRect(0, 0, WIDTH, HEIGHT);
for (let i = 0; i < bufferLength; i++) {
barHeight = dataArray[i];
ctx.fillStyle = color;
ctx.fillRect(x, HEIGHT - barHeight, barWidth, barHeight);
x += barWidth + 1;
}
}
musicPlay();
renderFrame();
}
And:
function musicPlay() {
status = 'playing';
audio.play();
}
So, I don't know if I'm doing something wrong on the audio analyzer, I've tried to make a global context and don't do the new AudioContext(); every time I enter on the function, also I've tried to specify a fixed volume with:
audio.volume = 1;
or with the GainNode as you can see on the function, but it's not working.
Where is my mistake and why the sound gets louder?
Regards!
--- Update 1 ---
The audio it's loaded from an URL:
function loadAudioElement(url) {
return new Promise(function (resolve, reject) {
let audio = new Audio();
audio.addEventListener('canplay', function () {
/* Resolve the promise, passing through the element. */
resolve(audio);
});
/* Reject the promise on an error. */
audio.addEventListener('error', reject);
audio.src = url;
});
}
And on my player I have:
let playButtonFunction = function () {
if (playstatus === 'pause') {
loadAudioElement(audio.src).then(
visualizer(audio)
);
} else if (playstatus === 'playing') {
musicPause();
}
};
I had a similar issue, did you try to set the audio context to a global object?
This is what I found here:
https://developer.mozilla.org/en-US/docs/Web/API/AudioContext
It's recommended to create one AudioContext and reuse it instead of initializing a new one each time
The AudioContext interface represents an audio-processing graph built from audio modules linked together, each represented by an AudioNode.
An audio context controls both the creation of the nodes it contains and the execution of the audio processing, or decoding. You need to create an AudioContext before you do anything else, as everything happens inside a context. It's recommended to create one AudioContext and reuse it instead of initializing a new one each time, and it's OK to use a single AudioContext for several different audio sources and pipeline concurrently.
Well, as Get Off My Lawn pointed, I was adding by mistake multiple audio elements.
The solution was taking the code of load the song outside the playButtonFunction and only do:
let playButtonFunction = function () {
if (playstatus === 'pause') {
musicPlay();
} else if (playstatus === 'playing') {
musicPause();
}
};
But I still had one problem, with the next/previous functions. In these cases I need call the loadAudioElement function because the song is changing (when you press play/pause no, it's the same song) but with this I have the same problem again.
Well, after a bit of digging, I found that if you want to play a playlist and visualize the music all the time, YOU HAVE TO RELEASE THE OLD CONTEXT BEFORE LOAD THE NEW SONG. Not only to avoid the increase of the song volume, the cpu and memory will also get increased after 3 - 4 songs and the browser will start to run slowly depending on the machine. So:
1 - I made a global variable called clearContextAudio = false;
2 - On my next/previous functions I added this code:
if (closeAudioContext) { //MANDATORY RELEASE THE PREVIOUS RESOURCES TO AVOID OBJECT OVERLAPPING AND CPU-MEMORY USE
context.close();
context = new AudioContext();
}
loadAudioElement(audio.src).then(
visualizer(audio)
);
3 - On my visualizer(audio) function I changed:
let context = new AudioContext();
to
closeAudioContext = true; //MANDATORY RELEASE THE PREVIOUS RESOURCES TO AVOID OBJECT OVERLAPPING AND CPU-MEMORY USE
The value it's initialized to false because the first time there is no song playing, and after play a song you will always need to release the old resources, so the variable will always set to true. Now, you can skip all the times you want a song and not concern about the memory and the overlapping issues.
Hope this helps someone else trying to achieve the same thing! Regards!
I have been looking at the Web Audio API and am not able to get the audio gain to work. I have a fiddle set up here, so you can understand the application of the function: http://jsfiddle.net/mnu70gy3/
I am hoping to dynamically create a tone on a click event, but am not able to have that tone fade out. Below is the relevant code:
var audioCtx = new AudioContext();
var osc = {}; // set up an object for all the oscillators
var gainNodes = {}; // set up an object for all the gains
var now;
function tone(id,freq) {
// create osc / set gain / connect osc
gainNodes.id = audioCtx.createGain();
osc.id = audioCtx.createOscillator();
osc.id.connect(audioCtx.destination);
// set frequency
osc.id.frequency.value = freq;
// set gain at 1 and fade to 0 in one second
gainNodes.id.gain.value = 1.0;
gainNodes.id.gain.setValueAtTime(0, audioCtx.currentTime + 1);
// start and connect
osc.id.start(0);
osc.id.connect(audioCtx.destination);
}
Any thoughts on if this can be done?
In your code you connect oscillator to the destination twice.
Instead of connecting oscillator -> gain -> destination
gainNodes.id = audioCtx.createGain();
osc.id = audioCtx.createOscillator();
osc.id.connect(gainNodes.id);
// set frequency and gain
osc.id.frequency.value = freq;
gainNodes.id.gain.value = 1.0;
gainNodes.id.gain.setValueAtTime(0, audioCtx.currentTime + 1);
// start and connect
osc.id.start(0);
gainNodes.id.connect(audioCtx.destination);
You need to disconnect your audioCtx.destination when you click on a tile again.
https://jsfiddle.net/2dyq2ajw/
function dismissTone(id,freq) {
gainNodes.id.gain.value = 0;
osc.id.disconnect(audioCtx.destination);
}
if($(this).hasClass('xx'))
tone(thisId,thisFreq);
else
dismissTone(thisId,thisFreq);