Send audio file over microphone using javascript - javascript

I want to send a audio file to a server (in my case discord) easly as if it was comming from the microphone
I found this code at Send sound through microphone in javascript and modified it to try to fit my use case, but I still cannot get it to work.
navigator.mediaDevices.getUserMedia = () => {
const audioContext = new AudioContext();
return fetch('http://127.0.0.1:8000/enemey.ogg',{mode: 'no-cors'})
.then((response) => response.arrayBuffer())
.then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer))
.then((audioBuffer) => {
const audioBufferSourceNode = audioContext.createBufferSource();
const mediaStreamAudioDestinationNode = audioContext.createMediaStreamDestination();
audioBufferSourceNode.buffer = audioBuffer;
// Maybe it makes sense to loop the buffer.
audioBufferSourceNode.loop = true;
audioBufferSourceNode.start();
audioBufferSourceNode.connect(mediaStreamAudioDestinationNode);
return mediaStreamAudioDestinationNode.stream;
});
};
any Ideas? I cannot find a fix for this, and the error is
[AudioActionCreators] unknown getUserMedia error: EncodingError
by discord
(all of this is done with the console, not a external program)

Related

How to access input/output of worklet node from main file?

I have created a real time voice chat application for a game I am making. I got it to work completely fine using audiocontext.createScriptProcessor() method.
Here's the code, I left out parts that weren't relevant
//establish websocket connection
const audioData = []
//websocket connection.onMessage (data) =>
audioData.push(decodeBase64(data)) //push audio data coming from another player into array
//on get user media (stream) =>
const audioCtx = new AudioContext({latencyHint: "interactive", sampleRate: 22050,})
const inputNode = audioCtx.createMediaStreamSource(stream)
var processor = audioCtx.createScriptProcessor(2048, 1, 1);
var outputNode = audioCtx.destination
inputNode.connect(tunerNode)
processor.connect(outputNode)
processor.onaudioprocess = function (e) {
var input = e.inputBuffer.getChannelData(0);
webSocketSend(input) //send microphone input to other sockets via a function set up in a different file, all it does is base 64 encode then send.
//if there is data from the server, play it, else, play nothing
var output
if(audioData.length > 0){
output = audioData[0]
audioData.splice(0,1)
}else output = new Array(2048).fill(0)
};
the only issue is that the createScriptProccessor() method is deprecated. As recommended, I attempted to do this using Audio Worklet Nodes. However I quickly ran into a problem. I can't access the user's microphone input, or set the output from the main file where the WebSocket connection is.
Here is my code for main.js:
document.getElementById('btn').onclick = () => {createVoiceChatSession()}
//establish websocket connection
const audioData = []
//webSocket connection.onMessage (data) =>
audioData.push(data) //how do I get this data to the worklet Node???
var voiceChatContext
function createVoiceChatSession(){
voiceChatContext = new AudioContext()
navigator.mediaDevices.getUserMedia({audio: true}).then( async stream => {
await voiceChatContext.audioWorklet.addModule('module.js')
const microphone = voiceChatContext.createMediaStreamSource(stream)
const processor = new AudioWorkletNode(voiceChatContext, 'processor')
microphone.connect(processor).connect(voiceChatContext.destination)
}).catch(err => console.log(err))
}
Here is my code for module.js:
class processor extends AudioWorkletProcessor {
constructor() {
super()
}
//copies the input to the output
process(inputList, outputList) { // how do I get the input list data (the data from my microphone) to the main file so I can send it via websocket ???
for(var i = 0; i < inputList[0][0].length; i++){
outputList[0][0][i] = inputList[0][0][i]
outputList[0][1][i] = inputList[0][1][i]
}
return true;
}
}
registerProcessor("processor", processor);
So I can record and process the input, but I can't send input via WebSocket or pass in data that is coming from the server to the worklet node because I can't access the input list or output list from the main file where the WebSocket connection is. Does anyone know a way to work around this? Or is there a better solution that doesn't use audio worklet nodes?
Thank you to all who can help!
I figured it out, all I needed to do was use the port.onmessage method to exchange data between the worklet and the main file.
processor.port.onmessage = (e) => {//do something with e.data}

JS: play multiple audio sources simultaneously when loaded

I have a web project (vanilla HTML/CSS/JS only) with three audio sources. The idea is for all three to play simultaneously, but I noticed on mobile that the files were playing out of sync (i.e. one source would start, then a few ms later the second would start, then the third). I believe they are playing due to the individual files playing as soon as they're loaded, so I would like to request that once all files have loaded that the play() method is called on all three at the same time,
What would be the best way to achieve this using vanilla JS?
Example: https://jacksorjacksor.xyz/soundblocks/
Repo: https://github.com/jacksorjacksor/jacksorjacksor/tree/master/soundblocks
TIA!
Rich
MediaElements are meant for normal playback of media and aren't optimized enough to get low latency. The best is to use the Web Audio API, and AudioBuffers.
You will first fetch each file's data in memory, then decode the audio data from these, and once all the audio data has been decoded, you'll be able to schedule playing all at the same precise moment:
(async() => {
const urls = [ "layer1_big.mp3", "layer2_big.mp3", "layer3_big.mp3" ]
.map( (url) => "https://cdn.jsdelivr.net/gh/jacksorjacksor/jacksorjacksor/soundblocks/audio/" + url );
// first, fetch each file's data
const data_buffers = await Promise.all(
urls.map( (url) => fetch( url ).then( (res) => res.arrayBuffer() ) )
);
// get our AudioContext
const context = new (window.AudioContext || window.webkitAudioContext)();
// decode the data
const audio_buffers = await Promise.all(
data_buffers.map( (buf) => context.decodeAudioData( buf ) )
);
// to enable the AudioContext we need to handle a user gesture
const btn = document.querySelector( "button" );
btn.onclick = (evt) => {
const current_time = context.currentTime;
audio_buffers.forEach( (buf) => {
// a buffer source is a really small object
// don't be afraid of creating and throwing it
const source = context.createBufferSource();
// we only connect the decoded data, it's not copied
source.buffer = buf;
// in order to make some noise
source.connect( context.destination );
// make it loop?
//source.loop = true;
// start them all 0.5s after we began, so we're sure they're in sync
source.start( current_time + 0.5 );
} );
};
btn.disabled = false;
})();
<button disabled>play</button>

How can we use an stream with Web Speech API in 2020?

I want to feed a recorded mp3 audio to Web Speech API (feed any stream to the SpeechRecognition).
I come up with some answers on web that tell us this is impossible but I found a comment on Stack overflow who says this is possible and provide a code to prove his claim.
The first comment here: https://stackoverflow.com/a/46448941/13849624
He said:
You can feed any stream to the SpeechRecognition, as the audio is
captured to the microphone of the computer.
But the fiddle he provided is not working...
Here is the code:
const recognition = new webkitSpeechRecognition();
const src = "https://audio00.forvo.com/phrases/mp3/c/8/c81c2618ae7246e5148df5892df485a2_9261923_39.mp3";
const transcript = document.getElementById("transcript");
const audio = new Audio();
recognition.continuous = true;
recognition.interimResults = true;
recognition.onresult = function(event) {
if (event.results[0].isFinal) {
console.log(event.results[0][0].transcript);
transcript.textContent = event.results[0][0].transcript;
recognition.stop();
}
}
recognition.onaudiostart = e => {
console.log("audio capture started");
}
recognition.onaudioend = e => {
console.log("audio capture ended");
}
audio.oncanplay = () => {
recognition.start();
audio.play();
}
fetch(`https://query.yahooapis.com/v1/public/yql?q=select * from data.uri where url="${src}"&format=json&callback=`)
.then(response => response.json())
.then(({
query: {
results: {
url
}
}
}) =>
audio.src = url
)
.catch(err => console.error(err));
<div id="transcript"></div>
I can't understand what he did...
How can we use an stream with Web Speech API?
as you can also see in the comments from your link to https://stackoverflow.com/a/46383699/4278324:
this does not actually stream audio to the Web Speech API implementation
it loads the audio and then plays the audio (through speakers) - so the recognition will only work in real time AND if there is a physical speaker AND if the microphone is able to record the speakers sound

WebRTC transmit high audio stream sample rate

Given a WebRTC PeerConnection between two clients, one client is trying to send an audio MediaStream to another.
If this MediaStream is an Oscillator at 440hz - everything works fine. The audio is very crisp, and the transmission goes through correctly.
However, if the audio is at 20000hz, the audio is very noisy and crackly - I expect to hear nothing, but I hear a lot of noise instead.
I believe this might be a problem of sample rate sent in the connection, maybe its not sending the audio at 48000samples/second like I expect.
Is there a way for me to increase the sample rate?
Here is a fiddle to reproduce the issue:
https://jsfiddle.net/mb3c5gw1/9/
Minimal reproduction code including a visualizer:
<button id="btn">start</button>
<canvas id="canvas"></canvas>
<script>class OscilloMeter{constructor(a){this.ctx=a.getContext("2d")}listen(a,b){function c(){g.getByteTimeDomainData(j),d.clearRect(0,0,e,f),d.beginPath();let a=0;for(let c=0;c<h;c++){const e=j[c]/128;var b=e*f/2;d.lineTo(a,b),a+=k}d.lineTo(canvas.width,canvas.height/2),d.stroke(),requestAnimationFrame(c)}const d=this.ctx,e=d.canvas.width,f=d.canvas.height,g=b.createAnalyser(),h=g.fftSize=256,j=new Uint8Array(h),k=e/h;d.lineWidth=2,a.connect(g),c()}}</script>
btn.onclick = e => {
const ctx = new AudioContext();
const source = ctx.createMediaStreamDestination();
const oscillator = ctx.createOscillator();
oscillator.type = 'sine';
oscillator.frequency.setValueAtTime(20000, ctx.currentTime); // value in hertz
oscillator.connect(source);
oscillator.start();
// a visual cue of AudioNode out (uses an AnalyserNode)
const meter = new OscilloMeter(canvas);
const pc1 = new RTCPeerConnection(),
pc2 = new RTCPeerConnection();
pc2.ontrack = ({
track
}) => {
const endStream = new MediaStream([track]);
const src = ctx.createMediaStreamSource(endStream);
const audio = new Audio();
audio.srcObject = endStream;
meter.listen(src, ctx);
audio.play()
};
pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
pc1.oniceconnectionstatechange = e => console.log(pc1.iceConnectionState);
pc1.onnegotiationneeded = async e => {
try {
await pc1.setLocalDescription(await pc1.createOffer());
await pc2.setRemoteDescription(pc1.localDescription);
await pc2.setLocalDescription(await pc2.createAnswer());
await pc1.setRemoteDescription(pc2.localDescription);
} catch (e) {
console.error(e);
}
}
const stream = source.stream;
pc1.addTrack(stream.getAudioTracks()[0], stream);
};
Looking around in the webrtc demo i found this: https://webrtc.github.io/samples/src/content/peerconnection/audio/ in the example they show a dropdown where you can setup the audio codec. I think this is your solution.

Obtain MediaStream from input device

Looking for experience working with media devices:
I'm working on recording on cache and playback from Microphone source; Firefox & Chrome using HTML5.
This is what I've so far:
var constraints = {audio: true, video: false};
var promise = navigator.mediaDevices.getUserMedia(constraints);
I've been checking on official documentation from MDN on getUserMedia
but nothing related to storage the audio from the constraint to cache.
No such question has been asked previously at Stackoverflow; I'm wondering if's possible.
Thanks you.
You can simply use the MediaRecorder API for such task.
In order to record only the audio from your video+audio gUM stream, you will need to create a new MediaStream, from the gUM's audioTrack:
// using async for brevity
async function doit() {
// first request both mic and camera
const gUMStream = await navigator.mediaDevices.getUserMedia({video: true, audio: true});
// create a new MediaStream with only the audioTrack
const audioStream = new MediaStream(gUMStream.getAudioTracks());
// to save recorded data
const chunks = [];
const recorder = new MediaRecorder(audioStream);
recorder.ondataavailable = e => chunks.push(e.data);
recorder.start();
// when user decides to stop
stop_btn.onclick = e => {
recorder.stop();
// kill all tracks to free the devices
gUMStream.getTracks().forEach(t => t.stop());
audioStream.getTracks().forEach(t => t.stop());
};
// export all the saved data as one Blob
recorder.onstop = e => exportMedia(new Blob(chunks));
// play current gUM stream
vid.srcObject = gUMStream;
stop_btn.disabled = false;
}
function exportMedia(blob) {
// here blob is your recorded audio file, you can do whatever you want with it
const aud = new Audio(URL.createObjectURL(blob));
aud.controls = true;
document.body.appendChild(aud);
document.body.removeChild(vid);
}
doit()
.then(e=>console.log("recording"))
.catch(e => {
console.error(e);
console.log('you may want to try from jsfiddle: https://jsfiddle.net/5s2zabb2/');
});
<video id="vid" controls autoplay></video>
<button id="stop_btn" disabled>stop</button>
And as a fiddle since stacksnippets don't work very well with gUM...

Categories

Resources