I have been having trouble taking audio data that is being recorded from a mic and sending it to the other clients in the room so that people can speak to each other in real time. I have a method of doing this, but it is inefficient and choppy...
setInterval(() => {
navigator.mediaDevices.getUserMedia({ audio: true })
.then(stream => {
const mediaRecorder = new MediaRecorder(stream);
mediaRecorder.start();
const audioChunks = [];
mediaRecorder.addEventListener("dataavailable", (event) => {
audioChunks.push(event.data);
});
mediaRecorder.addEventListener("stop", () => {
socket.emit('liveAudioToServer', audioChunks)
});
setTimeout(() => {
mediaRecorder.stop();
},2000);
});
}, 2000);
This snippet records audio and sends a buffer every two seconds so that the client side compiles it and plays it upon receiving the data. I know there's got to be another way to do this. I tried a different method, but just receive an error.
socket.on('active', () => {
if(navigator.getUserMedia) {
navigator.getUserMedia(
{audio: true},
function(stream) {
const audioContext3 = new AudioContext();
const audioSource3 = audioContext3.createMediaStreamSource(stream);
const analyser3 = audioContext3.createAnalyser();
audioSource3.connect(analyser3);
analyser3.fftSize = 256;
const bufferLength = analyser3.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
function sendAudioChunks(){
analyser3.getByteFrequencyData(dataArray);
requestAnimationFrame(sendAudioChunks);
socket.emit('liveAudioToServer', dataArray)
}
sendAudioChunks();
},
function(){ console.log("Error 003.")}
);
}
Can anyone help me?
Related
I am developing a simple webRTC application, using my own server for signaling.
the javascript code is as follow (I have removed the signaling process and unnecessary logic):
const configuration = {
iceServers: [
{
urls: [
"stun:stun1.l.google.com:19302",
"stun:stun2.l.google.com:19302",
],
},
],
iceCandidatePoolSize: 10,
};
const callerCandidatesString = "callerCandidates";
const calleeCandidatesString = "calleeCandidates";
var received_offer = null;
var offer = null;
var answer = null;
var peerConnection = null;
let localStream = null;
let remoteStream = null;
var constraints = {
optional: [],
mandatory: {
OfferToReceiveAudio: true,
OfferToReceiveVideo: true
}
}
async function startMedia(e) {
const localStream = await navigator.mediaDevices.getUserMedia({video: true, audio: true});
document.getElementById("video1").srcObject = localStream;
remoteStream = new MediaStream();
document.getElementById("video2").srcObject = remoteStream;
if (I am the caller) {
create_the_offer();
}
if (I am the callee) {
get_the_offer();
}
}
async function create_the_offer() {
peerConnection = new RTCPeerConnection(configuration);
registerPeerConnectionListeners();
localStream.getTracks().forEach(track => {
peerConnection.addTrack(track, localStream);
});
var offer = await peerConnection.createOffer(constraints);
peerConnection.setLocalDescription(offer);
peerConnection.onicecandidate = function(candidate) {
if (candidate.candidate == null) {
//save the offer in the server
--> offer: JSON.stringify(peerConnection.localDescription)},
}
}
check_if_there_is_an_answer();
}
async function get_the_offer() {
// --> retrieve the offer from the server, then
create_answer(offer_from_server);
}
async function create_answer(received_offer) {
console.log("Create PeerConnection with configuration: ", configuration);
peerConnection = new RTCPeerConnection(configuration);
registerPeerConnectionListeners();
localStream.getTracks().forEach(track => {
peerConnection.addTrack(track, localStream);
});
console.log('received offer:' + received_offer)
my_offer = new RTCSessionDescription(JSON.parse(received_offer));
peerConnection.setRemoteDescription(my_offer);
// collectIceCandidates(peerConnection, calleeCandidatesString, callerCandidatesString);
peerConnection.addEventListener("track", event => {
console.log("Got remote track:", event.streams[0]);
event.streams[0].getTracks().forEach(track => {
console.log("Add a track to the remoteStream:", track);
remoteStream.addTrack(track);
});
});
const answer = await peerConnection.createAnswer(constraints);
console.log("Created answer:", answer);
await peerConnection.setLocalDescription(answer);
peerConnection.onicecandidate = function (e) {
if (e.candidate == null) {
// --> send the answer to the server
}
}
function check_if_there_is_an_answer() {
// retrieve answer from server. this function is executed several times until the answer is received.
// when there is an aswer:
start_remote_connection(answer);
}
async function start_remote_connection(passed_answer) {
my_answer = new RTCSessionDescription(JSON.parse(passed_answer));
peerConnection.setRemoteDescription(my_answer);
peerConnection.addEventListener("track", event => {
console.log("Got remote track:", event.streams[0]);
event.streams[0].getTracks().forEach(track => {
console.log("Add a track to the remoteStream:", track);
remoteStream.addTrack(track);
});
console.log("stream remoto: " + JSON.stringify(remoteStream.getVideoTracks()));
});
document.getElementById("video1").srcObject = localStream;
document.getElementById("video2").srcObject = remoteStream;
}
async function hangUp(e) {
const tracks = document.getElementById("video1").srcObject.getTracks();
tracks.forEach(track => {
track.stop();
});
remoteStream.getTracks().forEach(track => track.stop());
peerConnection.close();
document.getElementById("video1").srcObject = null;
document.getElementById("video2").srcObject = null;
}
// collect ICE Candidates function below
async function collectIceCandidates(peerConnection, localName, remoteName) {
const candidatesCollection = null;
peerConnection.addEventListener("icecandidate", event => {
if (event.candidate) {
const json = event.candidate.toJSON();
candidatesCollection.add(json);
}
});
}
// collect ICE Candidates function above
function registerPeerConnectionListeners() {
peerConnection.addEventListener("icegatheringstatechange", () => {
console.log(
`ICE gathering state changed: ${peerConnection.iceGatheringState}`);
});
peerConnection.addEventListener("connectionstatechange", () => {
console.log(`Connection state change: ${peerConnection.connectionState}`);
});
peerConnection.addEventListener("signalingstatechange", () => {
console.log(`Signaling state change: ${peerConnection.signalingState}`);
});
peerConnection.addEventListener("iceconnectionstatechange ", () => {
console.log(
`ICE connection state change: ${peerConnection.iceConnectionState}`);
});
}
window.onload = startMedia();
If the caller uses chrome and the callee uses FireFox (on localhost, same PC) the code works fine and both users can share their screen.
output with chrome
If the caller uses FireFox and the callee uses Chrome (still on localhost) the code still works fine, but the connection is not established and users cannot see the screen of the other person. I get no error in the console.
output with FireFox
In particular, with FF I am not getting "connection state change: connecting" and then "connection state change: connected".
My guess is that FF and chrome manage the async/await differently, and somehow with FF some values are not ready when actually needed, but cannot figure out why ...
with safari (MacOS) it does not work!!
I then tried it between the computer (with chrome) and an Android phone (chrome browser). It worked the first time I test it, and then never again :(
Does anyone of you has a clue ?
I'm trying to build a website with Networked Aframe. I'm using the easyRTC adapter, which allows for audio and video streaming. I am trying to have the user initially join the room without their video or audio on. I've tried to turn off video and audio when entering, but the audio and video are still sent for a brief moment before it stops being sent. I tried to turn off the video stream but that yielded no luck as the webcam indicator is still on when the video is "off" and the video is still sent for a brief moment. Any help will be appreciated.
AFRAME.registerComponent('dynamic-room', {
init: function () {
const el = this.el;
const params = this.getUrlParams();
const networkedComp = {
room: params.room,
debug: true,
audio: true,
onConnect: onConnecth,
adapter: "easyrtc",
video: true
};
console.info('Init networked-aframe with settings:', networkedComp);
console.log(this.el);
console.log("setting it", this.el.setAttribute('networked-scene', networkedComp));
document.body.addEventListener('clientConnected', function (evt) {
onConnecth();
});
this.el.emit("connect", null, false);
},
let cameraEnabled = true;
let micEnabled = true;
let screenEnabled = false;
const cameraButton = document.getElementById('cameraaction');
const micButton = document.getElementById('micaction');
const screenButton = document.getElementById("screenshareaction");
let first = true;
function onConnecth() {
if(!first) {
console.log("something went wrong");
}
else {
document.getElementById('player').setAttribute('player-info', 'name', getUrlParams().username);
NAF.connection.adapter.enableCamera(!cameraEnabled);
NAF.connection.adapter.enableMicrophone(!micEnabled);
console.log("First Load");
cameraEnabled = !cameraEnabled;
micEnabled = !micEnabled;
disableVideo();
finishLoad();
}
}
function disableVideo(){
const stream = NAF.connection.adapter.getMediaStream();
stream.getTracks().forEach(track => {
if (track.kind === 'video') {
track.stop();
}
})
}
function finishLoad(){
cameraButton.onclick = function () {
NAF.connection.adapter.enableCamera(!cameraEnabled);
cameraEnabled = !cameraEnabled;
if(!cameraEnabled){
disableVideo();
}
};
micButton.onclick = function () {
NAF.connection.adapter.enableMicrophone(!micEnabled);
micEnabled = !micEnabled;
};
}
I am creating a chrome extension to record screen, facing an issue in converting the video recording blob into a video file, in background js video is getting recorded correctly but in content.js not able to convert the video blob to a video file
I am creating a chrome extension to record screen, facing an issue in converting the video recording blob into a video file, in background js video is getting recorded correctly but in content.js not able to convert the video blob to a video file
function startRecording() {
var constraints = {
audio: true,
video: true,
maxframeRate: fps,
};
navigator.mediaDevices.getDisplayMedia(constraints).then(function (stream) {
let output = new MediaStream();
if (output.getAudioTracks().length == 0) {
// Get microphone audio (system audio is unreliable & doesn't work on Mac)
if (micable) {
micsource.connect(destination);
output.addTrack(destination.stream.getAudioTracks()[0]);
}
} else {
syssource = audioCtx.createMediaStreamSource(stream);
if (micable) {
micsource.connect(destination);
}
syssource.connect(destination);
output.addTrack(destination.stream.getAudioTracks()[0]);
}
output.addTrack(stream.getVideoTracks()[0]);
mediaConstraints = {
audio: true,
video: true,
mimeType: "video/webm;codecs=vp8,opus",
};
mediaRecorder = new MediaRecorder(stream, mediaConstraints);
mediaRecorder.start(1000);
var recordedBlobs = [];
let writer = "";
mediaRecorder.ondataavailable = (event) => {
if (event.data && event.data.size > 0) {
recordedBlobs.push(event.data);
}
console.log("recordedBlobs", recordedBlobs);
};
mediaRecorder.onstop = () => {
chrome.tabs.getSelected(null, (tab) => {
chrome.tabs.sendMessage(tab.id, {
message: "download-video",
obj: {
blobs: recordedBlobs,
},
// camerasize: camerasize
});
});
endRecording(stream, writer, recordedBlobs);
};
stream.getVideoTracks()[0].onended = function () {
cancel = false;
mediaRecorder.stop();
};
});
}
content.js
function convertVideoBlobToVideo(obj) {
let chunk = obj.blobs;
// mediaRecorder.onstop = () => {
var superBuffer;
superBuffer = new Blob(chunks, {
type: "video/webm",
});
chunks = [];
// Create a video or audio element
// that stores the recorded media
const recordedMedia = document.createElement("video");
recordedMedia.controls = true;
const recordedMediaURL = URL.createObjectURL(superBuffer);
recordedMedia.src = recordedMediaURL;
const downloadButton = document.createElement("a");
downloadButton.download = "Recorded-Media";
downloadButton.href = recordedMediaURL;
downloadButton.innerText = "Download it!";
downloadButton.onclick = () => {
URL.revokeObjectURL(recordedMedia);
};
document.body.appendChild(recordedMedia, downloadButton);
// };
}
Im not sure what is wrong. The caller and callee both receive the same stream that is getting set in local video but it wont set the remote video.
I have seen it worked before when I clicked the call button twice but I changed the code and dont know how to recreate it.
Any help is appreciated. Thank you.
video.js
//Starts by calling the user
const callUser = (socketId, mySocket) => {
navigator.getUserMedia({ video: true, audio: true }, stream => {
console.log("My Stream: ", stream)
const localVideo = document.getElementById("local-video");
localVideo.srcObject = stream;
localStream = stream
createAndSendOffer(socketId, mySocket);
})
}
//Create Offer
const createAndSendOffer = async (socketId, mySocket) => {
peerConnection = new RTCPeerConnection()
peerConnection.ontrack = function(event) {
console.log("Remote Stream", event.streams[0])
const remoteVideo = document.getElementById("remoteVideo")
if(remoteVideo.srcObject !== event.stream[0]) {
remoteVideo.srcObject = event.streams[0];
}
}
localStream.getTracks().forEach(track => peerConnection.addTrack(track, localStream))
await peerConnection.createOffer( offer => {
peerConnection.setLocalDescription(new RTCSessionDescription(offer),
function() {
mySocket.emit("callUser", {offer,to: socketId});
})
})
}
//Callee Is getting called
const answerCall = (data, mySocket, inCall) => {
navigator.getUserMedia({ video: true, audio: true }, stream => {
console.log("My Stream: ", stream)
const localVideo = document.getElementById("local-video");
localVideo.srcObject = stream;
localStream = stream
createAndSendAnswer(data, mySocket, inCall);
})
}
//Create Answer
const createAndSendAnswer = async (data, mySocket, inCall) => {
peerConnection = new RTCPeerConnection()
peerConnection.ontrack = function(event) {
console.log("Remote Stream", event.streams[0])
const remoteVideo = document.getElementById("remoteVideo")
if(remoteVideo.srcObject !== event.stream[0]) {
console.log(event.streams[0])
remoteVideo.srcObject = event.streams[0];
}
inCall(true)
}
localStream.getTracks().forEach(track => peerConnection.addTrack(track, localStream))
await peerConnection.setRemoteDescription(new RTCSessionDescription(data.offer))
await peerConnection.createAnswer(answer => {
peerConnection.setLocalDescription(new RTCSessionDescription(answer),
function() {
mySocket.emit("makeAnswer", {answer, to: data.socket});
})
})
}
const startSockets = (socket, inCall) => {
socket = socket
socket.on("gettingCalled", data => {
console.log("You are getting called with socketId", data.socket)
answerCall(data, socket, inCall);
})
socket.on("answerMade", async data => {
await peerConnection.setRemoteDescription(new RTCSessionDescription(data.answer))
})
}
class Video extends Component {
constructor(props){
super(props);
this.state={
friends: [],
mySocket: [],
inCall: false
}
}
componentDidMount() {
if(this.props.user) {
startSockets(this.props.user.socket, this.inCallChange)
}
}
inCallChange = boolean => this.setState({inCall: boolean})
callUser = socketId => callUser(socketId, this.state.mySocket, this.state.inCall)
render() {
if(!this.props.user) {
return (<div>No User</div>)
} else {
return (
<div>`enter code here`
<div className="video-chat-container">
<div className="video-container">
<video
autoPlay
className="remote-video"
id="remoteVideo"
></video>
<video
autoPlay
muted
className="local-video"
id="local-video"
></video>
</div>
</div>
</div>
);
}
}
}
export default withRouter(connect(mapStateToProps)(Video))
Update
Im not sure why this is the case but I have to send just the offer and receive the answer again. For now, I resend an offer createAndSendOffer(), starting the process again. I dont think this is the best practice and if you find a better solution that would be great :)
first check your pc.ontrack() firing or not. pc.ontrack() event is fired after pc.addTrack() and remotedescription is set.and also make sure tracks are added before creating answer/offer
I want to record the video and background music without the microphone permission on Mobile devices.
It just like TikTok.
I know how to record stream by MediaRecorder.
But MediaRecorder only can record one stream, I don't know how to mix music in the stream.
My code:
record = (renderer, fps = 25, ms = 3000, mediaElement) => new Promise(resolve => {
if (this.recording) {
resolve(false);
}
this.recording = true;
try {
const stream = mediaElement.captureStream(fps);
const recorder = new MediaRecorder(stream);
const chunks = [];
recorder.ondataavailable = e => chunks.push(e.data);
recorder.onstop = () => resolve(new Blob(chunks));
recorder.start();
setTimeout(() => recorder.stop(), ms);
} catch (err) {
console.log('failed to record', err);
resolve(false);
}
this.recording = false;
});
Thank you for your appreciation.
I find the answer.
Need to change the audio source to stream and mix it to one stream.
It work!
record = (stream, fps = 25, ms = 3000) => new Promise(resolve => {
if (this.recording) {
resolve(false);
}
this.recording = true;
try {
const audio = new Audio();
audio.src = require('someMusic.mp3');
const context = new AudioContext();
const backgroundMusic = context.createMediaElementSource(audio);
audio.volume = 0.8;
audio.play();
const mixedOutput = context.createMediaStreamDestination();
backgroundMusic.connect(mixedOutput);
const videoTrack = stream.getVideoTracks()[0];
const mixedTracks = mixedOutput.stream.getAudioTracks()[0];
const streamN = new MediaStream([mixedTracks, videoTrack]);
const recorder = new MediaRecorder(streamN);
const chunks = [];
recorder.ondataavailable = e => chunks.push(e.data);
recorder.onstop = () => resolve(new Blob(chunks));
recorder.start();
setTimeout(() => recorder.stop(), ms);
} catch (err) {
console.log('failed to record', err);
resolve(false);
}
this.recording = false;
});