I'm trying to capture audio from a user's microphone and send it to a server where it will get sent to Google's Speech-to-Text-API for translation. I'm accessing the audio using navigator.mediaDevices.GetuserMedia() which I capture using a MediaRecorder object. When I run the following code I get an error from Google that says "INVALID_ARGUMENT: RecognitionAudio not set." I'm not sure how to set it as the relavant page (https://cloud.google.com/speech-to-text/docs/reference/rest/v1/RecognitionAudio) doesn't say much about it.
Relevant client side code that runs after user presses stop:
mediaRecorder.onstop = function(e) {
var blob = new Blob(chunks, { type : 'audio/flac' });
var reader = new FileReader();
reader.readAsBinaryString(blob);
reader.onloadend = function() {
base64data = reader.result;
writeBinaryFile(base64data)
}
chunks = []; //array to store recording
}
//asynchronous binary file write
function writeBinaryFile(content) {
$.ajax({
type: "POST",
url: "/voice_api",
data: { content: content }
}).done(function(data) {
// TODO: display success status somewhere
});
Server side code running node.js:
app.post("/voice_api", (req, res) => {
const audioBytes = req.body;
// The audio file's encoding, sample rate in hertz, and BCP-47 language code
const audio = {
content: audioBytes,
};
const config = {
languageCode: 'en-US'
};
const request = {
audio: audio,
config: config
};
// Detects speech in the audio file
client
.recognize(request)
.then(data => {
const response = data[0];
const transcription = response.results
.map(result => result.alternatives[0].transcript)
.join('\n');
console.log(`Transcription: ${transcription}`);
res.send(transcription);
})
.catch(err => {
console.error('ERROR:', err);
});
});
If I run the server code with the line "const audioBytes = req.body;" changed to "const audioBytes = req.body.content;" I get an error message that there is bad encoding. I'm not sure if I'm encoding it properly on the client side or if I'm accessing it properly on the server side. Any help would be appreciated. Thanks!
const config = {
// "enableAutomaticPunctuation": true,
"encoding": "LINEAR16",
"languageCode": "en-US",
"model": "default",
"sampleRateHertz": 44100,
audioChannelCount: 2,
enableSeparateRecognitionPerChannel: true,
};
Related
I would like to know how to share a base64 string as an image over WhatsApp primarily.
I have a react app that generates a QR code and renders it using <img src={base64String} />. Now that img is wrapped around an <a><a/> tag with download attribute, it is working fine. But what I want to accomplish is to be able to have a share button that can share the image on WhatsApp.
I don't want to store it locally or involve file reading if possible because I had a lot of issues implementing it since the QR code will be discarded after download and need not be stored as a file. But if that is the right way I'm ok to work with it.
I tried creating a blob and sharing by using atob(), but that returned an error saying "string not encoded properly" which I didn't want to dig into since it was able to render it as an image and also download by which I assumed the base64 string was correct.
code for Generator:
const GenerateQRCode = (e) => {
e.preventDefault()
QRCode.toDataURL(value, {
margin: 2,
color: {
dark: '#000000',
light: '#ffffff'
},
width: 2000,
height:2000
}, (err, value) => {
if (err) return console.error(err)
console.log(value)
setQr(value)
console.log(data);
})
}
const handleShare =(e)=>{
}
code for that renders the image:
{qr && <>
<img src={qr} alt='' className='qrimg' id='can'/>
<a href={qr} download={`${state.name}.png`} ><button value="Download" >Download</button></a>
</>
}
<button type="" onClick={handleShare}></button>
So, I want that share button to send an image over WhatsApp. My previous attempts for handleShare were messy so I wanted to get fresh inputs or a Solution to do this the right way.
Finally success, that was a hard one! You have to upload the file as a Media into the API and then use that uploaded Media ID to send. Check out the code and replace "xxx" with your values.
Note: Your QR code must be a PNG for this to work. This is not a snippet because it doesn't make sense to run
<img src="data:image/png;base64,iVBORw0KGg...VERY_LONG_STRING_HERE...">
<script>
// Your auth token. Keep it scret, keep it safe.
const Authorization = "Bearer xxx"
const type = 'image/png' // Image type (cannot be gif due to API restriction!)
const baseUrl = "https://graph.facebook.com/v15.0/xxx" // Phone number of sender
const recipient = "xxx" // Phone number of recipient
/** This function creates a File object from an img element which has a base64 image */
function fileFromImg(img) {
var byteString = atob(img.src.split(',')[1]);
var arrayBuffer = new ArrayBuffer(byteString.length);
var arr = new Uint8Array(arrayBuffer);
for (var i = 0; i < byteString.length; i++) {
arr[i] = byteString.charCodeAt(i);
}
const blob = new Blob([arrayBuffer], { type });
return new File([blob], "qr.png", { type })
}
/** This function will upload a File and call the given callback with the result */
function sendFile(f, callback) {
const url = baseUrl + "/media";
const formData = new FormData()
formData.append('file', f)
formData.append('type', type)
formData.append('messaging_product', "whatsapp")
fetch(url, {
method: 'POST',
body: formData,
headers: { Authorization }
})
.then(r => r.json())
.then(callback)
.catch(console.error)
}
/** This takes in a successful upload and sends that forward to receiver */
function handleResponse(response) {
// When it works response will look like {"id": "xxx"}
fetch(baseUrl + "/messages", {
method: 'POST',
body: JSON.stringify({
"messaging_product": "whatsapp",
"recipient_type": "individual",
"to": recipient,
"type": "image",
"image": response
}),
headers: { Authorization, "Content-Type": "application/json" }
})
.then(r => r.json())
.then(r => console.log(r))
.catch(console.error)
}
// I choose to trigger the sending when anything is clicked
document.addEventListener("click", () => {
const img = document.querySelector("img")
const file = fileFromImg(img)
sendFile(file, handleResponse)
})
</script>
The recipient will see this;
I was successful in storing images in mongodb using the base64 url. But when I tried to do so with PDFs, it gave an url which does not work. Wait, let me explain, when I put the image base64 url in the req.body of the POST request, the special signs would get disappeared, so I tried encodeURIComponent() method to make it error free. After that I found that storing the huge string in mongodb was too short to fit in the db, so I tried: app.use(express.json({limit: '50mb'})); app.use(express.urlencoded({limit: '50mb', extended: false })); and It worked! but when the client requests the base64 url, it would come encoded, so I put decodeURIComponent() to decode it and was not a great issue nd I got the desired result, yet with the Image one.
The main issue issue is when it comes to PDF. I don't know why it's happening with PDF only! when I make base64 url in CLIENT side and test it, it works fine, but when it comes to server side, all the mess happens. please help me deal with this.
Note: "I don't want to use Gridfs, formidabe, multer etc for file things"
here's my piece of code:
$('#seasonForm').submit(async function (e) {
e.preventDefault();
const form = $(this);
const ImgFile = document.getElementById('seasonThumbnail').files[0];
const PDFFile = document.getElementById('seasonPDF').files[0];
const imgurl = encodeURIComponent(await getBase64(ImgFile));
const PDFurl = encodeURIComponent(await getBase64(PDFFile));
const url = '/uploadSeason';
$.ajax({
type: "POST",
url: url,
data: form.serialize()+`&Version=<%- NxtSeasons %>&image=${imgurl}&PDF=${PDFurl}`,
success: data => {
console.log(data.message);
if (data.status == "error") {
showIt(".alert", data.message, "error");
} else {
showIt(".alert", data.message, "success");
}
}
});
})
wait, now don't get confused with getBase64() and showIt. these are my functions. getBase64() is a promice which returns base64 url of the file and showIt() is type of alert which I made. Now if you don't know what is base64 url, this is the getBase64 one's code:
const getBase64 = (file) => {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
reader.onerror = error => reject(error);
});
}
My nodejs code:
app.post("/uploadSeason", admin, async (req, res) => {
try {
const { Name, Desctiption, Version, image, PDF } = req.body;
const newSeason = new Season({
Name,
Desctiption,
Version,
image: encodeURIComponent(image),
PDF: encodeURIComponent(PDF),
});
await newSeason.save();
res.json({
status: "success",
message: "Season added successfully"
});
} catch (e) {
console.log(e);
res.json({
status: "error",
message: e
});
}
});
I will try to make this as concise as possible.
OVERVIEW
Using MediaRecorder I am sending blobs of my stream to my node server. From my node server, I am sending Uint8Arrays' back to the client. I then set my blob to a Uint8Array retrieved from my node server with a type of video/webm; codecs=vp9.
Here is the code in respects to my overview:
When the client starts the stream:
successCallback(stream) {
if (MediaRecorder.isTypeSupported('video/webm;codecs=vp9')) {
this.options = { mimeType: 'video/webm; codecs=vp9' };
}
this.video.srcObject = stream;
this.video.play();
this.mediaRecorder = new MediaRecorder(stream, this.options);
this.mediaRecorder.ondataavailable = this.handleDataAvailable.bind(this);
this.mediaRecorder.start(3000);
}
handleDataAvailable(blob) {
// POST/PUT "Blob" using FormData/XHR2
this.signalHub.sendStream(blob.data);
}
public sendStream(stream) {
this.socket.emit('send-stream', stream);
}
When the server receives the stream from the client after sendStream() method:
socket.on('send-stream', (newStreamBuff) => {
stream = Buffer.concat([stream, new Uint8Array(newStreamBuff)]);
socket.emit('stream-stream', new Uint8Array(newStreamBuff));
});
Notice, on send we then send the blob as a Uint8Array back to the client
Here is what happens in order when the client receives the Uint8Array:
public getHubStream = () => {
return Observable.create((observer) => {
this.socket.on('stream-stream', (data) => {
this.arrBuffer.next(data);
}, error => {
console.log(error);
});
});
}
this.signalHub.currentarrBuffer.subscribe((data: Uint8Array) => {
this.arrayBuffer = data;
if (this.arrayBuffer != undefined) {
this.blob = new Blob([data], { type: 'video/webm; codecs=vp9' });
this.recordedVideo.src = URL.createObjectURL(this.blob);
console.log(this.arrayBuffer);
console.log(this.blob);
console.log(this.recordedVideo.src);
console.log(this.recordedVideo.type);
this.recordedVideo.play().catch(function (error) {
console.log(error);
});
}
}, error => {
console.log(error);
});
The last set of code is Angular code or Typescript: THIS data
this.arrBuffer.next(data);
Is also this data:
this.arrayBuffer = data;
Just in case there is some confusion on how the data is being moved around.
Notice the console.logs throughout the code. Here is what it looks like in my console..
I've tried.. different array types, different codes, different "types", a lot. I'm not sure why my blob doesn't work correctly in when trying to play the video. DomException error is incredibly vague. I'm not having issues with receiving and sending from node server.
On Opera I get the original vague error.
On Firefox I get DomException error but a little more information.. which is kind of good.
DOMException: "The media resource indicated by the src attribute or assigned media provider object was not suitable."
My url for the page, in which the video is played is on: http://localhost:4200/#/intro which I couldn't see why this would be a problem anyway.
Using Node.js, I am trying to get an image from a URL and upload that image to another service without saving image to disk. I have the following code that works when saving the file to disk and using fs to create a readablestream. But as I am doing this as a cron job on a read-only file system (webtask.io) I'd want to achieve the same result without saving the file to disk temporarily. Shouldn't that be possible?
request(image.Url)
.pipe(
fs
.createWriteStream(image.Id)
.on('finish', () => {
client.assets
.upload('image', fs.createReadStream(image.Id))
.then(imageAsset => {
resolve(imageAsset)
})
})
)
Do you have any suggestions of how to achieve this without saving the file to disk? The upload client will take the following
client.asset.upload(type: 'file' | image', body: File | Blob | Buffer | NodeStream, options = {}): Promise<AssetDocument>
Thanks!
How about passing the buffer down to the upload function? Since as per your statement it'll accept a buffer.
As a side note... This will keep it in memory for the duration of the method execution, so if you call this numerous times you might run out of resources.
request.get(url, function (res) {
var data = [];
res.on('data', function(chunk) {
data.push(chunk);
}).on('end', function() {
var buffer = Buffer.concat(data);
// Pass the buffer
client.asset.upload(type: 'buffer', body: buffer);
});
});
I tried some various libraries and it turns out that node-fetch provides a way to return a buffer. So this code works:
fetch(image.Url)
.then(res => res.buffer())
.then(buffer => client.assets
.upload('image', buffer, {filename: image.Id}))
.then(imageAsset => {
resolve(imageAsset)
})
well I know it has been a few years since the question was originally asked, but I have encountered this problem now, and since I didn't find an answer with a comprehensive example I made one myself.
i'm assuming that the file path is a valid URL and that the end of it is the file name, I need to pass an apikey to this API endpoint, and a successful upload sends me back a response with a token.
I'm using node-fetch and form-data as dependencies.
const fetch = require('node-fetch');
const FormData = require('form-data');
const secretKey = 'secretKey';
const downloadAndUploadFile = async (filePath) => {
const fileName = new URL(filePath).pathname.split("/").pop();
const endpoint = `the-upload-endpoint-url`;
const formData = new FormData();
let jsonResponse = null;
try {
const download = await fetch(filePath);
const buffer = await download.buffer();
if (!buffer) {
console.log('file not found', filePath);
return null;
}
formData.append('file', buffer, fileName);
const response = await fetch(endpoint, {
method: 'POST', body: formData, headers: {
...formData.getHeaders(),
"Authorization": `Bearer ${secretKey}`,
},
});
jsonResponse = await response.json();
} catch (error) {
console.log('error on file upload', error);
}
return jsonResponse ? jsonResponse.token : null;
}
Background
I want to record students speaking and then upload their recordings automatically to Google drive.
Current state of affairs
I have client side code which can produce a blob containing .ogg recording.
var blob = new Blob(chunks, { 'type' : 'audio/ogg; codecs=opus' });
I can send the blob to a Google apps script standalone script in a doPost() request. In this case I'm using axios with a promise.
axios.post("https://script.google.com/macros/s/SOME_ID/exec", blob)
.then((response)=>{
console.log(response)
}).catch(error =>{
console.log(error)
})
The blob arrives at the standalone script.
The standalone Google apps script
function doPost(e) {
var params = JSON.stringify(e.parameters);
createNewSoundFile(params)
return ContentService.createTextOutput(params);
}
function createNewSoundFile(blob){
var title = 'Was created from a recording'
var folderId = 'SOME_FOLDER_ID'
var resource = {
title: title,
parents: [
{
"id": folderId,
"kind": "drive#fileLink"
}
],
mimeType: 'application/vnd.google-apps.audio',
};
try{
var newfile = Drive.Files.insert(resource, blob).id
} catch(e){
// Send error to Google sheet
// Exception: The mediaData parameter only supports blob types for uploads.
}
}
Problem
The above apps script code says the blob is not a support media type.
Question
How can I create become a new .ogg file in the Google drive from a .ogg blob created in the browser?
How about these modifications?
Modification points :
javascript side :
Encode blob to base64, and sent it as the string data.
GAS side :
Decode base64 to blob, and save it as a ogg file with the mimeType of audio/ogg.
Save the file as audio/ogg.
In my environment, it couldn't convert from audio/ogg to application/vnd.google-apps.audio.
Modified script : javascript side
From :
axios.post("https://script.google.com/macros/s/SOME_ID/exec", blob)
.then((response)=>{
console.log(response)
}).catch(error =>{
console.log(error)
})
To :
var reader = new FileReader();
reader.readAsDataURL(blob);
reader.onloadend = function() {
base64 = reader.result.replace(/^.*,/, "");
let data = new URLSearchParams();
data.append('data', base64);
axios.post(
"https://script.google.com/macros/s/SOME_ID/exec",
data,
{headers: {'Content-Type': 'application/x-www-form-urlencoded'}}
).then((response)=>{
console.log(response)
}).catch(error =>{
console.log(error)
});
}
Modified script : GAS side
To :
function createNewSoundFile(base64){
var data = Utilities.base64Decode(base64.parameters.data); // Added
var blob = Utilities.newBlob(data); // Added
var title = 'Was created from a recording'
var folderId = 'SOME_FOLDER_ID'
var resource = {
title: title,
parents: [
{
"id": folderId,
// "kind": "drive#fileLink" // I didn't know whether this is required.
}
],
mimeType: "audio/ogg", // Modified
};
try{
var newfile = Drive.Files.insert(resource, blob).id
} catch(e){
// Send error to Google sheet
// Exception: The mediaData parameter only supports blob types for uploads.
}
}
Note :
In my environment, {headers: {'Content-Type': 'application/x-www-form-urlencoded'}} was required for the script of javascript side. If it is not required in your environment, please remove it.
If I misunderstand your question, I'm sorry.