I have this code in js witch recod an audio from the browser and I need to send it back from js to flask
start: function () {
var options = {audio: true, video: false};
navigator.mediaDevices.getUserMedia(options).then(function (stream) {
myRecorder.objects.stream = stream;
myRecorder.objects.recorder = new Recorder(
myRecorder.objects.context.createMediaStreamSource(stream),
{numChannels: 1}
);
myRecorder.objects.recorder.record();
}).catch(function (err) {});
How I should do that while making the file in wav format?
The following example creates a limited time audio recording and uploads it when finished. A form containing a blob is used for this.
It would also be possible to transmit the pure blob to the server, but since there are differences in the audio format used depending on the browser, this is the more general variant.
(function() {
const uploadURL = "{{ url_for('upload') }}";
const startButton = document.getElementById("toggle-rec-btn");
startButton.addEventListener("click", function() {
if (!navigator.mediaDevices) {
console.error("getUserMedia not supported.")
return;
}
const constraints = { audio: true };
navigator.mediaDevices.getUserMedia(constraints)
.then(function(stream) {
let chunks = []
let recorder = new MediaRecorder(stream);
recorder.ondataavailable = event => {
// Collect all the chunks of the recording in an array.
chunks.push(event.data);
};
recorder.onstop = event => {
console.log("Recording stopped.")
// Create a blob with all the chunks of the recording.
let blob = new Blob(chunks, { type: recorder.mimeType });
chunks = [];
startButton.disabled = false;
// Create form data that contain the recording.
let formData = new FormData();
formData.append("audio_file", blob);
// Send the form data to the server.
fetch(uploadURL, {
method: "POST",
cache: "no-cache",
body: formData
}).then(resp => {
if (resp.status === 200) {
window.location.reload(true);
} else {
console.error("Error:", resp)
}
}).catch(err => {
console.error(err);
});
};
recorder.onstart = event => {
console.log("Recording started.");
startButton.disabled = true;
// Stop recording when the time is up.
setTimeout(function() { recorder.stop(); }, 10000);
};
recorder.start();
})
.catch(function(err) {
console.error(err);
});
});
})();
All recordings are saved on the server in a directory with the default name "var/app-instance/uploads".
import os
from flask import abort, current_app, make_response, request
from mimetypes import guess_extension
from werkzeug.utils import secure_filename
#app.route('/upload', methods=['POST'])
def upload():
if 'audio_file' in request.files:
file = request.files['audio_file']
# Get the file suffix based on the mime type.
extname = guess_extension(file.mimetype)
if not extname:
abort(400)
# Test here for allowed file extensions.
# Generate a unique file name with the help of consecutive numbering.
i = 1
while True:
dst = os.path.join(
current_app.instance_path,
current_app.config.get('UPLOAD_FOLDER', 'uploads'),
secure_filename(f'audio_record_{i}{extname}'))
if not os.path.exists(dst): break
i += 1
# Save the file to disk.
file.save(dst)
return make_response('', 200)
abort(400)
I wish you every success in implementing your project.
Related
I have an mp3 file that I am caching as such:
const request = URL_TO_MY_MP3_FILE
caches.open("my-cache").then(cache => {
cache.add(request);
});
I can see that the mp3 file is being cached in the Application tab:
By the way, how do I ensure that file is Content-Type cached as audio/mp3 and not audio/mpeg?
Later on, I would like to retrieve the mp3 file from cache so I can play it in the browser:
caches.open("my-cache").then(cache => {
const responsePromise = cache.match(request);
responsePromise.then(result => {
console.log(result.body)
this.audio = new Audio(result.body);
const playPromise = this.audio.play();
playPromise.then(function() {
console.log('success!')
}).catch(function(error) {
console.log(error)
}.bind(this), false);
});
})
This is the output of console.log(result.body):
After loading the mp3 file with new Audio(result.body) I try to play the file with this.audio.play() but this results in an error:
DOMException: Failed to load because no supported source was found.
How can I retrieve the mp3 file from cache and play it in the browser?
You have to call reader.read()
caches.open("my-cache").then(cache => {
const responsePromise = cache.match(request);
responsePromise.then(result => {
var reader = result.body.getReader();
reader.read().then((result) => {
const mimeType = 'audio/mpeg'
const url = URL.createObjectURL(new Blob([result.value.buffer], {type: mimeType}))
this.audio = new Audio(url);
const playPromise = this.audio.play();
playPromise.then(function() {
console.log('success!')
}).catch(function(error) {
console.log(error)
}.bind(this), false);
});
});
});
I'm trying to stream microphone from my browser to a server running a Python service connected to the google cloud speech-to-text. For the transfer I'm using socket.io. Everything seems to work but the speech recognition doesn't return any result. I suspect a problem with the format of the sent data.
On my browser I'm using the MediaRecorder with the mime type audio/webm;codecs=opus.
// simpleTest.js
'strict';
// Configuration
var language = 'fr-FR';
var mimeType = 'audio/webm;codecs=opus'; // Valid sample rates: 8000, 12000, 16000, 24000, 48000
var format = 'WEBM_OPUS'; // Valid sample rates: 8000, 12000, 16000, 24000, 48000
var sampleRate = 16000;
var recording = false;
var audioStream = null;
var mediaRecorder = null;
var audioChunks = [];
var namespace = '/ingestor'; // change to an empty string to use the global namespace
// Initialize socket
var socket = io(namespace);
socket.on('connect', function () {
console.log("connected to the SocketServer " + namespace);
});
socket.on('my_response', function (msg, callback) {
console.log("received message from the SocketServer " + namespace);
$('#log').append('<br>' + $('<div/>').text('logs #' + msg.count + ': ' + msg.data).html());
if (callback)
callback();
});
socket.on('connect_error', (error) => {
console.error("Socket connection error: " + error);
});
socket.on('disconnect', (reason) => {
console.log("Socket disconnected: " + reason);
if (reason === "io server disconnect") {
// the disconnection was initiated by the server, you need to reconnect manually
socket.connect();
}
});
const sendMessage = async (aSocket, msg) => {
if (aSocket.connected) {
aSocket.emit('my_event', {data: msg});
}
}
const initRecording = () => {
recording = false;
window.AudioContext = window.AudioContext || window.webkitAudioContext;
if (navigator.mediaDevices === undefined) {
navigator.mediaDevices = {};
}
if (navigator.mediaDevices.getUserMedia === undefined) {
navigator.mediaDevices.getUserMedia = function (constraints) {
// First get ahold of the legacy getUserMedia, if present
const getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
// Some browsers just don't implement it - return a rejected promise with an error
// to keep a consistent interface
if (!getUserMedia) {
return Promise.reject(new Error('getUserMedia is not implemented in this browser'));
}
// Otherwise, wrap the call to the old navigator.getUserMedia with a Promise
return new Promise(function (resolve, reject) {
getUserMedia.call(navigator, constraints, resolve, reject);
});
}
}
// Initialize audio stream
console.log("Creating audio stream");
navigator.mediaDevices.getUserMedia({audio: true})
.then((stream) => {
console.log("Audio stream successfully created");
audioStream = stream;
mediaRecorder = new MediaRecorder(stream, {
audioBitsPerSecond: sampleRate,
mimeType: mimeType
});
console.log('mimeType: ' + mediaRecorder.mimeType);
mediaRecorder.ondataavailable = handleDataAvailable;
}).catch((error) => {
console.log("Error while creating the audio stream");
console.log(error);
});
};
const startRecording = () => {
recording = true;
console.log('startRecording');
mediaRecorder.start(1000);
};
const stopRecording = () => {
recording = false;
console.log('stopRecording');
mediaRecorder.stop();
};
const handleDataAvailable = (event) => {
console.log('handleDataAvailable');
if (event.data && event.data.size > 0) {
console.log(event.data);
handleBlob(event.data);
}
};
const handleBlob = (blob) => {
console.log('handleBlob - blob type: ' + blob.type);
blob.arrayBuffer()
.then((buffer) => {
console.log(buffer);
console.log(audioChunks.length + '. ' + buffer);
sendMessage(socket, JSON.stringify({
type: 'audio',
content: {
command: 'stream',
audioData: new Uint8Array(buffer)
}
}));
})
.catch(function(err) {
console.log(err);
});
};
window.toggleRecording = () => {
if (!recording) {
startRecording();
} else {
stopRecording();
}
}
initRecording();
On the server side I specify in the google.cloud.speech.RecognitionConfig the encoding is google.cloud.speech.AudioEncoding.WEBM_OPUS. This way I suppose I'm using the same container and codec. Right?
The server is divided in two parts:
the ingestor reading the socket and writing the data as received to a redis queue
the transcriber reading the redis queue and transferring the data to the Google cloud text-to-speech
# Ingestor
import queue
import time
import eventlet
from flask import Flask, render_template, session, copy_current_request_context
from flask_socketio import SocketIO, emit, disconnect
import redis
eventlet.monkey_patch()
host = '127.0.0.1'
port = 5000
redisHost = 's2t_memory_store'
redisPort = 6379
redisQueue = 'livequeue'
id = 'ingestor'
maxPackets = 500
async_mode = None
app = Flask(__name__)
socketio = SocketIO(app, async_mode='eventlet')
thread = None
redisDatabase = redis.Redis(host=redisHost, port=redisPort, db=0,
health_check_interval=2, socket_timeout=3)
buffer = queue.Queue()
#socketio.on('connect', namespace='/ingestor')
def connect():
print('%s socket connected!' % id)
global thread
if thread is None:
thread = socketio.start_background_task(_enqueue_audio, redisQueue)
#socketio.on('my_event', namespace='/ingestor')
def handle_data(data):
"""Stores the received audio data in a local buffer."""
buffer.put(data['data'], block=False)
session['receive_count'] = session.get('receive_count', 0) + 1
emit('my_response',
{'data': data['data'], 'count': session['receive_count']})
def _enqueue_audio(redis_queue):
"""Blocking-reads data from the buffer and adds to Redis queue."""
print('%s enqueue_audio thread started!' % id)
while True:
try:
chunk = buffer.get(block=True)
print('Buffer read: {}'.format(chunk))
val = redisDatabase.lpush(redis_queue, chunk)
# debugging; under normal circumstances audio should not be accumulating
if val > 5:
print('Ingested audio queue length: %d' % val)
except redis.exceptions.RedisError as err:
print('Error pushing into Redis queue: %s' % err)
#app.route('/')
def index():
return render_template('test.html', sync_mode=socketio.async_mode)
if __name__ == '__main__':
print("Starting ingestor")
socketio.init_app(app)
# Transcriber
import redis
import json
from google.cloud import speech
encoding = speech.RecognitionConfig.AudioEncoding.WEBM_OPUS
sample_rate = 16000
language_code = 'fr-FR'
host = '127.0.0.1'
port = 5000
redis_host = 's2t_memory_store'
redis_port = 6379
redis_queue = 'livequeue'
id = 'transcriber'
class redisStream(object):
def __init__(self, host, port, queue):
self.host = host
self.port = port
self.queue = queue
self.redis_conn = redis.Redis(host=self.host, port=self.port)
def redis_generator(redis_conn, redis_queue):
while True:
yield redis_conn.blpop(redis_queue)[1]
def main():
redis_conn = redis.Redis(host=redis_host, port=redis_port, db=0,
health_check_interval=2,
socket_timeout=None,
socket_connect_timeout=None)
speech_client = speech.SpeechClient()
recognition_config = speech.RecognitionConfig(
encoding=encoding,
sample_rate_hertz=sample_rate,
language_code=language_code)
streaming_config = speech.StreamingRecognitionConfig(
config=recognition_config,
interim_results=True)
for message in redis_generator(redis_conn, redis_queue):
print(f'REDIS STREAM: {message}')
messageData = json.loads(message)
if messageData['content']['command'] == 'stream':
print('AUDIO DATA: %s' % messageData['content']['audioData'])
chunk = bytes(messageData['content']['audioData'].values())
print('CHUNK: %s' % chunk)
request = speech.StreamingRecognizeRequest(audio_content=chunk)
responses = speech_client.streaming_recognize(config=streaming_config, requests=[request])
print('RESPONSES: %s' % responses)
if responses:
for response in responses:
for i, result in response.results:
alternative = result.alternatives[0]
print("-" * 20)
print(u"First alternative of result {}".format(i))
print(u"Transcript: {}".format(alternative.transcript))
print(u"Confidence: {}".format(alternative.confidence))
print("-" * 20)
else:
print('No response')
if __name__ == "__main__":
print("Starting transcriber")
main()
What is wrong? Do you have somewhere an example of the best (right) way to realize the transfer of such a live stream?
I have read many threads and publications on the web but I was never able to make it run correctly.
Thanks for your answer.
You need to write some Python code or show your code that did not fix the issue you were seeing.
Background
Javascript library for Microsoft Office add-ins allows you to get raw content of the DOCX file through getFileAsync() api, which returns a slice of up to 4MB in one go. You keep calling the function using a sliding window approach till you have reed entire content. I need to upload these slices to the server and the join them back to recreate the original DOCX file.
My attempt
I'm using axios on the client-side and busboy-based express-chunked-file-upload middleware on my node server. As I call getFileAsync recursively, I get a raw array of bytes that I then convert to a Blob and append to FormData before posting it to the node server. The entire thing works and I get the slice on the server. However, the chunk that gets written to the disk on the server is much larger than the blob I uploaded, normally of the order of 3 times, so it is obviously not getting what I sent.
My suspicion is that this may have to do with stream encoding, but the node middleware does not expose any options to set encoding.
Here is the current state of code:
Client-side
public sendActiveDocument(uploadAs: string, sliceSize: number): Promise<boolean> {
return new Promise<boolean>((resolve) => {
Office.context.document.getFileAsync(Office.FileType.Compressed,
{ sliceSize: sliceSize },
async (result) => {
if (result.status == Office.AsyncResultStatus.Succeeded) {
// Get the File object from the result.
const myFile = result.value;
const state = {
file: myFile,
filename: uploadAs,
counter: 0,
sliceCount: myFile.sliceCount,
chunkSize: sliceSize
} as getFileState;
console.log("Getting file of " + myFile.size + " bytes");
const hash = makeId(12)
this.getSlice(state, hash).then(resolve(true))
} else {
resolve(false)
}
})
})
}
private async getSlice(state: getFileState, fileHash: string): Promise<boolean> {
const result = await this.getSliceAsyncPromise(state.file, state.counter)
if (result.status == Office.AsyncResultStatus.Succeeded) {
const data = result.value.data;
if (data) {
const formData = new FormData();
formData.append("file", new Blob([data]), state.filename);
const boundary = makeId(12);
const start = state.counter * state.chunkSize
const end = (state.counter + 1) * state.chunkSize
const total = state.file.size
return await Axios.post('/upload', formData, {
headers: {
"Content-Type": `multipart/form-data; boundary=${boundary}`,
"file-chunk-id": fileHash,
"file-chunk-size": state.chunkSize,
"Content-Range": 'bytes ' + start + '-' + end + '/' + total,
},
}).then(async res => {
if (res.status === 200) {
state.counter++;
if (state.counter < state.sliceCount) {
return await this.getSlice(state, fileHash);
}
else {
this.closeFile(state);
return true
}
}
else {
return false
}
}).catch(err => {
console.log(err)
this.closeFile(state)
return false
})
} else {
return false
}
}
else {
console.log(result.status);
return false
}
}
private getSliceAsyncPromise(file: Office.File, sliceNumber: number): Promise<Office.AsyncResult<Office.Slice>> {
return new Promise(function (resolve) {
file.getSliceAsync(sliceNumber, result => resolve(result))
})
}
Server-side
This code is totally from the npm package (link above), so I'm not supposed to change anything in here, but still for reference:
makeMiddleware = () => {
return (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
busboy.on('file', (fieldName, file, filename, _0, _1) => {
if (this.fileField !== fieldName) { // Current field is not handled.
return next();
}
const chunkSize = req.headers[this.chunkSizeHeader] || 500000; // Default: 500Kb.
const chunkId = req.headers[this.chunkIdHeader] || 'unique-file-id'; // If not specified, will reuse same chunk id.
// NOTE: Using the same chunk id for multiple file uploads in parallel will corrupt the result.
const contentRangeHeader = req.headers['content-range'];
let contentRange;
const errorMessage = util.format(
'Invalid Content-Range header: %s', contentRangeHeader
);
try {
contentRange = parse(contentRangeHeader);
} catch (err) {
return next(new Error(errorMessage));
}
if (!contentRange) {
return next(new Error(errorMessage));
}
const part = contentRange.start / chunkSize;
const partFilename = util.format('%i.part', part);
const tmpDir = util.format('/tmp/%s', chunkId);
this._makeSureDirExists(tmpDir);
const partPath = path.join(tmpDir, partFilename);
const writableStream = fs.createWriteStream(partPath);
file.pipe(writableStream);
file.on('end', () => {
req.filePart = part;
if (this._isLastPart(contentRange)) {
req.isLastPart = true;
this._buildOriginalFile(chunkId, chunkSize, contentRange, filename).then(() => {
next();
}).catch(_ => {
const errorMessage = 'Failed merging parts.';
next(new Error(errorMessage));
});
} else {
req.isLastPart = false;
next();
}
});
});
req.pipe(busboy);
};
}
Update
So it looks like I have found the problem at least. busboy appears to be writing my array of bytes as text in the output file. I get 80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25 (as text) when I upload the array of bytes [80,75,3,4,20,0,6,0,8,0,0,0,33,0,44,25]. Now need to figure out how to force it to write it as a binary stream.
Figured out. Just in case it helps anyone, there was no problem with busboy or office.js or axios. I just had to convert the incoming chunk of data to Uint8Array before creating a blob from it. So instead of:
formData.append("file", new Blob([data]), state.filename);
like this:
const blob = new Blob([ new Uint8Array(data) ])
formData.append("file", blob, state.filename);
And it worked like a charm.
I am using react-media-recorder to record audio. It works fine on localhost but when deployed to a server and accessed, the onStop method stops working. It doesn't get invoked.
this is my file:
import { useReactMediaRecorder } from "react-media-recorder"
const { status, startRecording, stopRecording, error, mediaBlobUrl, clearBlobUrl } = useReactMediaRecorder({
audio: true,
type: "audio/wav",
onStop: (blobUrl, blob) => {
console.log("onStop recording")
const url = URL.createObjectURL(blob)
let formData = new FormData()
//person_name-person_id-language_id-sentence.id-date
const today = new Date()
// const file_name = `${id}-${language_id}-${sentence.id}-${today.toISOString()}.wav`
const file_name = `${name}-${id}-${language_id}-${
sentence.id
}-${today.toDateString()}-${language_name}.wav`
console.log("-------------------------------------------------------")
console.log("file_name :>>", file_name)
console.log("-------------------------------------------------------")
formData.append("file", blob, file_name)
let upload_url
if (sample) {
upload_url = "sentence/upload_audio_sample"
} else {
upload_url = "sentence/upload_audio"
}
console.log(`upload_url`, upload_url)
axios
.post(upload_url, formData)
.then((d) => console.log("after post blob :>>", d))
.catch((e) => console.log("error in post blob :>>", e))
},
})
const handleStartRecording = () => {
setRecording(!recording)
if (!recording) {
clearBlobUrl()
startRecording()
} else {
stopRecording()
}
}
any help is much appreciated
for this to work both the frontend and backend need to be on https.
Make sure your backend server is running on https
I am working on VUI interface with Reactjs frontend. I got a BLOB file that I can play but I want to convert it to .WAV file using REACT or Javascript to send it to my server.
I tried lot of things, but found no solution
toggleRecording() {
if (this.state.start === 1) {
console.log("we start recording", this.state.start)
this.setState({ start: 0, recognition: "" })
const constraints = {
audio: {
sampleRate: 16000,
channelCount: 1,
}
}
navigator.mediaDevices.getUserMedia(constraints)
.then(stream => {
console.log(this);
this.recorder = new MediaRecorder(stream);
this.recorder.start();
const audioChunks = [];
this.recorder.addEventListener("dataavailable", event => {
audioChunks.push(event.data);
});
this.recorder.addEventListener("stop", () => {
const audioBlob = new Blob(audioChunks, { 'type': 'audio/wav' });
const audioUrl = URL.createObjectURL(audioBlob);
console.log("test: ", audioUrl)
console.log(audioBlob.type)
fetch('http://127.0.0.1:6060/api/sendaudio', {
method: "post",
headers: { 'Content-Type': 'audio/wav' },
body: audioBlob
})
.then(response => {
return response.text()
}).then(text => {
console.log(text);
this.setState({ recognition: text })
});
//to play the audio file:
const audio = new Audio(audioUrl);
audio.play();
});
});
}
I expect to get a Wav file to post to my server but don't know how to do that ....
You can try this package if you don't have a problem to add new dependency: https://www.npmjs.com/package/audiobuffer-to-wav
Hope it will work for you