How to use Azure Custom Commands API with react native (expo) - javascript

I am trying to get Azure Speech Services Custom Commands to work with react native. I am unable to find examples in javascript that use Custom Commands. Has anyone created an application that I can refer to? I found the following resources but they do not show how to use Custom Commands specifically.
https://github.com/microsoft/cognitive-services-sdk-react-native-example/tree/main/ExampleTSProject
This one has JS code but does not show how to use microphone to send audio to the API.
https://learn.microsoft.com/en-us/azure/cognitive-services/speech-service/how-to-custom-commands-setup-speech-sdk
This one has example in C# for which I wrote JS equivalent code which does not work.
This is my code right now, I am able to record audio using expo-av but don't know how to convert it to audio stream and pass it to SDK.
import 'react-native-get-random-values';
import { useState, useEffect } from 'react';
import { Audio } from 'expo-av';
import { StyleSheet, Button, View } from 'react-native';
import * as SpeechSDK from 'microsoft-cognitiveservices-speech-sdk';
const APPLICATION_ID = '6e3b5880-918e-11ed-a04e-cf28b0773cf7';
const SPEECH_KEY = '51ecddfd91564074b3cb2cee2f9e4f52';
const SPEECH_RESOURCE_REGION = 'centralindia'
export default function App() {
const [recording, setRecording] = useState();
const [dialogueServiceConnector, setDialogueServiceConnector] = useState();
const [lastRecordedSound, setLastRecordedSound] = useState();
function initializeDialogueServiceConnector() {
const commandsConfig = new SpeechSDK.CustomCommandsConfig.fromSubscription(APPLICATION_ID, SPEECH_KEY, SPEECH_RESOURCE_REGION);
const connector = new SpeechSDK.DialogServiceConnector(commandsConfig);
connector.activityReceived = (sender, arg) => {
console.log(`Activity received, activity=${arg.activity}`);
let buff = [];
arg.audioStream.read(buff);
console.log(`Buffer: ${buff}`);
}
connector.canceled = (sender, arg) => {
console.log(`Cancelled, reason=${arg.reason}`);
if (arg.reason === SpeechSDK.CancellationReason.Error) {
console.log(`Error: code=${arg.errorCode}, details=${arg.errorDetails}`);
}
}
connector.recognizing = (sender, arg) => {
console.log(`Recognizing! in-progress text=${arg.result.text}`);
}
connector.recognized = (sender, arg) => {
console.log(`Final speech-to-text result: ${arg.result.text}`);
}
connector.sessionStarted = (sender, arg) => {
console.log(`Now listening! session started, id=${arg.sessionId}`);
}
connector.sessionStopped = (sender, arg) => {
console.log(`Listening complete. session ended, id=${arg.sessionId}`);
}
setDialogueServiceConnector(connector);
console.log("initialized");
}
useEffect(() => {
initializeDialogueServiceConnector();
}, [])
async function playSound() {
if (lastRecordedSound) {
await lastRecordedSound.playAsync()
}
}
async function startRecording() {
/*
console.log(dialogueServiceConnector)
dialogueServiceConnector.listenOnceAsync(() => {
console.log("Recording complete")
});
setRecording(true);
*/
try {
await Audio.requestPermissionsAsync();
await Audio.setAudioModeAsync({
allowsRecordingIOS: true,
playsInSilentModeIOS: true,
});
const { recording } = await Audio.Recording.createAsync(
Audio.RecordingOptionsPresets.HIGH_QUALITY
);
setRecording(recording);
} catch (err) {
console.error('Failed to start recording', err);
}
}
async function stopRecording() {
/*
setRecording(false);
*/
await recording.stopAndUnloadAsync();
await Audio.setAudioModeAsync({
allowsRecordingIOS: false,
playsInSilentModeIOS: true,
});
const { sound } = await recording.createNewLoadedSoundAsync();
setLastRecordedSound(sound)
}
return (
<View style={styles.container}>
<Button
title={recording ? 'Stop Recording' : 'Start Recording'}
onPress={recording ? stopRecording : startRecording}
/>
<Button
title='Play recording'
onPress={playSound}
/>
</View>
);
}
const styles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: '#fff',
alignItems: 'center',
justifyContent: 'center',
},
});

Related

Electron send data to react with preload.js

I am trying achieve something where I can directly send data to react and where in react whenever it receives it does something.
So my electron.js is in below format.
// ./public/electron.js
const path = require("path");
const { app, BrowserWindow, ipcMain} = require("electron");
const isDev = false; //require("electron-is-dev"); //false
let splash = null;
let win = null;
let etmf_obj = null;
function createWindow() {
// Create the browser window.
win = new BrowserWindow({
width: 1920,
height: 1080,
webPreferences: {
nodeIntegration: true,
contextIsolation: true,
enableRemoteModule: true,
preload: path.join(__dirname, "./preloadDist.js"),
},
});
// win.loadFile("index.html");
win.loadURL(
isDev
? "http://localhost:3000"
: `file://${path.join(__dirname, "../build/index.html")}`
);
// Open the DevTools.
if (!isDev) {
win.webContents.openDevTools({ mode: "undocked" });
}
}
app.whenReady().then(createWindow);
app.on("window-all-closed", () => {
if (process.platform !== "darwin") {
app.quit();
}
});
app.on("activate", () => {
if (BrowserWindow.getAllWindows().length === 0) {
createWindow();
}
});
function restartApp() {
console.log("restarting app..");
app.exit();
app.relaunch();
}
//IPC SECTION
ipcMain.handle("notify", (event, args) => {
console.log("from react I got" + args);
console.log("hello from electron via react"); //this one works as expected
});
ipcMain.on("splashDone", function () {
console.log("splash done");
});
ipcMain.on("relaunchApp", function () {
restartApp();
});
ipcMain.on("closeAll", function () {
app.quit();
});
ipcMain.on("callAnim", function (args) {
win.webContents.send("showAnimation", args);// trying to send data directly to
//react here but don't know whether its right way or not
});
and my preload file preloadDist.js is in below format
const { ipcRenderer, contextBridge } = require("electron");
contextBridge.exposeInMainWorld("electron", {
notificationApi: {
sendNotification(message) {
ipcRenderer.invoke("notify", message);
},
},
batteryApi: {},
filesApi: {},
splashStatus: {
splashDone() {
ipcRenderer.invoke("splashDone")
}
},
});
and react to call function of or to send data I am do this
for example to send notification data :
<button
className="speak_border"
onMouseEnter={() => setHovermic(true)}
onMouseLeave={() => setHovermic(false)}
onClick={() => {
soundwave();
window.electron.notificationApi.sendNotification(
"From react Hi!");
}}
>
and to receive data I am not able to figure out but as I am doing win.webContents.send("showListenAnimation", args);
I am not able to understand how this will be received at the react end
what I tried is:
useEffect(() => {
try {
window.electron.on(
"showAnimation",
function (event, data) {
if (data) {
setAnim(true);
}
if (!data) {
setAnim(false);
}
}
);
} catch (e) {
console.log("issue with on getting data");
}
});
But this way I am getting error and not able to figure out the way to receive, but sending data from react to electron is working perfectly fine!
Please guide on this and how to achieve with about preload.js and electron.js
format.

React Native app has low performance while fetching data from firebase

Hello I have a react native app that gets slow/freeze while fetching data from realtime db.
This is only on android devices, on IOs the app works fine.
for example, by using the performance monitor in my physical device I get:
UI: 59 fps
79 DROPEDD
3 Stuters fo far
JS: -2.1 fps
I get the data in the App.js as the following code, which it is inside an useEffect with an empty dependency array:
async function fetchData() {
try {
const [
postFetched,
categoriesFetched,
assetsFetched,
] = await Promise.all([getAllPosts(), getCategories(), getAssets()]);
const email = await AsyncStorage.getItem('#email');
//Alert.alert(`whats uid ? ${email}`);
if (email) {
const userFetched = await searchUserByEmail(email);
setUser(Object.values(userFetched.val())[0]);
}
// eslint-disable-next-line no-undef
const t2 = performance.now();
//Alert.alert(`time ${(t2 - t1) / 1000} seg`);
return {postFetched, categoriesFetched, assetsFetched};
} catch (error) {
throw new Error(error.toString());
}
}
fetchData()
.then((data) => {
const {postFetched, categoriesFetched, assetsFetched} = data;
if (postFetched.exists()) {
setPosts(
orderBy(
toArray(postFetched.val()),
['isTop', 'created'],
['desc', 'desc'],
),
);
//setLoading(false);
}
if (categoriesFetched.exists()) {
setCategories(toArray(categoriesFetched.val()));
}
if (assetsFetched.exists()) {
setAssetsFirebase(assetsFetched.val());
// SplashScreenNative.hide();
}
// SplashScreenNative.hide();
/*if (Platform.OS === 'android') {
setIsLoading(false);
}*/
})
.catch((error) => {
Toast.show({
type: 'error',
text1: 'error ' + error.toString(),
});
console.log('error', error);
});
listenPostsChange(); // -> this also inside the same use effect
the listenPostChange function:
const listenPostsChange = () => {
listenPosts((data) => {
data.exists() &&
setPosts(
orderBy(toArray(data.val()), ['isTop', 'created'], ['desc', 'desc']),
);
});
};
the firebase methods are defined in a separate js file for each reference:
post.js
import database from '#react-native-firebase/database';
const PostsRef = database().ref('/posts');
export const listenPosts = (callbackFunction) => {
return PostsRef.on('value', callbackFunction);
};
export const listenPosts = (callbackFunction) => {
return PostsRef.on('value', callbackFunction);
};
category.js
import database from '#react-native-firebase/database';
const CategoriesRef = database().ref('/categories');
export const addCategory = (label, category) => {
return CategoriesRef.child(label).update({...category});
};
export const getCategories = () => {
return CategoriesRef.once('value', (snapshot) =>
snapshot.exists() ? snapshot.val() : [],
);
};
assets.js
import database from '#react-native-firebase/database';
const AssetsRef = database().ref('/assets');
export const getAssets = () => {
return AssetsRef.once('value', (snapshot) =>
snapshot.exists() ? snapshot.val() : [],
);
};
I have the following dependencies installed in my app:
"#react-native-firebase/app": "^14.2.2",
"#react-native-firebase/auth": "^14.2.2",
"#react-native-firebase/database": "^14.11.0",
"#react-native-firebase/storage": "^14.11.0",
"firebase": "8.10.1",
I think the main problem is related to firebase since when I deleted the fetchData and the listenPostsChange functions, the app worked smoothly but without data to show.
I hope someone can help me
thanks.
edit: I just have like 15 post and 6 categories, and the assets are just 3 strings values.

Trying to figure out how to use socket.io the correct way in a useEffect that is using an axios get request to fetch messages

So far i'm stuck on my useEffect that fetches all the current messages and renders the state accordingly. as of right now it doesn't render the new state until page is refreshed.
const Home = ({ user, logout }) => {
const history = useHistory();
const socket = useContext(SocketContext);
const [conversations, setConversations] = useState([]);
const [activeConversation, setActiveConversation] = useState(null);
const classes = useStyles();
const [isLoggedIn, setIsLoggedIn] = useState(false);
const addSearchedUsers = (users) => {
const currentUsers = {};
// make table of current users so we can lookup faster
conversations.forEach((convo) => {
currentUsers[convo.otherUser.id] = true;
});
const newState = [...conversations];
users.forEach((user) => {
// only create a fake convo if we don't already have a convo with this user
if (!currentUsers[user.id]) {
let fakeConvo = { otherUser: user, messages: [] };
newState.push(fakeConvo);
}
});
setConversations(newState);
};
const clearSearchedUsers = () => {
setConversations((prev) => prev.filter((convo) => convo.id));
};
const saveMessage = async (body) => {
const { data } = await axios.post("/api/messages", body);
return data;
};
const sendMessage = (data, body) => {
socket.emit("new-message", {
message: data.message,
recipientId: body.recipientId,
sender: data.sender,
});
};
const postMessage = async (body) => {
try {
const data = await saveMessage(body);
if (!body.conversationId) {
addNewConvo(body.recipientId, data.message);
} else {
addMessageToConversation(data);
}
sendMessage(data, body);
} catch (error) {
console.error(error);
}
};
const addNewConvo = useCallback(
(recipientId, message) => {
conversations.forEach((convo) => {
if (convo.otherUser.id === recipientId) {
convo.messages.push(message);
convo.latestMessageText = message.text;
convo.id = message.conversationId;
}
});
setConversations(conversations);
},
[setConversations, conversations],
);
const addMessageToConversation = useCallback(
(data) => {
// if sender isn't null, that means the message needs to be put in a brand new convo
const { message, sender = null } = data;
if (sender !== null) {
const newConvo = {
id: message.conversationId,
otherUser: sender,
messages: [message],
};
newConvo.latestMessageText = message.text;
setConversations((prev) => [newConvo, ...prev]);
}
conversations.forEach((convo) => {
console.log('hi', message.conversationId)
if (convo.id === message.conversationId) {
const convoCopy = { ...convo }
convoCopy.messages.push(message);
convoCopy.latestMessageText = message.text;
console.log('convo', convoCopy)
} else {
return convo
}
});
setConversations(conversations);
},
[setConversations, conversations],
);
const setActiveChat = useCallback((username) => {
setActiveConversation(username);
}, []);
const addOnlineUser = useCallback((id) => {
setConversations((prev) =>
prev.map((convo) => {
if (convo.otherUser.id === id) {
const convoCopy = { ...convo };
convoCopy.otherUser = { ...convoCopy.otherUser, online: true };
return convoCopy;
} else {
return convo;
}
}),
);
}, []);
const removeOfflineUser = useCallback((id) => {
setConversations((prev) =>
prev.map((convo) => {
if (convo.otherUser.id === id) {
const convoCopy = { ...convo };
convoCopy.otherUser = { ...convoCopy.otherUser, online: false };
return convoCopy;
} else {
return convo;
}
}),
);
}, []);
// Lifecycle
useEffect(() => {
// Socket init
socket.on("add-online-user", addOnlineUser);
socket.on("remove-offline-user", removeOfflineUser);
socket.on("new-message", addMessageToConversation);
return () => {
// before the component is destroyed
// unbind all event handlers used in this component
socket.off("add-online-user", addOnlineUser);
socket.off("remove-offline-user", removeOfflineUser);
socket.off("new-message", addMessageToConversation);
};
}, [addMessageToConversation, addOnlineUser, removeOfflineUser, socket]);
useEffect(() => {
// when fetching, prevent redirect
if (user?.isFetching) return;
if (user && user.id) {
setIsLoggedIn(true);
} else {
// If we were previously logged in, redirect to login instead of register
if (isLoggedIn) history.push("/login");
else history.push("/register");
}
}, [user, history, isLoggedIn]);
useEffect(() => {
const fetchConversations = async () => {
try {
const { data } = await axios.get("/api/conversations");
setConversations(data);
} catch (error) {
console.error(error);
}
};
if (!user.isFetching) {
fetchConversations();
}
}, [user]);
const handleLogout = async () => {
if (user && user.id) {
await logout(user.id);
}
};
return (
<>
<Button onClick={handleLogout}>Logout</Button>
<Grid container component="main" className={classes.root}>
<CssBaseline />
<SidebarContainer
conversations={conversations}
user={user}
clearSearchedUsers={clearSearchedUsers}
addSearchedUsers={addSearchedUsers}
setActiveChat={setActiveChat}
/>
<ActiveChat
activeConversation={activeConversation}
conversations={conversations}
user={user}
postMessage={postMessage}
/>
</Grid>
</>
);
};
this is the main part im working on, the project had starter code when i began and was told not to touch the backend so i know its something wrong with the front end code. i feel like im missing something important for the socket.io
import { io } from 'socket.io-client';
import React from 'react';
export const socket = io(window.location.origin);
socket.on('connect', () => {
console.log('connected to server');
});
export const SocketContext = React.createContext();
this is how i have the socket.io setup, if anyone could point me in the right direction that would be cool. I have been reading up on socket.io as much as I can but am still pretty lost on it.
Based on the assumption the backend is working properly...
const addNewConvo = useCallback(
(recipientId, message) => {
conversations.forEach((convo) => {
if (convo.otherUser.id === recipientId) {
convo.messages.push(message);
convo.latestMessageText = message.text;
convo.id = message.conversationId;
}
});
setConversations(conversations);
},
[setConversations, conversations],
);
setConversations(conversations);
This is an incorrect way to set a state using the state's variable, and such it wont do anything. Likely why your code wont change until refresh.
Suggested fix:
const addNewConvo = useCallback(
(recipientId, message) => {
setConversations(previousState => previousState.map(convo => {
if (convo.otherUser.id === recipientId) {
convo.messages.push(message)
convo.latestMessageText = message.text;
convo.id = message.conversationId;
return convo
}
return convo
}))
},
[setConversations, conversations],
);
note: even above could be done more efficiently since I made a deep copy of messages

audio is shown disable in twilio video chat

I am trying to create a video chat with twilio. I could turn the webcam and run the video, however i could not make the audio work. When i select the control, i get to enlarge the video and picture to picture mode but not control the audio.
This is how seen
Here is the code
function App() {
let localMediaRef = React.useRef(null);;
const [data, setIdentity] = React.useState({
identity: null,
token: null
});
const [room, setRoom] = React.useState({
activeRoom: null,
localMediaAvailable: null,
hasJoinedRoom: null
});
async function fetchToken() {
try {
const response = await fetch("/token");
const jsonResponse = await response.json();
const { identity, token } = jsonResponse;
setIdentity({
identity,
token
});
} catch (e) {
console.error("e", e);
}
}
React.useEffect(() => {
fetchToken();
}, []);
const attachTracks = (tracks, container) => {
tracks.forEach(track => {
container.appendChild(track.attach());
});
};
// Attaches a track to a specified DOM container
const attachParticipantTracks = (participant, container) => {
const tracks = Array.from(participant.tracks.values());
attachTracks(tracks, container);
};
const roomJoined = room => {
// Called when a participant joins a room
console.log("Joined as '" + data.identity + "'");
setRoom({
activeRoom: room,
localMediaAvailable: true,
hasJoinedRoom: true
});
// Attach LocalParticipant's Tracks, if not already attached.
const previewContainer = localMediaRef.current;
if (!previewContainer.querySelector("video")) {
attachParticipantTracks(room.localParticipant, previewContainer);
}
};
const joinRoom = () => {
let connectOptions = {
name: "Interview Testing"
};
let settings = {
audio: true
}
console.log('data', data, data.token)
Video.connect(
data.token,
connectOptions,
settings
).then(roomJoined, error => {
alert("Could not connect to Twilio: " + error.message);
});
};
return (
<div className="App">
<FeatureGrid>
<span onClick={joinRoom}>Webcam</span>
</FeatureGrid>
<PanelGrid>
{room.localMediaAvailable ? (
<VideoPanels>
<VideoPanel ref={localMediaRef} />
</VideoPanels>
) : (
""
)}
</PanelGrid>
</div>
);
}
export default App;
How do i enable audio too? Also the settings of video is shown only after right click. can't we show this by default?
UPDATE
its a LocalAudioTrack
this is remoteaudiotrack

Repurpose Firebase's Facebook Login ID as a Data Key?

How can I utilize a user's Firebase ID ("id" : "1234567890") as a key for a separate data set within the same db?
the following would be my login detail where I would pull the ID from:
"users": {
"1234567890abcdefghijklmnopqrstuvwxyz" : {
"dp" : "https://www.profilePic.com",
"first_name" : "John",
"id" : "1234567890",
"last_name" : "Doe",
"token" : "abcdefghijklmnopqrstuvwxyz1234567890",
"uid" : "987654321"
}
}
Within Firebase Functions I currently have my code as:
admin.database().ref('location_config/{id}').set({current_location:[34.047220, -118.443534]})
The result currently comes out as:
"location_config": {
"{id}": {
"current_location": [34.047220, -118.443534]
}
}
But this is how I would like the data to appear with the ID being the key:
"location_config": {
"1234567890": {
"current_location": [34.047220, -118.443534]
}
}
The screenshot below illustrates how the UID is dynamic while the ID is constant.
Here is the code within Firebase:
let fbLocation;
module.exports = (event) => {
event.geoFire = functions.database.ref('users').onUpdate(event => {
admin.database().ref('/portal_coordinates_all').once('value', snapshot =>{
fbLocation = snapshot.val();
console.log ("snapshot", fbLocation);
}).then(() => {
// Create a Firebase reference where GeoFire will store its information
let firebaseRef = admin.database().ref('geofire');
// Create a GeoFire index
let geoFire = new GeoFire(firebaseRef);
geoFire.set(fbLocation)
.then(() => {
console.log("Provided key has been added to GeoFire");
}).catch(err => console.log(err))
.then(() => {
let geoQuery = geoFire.query({
center: [34.047220, -118.443534],
radius: 2
});
let locations = [];
let onKeyEnteredRegistration = geoQuery.on("key_entered", function(key, location, distance) {
locations.push(location);
});
// fires once when this query's initial state has been loaded from the server.
let onReadyRegistration = geoQuery.on("ready", function() {
console.log("GeoQuery has loaded and fired all other events for initial data");
console.log(locations);
// ******* here is where I'm having the issue *******
admin.database().ref( 'location_config/`${id}`' ).set( {current_location: locations} )
// **************************************************
// Cancel the "key_entered" callback
onKeyEnteredRegistration.cancel();
});
}).catch(err => console.log(err))
})
})
}
And here is the code within React Native:
import React, { Component } from 'react';
import {
StyleSheet,
View,
ActivityIndicator,
Button
} from 'react-native';
import firebase from 'firebase';
import { connect } from 'react-redux';
import { loginSuccess } from '../actions/AuthActions';
const FBSDK = require('react-native-fbsdk');
const { LoginManager, AccessToken } = FBSDK;
class Login extends Component {
constructor(props) {
super(props);
this.state = {
showSpinner: true,
};
}
componentDidMount() {
this.fireBaseListener = firebase.auth().onAuthStateChanged(auth => {
if (auth) {
this.firebaseRef = firebase.database().ref('users');
this.firebaseRef.child(auth.uid).on('value', snap => {
const user = snap.val();
if (user != null) {
this.firebaseRef.child(auth.uid).off('value');
this.props.loginSuccess(user);
}
});
} else {
this.setState({ showSpinner: false });
}
});
}
onPressLogin() {
this.setState({ showSpinner: true })
LoginManager.logInWithReadPermissions([
'public_profile',
'user_birthday',
'email',
'user_photos'
])
.then((result) => this.handleCallBack(result),
function(error) {
alert('Login fail with error: ' + error);
}
);
}
handleCallBack(result) {
let that = this;
if (result.isCancelled) {
alert('Login canceled');
} else {
AccessToken.getCurrentAccessToken().then(
(data) => {
const token = data.accessToken
fetch('https://graph.facebook.com/v2.8/me? fields=id,first_name,last_name&access_token=' + token)
.then((response) => response.json())
.then((json) => {
const imageSize = 120
const facebookID = json.id
const fbImage = `https://graph.facebook.com/${facebookID}/picture?height=${imageSize}`
this.authenticate(data.accessToken)
.then(function(result) {
const { uid } = result;
that.createUser(uid, json, token, fbImage)
});
})
.catch(function(err) {
console.log(err);
});
}
);
}
}
authenticate = (token) => {
const provider = firebase.auth.FacebookAuthProvider;
const credential = provider.credential(token);
return firebase.auth().signInWithCredential(credential);
}
createUser = (uid, userData, token, dp) => {
const defaults = {
uid,
token,
dp
};
firebase.database().ref('users').child(uid).update({ ...userData, ...defaults });
}
render() {
return (
this.state.showSpinner ? <View style={styles.container}><ActivityIndicator animating={this.state.showSpinner} /></View> :
<View style={styles.container}>
<Button
onPress={this.onPressLogin.bind(this)}
title="Login with Facebook"
color="#841584"
/>
</View>
);
}
}
const styles = StyleSheet.create({
container: {
flex: 1,
justifyContent: 'center',
alignItems: 'center',
backgroundColor: '#F5FCFF',
}
});
const mapStateToProps = (state) => {
console.log('mapStateToProps', state);
return {
logged: state.auth.loggedIn,
user: state.auth.user
};
};
export default connect(mapStateToProps, { loginSuccess })(Login);
After banging my head against the wall as to why I couldn't grab the Facebook ID, it turned out that there isn't a need to grab this ID since Firebase's UID stays constant. I was unaware that the Firebase ID didn't change because in my test environment I would always login resulting in a new UID being created.

Categories

Resources