I am making an API that gets a list of image names, then it has to download them one by one from S3 bucket and then send them all as a response.
The issue is that my images are being uploaded but it seems that when I put them in a list as base64 and then try to send the list then the list just comes up empty.
const getImagesById = async (req, res) => {
const { id } = req.params;
const imagesSet = new Map();
try {
const documentFromDB = await document.findOne({ id });
documentFromDB.devices.forEach((device) => {
const images = new Set();
device.images.forEach(item => images.add(downloadFromS3(item)))
imagesSet.set(device.name, JSON.stringify(mapToObj(images))) // tried adding just images also but neither works
});
res.status(200).json(JSON.stringify(mapToObj(imagesSet)));
} catch (e) {
console.log(`An error occurred : ${e.message}`);
res.status(500)
.send(e.message);
}
};
function mapToObj(inputMap) {
let obj = {};
inputMap.forEach(function(value, key){
obj[key] = value
});
return obj;
}
And this is how I get images from S3:
const downloadFromS3 = async (imageName) => {
try {
const image = await S3Utils.downloadFile(BUCKET_NAME, imageName);
if (image.stack) {
return null;
}
const imageBase64 = image.Body.toString('base64');
return imageBase64;
} catch (e) {
console.log(`An error occurred while downloading : ${e.message}`);
throw e;
}
};
This is the response I am getting at the moment:
"{\"{ name: 'Martin'}\":\"{\\\"[object Promise]\\\":{}}\"}"
What I am trying to do is get a lits of device names, map them in a Map as key with value as the base64 list of images and then send it all in a response to the UI to show the images with the names.
What am I doing wrong here?
You just need to add await before call the downloadFromS3 function, consequently changing all the above functions.
const getImagesById = async (req, res) => {
const { id } = req.params;
const imagesSet = new Map();
try {
const documentFromDB = await document.findOne({ id });
await Promise.all(documentFromDB.devices.map(async (device) => {
const images = new Set();
await Promise.all(device.images.map(async item => images.add(await downloadFromS3(item))))
imagesSet.set(device.name, JSON.stringify(mapToObj(images))) // tried adding just images also but neither works
}));
res.status(200).json(JSON.stringify(mapToObj(imagesSet)));
} catch (e) {
console.log(`An error occurred : ${e.message}`);
res.status(500)
.send(e.message);
}
};
function mapToObj(inputMap) {
let obj = {};
inputMap.forEach(function(value, key){
obj[key] = value
});
return obj;
}
Related
I'm a new programmer and very new to firebase and I'm trying to get the current user files info to display on the screen, it seems that my problem is that I can get the URL and the metadata separately, how do I combine them? how can I take everything at once?
I need to show the file name, date, time, link to download.
const getUserFiles = async () => {
if (!userUID) {
return null;
}
let listRef = storageRef.child(userUID);
listRef.listAll().then(res => {
// res.prefixes.forEach((item) => {
// });
res.items.forEach(item => {
item.getMetadata().then(item => {
var file = {
name: item.name.toString(),
timeCreated: item.timeCreated.toString(),
link: '',
};
myFiles.push(file);
});
});
res.items.forEach(item => {
let counter = 0;
item.getDownloadURL().then(url => {
myFiles[counter].link = url.toString();
});
});
});
console.log(myFiles);
};
the current method don't work! and notice that the userUID its only the uid without the user (local state)
Thanks!
The problem is with the asynchronous calls. You're making an async call in forEach and forEach expects a synchronous function.
You can change the logic to use for-of instead.
See below:
const getUserFiles = async () => {
if (!userUID) {
return null;
}
let listRef = storageRef.child(userUID);
const res = await listRef.listAll();
for (const itemRef of res.items) {
const itemMetadata = await itemRef.getMetadata();
const url = await itemRef.getDownloadUrl();
var file = {
name: itemMetadata.name.toString(),
timeCreated: itemMetadata.timeCreated.toString(),
link: url,
};
myFiles.push(file);
}
console.log(myFiles);
}
The getURL() function creates an array of scraped URLs from the original URL. getSubURL() then loops through that array and scrapes all of those pages' URLs. Currently, this code outputs just fine to the console, but I don't know how to wait for my data to resolve so I can push all gathered data to a single array. Currently, when I try and return sites and then push to array, it only pushes the last value. I believe it's a promise.all(map) situation, but I don't know how to write one correctly without getting an error. Ideally, my completed scrape could be called in another function. Please take a look if you can
const cheerio = require('cheerio');
const axios = require('axios');
let URL = 'https://toscrape.com';
const getURLS = async () => {
try {
const res = await axios.get(URL);
const data = res.data;
const $ = cheerio.load(data);
const urlQueue = [];
$("a[href^='http']").each((i, elem) => {
const link = $(elem).attr('href');
if (urlQueue.indexOf(link) === -1) {
urlQueue.push(link);
}
});
return urlQueue;
} catch (err) {
console.log(`Error fetching and parsing data: `, err);
}
};
const getSubURLs = async () => {
let urls = await getURLS();
try {
//loop through each url in array
for (const url of urls) {
//fetch all html from the current url
const res = await axios.get(url);
const data = res.data;
const $ = cheerio.load(data);
//create object and push that url into that object
let sites = {};
sites.url = url;
let links = [];
//scrape all links and save in links array
$("a[href^='/']").each((i, elem) => {
const link = $(elem).attr('href');
if (links.indexOf(link) === -1) {
links.push(link);
}
//save scraped data in object
sites.links = links;
});
// returns list of {url:'url', links:[link1,link2,link3]}
console.log(sites);
}
} catch (err) {
console.log(`Error fetching and parsing data: `, err);
}
};
Don't think this is a Promise related issue at heart.
You'll need to collect your sites into an array that is initialized outside the loop. Then when getSubURLs() resolves, it will resolve to your array:
const getSubURLs = async() => {
let urls = await getURLS();
let siteList = [];
try {
for (const url of urls) {
// :
// :
// :
siteList.push(sites);
}
} catch (err) {
console.log(`Error fetching and parsing data: `, err);
}
return siteList; // array of objects
};
getSubURLs().then(console.log);
I have a script that reads an excel file and gets data from a specific column to perform a search on the Google Maps API where I use axios. For each request made, I need to save it in the newFileList variable. After completing all the requests, I must save the contents of this variable in a file. However, whenever I run my code, the file is being saved without the content of the newFileList variable. How do I wait for all requests to finish before being able to save the content in the file?
Note: the reading, writing and requesting data are working. I just need the rescue to happen only after all the loop requests are finished. I tried to solve by placing the loop inside a promisse and at the end of the execution of this loop I used resolve.
const xlsx = require("node-xlsx");
const fs = require("fs");
const coordinate = require("./coordinate");
const resourcePath = `${__dirname}/resources`;
const contentFile = xlsx.parse(`${resourcePath}/file-2.xlsx`)[0].data;
const newFile = [[...contentFile, ...["Latitude", "Longitude"]]];
for (let i = 1; i < contentFile.length; i++) {
const data = contentFile[i];
const address = data[2];
coordinate
.loadCoordinates(address)
.then((response) => {
const { lat, lng } = response.data.results[0].geometry.location;
newFile.push([...data, ...[lat.toString(), lng.toString()]]);
})
.catch((err) => {
console.log(err);
});
}
console.log(newFile);
//The code below should only be executed when the previous loop ends completely
var buffer = xlsx.build([{ name: "mySheetName", data: newFile }]); // Returns a buffer
fs.writeFile(`${resourcePath}/file-3.xlsx`, buffer, function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
The coordinate file:
const axios = require("axios");
module.exports = {
loadCoordinates(address) {
const key = "abc";
return axios
.get(`https://maps.googleapis.com/maps/api/geocode/json`, {
params: {
address,
key,
},
})
},
};
Will using an async IIFE help?
const xlsx = require("node-xlsx");
const fs = require("fs");
const coordinate = require("./coordinate");
const resourcePath = `${__dirname}/resources`;
const contentFile = xlsx.parse(`${resourcePath}/file-2.xlsx`)[0].data;
const newFile = [[...contentFile, ...["Latitude", "Longitude"]]];
(async() => {
try{
for (let i = 1; i < contentFile.length; i++) {
const data = contentFile[i];
const address = data[2];
await coordinate
.loadCoordinates(address)
.then((response) => {
const { lat, lng } = response.data.results[0].geometry.location;
newFile.push([...data, ...[lat.toString(), lng.toString()]]);
})
.catch((err) => {
console.log(err);
});
}
console.log(newFile);
//The code below should only be executed when the previous loop ends completely
var buffer = xlsx.build([{ name: "mySheetName", data: newFile }]); // Returns a buffer
fs.writeFile(`${resourcePath}/file-3.xlsx`, buffer, function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
} catch(e) {
console.log(e)
}
})();
Do note that I added await before coordinate.loadCoordinates, in order to make sure the first axios request is finished before we proceed to the next one.
You need to use Promise.all() to wait until all the promises are resolved. After that execute the writeToFile part. For more info on Promise.all(), you can refer https://www.javascripttutorial.net/es6/javascript-promise-all/
const requestPromiseArray = [];
for (let i = 1; i < contentFile.length; i++) {
const data = contentFile[i];
const address = data[2];
requestPromiseArray.push(coordinate
.loadCoordinates(address))
}
Promise.all(requestPromiseaArray).then(results=>{
// Handle "results" which contains the resolved values.
// Implement logic to write them onto a file
var buffer = xlsx.build([{ name: "mySheetName", data: results }]);
fs.writeFile(`${resourcePath}/file-3.xlsx`, buffer, function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
})
I am using following NodeJS module for getting onvif device in the network https://github.com/futomi/node-onvif, and which works fine.
But using below code the list of device found is always empty. I checked the getDeviceData function and which is getting the device and print data.
But using below code the line console.log(JSON.stringify(list)); execute before all scan process completed. How can I fix it.
async function getDeviceData(info){
var device = new onvif.OnvifDevice({
xaddr: info.xaddrs[0],
user : 'admin',
pass : '123456'
});
await device.init();
var dev_info = device.getInformation();
var rtsp_url = device.getUdpStreamUrl();
var data = {"Manufacturer":dev_info.Manufacturer,"Model":dev_info.Model};
console.log(data); // this print last
return data;
}
const onvif = require('node-onvif');
onvif.startProbe().then((device_info_list) => {
var list = [];
device_info_list.forEach((info) => {
var data = getDeviceData(info)
list.push(data);
});
console.log(JSON.stringify(list)); // this print first
}).catch((error) => {
console.error(error);
});
Here is what you want:
You need to use await to wait the promise to resolve with the data.
onvif.startProbe().then(async (device_info_list) => {
var list = [];
await Promise.all(device_info_list.map(async(info) => {
var data = await getDeviceData(info)
list.push(data);
}));
console.log(JSON.stringify(list)); // this print first
}).catch((error) => {
console.error(error);
});
The data const in the first function returns a promise so would have yo await that. So this should work:
async function getDeviceData(info) {
const device = new onvif.OnvifDevice({
xaddr: info.xaddrs[0],
user: 'admin',
pass: '123456',
});
await device.init();
const dev_info = device.getInformation();
const rtsp_url = device.getUdpStreamUrl();
const data = { Manufacturer: dev_info.Manufacturer, Model: dev_info.Model };
await Promise.resolve(data);
console.log(data); // this print last
return data;
}
onvif
.startProbe()
.then(device_info_list => {
const list = [];
device_info_list.forEach(async info => {
const data = await getDeviceData(info);
await Promise.resolve(list.push(data));
});
console.log(JSON.stringify(list)); // this print first
})
.catch(error => {
console.error(error);
});
const getUser = async (user) => {
const body = await snekfetch.get('https://www.website.com/api/public/users?name=' + user);
const userInfo = JSON.parse(body.text);
const r = await snekfetch.get('https://www.website.com/api/public/users/' + userInfo.uniqueId + '/profile');
const extraUserInfo = JSON.parse(r.text);
const _message = await client.users.get('437502925019807744').send({ files: ['https://www.website.nl/avatar-imaging/avatarimage?figure=h' + userInfo.figureString + '.png'] });
const avatarImage = _message.attachments.first().url;
return { userInfo, extraUserInfo, avatarImage };
};
getUser(args[0]).then((result) => {
message.channel.send(`${result.userInfo.name}`);
}).catch(function(result) {
console.log(result.userInfo.name);
});
Here I am trying to use 3 API's, however it always goes to the catch, even if one exists and other don't, I tried to only use result.userInfo.name to only use the first API, also in the catch I use the first one, then I tried a name that only has the first API but not the second one however I still get: TypeError: Cannot read property 'name' of undefined because it looks at the second API as well, what else can I do to handle with this situation? So basically how can I only catch errors for the first API
edit: I also tried:
if (extraUserInfo.user.name) {return { userInfo, extraUserInfo, avatarImage };}
else {return { userInfo, avatarImage };}
Fixed it with a try catch
try {
const r = await snekfetch.get('https://www.website.com/api/public/users/' + userInfo.uniqueId + '/profile');
const extraUserInfo = JSON.parse(r.text);
return {
userInfo,
extraUserInfo,
avatarImage
};
} catch (error) {
const extraUserInfo = {
'error': 'not-found'
};
return {
userInfo,
extraUserInfo,
avatarImage
};
}
};
getUser(args[0]).then((result) => {
console.log(result.extraUserInfo.error === 'not-found' ? result.userInfo.name : result.extraUserInfo.user.name);
}).catch((error) => {
console.log(error);
});