I want to fetch icon PNGS from gridfs out of our mongodb database with mongoose. These icons then should be zipped and served at a specific route.
My current code is as follows:
var zip = require("node-native-zip");
async function getZipFile() {
//get the events out of the DB
db.Category.find({}).populate('icons.file').exec(async function (err, cats) {
if (err) {
//oh oh something went wrong, better pass the error along
return ({
"success": "false",
message: err
});
}
else {
//all good, build the message and return
try {
const result = await buildZip(cats);
return ({
"success": "true",
message: result
});
}
catch (err) {
console.log("ZIP Build Failed")
}
}
});
}
async function buildZip(cats) {
let archive = new zip();
for (let i = 0; i < cats.length; i++) {
cats[i].icons.forEach(function (icon) {
if (icon.size === "3x") {
db.Attachment.readById(icon.file._id, function (err, buffer) {
if (err)
return;
archive.add(cats[i]._id + ".png", buffer);
});
}
});
//return when everything is done
if (i === cats.length - 1) {
return archive.toBuffer();
}
}
}
module.exports =
{
run: getZipFile
};
I don't want to build the zip before runtime, as I want to rename the icons acording to the category ID. I tried going for a async/await structure, but my callback is being returned before the building of the zip file even started.
I'm calling the function with
case 'categoryZip':
categoryHelper.getZipFile.run().then((result) => {
callback(result);
});
break;
This should (as far as I understood it) fire the callback when the zipping is done, but I think I'm missing something essential here.
I wrapped both your callback methods into promises, and also awaited your double for-loop of callbacks in parallel using Promise.all() since they don't rely on each other and I assume they don't need to be in any particular order in the zip file:
async function getZipFile() {
//get the events out of the DB
return new Promise((resolve, reject) => {
db.Category.find({}).populate('icons.file').exec(async function(err, cats) {
if (err) {
//oh oh something went wrong, better pass the error along
reject({
success: false,
message: err
});
} else {
//all good, build the message and return
try {
const result = await buildZip(cats);
resolve({
success: true,
message: result
});
} catch (err) {
console.log("ZIP Build Failed")
reject({
success: false,
message: err
});
}
}
});
});
}
async function buildZip(cats) {
let archive = new zip();
await Promise.all(
cats.map(cat => Promise.all(cat.icons
.filter(icon => icon.size === '3x')
.map(icon => new Promise((resolve, reject) => {
db.Attachment.readById(icon.file._id, function(err, buffer) {
if (err) return reject(err);
archive.add(cat._id + ".png", buffer);
resolve();
});
}))
))
);
return archive.toBuffer()
}
Related
The problem:
I have a function that maps over countries and regions and creates an array of urls, then makes a GET request to each one. I want to save the responses in a single json file, and I want this function to handle that as well.
Expected results:
I expected to be able to run the function as needed (like when source data is updated), and get a new or updated local json file with all the data objects in one array.
Actual results:
A file with only one record, an array with the last response object.
What I've tried:
I tried using fs.writeFile and fs.readFile. I did not get any errors, but the resulting file had only one record, even though console showed all the requests being made. It seemed that each response was being written over the previous.
Minimum reproducable (node.js) example:
const fs = require('fs')
// subset of countries and codes for demo purposes
const countryDirList = [
'antarctica',
'central-asia',
]
const fbCountryCodes = [
{ "region": "antarctica", "codes": ["ay", "bv"] },
{ "region": "central-asia", "codes": ["kg", "kz"] },
]
const callingUrlsInSequence = async () => {
fs.writeFile('./test.json', '[]', function (err) {
if (err) throw err
console.log('File - test.json - was created successfully.')
})
try {
const urlConstructor = countryDirList.map(async (country) => {
console.log('countries mapped', country)
return fbCountryCodes.filter(async (f) => {
if (country === f.region) {
const urls = f.codes.map(async (c) => {
const response = await axios({
method: 'get',
url: `https://raw.githubusercontent.com/factbook/factbook.json/master/${country}/${c}.json`,
responseType: 'json',
headers: {
'Content-Type': 'application/json',
},
})
fs.readFile('./test.json', function (err, data) {
let json = JSON.parse(data)
json.push(response.data)
setTimeout(() => {
fs.writeFile('./test.json', JSON.stringify(json), function (err) {
if (err) throw err
console.log('The "data to append" was appended to file!')
})
}, 1000)
})
return response.data
})
const dataArr = await Promise.all(urls)
dataArr.map((item) =>
console.log(
'dataArr',
item.Government['Country name']['conventional short form']
)
)
}
})
})
} catch (err) {
console.log('axios error: ', err)
}
}
callingUrlsInSequence()
I'm re-writing this question now because it kept getting downvoted, and I could see that it was not very concise.
I can also see now, that obviously, the fs.readFile inside the fs.writeFile is not going to work in the code I provided, but I'm leaving it there in case it might help someone else, combined with the solution I provided in response to my own question.
I ended up learning how to solve this problem with both node-fetch and axios. They are not exactly the same.
For both:
First, check for existence of destination file, and create one if it's not already there.
const createNew = () => {
try {
if (existsSync('./data.json')) {
console.log('file exists')
return
} else {
writeFile('./data.json', '[]', (error, data) => {
if (error) {
console.log('fs.writeFile - create new - error: ', error)
return
}
})
}
} catch (err) {
console.log('fs.existsSync error: ', err)
}
}
createNew()
Then make the array of urls:
const countryDirList = [...countries]
const fbCountryCodes = [...codes]
const urls = []
// maybe a reducer function would be better, but my map + filter game is much stronger X-D
const makeUrls = (countriesArr, codesArr) =>
countriesArr.map((country) => {
return codesArr.filter((f) => {
if (country === f.region) {
return f.codes.map((code) => {
return urls.push(
`https://raw.githubusercontent.com/factbook/factbook.json/master/${country}/${code}.json`
)
})
}
})
})
makeUrls(countryDirList, fbCountryCodes)
Next, make the requests.
Axios:
fs.readFile('./data.json', (error, data) => {
if (error) {
console.log(error)
return
}
Promise.all(
urls.map(async (url) => {
let response
try {
response = await axios.get(url)
} catch (err) {
console.log('axios error: ', err)
return err
}
return response
})
)
.then((res) => {
const responses = res.map((r) => r.data)
fs.writeFile('./data.json', JSON.stringify(responses, null, 2), (err) => {
if (err) {
console.log('Failed to write data')
return
}
console.log('Updated data file successfully')
})
})
.catch((err) => {
console.log('axios error: ', err)
})
})
Node-fetch:
//same basic structure, readFile with fetch and write file inside
fs.readFile('./data2.json', (error, data) => {
if (error) {
console.log(error)
return
}
async function fetchAll() {
const results = await Promise.all(
urls.map((url) => fetch(url).then((r) => r.json()))
)
fs.writeFile('./data2.json', JSON.stringify(results, null, 2), (err) => {
if (err) {
console.log('Failed to write data')
return
}
console.log('Updated data file successfully')
})
}
fetchAll()
})
Both methods produce exactly the same output: a json file containing a single array with however many response objects in it.
I am constantly getting the printed out message of "No File Upload" Failed when I select my image and hit upload. It never goes to true..
As you can see, I am not actually uploading here. Just testing the req.files is there something wrong in my router.post? Any input would be appreciated.
router.post('/upload', async (req, res) => {
try {
if(!req.files) {
res.send({
status: false,
message: 'No file uploaded'
});
} else {
res.send({
status: true,
message: 'Files are uploaded',
data: data
});
}
} catch (err) {
res.status(500).send(err);
}
})
module.exports=router
can you share your controller file of it and also you are using async function but not defining await inside of that function,must have to use 'await' when executing async function.
here is the controller file AAmir
let fs = require('fs');
let async = require('async');
function uploaddownFiles(connection, fromFolder, toFolder, sftpmethod) {
return new Promise((resolve, reject) => {
// Getting all file list in given folder of local machine
let fileList = fs.readdirSync(fromFolder);
// filter only files not folders
fileList = fileList.filter(file => {
if (file.includes('.')) return true;
return false;
});
console.log('Total files: ', fileList.length)
if (!fileList.length) return reject('No file to send')
connection.sftp(function (err, sftp) {
if (err) return;
async.eachOfSeries(fileList, (file, key, cb) => {
let moveFrom = `${fromFolder}/${file}`;
let moveTo = `${toFolder}/${file}`;
if (sftpmethod=== 'put')
sftp.fastPut(moveFrom, moveTo, {}, function (uploadError) {
if (uploadError) return cb(uploadError);
console.log("Successfully Uploaded", file);
cb();
});
else if (sftpmethod === 'get')
sftp.fastGet(moveFrom, moveTo, {}, function (uploadError) {
if (uploadError) return cb(uploadError);
console.log("Successfully Downloaded", file);
cb();
});
}, function (err) {
if (err) {
console.log(err);
return reject(err);
} else {
console.log('all files have been uploaded/downloaded');
return resolve();
}
})
});
});
}
I'm working on some code for an express API that essentially reaches out to an external REST service for an authentication token, and then uses that token to do stuff. I'm very new to NodeJS, so I think I'm having a lot of trouble with the whole sync/async thing.
Right now my main problem seems to be that I'm getting ReferenceError: err is not defined, which seems to be related to the line in library.js, but I expect there are a lot of problems here, and will appreciate anything that can get me back on the right track.
index.js
library = require('library.js');
module.exports = async (req,res) => {
// This is a test endpoint for prototyping code and testing calls.
URI = "/some/uri";
method = "GET";
body = "";
try {
restResponse = await library.RESTCall(URI,method,body);
res.send(data);
} catch (e) {
return res.status(500).json({ errors: err});
}
};
library.js
exports.RESTCall = async function(URI,method,body) {
return new Promise((resolve, reject) => {
getAuthToken().then((token) => {
console.log("Token: " + token);
try {
// Do stuff with the token to make another call
resolve(data);
} catch (e) {
reject(e);
}
}).catch((err) => {
reject(err);
});
});
}
exports.getAuthToken = () => {
return new Promise((resolve, reject) => {
try {
// Do stuff to get an authentication token
resolve(authToken);
} catch(e) {
reject("Failed to get Facets Authentication token. Error: " + e);
}
});
}
This looks like just a typo:
return res.status(500).json({ errors: e});
FYI this:
exports.RESTCall = async function(URI,method,body) {
return new Promise((resolve, reject) => {
getAuthToken().then((token) => {
console.log("Token: " + token);
try {
// Do stuff with the token to make another call
resolve(data);
} catch (e) {
reject(e);
}
}).catch((err) => {
reject(err);
});
});
}
Is mostly equivalent, but slightly worse as:
exports.RESTCall = function(URI,method,body) {
return getAuthToken().then((token) => {
console.log("Token: " + token);
// Do stuff with the token to make another call
return data;
}
}
But because you have async/await, can be simplified further:
exports.RESTCall = async function(URI,method,body) {
const token = await getAuthToken();
console.log("Token: " + token);
// Do stuff with the token to make another call
return data;
}
Every time you see yourself type new Promise, consider it a red flag. I'd really suggest you take the time to learn how promises work.
Having issues with using async and await. I'm executing two queries and then saving the result to a temp variable. After I have collected the response from all executed queries, I'm going to send that to the client.
Here is my current example code.
module.exports = (app) => {
app.get('/api/stats', (req, res) => {
let fetch1 = '';
let fetch2 = '';
conn.query('query here', [], async (error, results) => {
if (error) {
return res.send({
success: false,
message: 'There was an error.'
});
} else {
fetch1 = results;
}
});
conn.query('query here', [], async (error, results) => {
if (error) {
return res.send({
success: false,
message: 'There was an error.'
});
} else {
fetch2 = results;
}
});
// I need to wait until the queries have resolved so that I can send the correct data
return res.send({
success: true,
fetch1: fetch1,
fetch2: fetch2
});
});
};
I basically need to wait until the queries have been resolved so that I can send the correct data towards the end.
Can anyone explain how I can use await and async to accomplish this?
Thanks.
You can only await a Promise, so for functions that don't return Promises you need to create a Promise wrapper. This needs to be done per call that would previously use a callback, but you can make a helper function per function you need to wrap.
function queryPromise(query, parameters) {
return new Promise((resolve, reject) => {
conn.query(query, parameters, (err, results) => {
if(err) {
reject(err);
} else {
resolve(results);
}
});
});
}
module.exports = (app) => {
app.get('/api/stats', async (req, res) => {
try {
let fetch1 = await queryPromise('query here', []);
let fetch2 = await queryPromise('query here', []);
res.send({
success: true,
fetch1: fetch1,
fetch2: fetch2
});
} catch {
res.send({
success: false,
message: 'There was an error.'
});
}
});
};
From my knowledge, I usually apply async to functions and perform await on certain variables (inside the function) that need to be acquired from a specific database.
So to implement this into your function containing the async tag, you could possibly do:
fetch1 = await results;
fetch2 = await results;
This will wait until the data is attached onto the variable fetch1 and fetch2 before continuing on with the code.
Sorry if this was very vague, hopefully this was somewhat helpful.
I'm very new to node.js so I think I'm missing something obvious here.
I'm simply trying to get a list of SQS queues using aws-sdk and return them from a module to be accessible to other code. list_queues is the function in question.
The code below works to an extent, I see a "success" log and a log of a string array of all my queues, however, the function does not return that array to the caller and I don't understand why.
const AWS = require('aws-sdk');
AWS.config.update({region: 'eu-west-1'});
var sqs;
var sts = new AWS.STS();
sts.assumeRole({
RoleArn: 'arn:aws:iam::xxxxx:role/UserRole',
RoleSessionName: 'NodeDeveloperRoleSession'
}, function(err, data) {
if (err) { // an error occurred
console.log('Cannot assume role :(');
console.log(err, err.stack);
} else { // successful response
console.log('Assumed role success :)');
AWS.config.update({
accessKeyId: data.Credentials.AccessKeyId,
secretAccessKey: data.Credentials.SecretAccessKey,
sessionToken: data.Credentials.SessionToken
});
sqs = new AWS.SQS({apiVersion: '2012-11-05'});
}
});
exports.list_queues = function() {
sqs.listQueues({}, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("success");
console.log(data.QueueUrls);
return data.QueueUrls;
}
});
}
Any help is appreciated
exports.list_queues = function() { // 2. but you actually want to return from this one
sqs.listQueues({}, function(err, data) { <-----------------
if (err) { |
console.log("Error", err); |
} else { |
console.log("success"); |
console.log(data.QueueUrls); |
return data.QueueUrls; // 1. you are returning from this one
}
});
}
there are two ways you can make it work
Promise based
exports.list_queues = function() {
return sqs.listQueues({}).promise().then((data) => data.QueueUrls);
}
// and in another file you would:
const {list_queues} = require('./list_queues.js');
list_queues.then((queues) => console.log(queues));
Callback based
exports.list_queues = function(cb) { // notice I added callback here
sqs.listQueues({}, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("success");
console.log(data.QueueUrls);
cb(data.QueueUrls);
}
});
}
// and in another file you would:
const {list_queues} = require('./list_queues.js');
list_queues(function(queues) {
console.log(queues);
});
I strongly recommend you to use promise based approach, since it's much more readable and you can make use of async/await on it, which is great.