Develop Reference

NodeJS: Wait for all foreach with Promises to finish but never actually finishes

I am working with Nodejs. I have a forEach which is async as I have to wait for a result inside the forEach. As a result, I need to wait for the forEach to finish and then carry on with the result of the loop. I found several solutions for waiting for the forEach, one of them is using Promises. I did though, and these promises are created, however, the code after the forEach (and therefore the promises) are finished, is never actually executed (console.log is not printed). And the NodeJS function just ends without any errors.
Here is my Code:
var Client = require('ssh2').Client;
// eslint-disable-next-line no-undef
var csv = require("csvtojson");
// eslint-disable-next-line no-undef
var fs = require("fs");
// eslint-disable-next-line no-undef
const config = require('./config.json');
// eslint-disable-next-line no-undef
const os = require('os');
let headerRow = [];
let sumTxAmount = 0;
const filenameShortened = 'testFile';
let csvLists = [];
let csvFile;
const options = {
flags: 'r',
encoding: 'utf8',
handle: null,
mode: 0o664,
autoClose: true
}
var conn = new Client();
async function start() {
const list = await getCSVList();
let content = fs.readFileSync('./temp.json', 'utf8');
content = JSON.parse(content);
var promises = list.map(function(entry) {
return new Promise(async function (resolve, reject) {
if (!content['usedFiles'].includes(entry.filename)) {
const filename = entry.filename;
csvFile = await getCsv(filename);
csvLists.push(csvFile);
console.log('here');
resolve();
} else {
resolve();
}
})
});
console.log(promises)
Promise.all(promises)
.then(function() {
console.log(csvLists.length, 'length');
})
.catch(console.error);
}
start();
The "here" is printed once (not 8 times as the arrays length is 8), but there are 8 promises created. The lower part where I am printing the length of the array is not executed.
Can anyone tell me what I am doing wrong? Am I using Promises and forEach falsely as I have to do an await inside the forEach?
Note: getCSVList() and getCsv() are functions to get Csvs from an sftp server:
function getCSVList() {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) throw err;
sftp.readdir(config.development.pathToFile, function (err, list) {
if(err) {
console.log(err);
conn.end();
reject(err);
} else {
console.log('resolved');
conn.end();
resolve(list);
}
})
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
})
}
function getCsv(filename) {
return new Promise((resolve, reject) => {
conn.on('ready', function () {
conn.sftp(function (err, sftp) {
if (err) reject(err);
let csvFile = sftp.createReadStream(`${config.development.pathToFile}/${filename}`, options);
// console.log(csvFile);
conn.end();
resolve(csvFile);
})
}).connect({
host: config.development.host,
port: config.development.port, // Normal is 22 port
username: config.development.username,
password: config.development.password
// You can use a key file too, read the ssh2 documentation
});
});
}
The output in my console from all the console logs is:
`➜ node server.js
resolved
[ Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> },
Promise { <pending> } ]
here`

Break up your problem into pieces, confirming they work along the way.
You are not using the stream correctly, among other things.
I made a working example with ssh2-sftp-client so you can maybe use it as a starting point.
Working example :
var fs = require('fs'); var _ = require('underscore');
var SFTPClient = require('ssh2-sftp-client');
const CONFIG = {
"SSH_CONN_OPTS":{"host":"XXXXXXXX","port":22,"username":"XXXXXXXX","password":"XXXXXXXX"},
"CSV_DIRECTORY":"/var/www/html"
}
//---------------
//.:The order-logic of the script is here
function StartScript(){
console.log("[i] SSH Connection")
LoadValidationFile(()=>{
InitializeSFTP(()=>{ console.log("[+] SSH Connection Established")
ListRemoteDirectory((list)=>{ console.log(`[i] Total Files # ${CONFIG.CSV_DIRECTORY} : ${list.length}`)
//console.log(list) //:now you have a 'list' of file_objects, you can iterate over to check the filename
var csvFileList = [] //store the names of the files you will request after
_.each(list,(list_entry)=>{ console.log(list_entry)
if(!CONFIG.USED_FILES.includes(list_entry.name)){ csvFileList.push(list_entry.name) }
})
//:now loop over the new final list of files you have just validated for future fetch
GenerateFinalOutput(csvFileList)
})
})
})
}
//.:Loads your validation file
function LoadValidationFile(cb){
fs.readFile(__dirname+'/temp.json','utf8',(err,data)=>{ if(err){throw err}else{
var content = JSON.parse(data)
CONFIG.USED_FILES = content.usedFiles
cb()
}})
}
//.:Connects to remote server using CONFIG.SSH_CONN_OPTS
function InitializeSFTP(cb){
global.SFTP = new SFTPClient();
SFTP.connect(CONFIG.SSH_CONN_OPTS)
.then(()=>{cb()})
.catch((err)=>{console.log("[!] InitializeSFTP :",err)})
}
//.:Get a list of files from a remote directory
function ListRemoteDirectory(cb){
SFTP.list(`${CONFIG.CSV_DIRECTORY}`)
.then((list)=>{cb(list)})
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//.:Get target file from remote directory
function GetRemoteFile(filename,cb){
SFTP.get(`${CONFIG.CSV_DIRECTORY}/${filename}`)
.then((data)=>{cb(data.toString("utf8"))}) //convert it to a parsable string
.catch((err)=>{console.log("[!] ListRemoteDirectory :",err)})
}
//-------------------------------------------
var csvLists = []
function GenerateFinalOutput(csv_files,current_index){ if(!current_index){current_index=0}
if(current_index!=csv_files.length){ //:loop
var csv_file = csv_files[current_index]
console.log(`[i] Loop Step #${current_index+1}/${csv_files.length} : ${csv_file}`)
GetRemoteFile(csv_file,(csv_data)=>{
if(csv_data){csvLists.push(csv_data)}
current_index++
GenerateFinalOutput(csv_files,current_index)
})
}else{ //:completed
console.log("[i] Loop Completed")
console.log(csvLists)
}
}
//------------
StartScript()
Good luck!

Promise.all is a method that will return a promise object, but you are not waiting for your start method to execute.
function getCSVList() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve([1, 2, 3, 4]);
}, 1000);
});
}
function getCsv(params) {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve(params);
}, 1000);
});
}
async function start() {
const list = await getCSVList();
const promises = list.map(item => {
return new Promise(async function (resolve, reject) {
const csvFile = await getCsv(item);
console.log('here');
resolve(csvFile);
});
});
return Promise.all(promises);
}
start().then(res => {
console.log(res);
});

Is there a mock way in jest to test all dynamo low-level services with my controllers?

I'm writing jets tests and fully support unit test for dynamo express app. Is there a good example how mocking dynamo from my controller dependencies that it returns same response and verify the type of input data at dynamoDB low level api through the following functions attributes: get, query, batchGet, batchWrite as well as which data needs to be compared. The mock structure should behave as follows: controller -> db services -> json -> wrapper and thus be able to start asserting each case.
I tried aws-sdk-mock but unsuccessfully didn't work. Below is my code snippet:
function getRecordIdsFromDynamo(message) {
return new Promise((resolve, reject) => {
let is = message.id;
let params = {
Key: { id: id },
TableName: LOOKUP,
ConsistentRead: true,
ProjectionExpression: 'activeRecord'
}
dynamoClient.get(params, function(err, data) {
try {
let activeRecords = [];
if(data.Item.activeRecord)
activeRecords = data.Item.activeRecord.values;
resolve(activeRecords);
} catch(){}
})
})
}
function getRecordsFromDynamo(keys, id) {
return new Promise((resolve, reject) => {
let results = [];
let params = { RequestItems: {} };
if(keys.length == 0)
return resolve(results);
// unprocessed logic...
dynamoClient.batchGet(params, function(err, data) {
// err logic...
const responses = data.Responses[RECORD],
newUnprocessedKeys = data.UnprocessedKeys[RECORD];
results = results.concat(responses);
resolve(results)
})
})
}
exports.getRecords = function(message) {
return new Promise((resolve, reject) => {
getRecordsIdsFromDynamo(message)
.then((records) => {
let keys = records.map((id) => {
return { RecordId: id }
});
getRecordsFromDynamo(keys, message.id)
.then((records) => {})
.catch((err) => {
reject(err);
});
});
}
// test code
exports.getActiveRecordIds = async (message, docClient) => {
let data = await getActiveRecordsIdsfromDB(message, docClient);
console.log("data.Items =========>", data.Items);
return {
"Items": {
activeRecordIds: {
values: data.Items
}
}
}
};
/**
*
* #param {String} messageDevice
* #param {AWS} docClient
* #returns {Promise<object>} return promise with activeRuleIds
*/
function getActiveRecordsIdsfromDB(message, docClient) {
const activeRecords = db.getRecords(message);
console.log("In getActiveRecordsIdsFromDB " + activeRecords);
const params = {
Key: { id: message.id },
TableName: LOOKUP,
ConsistentRead: true,
ProjectionExpression: 'activeRecordIds'
};
return docClient.get(params).promise();
};
beforeEach(() => {
// Clear all instances and calls to constructor and all methods:
Message.mockClear();
event = record.event;
recordId = record.id;
ts = record.ts;
context = '1357';
message = new Message({
id: id,
ts: ts,
event: event
}, context);
});
it('should get all active records from dynamo', () => {
AWS_mock.mock('DynamoDB.DocumentClient', 'get', function(params, callback) {
callback(null, { Items: ['1234', '4321'] });
});
let docClient = new aws.DynamoDB.DocumentClient();
return getActiveRecordsIds(message, docClient)
.then(recordIds => {
expect(recordIds).toEqual({ Items: { activeRecordsIds: { values: ['1234', '4321'] } } });
});
});

why my array is empty?

i can not figure out why my array is empty while the console is showing output:
I'm fetching data from dynamodb and trying to make a modified response json array i can print the values of dynamodb response but when i push it into array the returned array is empty here is my code.
Here is Util.js file
const dynamo = require('./dynamo.js')
var myjson = [];
exports.myjson = [];
exports.maketimetabledata = function(callback) {
var table = "timetable";
for (i = 1; i <= 2; i++) {
dynamo.docClient.get({
TableName: table,
Key: {
"id": i
}
}, function(err, data) {
if (err) {
console.error("Unable to read item. Error JSON:", JSON.stringify(err, null, 2));
return err;
} else {
dynamores = JSON.parse(JSON.stringify(data));
myjson.push(i);
}
});
console.log(myjson);
}
callback("done");
};
and here is my partial index file:
app.get('/', (req, res) => {
var myjson = [];
util.maketimetabledata(function(returnvalue) {
if (returnvalue) {
console.log(util.myjson);
}
});
});
output :
[] [] []
undefined
{ response from dynamo db for debugging purpose}

Assuming dynamo.docClient.get() returns a Promise, You can use Promise.all() to wait for all the promise to complete then invoke the callback method.
//Use Promise.all()
exports.maketimetabledata = function (callback) {
var table = "timetable";
for (i = 1; i <= 2; i++) {
var table = "timetable";
var promiseArray = [];
for (i = 1; i <= 2; i++) {
var promise = dynamo.docClient.get({
//Your code
});
promiseArray.push(promise)
}
console.log(myjson);
}
Promise.all(promiseArray).then(function () {
callback(myjson);
});
};
//Usage
util.maketimetabledata(function (myjson) {
if (returnvalue) {
console.log(myjson);
}
});
If the method is not returning Promise
var promise = new Promise(function (resolve, reject) {
dynamo.docClient.get({
TableName: table,
Key: {
"id": i
}
}, function (err, data) {
if (err) {
console.error("Unable to read item. Error JSON:", JSON.stringify(err, null, 2));
reject();
return err;
} else {
dynamores = JSON.parse(JSON.stringify(data));
myjson.push(i);
resolve();
}
});
});

How to return string value of items inside file using fs and promises?

i have a doubt, i have this code in js:
var fs = require('fs');
var Promise = require('bluebird');
fs.readFileAsync = function(filename) {
return new Promise(function(resolve, reject) {
fs.readFile(filename, function(err, data){
if (err)
reject(err);
else
resolve(data);
});
});
};
function constructStringArray(nameArray) {
var i = 1;
fs.readFileAsync("./input.txt").then((value) => {
console.log(value);
}).catch((err) => {
console.log("Something goes wrong");
});
}
constructStringArray("test");
And this code returns me this value: .
How can i make to return strings like this:
Adria
Joan
Papo
Thank you.

Using promises in my mean stack app

I've created a controller that does a bing search based on the user's input into the url. Based on my results from doing a console.log the controller is working correctly and I have set that variable to return. In the route file the information is not displaying to the page. I thought it might be an asynchronous issue so I am trying to use promises to make sure the controller has returned before it tries to do the res.json but I'm not very familiar with promises so my syntax might be off or I might be going about this the wrong way. Will someone take a look at this syntax and see if there is an issue.Currently only an empty object is displaying on the page.
app.route('/imagesearch/:keyword')
.get(function (req, res) {
var resObj = [];
resObj = new Promise (function(resolve, reject){
resolve(bingSearchHandler.findImages(req.params));
});
resObj.then(res.json(resObj));
});
//BINGSEARCHHANDLER
'use strict';
var bingAPPID = 'fwHyQAoJMJYmK8L4a3dIV2GAEUfXAlFRjCnBx0YbfPE=';
var Search = require('bing.search');
var util = require('util');
var search = new Search(bingAPPID);
function bingSearchHandler () {
this.findImages = function(userInput){
var keyword = userInput.keyword;
search.images(keyword,
{top: 10},
function(err, results) {
if(err)
{
console.log(err);
}
else
{
var resArr = [];
(util.inspect(results,
{colors: true, depth: null}));
for(var i=0;i<results.length;i++)
{
var tempObj = {};
tempObj.url = results[i].url;
tempObj.snippet = results[i].title;
tempObj.thumbnail = results[i].thumbnail.url;
tempObj.context = results[i].sourceUrl;
resArr.push(tempObj);
}
console.log(resArr);
return resArr;
}
}
);
}
}
module.exports = bingSearchHandler;

Could you please try this code? Here you have the bing.search documentation, https://www.npmjs.com/package/bing.search Always try to use callbacks instead of promises in NodeJs, remember that the first parameter of a callback is always the error (if there is any), then the response
app.route('/imagesearch/:keyword')
.get(function (req, res) {
bingSearchHandler.findImages(req.params, function (err, response) {
if (err) return res.status(400)
res.json(response)
})
});
//BINGSEARCHHANDLER
'use strict';
var bingAPPID = 'fwHyQAoJMJYmK8L4a3dIV2GAEUfXAlFRjCnBx0YbfPE=';
var Search = require('bing.search');
var util = require('util');
var search = new Search(bingAPPID);
function bingSearchHandler () {
this.findImages = function(userInput, callback){
var keyword = userInput.keyword;
search.images(keyword,
{top: 10},
function(err, results) {
if(err) callback(err)
else
{
var resArr = [];
(util.inspect(results,
{colors: true, depth: null}));
for(var i=0;i<results.length;i++)
{
var tempObj = {};
tempObj.url = results[i].url;
tempObj.snippet = results[i].title;
tempObj.thumbnail = results[i].thumbnail.url;
tempObj.context = results[i].sourceUrl;
resArr.push(tempObj);
}
console.log(resArr);
return callback(null, resArr);
}
}
);
}
}
module.exports = bingSearchHandler;

Something like this should work.
Using callbacks
app.route('/imagesearch/:keyword')
.get(function (req, res) {
// Make the async request, pass the callback function
bingSearchHandler.findImages(req.params, (error, response) => {
if (error === null) {
res.json(response);
}
});
});
Additionally, you'll need to rework your findImages function.
this.findImages = (userInput, callback) => {
var keyword = userInput.keyword;
search.images(keyword, {top: 10}, function (err, results) {
if (err) {
callback(err);
}
else {
var resArr = [];
util.inspect(results, {colors: true, depth: null});
for(var i = 0; i < results.length; i++) {
var tempObj = {};
tempObj.url = results[i].url;
tempObj.snippet = results[i].title;
tempObj.thumbnail = results[i].thumbnail.url;
tempObj.context = results[i].sourceUrl;
resArr.push(tempObj);
}
callback(null, resArr);
}
});
}
Using promises
app.route('/imagesearch/:keyword')
.get(function (req, res) {
// Make the async request, pass the callback function
bingSearchHandler.findImages(req.params).then(response =>
res.json(response);
});
});
// Images function
this.findImages = (userInput) => {
return new Promise((resolve, reject) => {
var keyword = userInput.keyword;
search.images(keyword, {top: 10}, function (err, results) {
if (err && typeof reject === 'function') {
reject(err);
}
else {
var resArr = [];
util.inspect(results, {colors: true, depth: null});
for(var i = 0; i < results.length; i++) {
var tempObj = {};
tempObj.url = results[i].url;
tempObj.snippet = results[i].title;
tempObj.thumbnail = results[i].thumbnail.url;
tempObj.context = results[i].sourceUrl;
resArr.push(tempObj);
}
if (typeof resolve === 'function') {
resolve(resArr);
}
}
});
});
}