I have been learning JavaScript and started exploring Node. I have spent time understanding callback hell and how to fix it, but now I am more lost than ever and I'm starting to think I'm looking at it backwards (this might be a stupid question). I have read a guide on callbackhell.com and number 1 rule is to keep the code shallow.
Here is an example of connecting to Database, then reading a file, then inserting records to MongoDB and then logging:
MongoClient.connect(url, (err, db)=>{
let dbo = db.db('test');
fs.readFile('./data.json', (err, data)=>{
let records = JSON.parse(data);
dbo.collection('people').insertMany(records, (err, result) =>{
console.log("result");
db.close();
})
});
});
In the example, I have 3 anonymous callback functions and I have no idea how to refactor as the db is used throughout the callbacks. From what I understand I am should be aiming for 3 named functions (or am I missing something?) and call it like this:
MongoClient.connect(url, cb1);
and the functions something like this:
function cb1(err, db){
let dbo = db.db('test');
fs.readFile('./data.json', cb2);
}
function cb2(err, data){
let records = JSON.parse(data);
// ??? dbo.collection('people).insertMany(records, cb3)
//now what?
}
function cb3(err, result){
console.log(result);
// ??? db.close?
}
Looking for any insight on Callback hell. Is this callback hell? How would I go about this? Am I missing something conceptually? Should it even be refactored?
PS. Promises and Async/Await can wait till I understand how to go about async programming using callbacks
Thank you!
The nested callbacks are indeed what is commonly called callback hell.
Your attempt at separating the callbacks into named functions is fine, but you have realised one issue: that you need a reference to the db object in all callbacks, and it is not there.
You can solve this by either binding this to db, or else binding a (first) parameter for passing on that db object as argument. The principle is really the same.
Here is how it would look with binding this to the db object:
function cb1(err, db) {
let dbo = db.db('test');
fs.readFile('./data.json', cb2.bind(db));
}
function cb2(err, data) {
let records = JSON.parse(data);
this.collection('people').insertMany(records, cb3.bind(this));
}
function cb3(err, result) {
console.log(result);
this.close();
}
And here is how it would look with an extra parameter:
function cb1(err, db) {
let dbo = db.db('test');
fs.readFile('./data.json', cb2.bind(null, db));
}
function cb2(db, err, data) {
let records = JSON.parse(data);
db.collection('people').insertMany(records, cb3.bind(null, db));
}
function cb3(db, err, result) {
console.log(result);
db.close();
}
The next step would be to embrace promises and async await syntax. See for instance "How to use MongoDB with promises in Node.js?".
Related
I am listing all files from all directories in /home/myComputer/Desktop/Research, and then filtering them with an if statement to only get the .txt files that I would like to read and store into arrays. All works fine, but pushing the data into the arrays is not functioning. When I console log them, they return no value [].
I tried promise as well as call back function, but they didn't work for me because I didn't know how to implement them properly.
app.get('/jsonData', function(req, res) {
/* Define Arrays */
var theFile = [];
var theCategory = [];
var theContent = [];
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
};
//walk(process.env.HOME, function(err, results) {
walk("/home/myComputer/Desktop/Research", function(err, results) {
if (err) throw err;
//console.log(results);
results.map(function(val) {
//Get the filename
var fileName = val.match(/[^\/]+$/).join();
//Get the category
var category = val.substr(48).match(/[^\/]+/);
if (fileName == 'written-speech.txt') {
console.log('FOUND!: ' + fileName + ' Category: ' + category) //this works
fs.readFile(val, 'utf8', function(err, contents) {
console.log(contents); // this works
theFile.push(fileName);
theCategory.push(category);
theContent.push(contents);
});
}
})
});
console.log(theFile); // The problem: This returns an empty Array []
console.log(theCategory); // The problem: This returns an empty Array []
console.log(theContent); // The problem: This returns an empty Array []
});
I expect console.log(theFile); console.log(theCategory); and console.log(theContent); to return the data pushed in them.
The reason for this is that many callbacks in Javascript are asynchronous, which means both fs.readdir and fs.readFile are asynchronous and their callbacks are not called immediately but slightly later (please read about Event Loop in javascript). So at the moment, when you log your arrays they are empty and data to them will be pushed later, e.g. in future. To avoid this you can either use synchronous methods (fs.readdirSync and fs.readFileSync) which is ugly and can cause performance issues if the app has a lot of other asynchronous operations. If in your case it is just a simple script to read some data, it might be fine.
And the other, preferred way is to use promises or some library for managing callbacks, e.g. async. Please read some articles regarding managing async code if these concepts are fully unfamiliar for you, e.g. https://dev.to/mrm8488/from-callbacks-to-fspromises-to-handle-the-file-system-in-nodejs-56p2 to get a basic understanding and see some use case examples.
Regarding your current version, there is no easy way to make it work without a lot of changes. It is better to rewrite it to use the concepts I described earlier.
walk is an asynchronous function because fs.readdir is an asynchronous method and the console.log statements are running (in a synchronous manner) before the callback of fs.readdir getting invoked.
You can console the values of these variables at the end inside the callback of walk.
EDIT
OK I read here."You can't usefully return with asynchronous functions. You'll have to work with the result within the callback. This is due to the nature of asynchronous programming: "exit immediately, setting up a callback function to be called sometime in the future. And, at least with the current standard of ECMAScript 5, you can't get around this. As JavaScript is single-threaded, any attempt to wait for the callback will only lock up the single thread, keeping the callback and the return user forever pending in the event queue."
Is this still the case today?
ORIGINAL QUESTION
I have a problem accessing my variable outside the function in my node.js application.
const url = "mongodb://localhost:27017/";
getAllSampleTypes();
// I would like to have the variable "requested" accessible here
function getAllSampleTypes() {
MongoClient.connect(url, function (err, db) {
var dbo = db.db("myDb");
dbo.collection("data").distinct("sample_type", {}, (function (err, requested) {
// variable "requested" is accessible here
})
);
});
}
I tried with async/await but I still have the same problem.
function getTypes() {
MongoClient.connect(url, async function (err, db) {
let dbo = db.db("myDb");
return await dbo.collection("data").distinct("sample_type", {});
});
}
console.log(getTypes()); //Promise { undefined }
I don't think you are going to be able to achieve what you are looking for. Async await only works once you are in scope of an async function. Your top level calls are not inside an async function so you are forced to handle the returned Promise or callback.
e.g. getAllSampleTypes().then(function(response){});
Here are a couple of samples that are similar to what you want, but either way, the top level call into an async function will have to handle the response as a Promise.
const url = "mongodb://localhost:27017/";
getAllSampleTypes().then(function(sample_types){
// Do something here.
});
async function getAllSampleTypes() {
var db = await mongo.connect(url);
var dbo = db.db("myDb");
return await dbo.collection("data").distinct("sample_type", {});
}
It's important to understand that async await really isn't anything magical, behind the scenes it's translated to Promises really. That's why your top level call into an async function can handle the response with a .then(). It's just really much cleaner to read. The code above would roughly get translated and executed as:
const url = "mongodb://localhost:27017/";
getAllSampleTypes().then(function(sample_types){
// Do something here.
});
function getAllSampleTypes() {
return new Promise(function(resolve, reject){
mongo.connect(url).then(function(db){
var dbo = db.db("myDb");
dbo.collection("data").distinct("sample_type", {}).then(function(results) {
resolve(results);
});
});
});
}
getTypes doesn't return anything. You've gotta pass it up
If you're gonna use async/await try something like
async function getTypes() {
const db = MongoClient.connect(url);
const dbo = db.db("myDb");
return await dbo.collection("data").distinct("sample_type", {});
}
console.log(await getTypes());
These might be helpful:
How can I use asyn-await with mongoclient and how-to-use-mongodb-with-promises-in-node-js
Also, you should probably close the connection with db.close() somewhere
I have a for loop and I want to call an async function on each iteration. But I am getting a JS stack trace error. Below is the prototype of my code. I have also used IIFE pattern but it's not working.
for(let i = 0; i<99999;i++){
getData(i, function(err, result){
if(err) return err;
else{
console.log(result);
}
});
}
function getData(number, callback){
request('http://someapiurl'+number, function(err, response){
if(err) callback(err, null);
else{
callback(null, response)
}
})
}
You re trying to make 99999 at the same time, this just won t work on most browsers.
Use promises instead, push them in an array, then use Promise.all to let the browser handle the request, simply handle the final resolved response.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all
var p1 = Promise.resolve(3);
var p2 = 1337;
var p3 = new Promise((resolve, reject) => {
setTimeout(resolve, 100, 'foo');
});
Promise.all([p1, p2, p3]).then(values => {
console.log(values); // [3, 1337, "foo"]
});
First of all I highly suggest that you debug your code to see exactly what happens with the for loop.
Its very likely that the loop runs completely before its contents (the getData function) executes, asynchronous behavior of via a fetching data request. If you tried using the IIFE pattern you'd have at least wrapped the contents inside your for loop in a way that guarantees its contents to execute. An example of an implementation looks like so:
for(let i = 0; i<99999;i++){
(function(){ // <-- IIFE used to represent a closure.
getData(i, function(err, result){
if(err) return err;
else{
console.log(result);
}
});
})(i);
}
To let #mika sit on the subject of promises for you, which you can read on their answer; just keep in mind the promise anti-pattern and you'll be fine. It is especially tricky if you are not used to promises and want to use them in a for loop.
My goal is to create a JSON object, from a paragraph of text, that I can then insert as a document into MongoDB. I'm using nodejs and wanted to go for the async approach.
My JSON has parameters like so
{
height:height,
weight:weight
}
My logic is this
create a module with async functions that parse the text and extract weight and height using regex.
but then how would I combine all the responses from these functions into one JSON that I can import at once?
I'm thinking something like this
var get_height = require().height;
var get_weight = require().weight;
exports.contr = function(){
var height,
weight;
get_height(text, function(err, res){
if(err)
throw(err)
height=res;
});
get_weight(text, function(err, res){
if(err)
throw(err)
weight=res;
});
//All other async functions
combine_json(height, weight, ... , function(err, res){
if(err)
throw(err);
console.log(res); //the json was successfully inserted into mongoDB
});
}
I find async confusing and in the above example I'm not sure about two things
wouldn't combine_json run without waiting for the data from the previous two functions (weight, height)
what is the best practice to handle such cases? Should i just use sync functions and wait top-to-bottom for each one to do its thing and then run the final one or I can leverage async?
The simplest way to wait for the results of two independent asynchronous functions is to use promises and Promise.all. For this we'll assume get_height and get_weight return a Promise and can be used as such:
get_height().then(function (height) { console.log(height); });
Then it's trivial to combine two of those promises:
Promise.all([get_height(), get_weight()]).then(function (results) {
combine_json(results[0], results[1]);
});
See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise for documentation and details.
If you do not know nothing of Promises, you first should know how callbacks works. If you do not want a elegant solution with Promise.all() and you just want your code working, you need nested functions. When you are inside get_height callback you should to call get_weight and same when you are inside get_weight callback you should call combine_json() . The only problem is that you have to wait for get_height to call get_weight. This is solved with Promise.all() as it's said.
get_height(text, function(err, height){
if(err)
throw(err);
get_weight(text, function(err, weight){
if(err)
throw(err);
//All other async functions
combine_json(height, weight, ... , function(err, res){
if(err)
throw(err);
console.log(res); //the json was successfully inserted into mongoDB
});
});
});
Promises are your best bet, but if you don't want to use them for some reason and prefer the callback style, then
function get_height_and_weight(text, callback) {
var have_height = false;
var have_weight = false;
var result = {};
get_height(text, function(err, height) {
if (err) callback(err);
have_height = true;
result.height = height;
if (have_weight) callback(null, result);
});
get_weight(text, function(err, weight) {
if (err) callback(err);
have_weight = true;
result.weight = weight;
if (have_height) callback(null, result);
});
}
This is a special case of the parallel async call case which could be handled better by async.parallel.
I have frustrating problem with learning to work with callback style of programming in Node.js. I have a query to a MongoDB database. If I pass in a function to execute on the result it works but I'd rather flatten it out and have it return the value. Any help or direction on how to do this correctly is appreciated. Here's my code:
var getLots = function(response){
db.open(function(err, db){
db.collection('lots', function(err, collection){
collection.find(function(err, cursor){
cursor.toArray(function(err, items){
response(items);
})
})
})
})
}
I want something more like this:
lots = function(){
console.log("Getting lots")
return db.open(openCollection(err, db));
}
openCollection = function(err, db){
console.log("Connected to lots");
return (db.collection('lots',findLots(err, collection))
);
}
findLots = function(err, collection){
console.log("querying 2");
return collection.find(getLots(err, cursor));
}
getLots = function(err, cursor) {
console.log("Getting lots");
return cursor.toArray();
}
Where the final set of data would bubble back up through the function calls.
The problem is that I get an error from Node.js saying that err is not defined or that the collection is not defined. For some reason when I nest the callbacks the correct object is getting passed down. When I try going to this flattened style it complains that things are not defined. I don't know how to get it to pass the necessary objects.
What you need is one of the many control flow libraries available for node via npm and catalogued on the Node.js wiki. My specific recommendation is caolan/async, and you would use the async.waterfall function to accomplish this type of flow where each async operation must be executed in order and each requires the results from the previous operation.
Pseudocode example:
function getLots(db, callback) {
db.collection("lots", callback);
}
function findLots(collection, callback) {
collection.find(callback);
}
function toArray(cursor, callback) {
cursor.toArray(callback);
}
async.waterfall([db.open, getLots, find, toArray], function (err, items) {
//items is the array of results
//Do whatever you need here
response(items);
});
async is a good flow control library. Frame.js offers some specific advantages like better debugging, and better arrangement for synchronous function execution. (though it is not currently in npm like async is)
Here is what it would look like in Frame:
Frame(function(next){
db.open(next);
});
Frame(function(next, err, db){
db.collection('lots', next);
});
Frame(function(next, err, collection){
collection.find(next);
});
Frame(function(next, err, cursor){
cursor.toArray(next);
});
Frame(function(next, err, items){
response(items);
next();
});
Frame.init();