Based on the example of processLineByLine() I noticed that we cannot catch the error if the given filename does not exist. In that case the program finishes with something like:
UnhandledPromiseRejectionWarning: Error: ENOENT: no such file or directory
So the most simpler approach that I followed to raise a catchable error was to make 2 modifications to the processLineByLine() function:
turn it in a generator such as function*
await on file exist check await access(filename, fs.constants.F_OK)
Finally I had to convert the readline.Interface instance to an async generator. I do not like this last part in particularly. The resulting lines() function is like:
export async function* lines(filename) {
await access(filename, fs.constants.F_OK)
const lines = readline.createInterface({
input: fs.createReadStream(filename),
crlfDelay: Infinity
})
for await (const l of lines) {
yield l
}
}
Question: Is there a better approach to make lines() either return an async iterator or throw an error if the filename does not exist?
BUG report: Regarding #jfriend00 observations I have opened a Bug issue on nodejs: https://github.com/nodejs/node/issues/30831
Hmm, this is a tricky one. Even detecting whether the file exists as a pre-flight doesn't guarantee that you can successfully open it (it could be locked or have permission issues) and detecting if it exists before opening is a classic race condition in server development (small window, but still a race condition).
I'm still thinking there must be a better way to get an error out of a fs.createReadStream(), but the only way I could find was to wrap it in a promise that only resolves when the file is successfully open. That lets you get the error from opening the file and propagate it back to the caller of your async function. Here's what that would look like:
const fs = require('fs');
const readline = require('readline');
function createReadStreamSafe(filename, options) {
return new Promise((resolve, reject) => {
const fileStream = fs.createReadStream(filename, options);
fileStream.on('error', reject).on('open', () => {
resolve(filestream);
});
});
}
async function processLineByLine(f) {
const fileStream = await createReadStreamSafe(f);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
for await (const line of rl) {
// Each line in input.txt will be successively available here as `line`.
console.log(`Line from file: ${line}`);
}
}
processLineByLine("nofile").catch(err => {
console.log("caught error");
});
This makes it so that the promise that processLineByLine() returns will reject and you can handle the error there which is what I think you were asking for. If I misunderstood what you were asking for, then please clarify.
FYI, this seems to me to be a bug in readline.createInterface() because it seems like it should reject on the first iteration of for await (const line of rl), but that doesn't appear to be what happens.
So, as a consequence of that, even this work-around won't detect read errors on the stream after it's opened. That really needs to be fixed internal to createInterface(). I agree both a file open error or a read error should show up as a reject on for await (const line of rl).
Another work-around for the file open issue would be to pre-open the file using await fs.promises.open(...) and pass the fd to fs.createReadStream and then you would see the error on the open yourself.
A Different Solution - Wrapping the readLine iterator to add error handling
Warning, this ends up looking like a bit of hack, but it's a really interesting learning project because I ended up having to wrap an the readline asyncIterator with my own in order to reject when I detected an error on the readStream (the error handling that the readline library is missing).
I set out on a mission to figure out how to write a processLineByLine() function that would return an asyncIterator that would properly reject on stream errors (even though the readline code has bugs in this regard) while still using the readline library internally.
The goal was to be able to write code like this:
for await (let line of processLineByLine("somefile1.txt")) {
console.log(line);
}
that properly handles errors on the readStream used internally, whether the file doesn't exist, exists but can't be opened or even encounters a read error later while reading. Since I'm not changing/fixing the readline interface code internally, I had to install my own error listener on the readStream and when I see an error there, I need to cause any pending or future promises from the readline interface to reject.
Here's what I ended up with:
// This is an experiment to wrap the lines asyncIterator with our own iterator
// so we can reject when there's been an error on the readStream. It's really
// ugly, but does work.
const fs = require('fs');
const readline = require('readline');
function processLineByLine(filename, options = {}) {
const fileStream = fs.createReadStream(filename, options);
let latchedError = null;
let kill = new Set();
fileStream.on('error', (err) => {
latchedError = err;
// any open promises waiting on this stream, need to get rejected now
for (let fn of kill) {
fn(err);
}
});
const lines = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
// create our own little asyncIterator that wraps the lines asyncIterator
// so we can reject when we need to
function asyncIterator() {
const linesIterator = lines[Symbol.asyncIterator]();
return {
next: function() {
if (latchedError) {
return Promise.reject(latchedError);
} else {
return new Promise((resolve, reject) => {
// save reject handlers in higher scope so they can be called
// from the stream error handler
kill.add(reject);
let p = linesIterator.next();
// have our higher level promise track the iterator promise
// except when we reject it from the outside upon stream error
p.then((data => {
// since we're resolving now, let's removing our reject
// handler from the kill storage. This will allow this scope
// to be properly garbage collected
kill.delete(reject);
resolve(data);
}), reject);
});
}
}
}
}
var asyncIterable = {
[Symbol.asyncIterator]: asyncIterator
};
return asyncIterable;
}
async function runIt() {
for await (let line of processLineByLine("xfile1.txt")) {
console.log(line);
}
}
runIt().then(() => {
console.log("done");
}).catch(err => {
console.log("final Error", err);
});
Some explanation on how this works...
Our own error monitoring on the stream
First, you can see this:
fileStream.on('error', (err) => {
latchedError = err;
// any open promises waiting on this stream, need to get rejected now
for (let fn of kill) {
fn(err);
}
});
This is our own error monitoring on the readStream to make up for the missing error handling inside of readline. Anytime we see an error, we save it in a higher scoped variable for potential later use and, if there are any pending promises registered from readline for this stream, we "kill" them (which rejects them, you will see later how that works).
No special handling for file open errors
Part of the goal here was to get rid of the special handling in the previous solution for file open errors. We want ANY error on the readStream to trigger a rejection of the asyncIterable so this is a much more general purpose mechanism. the file open error gets caught in this error handling just the same way any other read error would.
Our own asyncIterable and asyncIterator
Calling readline.createInterace() returns an asyncIterable. It's basically the same as a regular iterable in that you call a special property on it to get an asyncIterator. That asyncIterator has a .next() property on it just like a regular iterator except when asyncIterator.next() is called, it returns a promise that resolves to an object instead of an object.
So, that's how for await (let line of lines) works. It first calls lines[Symbol.asyncIterator]() to get an asyncIterator. Then, on that asyncIterator that it gets back, it repeatedly does await asyncIterator.next() waiting on the promise that asyncIterator.next() returns.
Now, readline.createInterface() already returns such an asyncIterable. But, it doesn't work quite right. When the readStream gets an error, it doesn't reject the promise returned by .next() on each iteration. In fact, that promise never gets rejected or resolved. So, things get stalled. In my test app, the app would just exit because the readStream was done (after the error) and there was no longer anything keeping the app from exiting, even though a promise was still pending.
So, I needed a way to force that promise that readlineIterator.next() had previously returned and was currently being awaited by for await (...) to be rejected. Well, a promise doesn't provide an outward interface for rejecting it and we don't have access to the internals to the readline implementation where there is access to reject it.
My solution was to wrap the readlineIterator with my own as a sort of proxy. Then, we my own error detector sees an error and there are promise(s) outstanding from readline, I can use my proxy/wrapper to force a rejection on those outstanding promise(s). This will cause the for await (...) to see the reject and get a proper error. And, it works.
It took me awhile to learn enough about how asyncIterators work to be able to wrap one. I owe a lot of thanks to this Asynchronous Iterators in JavaScript article which provided some very helpful code examples for constructing your own asyncIterable and asyncIterator. This is actually where the real learning came about in this exercise and where others might learn by understanding how this works in the above code.
Forcing a wrapped promise to reject
The "ugliness" in this code comes in forcing a promise to reject from outside the usual scope of the reject handler for that promise. This is done by storing the reject handler in a higher level scope where an error handling for the readStream can call trigger that promise to reject. There may be a more elegant way to code this, but this works.
Making our own asyncIterable
An async iterable is just an object that has one property on it named [Symbol.asyncIterator]. That property must be a function that, when called with no arguments, returns an asyncIterator. So, here's our asyncIterable.
var asyncIterable = {
[Symbol.asyncIterator]: asyncIterator
};
Making our own asyncIterator
An asyncIterator is a function that when called returns an object with a next() property on it. Each time obj.next() is called, it returns a promise that resolves to the usual iterator tuple object {done, value}. We don't have to worry about the resolved value because we'll just get that from the readline's iterator. So, here's our asyncIterator:
// create our own little asyncIterator that wraps the lines asyncIterator
// so we can reject when we need to
function asyncIterator() {
const linesIterator = lines[Symbol.asyncIterator]();
return {
next: function() {
if (latchedError) {
return Promise.reject(latchedError);
} else {
return new Promise((resolve, reject) => {
// save reject handlers in higher scope so they can be called
// from the stream error handler
kill.push(reject);
let p = linesIterator.next();
// have our higher level promise track the iterator promise
// except when we reject it from the outside upon stream error
p.then(resolve, reject);
});
}
}
}
}
First, it gets the asyncIterator from the readline interface (the one we're proxying/wrapping) and stores it locally in scope so we can use it later.
Then, it returns the mandatory iterator structure of the form {next: fn}. Then, inside that function is where our wrapping logic unfolds. If we've seen a previous latched error, then we just always return Promise.reject(latchedError);. If there's no error, then we return a manually constructed promise.
Inside the executor function for that promise, we register our reject handling by adding it into a higher scoped Set named kill. This allows our higher scoped filestream.on('error', ....) handler to reject this promise if it sees an error by calling that function.
Then, we call linesIterator.next() to get the promise that it returns. We register an interest in both the resolve and reject callbacks for that promise. If that promise is properly resolved, we remove our reject handler from the higher level scope (to enable better garbage collection of our scope) and then resolve our wrap/proxy promise with the same resolved value.
If that linesIterator promise rejects, we just pass the reject right through our wrap/proxy promise.
Our own filestream error handling
So, now the final piece of explanation. We have this error handler watching the stream:
fileStream.on('error', (err) => {
latchedError = err;
// any open promises waiting on this stream, need to get rejected now
for (let fn of kill) {
fn(err);
}
});
This does two things. First, it stores/latches the error so any future calls to the lines iterator will just reject with this previous error. Second, if there are any pending promises from the lines iterator waiting to be resolved, it cycles through the kill Set and rejects those promises. This is what gets the asyncIterator promise to get properly rejected. This should be happening inside the readline code, but since it isn't doing it properly, we force our wrap/proxy promise to reject so the caller sees the proper rejection when the stream gets an error.
In the end, you can just do this as all the ugly detail is hidden behind the wrapped asyncIterable:
async function runIt() {
for await (let line of processLineByLine("xfile1.txt")) {
console.log(line);
}
}
runIt().then(() => {
console.log("done");
}).catch(err => {
console.log("final Error", err);
});
Readline error handling
I also struggled to get readline to be able to throw any error states.
I did mange to get readstream to throw errors, if the file was not found. (shown in the code below)
But writestream never does.
My use case was, I did not want to use up loads of memory reading a whole file then converting from JSON to an object. Just line by line - hence readline.
Which I now see Node.js has done some work on ver 19.xxx? but its not production ready yet.(As of 25/OCT/2022)
I tried to convert readline to promise based but its has to many hoops in the current version.
This is a working shell that shows my structure if it helps others.
It needs more work to make it async/await but great care is needed here to
avoid the readstream or writestream race conditions.
////////////////////////////////////////////////////////////////////////
// Simple file stream that can be used to find/edit/remove/filter data
// Example; User name password email
////////////////////////////////////////////////////////////////////////
const readline = require('readline');
const fs = require('fs');
Let obj={};
// A file called json_users.txt it has JSON strings terminated with line feed
//{"id":1,"username":"","password":"","email":""}\n
const readStream2 = fs.createReadStream( "json_users.txt" );
const writeStream2 = fs.createWriteStream( "update_users.txt", { encoding: "utf8"} );
// Some sort of Read error handler - works if no file
readStream2.on('error', function (err) {
console.log("This is a read stream error "+ err);
});
// Some sort of Write error handler -
// but never called even with a file name like "$$--!!.$$$"
writeStream2.on('error', function (err) {
console.log("This is a write stream error "+ err);
});
// Create readline with input read stream and output write stream
const rl = readline.createInterface({
input: readStream2,
output: writeStream2,
terminal: false,
crlfDelay: Infinity,
historySize: 0
});
// readline is event driven on line feed
rl.on('line', (line) => {
obj =JSON.parse(line); // convert line into an object
// Any Filter work goes here e.g. Remove user - find user edit user
if(obj.id==20) { // test if id=20 make username ="
obj.username="Douglas Crockford";
}
// Write object and \n back to stream
writeStream2.write(JSON.stringify(obj)+'\n');
});
// much better way to close stream do this but for now
// await new Promise((res) => rl.once('close', res));
rl.once("close",()=>{
console.log("done");
rl.close;
writeStream2.close;
});
Related
I'm having trouble understanding how to properly end a firestore trigger. From what I read from this and this, it seems you should only return null to end a function if there's no async code such as to quickly end if a condition isn't met. When I return null in my scenario below, it seems to work fine. Is there a better practice for what I'm doing that I'm missing?
I need to log my own custom error message which why I need the catch. I know I could return Promise.all here instead of null in the try block, but this is just sudo code for my scenario.
export const delacc = functions.auth.user().onDelete(async (user) => {
const userUID = user.uid;
try{
await admin.firestore().collection("users").doc(userUID).delete();
await admin.firestore().collection("spam").doc(userUID).delete();
await admin.firestore().collection("photos").doc(userUID).delete();
return null;
}catch(error){
functions.logger.error(error)
return null
}
});
There's no hard requirement to return null. In fact, async functions always return a promise, no matter what you do inside the function. The promise it returns is based on the completion of any other promises that you await during processing. Even if you explicitly return null, the function is still actually just returning a promise that is immediately fulfilled with the value null. It has no effect on the final outcome, since Cloud Functions onDelete triggers don't use the fulfilled value in any way. The important thing is that the function indeed returns a promise that indicates when all the work is complete (and, as I stated, async functions always do that if you use await correctly inside the function on all async work).
When you're not using async/await, I advise programmers to always return a promise, or null if there is no async work. The null there is an explicit way to tell the reader of your code that you do not intend for Cloud Functions to wait for any async work before fully terminating the function. It helps also helps satisfy eslint or TypeScript warnings, which will suggest to you that you should return something. The value itself isn't really important - it's what you're communicating to others about the termination of your function. Code readability is important if you work with others.
What I'd do is make the delete operations atomic—delete all of the documents or none of them—and return the promise returned by the batch since there isn't any other task performed in this function (which makes me think returning null isn't a necessary abstraction). If the batch throws an error, throw the client a new HTTPS error (along with the batch error) which (a) automatically terminates the cloud function and (b) gives the client the batch error to evaluate, which, depending on the reason, could warrant a retry or an error message to the end user.
export const delacc = functions.auth.user().onDelete(async (user) => {
const userUID = user.uid;
const db = admin.firestore();
const batch = db.batch();
try {
batch.delete(db.collection("users").doc(userUID));
batch.delete(db.collection("spam").doc(userUID));
batch.delete(db.collection("photos").doc(userUID));
const writeResult = await batch.commit();
return writeResult;
} catch(error) {
functions.logger.error(error);
throw new functions.https.HttpsError("unknown", "batch-error", error);
}
});
So I am reading about callbacks because I´m learning backend development on Node.js, and in several webs they say this good practice about writing callbacks with error argument as its first argument:
For example:
fs.readFile('/foo.txt', function(err, data) {
// If an error occurred, handle it (throw, propagate, etc)
if(err) {
console.log('Unknown Error');
return;
}
// Otherwise, log the file contents
console.log(data);
});
Ok sure, I think I understand it clearly what is happening. If once the module fs finishes reading the file "foo.text" there is an error, then the callback function executes console.log("Uknown error") but how Javascript / Node knows that the variable err corresponds to an error in the code??
Because If i name it error instead of err , I imagine it also works right? And what If put it in the second argument? I imagine then it wouldn´t work. Is that it? If its why it is called a good practice if there is no other way to put the error argument but in the first place.
but how Javascript / Node knows that the variable err corresponds to an error in the code??
By convention. The way readFile (and other Node.js callback-style functions) are written, they call their callback with the error as the first argument, or null as the first argument. The name of that parameter in the function signature is irrelevant (you can call it anything you like; err, e, and error are all common). It's the fact it's the first parameter that matters, because it will receive the first argument when called.
In these modern times, though, things are moving away from Node.js callback-style APIs and toward APIs using Promises, which make the error and success paths much more distinct. Then async/await syntax is layered on top of promises to make it possible to write asynchronous code using the standard logical flow control structures.
Node.js callback style (like your code):
const fs = require("fs");
// ...
fs.readFile('/foo.txt', function(err, data) {
// If an error occurred, handle it (throw, propagate, etc)
if (err) {
// It failed
console.log(err);
return;
}
// It worked
console.log(data);
});
With promises via the fs.promises API:
const fsp = require("fs").promises;
// ...
fsp.readFile('/foo.txt')
.then(data => {
// It worked
console.log(data);
})
.catch(err => {
console.log(err);
});
Of course, you may not handle errors at that level; you might instead return the result of calling then so that the caller can chain off it and handle errors (or pass it off to its caller, etc.).
With async/await (this must be inside an async function, although in modules top-level await is coming):
const fsp = require("fs").promises;
// ...inside an `async` function:
try {
const data = await fsp.readFile('/foo.txt');
} catch (err) {
console.log(err);
}
And again, you might not handle errors at that level; you might let them propagate and have the caller handle them (or the caller might let them propgate to its caller to handle them, etc.).
Not all of Node.js's API has promises yet. You can wrap a single callback-style API function with promises via util.promisify; you can wrap an entire API via various npm modules like promisify.
This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
I have an es6 class, with an init() method responsible for fetching data, transforming it, then update the class's property this.data with newly transformed data.
So far so good.
The class itself has another getPostById() method, to just do what it sounds like. Here is the code for the class:
class Posts {
constructor(url) {
this.ready = false
this.data = {}
this.url = url
}
async init() {
try {
let res = await fetch( this.url )
if (res.ok) {
let data = await res.json()
// Do bunch of transformation stuff here
this.data = data
this.ready = true
return data
}
}
catch (e) {
console.log(e)
}
}
getPostById(id){
return this.data.find( p => p.id === id )
}
}
Straightforward, except I have an async/await mechanism in the init() method.
Now, this code will work correctly:
let allPosts = new Posts('https://jsonplaceholder.typicode.com/posts')
allPosts.init()
.then( d => console.log(allPosts.getPostById(4)) )
// resulting Object correctly logged in console
but it only gets printed into the console:
How could I use allPosts.getPostById(4) as a return of a function ?
Like:
let myFunc = async () => {
const postId = 4
await allPosts.init() // I need to wait for this to finish before returning
// This is logging correct value
console.log( 'logging: ' + JSON.stringify(allPosts.getPostById( postId ), null, 4) )
// How can I return the RESULT of allPosts.getPostById( postId ) ???
return allPosts.getPostById( postId )
}
myFunc() returns a Promise but not the final value. I have read several related posts on the subject but they all give example of logging, never returning.
Here is a fiddle that includes two ways of handling init(): using Promise and using async/await. No matter what I try, I can't manage to USE the FINAL VALUE of getPostById(id).
The question of this post is: how can I create a function that will RETURN the VALUE of getPostById(id) ?
EDIT:
A lot of good answers trying to explain what Promises are in regards to the main execution loop.
After a lot of videos and other good reads, here is what I understand now:
my function init() correctly returns. However, within the main event loop: it returns a Promise, then it is my job to catch the result of this Promise from within a kinda parallel loop (not a new real thread). In order to catch the result from the parallel loop there are two ways:
use .then( value => doSomethingWithMy(value) )
use let value = await myAsyncFn(). Now here is the foolish hiccup:
await can only be used within an async function :p
thus itself returning a Promise, usable with await which should be embed in an async function, which will be usable with await etc...
This means we cannot really WAIT for a Promise: instead we should catch parallel loop indefinitely: using .then() or async/await.
Thanks for the help !
As for your comment; I'll add it as answer.
The code you write in JavaScript is run on one thread, that means that if your code could actually wait for something it will block any of your other code from getting executed. The event loop of JavaScript is explained very well in this video and if you like to read in this page.
A good example of blocking code in the browser is alert("cannot do anything until you click ok");. Alert blocks everything, the user can't even scroll or click on anything in the page and your code also blocks from executing.
Promise.resolve(22)
.then(x=>alert("blocking")||"Hello World")
.then(
x=>console.log(
"does not resolve untill you click ok on the alert:",
x
)
);
Run that in a console and you see what I mean by blocking.
This creates a problem when you want to do something that takes time. In other frameworks you'd use a thread or processes but there is no such thing in JavaScript (technically there is with web worker and fork in node but that's another story and usually far more complicated than using async api's).
So when you want to make a http request you can use fetch but fetch takes some time to finish and your function should not block (has to return something as fast as possible). This is why fetch returns a promise.
Note that fetch is implemented by browser/node and does run in another thread, only code you write runs in one thread so starting a lot of promises that only run code you write will not speed up anything but calling native async api's in parallel will.
Before promises async code used callbacks or would return an observable object (like XmlHttpRequest) but let's cover promises since you can convert the more traditional code to a promise anyway.
A promise is an object that has a then function (and a bunch of stuff that is sugar for then but does the same), this function takes 2 parameters.
Resolve handler: A function that will be called by the promise when the promise resolves (has no errors and is finished). The function will be passed one argument with the resolve value (for http requests this usually is the response).
Reject handler: A function that will be called by the promise when the promise rejects (has an error). This function will be passed one argument, this is usually the error or reason for rejection (can be a string, number or anything).
Converting callback to promise.
The traditional api's (especially nodejs api's) use callbacks:
traditionalApi(
arg
,function callback(err,value){
err ? handleFail(err) : processValue(value);
}
);
This makes it difficult for the programmer to catch errors or handle the return value in a linear way (from top to bottom). It gets even more impossible to try and do things parallel or throttled parallel with error handling (impossible to read).
You can convert traditional api's to promises with new Promise
const apiAsPromise = arg =>
new Promise(
(resolve,reject)=>
traditionalApi(
arg,
(err,val) => (err) ? reject(err) : resolve(val)
)
)
async await
This is what's called syntax sugar for promises. It makes promise consuming functions look more traditional and easier to read. That is if you like to write traditional code, I would argue that composing small functions is much easier to read. For example, can you guess what this does?:
const handleSearch = search =>
compose([
showLoading,
makeSearchRequest,
processRespose,
hideLoading
])(search)
.then(
undefined,//don't care about the resolve
compose([
showError,
hideLoading
])
);
Anayway; enough ranting. The important part is to understand that async await doesn't actually start another thread, async functions always return a promise and await doesn't actually block or wait. It's syntax sugar for someFn().then(result=>...,error=>...) and looks like:
async someMethod = () =>
//syntax sugar for:
//return someFn().then(result=>...,error=>...)
try{
const result = await someFn();
...
}catch(error){
...
}
}
The examples allways show try catch but you don't need to do that, for example:
var alwaysReject = async () => { throw "Always returns rejected promise"; };
alwaysReject()
.then(
x=>console.log("never happens, doesn't resolve")
,err=>console.warn("got rejected:",err)
);
Any error thrown or await returning a rejected promise will cause the async function to return a rejected promise (unless you try and catch it). Many times it is desirable to just let it fail and have the caller handle errors.
Catching errors could be needed when you want the promise to succeed with a special value for rejected promises so you can handle it later but the promise does not technically reject so will always resolve.
An example is Promise.all, this takes an array of promises and returns a new promise that resolves to an array of resolved values or reject when any one of them rejects. You may just want to get the results of all promises back and filter out the rejected ones:
const Fail = function(details){this.details=details;},
isFail = item => (item && item.constructor)===Fail;
Promise.all(
urls.map(//map array of urls to array of promises that don't reject
url =>
fetch(url)
.then(
undefined,//do not handle resolve yet
//when you handle the reject this ".then" will return
// a promise that RESOLVES to the value returned below (new Fail([url,err]))
err=>new Fail([url,err])
)
)
)
.then(
responses => {
console.log("failed requests:");
console.log(
responses.filter(//only Fail type
isFail
)
);
console.log("resolved requests:");
console.log(
responses.filter(//anything not Fail type
response=>!isFail(response)
)
);
}
);
Your question and the comments suggest you could use a little intuition nudge about the way the event loop works. It really is confusing at first, but after a while it becomes second nature.
Rather than thinking about the FINAL VALUE, think about the fact that you have a single thread and you can't stop it — so you want the FUTURE VALUE -- the value on the next or some future event loop. Everything you write that is not asynchronous is going to happen almost immediately — functions return with some value or undefined immediately. There's nothing you can do about. When you need something asynchronously, you need to setup a system that is ready to deal with the async values when they return sometime in the future. This is what events, callbacks, promises (and async/await) all try to help with. If some data is asynchronous, you simply can not use it in the same event loop.
So what do you do?
If you want a pattern where you create an instance, call init() and then some function that further process it, you simply need to setup a system that does the processing when the data arrives. There are a lot of ways to do this. Here's one way that's a variation on your class:
function someAsync() {
console.log("someAsync called")
return new Promise(resolve => {
setTimeout(() => resolve(Math.random()), 1000)
})
}
class Posts {
constructor(url) {
this.ready = false
this.data = "uninitilized"
this.url = url
}
init() {
this.data = someAsync()
}
time100() {
// it's important to return the promise here
return this.data.then(d => d * 100)
}
}
let p = new Posts()
p.init()
processData(p)
// called twice to illustrate point
processData(p)
async function processData(posts) {
let p = await posts.time100()
console.log("randomin * 100:", p)
}
init() saves the promise returned from someAsync(). someAsync() could be anything that returns a promise. It saves the promise in an instance property. Now you can call then() or use async/await to get the value. It will either immediately return the value if the promise has already resolved or it will deal with it when it has resolved. I called processData(p) twice just to illustrate that it doesn't calle the someAsync() twice.
That's just one pattern. There are a lot more — using events, observables, just using then() directly, or even callbacks which are unfashionable, but still can be useful.
NOTE: Wherever you use await it has to be inside an async function.
Check out the UPDATED FIDDLE
You need to use await myFunc() to get the value you expect from getPostById because an async function always returns a promise.
This sometimes is very frustrating as the whole chain needs to be converted into async functions but that's the price you pay for converting it to a synchronous code, I guess. I am not sure if that can be avoided but am interested in hearing from people who have more experience on this.
Try out the below code in your console by copying over the functions and then accessing final and await final.
NOTE:
An async function CAN contain an await expression.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
There is no rule that is must have await in order to even declare an async function.
The example below uses an async function without await just to show that an async function always returns a promise.
const sample = async () => {
return 100;
}
// sample() WILL RETURN A PROMISE AND NOT 100
// await sample() WILL RETURN 100
const init = async (num) => {
return new Promise((resolve, reject) => {
resolve(num);
});
}
const myFunc = async (num) => {
const k = await init(num);
return k;
}
// const final = myFunc();
// final; This returns a promise
// await final; This returns the number you provided to myFunc
I am have a problem understanding why rejections are not passed on through a promise chain and I am hoping someone will be able to help me understand why. To me, attaching functionality to a chain of promises implies an intent that I am depending on an original promise to be fulfilled. It's hard to explain, so let me show a code example of my problem first. (Note: this example is using Node and the deferred node module. I tested this with Dojo 1.8.3 and had the same results)
var d = require("deferred");
var d1 = d();
var promise1 = d1.promise.then(
function(wins) { console.log('promise1 resolved'); return wins;},
function(err) { console.log('promise1 rejected'); return err;});
var promise2 = promise1.then(
function(wins) { console.log('promise2 resolved'); return wins;},
function(err) { console.log('promise2 rejected'); return err;});
var promise3 = promise2.then(
function(wins) { console.log('promise3 resolved'); return wins;},
function(err) { console.log('promise3 rejected'); return err;});
d1.reject(new Error());
The results of running this operation is this output:
promise1 rejected
promise2 resolved
promise3 resolved
Okay, to me, this result doesn't make sense. By attaching to this promise chain, each then is implying the intent that it will be dependant upon the successful resolution of d1 and a result being passed down the chain. If the promise in promise1 doesn't receive the wins value, but instead gets an err value in its error handler, how is it possible for the next promise in the chain to have its success function called? There is no way it can pass on a meaningful value to the next promise because it didn't get a value itself.
A different way I can describe what I'm thinking is: There are three people, John, Ginger, and Bob. John owns a widget shop. Ginger comes into his shop and requests a bag of widgets of assorted colours. He doesn't have them in stock, so he sends in a request to his distributor to get them shipped to him. In the mean time, he gives Ginger a rain check stating he owes her the bag of widgets. Bob finds out Ginger is getting the widgets and requests that he get the blue widget when she's done with them. She agrees and gives him a note stating she will. Now, John's distributor can't find any widgets in their supply and the manufacturer doesn't make them any more, so they inform John, who in turn informs Ginger she can't get the widgets. How is Bob able to get a blue widget from Ginger when didn't get any herself?
A third more realistic perspective I have on this issue is this. Say I have two values I want updated to a database. One is dependant on the id of the other, but I can't get the id until I have already inserted it into a database and obtained the result. On top of that, the first insert is dependant on a query from the database. The database calls return promises that I use to chain the two calls into a sequence.
var promise = db.query({parent_id: value});
promise.then(function(query_result) {
var first_value = {
parent_id: query_result[0].parent_id
}
var promise = db.put(first_value);
promise.then(function(first_value_result) {
var second_value = {
reference_to_first_value_id: first_value_result.id
}
var promise = db.put(second_value);
promise.then(function(second_value_result) {
values_successfully_entered();
}, function(err) { return err });
}, function(err) { return err });
}, function(err) { return err });
Now, in this situation, if the db.query failed, it would call the err function of the first then. But then it would call the success function of the next promise. While that promise is expecting the results of the first value, it would instead get the error message from its error handler function.
So, my question is, why would I have an error handing function if I have to test for errors in my success function?
Sorry for the length of this. I just didn't know how to explain it another way.
UPDATE and correction
(Note: I removed a response I had once made to some comments. So if anyone commented on my response, their comments might seem out of context now that I removed it. Sorry for this, I am trying to keep this as short as possible.)
Thank you everybody who replied. I would like to first apologize to everybody for writing out my question so poorly, especially my pseudo code. I was a little too aggressive in trying to keep it short.
Thanks to Bergi's response, I think I found the error in my logic. I think I might have overlooked another issue that was causing the problem I was having. This is possibly causing the promise chain work differently than I thought it should. I am still testing different elements of my code, so I can't even form a proper question to see what I'm doing wrong yet. I did want to update you all though and thank you for your help.
To me, this result doesn't make sense. By attaching to this promise chain, each then is implying the intent that it will be dependant upon the successful resolution of d1 and a result being passed down the chain
No. What you are describing is not a chain, but just attaching all the callbacks to d1. Yet, if you want to chain something with then, the result for promise2 is dependent on the resolution of promise1 and how the then callbacks handled it.
The docs state:
Returns a new promise for the result of the callback(s).
The .then method is usually looked upon in terms of the Promises/A specification (or the even stricter Promsises/A+ one). That means the callbacks shell return promises which will be assimilated to become the resolution of promise2, and if there is no success/error handler the respective result will in case be passed directly to promise2 - so you can simply omit the handler to propagate the error.
Yet, if the error is handled, the resulting promise2 is seen as fixed and will be fulfilled with that value. If you don't want that, you would have to re-throw the error, just like in a try-catch clause. Alternatively you can return a (to-be-)rejected promise from the handler. Not sure what Dojo way to reject is, but:
var d1 = d();
var promise1 = d1.promise.then(
function(wins) { console.log('promise1 resolved'); return wins;},
function(err) { console.log('promise1 rejected'); throw err;});
var promise2 = promise1.then(
function(wins) { console.log('promise2 resolved'); return wins;},
function(err) { console.log('promise2 rejected'); throw err;});
var promise3 = promise2.then(
function(wins) { console.log('promise3 resolved'); return wins;},
function(err) { console.log('promise3 rejected'); throw err;});
d1.reject(new Error());
How is Bob able to get a blue widget from Ginger when didn't get any herself?
He should not be able. If there are no error handlers, he will just perceive the message (((from the distributor) from John) from Ginger) that there are no widgets left. Yet, if Ginger sets up an error handler for that case, she still might fulfill her promise to give Bob a widget by giving him a green one from her own shack if there are no blue ones left at John or his distributor.
To translate your error callbacks into the metapher, return err from the handler would just be like saying "if there are no widgets left, just give him the note that there are no ones left - it's as good as the desired widget".
In the database situation, if the db.query failed, it would call the err function of the first then
…which would mean that the error is handled there. If you don't do that, just omit the error callback. Btw, your success callbacks don't return the promises they are creating, so they seem to be quite useless. Correct would be:
var promise = db.query({parent_id: value});
promise.then(function(query_result) {
var first_value = {
parent_id: query_result[0].parent_id
}
var promise = db.put(first_value);
return promise.then(function(first_value_result) {
var second_value = {
reference_to_first_value_id: first_value_result.id
}
var promise = db.put(second_value);
return promise.then(function(second_value_result) {
return values_successfully_entered();
});
});
});
or, since you don't need the closures to access result values from previous callbacks, even:
db.query({parent_id: value}).then(function(query_result) {
return db.put({
parent_id: query_result[0].parent_id
});
}).then(function(first_value_result) {
return db.put({
reference_to_first_value_id: first_value_result.id
});
}.then(values_successfully_entered);
#Jordan firstly as commenters noted, when using deferred lib, your first example definitely produces result you expect:
promise1 rejected
promise2 rejected
promise3 rejected
Secondly, even if it would produce output you suggest, it wouldn't affect execution flow of your second snippet, which is a bit different, more like:
promise.then(function(first_value) {
console.log('promise1 resolved');
var promise = db.put(first_value);
promise.then(function (second_value) {
console.log('promise2 resolved');
var promise = db.put(second_value);
promise.then(
function (wins) { console.log('promise3 resolved'); },
function (err) { console.log('promise3 rejected'); return err; });
}, function (err) { console.log('promise2 rejected'); return err;});
}, function (err) { console.log('promise1 rejected'); return err});
and that, in case of first promise being rejected will just output:
promise1 rejected
However (getting to the most interesting part) even though deferred library definitely returns 3 x rejected, most of other promise libraries will return 1 x rejected, 2 x resolved (that leads to assumption you got those results by using some other promise library instead).
What's additionally confusing, those other libraries are more correct with their behavior. Let me explain.
In a sync world counterpart of "promise rejection" is throw. So semantically, async deferred.reject(new Error()) in sync equals to throw new Error().
In your example you're not throwing errors in your sync callbacks, you just returning them, therefore you switch to success flow, with an error being a success value. To make sure rejection is passed further, you need to re-throw your errors:
function (err) { console.log('promise1 rejected'); throw err; });
So now question is, why do deferred library took returned error as rejection?
Reason for that, is that rejection in deferred works a bit different. In deferred lib the rule is: promise is rejected when it's resolved with an instance of error, so even if you do deferred.resolve(new Error()) it will act as deferred.reject(new Error()), and if you try to do deferred.reject(notAnError) it will throw an exception saying, that promise can be rejected only with instance of error. That makes clear why error returned from then callback rejects the promise.
There is some valid reasoning behind deferred logic, but still it's not on par with how throw works in JavaScript, and due to that this behavior is scheduled for change with version v0.7 of deferred.
Short summary:
To avoid confusion and unexpected results just follow the good practice rules:
Always reject your promises with an error instances (follow rules of sync world, where throwing value that's not an error is considered a bad practice).
Reject from sync callbacks by throwing errors (returning them doesn't guarantee rejection).
Obeying to above, you'll get both consistent and expected results in both deferred and other popular promise libraries.
Use can wrap the errors at each level of the Promise. I chained the errors in TraceError:
class TraceError extends Error {
constructor(message, ...causes) {
super(message);
const stack = Object.getOwnPropertyDescriptor(this, 'stack');
Object.defineProperty(this, 'stack', {
get: () => {
const stacktrace = stack.get.call(this);
let causeStacktrace = '';
for (const cause of causes) {
if (cause.sourceStack) { // trigger lookup
causeStacktrace += `\n${cause.sourceStack}`;
} else if (cause instanceof Error) {
causeStacktrace += `\n${cause.stack}`;
} else {
try {
const json = JSON.stringify(cause, null, 2);
causeStacktrace += `\n${json.split('\n').join('\n ')}`;
} catch (e) {
causeStacktrace += `\n${cause}`;
// ignore
}
}
}
causeStacktrace = causeStacktrace.split('\n').join('\n ');
return stacktrace + causeStacktrace;
}
});
// access first error
Object.defineProperty(this, 'cause', {value: () => causes[0], enumerable: false, writable: false});
// untested; access cause stack with error.causes()
Object.defineProperty(this, 'causes', {value: () => causes, enumerable: false, writable: false});
}
}
Usage
throw new TraceError('Could not set status', srcError, ...otherErrors);
Output
Functions
TraceError#cause - first error
TraceError#causes - list of chained errors
a simple explanation from here:
In a regular try..catch we can analyze the error and maybe rethrow it if it can’t be handled. The same thing is possible for promises.
If we throw inside .catch, then the control goes to the next closest error handler. But if we handle the error and finish normally, then it continues to the next closest successful .then handler.
In the example below the .catch successfully handles the error:
new Promise((resolve, reject) => {
throw new Error("Whoops!");
}).catch(function(error) {
alert("The error is handled, continue normally");
}).then(() => alert("Next successful handler runs"));
Here the catch block finishes normally. So the next successful then handler is called.
note that we may have as many .then handlers as we want, and then use a single .catch at the end to handle errors in all of them.
If you have mid catch blocks and you want to break the next chain functions for errors, you shall re-throw the errors inside the catch blocks to signal this error is not handled completely.
new Promise((resolve, reject) => {
throw new Error("Whoops!");
}).catch(function(error) { // (*) first catch
if (error instanceof URIError) { //just as example
// handle it...
} else {
alert("Can't handle such error");
throw error; // throwing this jumps to the next catch
}
}).then(function() {
// our error is other than URIError, so:
// the code doesn't reach here (jump to next catch)
}).catch(error => { // (**) second catch
alert(`The unknown error has occurred: ${error}`);
// don't return anything => execution goes the normal way
});
In the above example we see the first catch (*) will catch the error but can’t handle it (e.g. it only knows how to handle URIError), so it throws it again. The execution jumps from the first catch (*) to the next one (**) down the chain.
I have a function that reads a directory and copies and creates a new file within that directory.
function createFiles (countryCode) {
fs.readdir('./app/data', (err, directories) => {
if (err) {
console.log(err)
} else {
directories.forEach((directory) => {
fs.readdir(`./app/data/${directory}`, (err, files) => {
if (err) console.log(err)
console.log(`Creating ${countryCode}.yml for ${directory}`)
fs.createReadStream(`./app/data/${directory}/en.yml`).pipe(fs.createWriteStream(`./app/data/${directory}/${countryCode}.yml`))
})
})
}
})
}
How do I do this using promises or Promise All to resolve when it's complete?
First, you need to wrap each file stream in a promise that resolves when the stream emits the finish event:
new Promise((resolve, reject) => {
fs.createReadStream(`./app/data/${directory}/en.yml`).pipe(
fs.createWriteStream(`./app/data/${directory}/${countryCode}.yml`)
).on('finish', resolve);
});
The you need to collect these promises in an array. This is done by using map() instead of forEach() and returning the promise:
var promises = directories.map((directory) => {
...
return new Promise((resolve, reject) => {
fs.createReadStream( ...
...
});
});
Now you have a collection of promises that you can wrap with Promise.all() and use with a handler when all the wrapped promises have resolved:
Promise.all(promises).then(completeFunction);
In recent versions of Node (8.0.0 and later), there's a new util.promisify function you can use to get a promise. Here's how we might use it:
// Of course we'll need to require important modules before doing anything
// else.
const util = require('util')
const fs = require('fs')
// We use the "promisify" function to make calling promisifiedReaddir
// return a promise.
const promisifiedReaddir = util.promisify(fs.readdir)
// (You don't need to name the variable util.promisify promisifiedXYZ -
// you could just do `const readdir = util.promisify(fs.readdir)` - but
// I call it promisifiedReaddir here for clarity.
function createFiles(countryCode) {
// Since we're using our promisified readdir function, we'll be storing
// a Promise inside of the readdirPromise variable..
const readdirPromise = promisifiedReaddir('./app/data')
// ..then we can make something happen when the promise finishes (i.e.
// when we get the list of directories) by using .then():
return readdirPromise.then(directories => {
// (Note that we only get the parameter `directories` here, with no `err`.
// That's because promises have their own way of dealing with errors;
// try looking up on "promise rejection" and "promise error catching".)
// We can't use a forEach loop here, because forEach doesn't know how to
// deal with promises. Instead we'll use a Promise.all with an array of
// promises.
// Using the .map() method is a great way to turn our list of directories
// into a list of promises; read up on "array map" if you aren't sure how
// it works.
const promises = directory.map(directory => {
// Since we want an array of promises, we'll need to `return` a promise
// here. We'll use our promisifiedReaddir function for that; it already
// returns a promise, conveniently.
return promisifiedReaddir(`./app/data/${directory}`).then(files => {
// (For now, let's pretend we have a "copy file" function that returns
// a promise. We'll actually make that function later!)
return copyFile(`./app/data/${directory}/en.yml`, `./app/data/${directory}/${countryCode}.yml`)
})
})
// Now that we've got our array of promises, we actually need to turn them
// into ONE promise, that completes when all of its "children" promises
// are completed. Luckily there's a function in JavaScript that's made to
// do just that - Promise.all:
const allPromise = Promies.all(promises)
// Now if we do a .then() on allPromise, the function we passed to .then()
// would only be called when ALL promises are finished. (The function
// would get an array of all the values in `promises` in order, but since
// we're just copying files, those values are irrelevant. And again, don't
// worry about errors!)
// Since we've made our allPromise which does what we want, we can return
// it, and we're done:
return allPromise
})
}
Okay, but, there's probably still a few things that might be puzzling you..
What about errors? I kept saying that you don't need to worry about them, but it is good to know a little about them. Basically, in promise-terms, when an error happens inside of a util.promisify'd function, we say that that promise rejects. Rejected promises behave mostly the same way you'd expect errors to; they throw an error message and stop whatever promise they're in. So if one of our promisifiedReaddir calls rejects, it'll stop the whole createFiles function.
What about that copyFile function? Well, we have two options:
Use somebody else's function. No need to re-invent the wheel! quickly-copy-file looks to be a good module (plus, it returns a promise, which is useful for us).
Program it ourselves.
Programming it ourselves isn't too hard, actually, but it takes a little bit more than simply using util.promisify:
function copyFile(from, to) {
// Hmm.. we want to copy a file. We already know how to do that in normal
// JavaScript - we'd just use a createReadStream and pipe that into a
// createWriteStream. But we need to return a promise for our code to work
// like we want it to.
// This means we'll have to make our own hand-made promise. Thankfully,
// that's not actually too difficult..
return new Promise((resolve, reject) => {
// Yikes! What's THIS code mean?
// Well, it literally says we're returning a new Promise object, with a
// function given to it as an argument. This function takes two arguments
// of its own: "resolve" and "reject". We'll look at them separately
// (but maybe you can guess what they mean already!).
// We do still need to create our read and write streams like we always do
// when copying files:
const readStream = fs.createReadStream(from)
const writeStream = fs.createWriteStream(to)
// And we need to pipe the read stream into the write stream (again, as
// usual):
readStream.pipe(writeStream)
// ..But now we need to figure out how to tell the promise when we're done
// copying the files.
// Well, we'll start by doing *something* when the pipe operation is
// finished. That's simple enough; we'll just set up an event listener:
writeStream.on('close', () => {
// Remember the "resolve" and "reject" functions we got earlier? Well, we
// can use them to tell the promise when we're done. So we'll do that here:
resolve()
})
// Okay, but what about errors? What if, for some reason, the pipe fails?
// That's simple enough to deal with too, if you know how. Remember how we
// learned a little on rejected promises, earlier? Since we're making
// our own Promise object, we'll need to create that rejection ourself
// (if anything goes wrong).
writeStream.on('error', err => {
// We'll use the "reject" argument we were given to show that something
// inside the promise failed. We can specify what that something is by
// passing the error object (which we get passed to our event listener,
// as usual).
reject(err)
})
// ..And we'll do the same in case our read stream fails, just in case:
readStream.on('error', err => {
reject(err)
})
// And now we're done! We've created our own hand-made promise-returning
// function, which we can use in our `createFiles` function that we wrote
// earlier.
})
}
..And here's all the finished code, so that you can review it yourself:
const util = require('util')
const fs = require('fs')
const promisifiedReaddir = util.promisify(fs.readdir)
function createFiles(countryCode) {
const readdirPromise = promisifiedReaddir('./app/data')
return readdirPromise.then(directories => {
const promises = directory.map(directory => {
return promisifiedReaddir(`./app/data/${directory}`).then(files => {
return copyFile(`./app/data/${directory}/en.yml`, `./app/data/${directory}/${countryCode}.yml`)
})
})
const allPromise = Promies.all(promises)
return allPromise
})
}
function copyFile(from, to) {
return new Promise((resolve, reject) => {
const readStream = fs.createReadStream(from)
const writeStream = fs.createWriteStream(to)
readStream.pipe(writeStream)
writeStream.on('close', () => {
resolve()
})
writeStream.on('error', err => {
reject(err)
})
readStream.on('error', err => {
reject(err)
})
})
}
Of course, this implementation isn't perfect. You could improve it by looking at other implementations - for example this one destroys the read and write streams when an error occurs, which is a bit cleaner than our method (which doesn't do that). The most reliable way would probably to go with the module I linked earlier!
I highly recommend you watch funfunfunction's video on promises. It explains how promises work in general, how to use Promise.all, and more; and he's almost certainly better at explaining this whole concept than I am!
First, create a function that returns a promise:
function processDirectory(directory) {
return new Promise((resolve, reject) => {
fs.readdir(`./app/data/${directory}`, (err, files) => {
if (err) reject(err);
console.log(`Creating ${countryCode}.yml for ${directory}`);
fs.createReadStream(`./app/data/${directory}/en.yml`)
.pipe(fs.createWriteStream(`./app/data/${directory}/${countryCode}.yml`))
.on('finish', resolve);
});
});
}
Then use Promise.all:
Promise.all(directories.map(processDirectory))
.then(...)
.catch(...);