Develop Reference

jszip wait for creation of zip before working with data

Working on a nodejs implementation of JSZip to create a zip file in binary string format and store its value in a global variable however in spite of using async/await the zipstring is being printed before the zipping process is finished,
Here is my code so far:
const JSZip = require("jszip");
const zip = new JSZip();
let s="not yet done";
let zipstring = "";
async function dozipping() {
zip.file("Hello.txt", "Hello World\n");
zip.generateAsync({ type: "binarystring" })
.then(function(content) {
console.log("doing");
zipstring = content;
});
}
async function status() {
console.log(s)
await dozipping();
console.log(zipstring)
s = "done";
console.log(s)
}
status();
Expected Output
not yet done
doing
*zipstring value*
done
Output I am currently getting
not yet done
done
doing
I am new to node and working with jszip for the 1st time and the documentation is a little confusing, sorry in advance if the question is too trivial

The reason you got nothing is promises are async and they are called once the async task is completed with the status as fulfilled or rejected. I have made some changes to it:
const zip = new JSZip();
let s="not yet done";
let zipstring = "";
async function dozipping(){
zip.file("Hello.txt", "Hello World\n");
try {
console.log("doing");
const response = await zip.generateAsync({type:"binarystring"});
zipstring = response;
return response;
} catch (error) {
console.error(error);
}
}
async function status(){
try {
console.log(s)
await dozipping();
console.log(zipstring)
s="done";
console.log(s)
} catch (error) {
console.error(error);
}
}
await status();
I would suggest you to first go through how promises work and how async\await should be used. They are just a syntactic sugar on it. I am attaching the link to through them.

This is because the .then from zip.generateAsync does not halt the execution. You are gonna need to use
zipstring = await zip.generateAsync(...);
Await halts the execution of an asynchronous function until the promise is resolved.

const JSZip = require('jszip');
const zip = new JSZip();
const content = fs.readFileSync("something.txt");
zip.file("something.txt", content);
let zip_file_path = "something.zip";
zip.generateNodeStream({ type: 'nodebuffer', streamFiles: true })
.pipe(fs.createWriteStream(zip_file_path))
.on('finish', () => {
/**/
});

Node does not wait for loop to complete

I have tried async/await & using promises however I cannot get this code to execute in order.
The code iterates through a document and parses it before saving it to an array, then saving the array to .json file.
The code continues to run before the loop finishes however which means it writes an empty file as the parsing has not been completed.
Turning it into an async function to await does not solve the issue. Nor does returning a promise and then using .then() to execute final code. It still runs straight away.
const fs = require('fs');
const cheerio = require('cheerio');
const mammoth = require("mammoth");
const articleFolder = './Articles/';
var allArticles = [];
const extractDocuments = async () => {
let files = fs.readdirSync(articleFolder);
for(const file of files) {
await convertToHTML(file);
}
completedExtraction();
}
async function convertToHTML(filename) {
var filepath = articleFolder + filename;
mammoth.convertToHtml({path: filepath})
.then(function(result){
let html = result.value; // The generated HTML
let messages = result.messages; // Any messages, such as warnings during conversion
updateArticles(filename, html);
})
.done();
}
function updateArticles (filename, html) {
var article = {
file: filename,
content: parseHTML(html)
}
allArticles.push(article);
}
function parseHTML (html) {
let $ = cheerio.load(html);
let title = $('h3').first().text();
let date = $('h3:eq(1)').text();
$('h3').slice(0,2).remove()
let content = $('body').html();
let parsedArticle = {
title: title,
date: date,
content: content
}
return parsedArticle;
}
function completedExtraction() {
fs.writeFile('./articles.json', JSON.stringify(allArticles), (err)=>{
if (err) throw err;
console.log('File Written.');
});
console.log('Finished.');
}
extractDocuments();

To solve with map I would do something similar to:
const extractDocuments = async () => {
let files = fs.readdirSync(articleFolder);
const articlePromises = files.map(async file => {
const html = await convertToHTML(file)
return {
filename: file,
html: html
}
})
allArticles = await Promise.all(articlePromises)
completedExtraction();
}
async function convertToHTML(filename) {
var filepath = articleFolder + filename;
return mammoth.convertToHtml({path: filepath})
.then(function(result){
let html = result.value; // The generated HTML
let messages = result.messages; // Any messages, such as warnings during conversion
return html
})
.done();
}
So to wrap up extractDocuments uses a map to iterate and create articles. convertToHTML only returns the created HTML and nothing more. We no longer use the updateArticles since this is now handled in the extractDocuments
Hopes this helps a bit. Hope it points you in the right direction

Resolving promises in JavaScript

I am having trouble getting promises to work the way I need. I have tried many different ways to resolve the promise but nothing I have done will work as I need. I am trying to get drag and drop of file working on a web page. I need a list of all the files in a Set (this.files) that is passed to a call to upload the files. The problem is that the Promise.all is being run before the promises are completed.
I am still struggling to wrap my mind around promises so maybe I have it all wrong but it seems from all my research this should work. Any help would be appreciated.
async dndDropFiles(event) {
event.preventDefault();
let ptable = [];
this.files = new Set();
if (event.dataTransfer.types[0] === "Files") {
var items = event.dataTransfer.items;
for (var i=0; i<items.length; i++) {
// webkitGetAsEntry is where the magic happens
var item = await items[i].webkitGetAsEntry();
if (item) {
ptable.push(new Promise(async resolve => {
resolve(await this.dndTraverseFileTree(item, ""));
}))
}
}
Promise.all(ptable)
.then( (results) => {
if (this.files.size > 0) {
this.progress = this.uploadService.upload(this.files, this.currentDir, this.currentProject);
}
})
}
}
async dndTraverseFileTree(item, path) {
if (item.isFile) {
// Get file
item.file((file) => {
this.files.add(file);
});
} else if (item.isDirectory) {
// Get folder contents
let ptable = [];
var dirReader = await item.createReader();
dirReader.readEntries((entries) => {
for (var i=0; i<entries.length; i++) {
ptable.push(new Promise( async resolve => {
resolve(await this.dndTraverseFileTree(entries[i], path + item.name + "/"));
}));
}
Promise.all(ptable)
.then (results => {});
});
}
}

It feels like you are making things a bit too difficult, and I suggest another good read into coding standards and the way Promises work :) The async/await construct has been introduced to increase code readability.
Anyways, I have some untested code here. But it should do the trick. Also I strongly advise you to add typings. You are using angular, so I can only assume you are using TypeScript. With typings you will make less errors, and the compiler helps you along the way.
Before I give the code, this webkitGetAsEntry is non standard. And should only be used if you really don't want to target old browsers or safari/ios:
Non-standard
This feature is non-standard and is not on a standards track. Do not use it on production sites facing the Web: it will not work for every user. There may also be large incompatibilities between implementations and the behavior may change in the future.
But, you could go about it like this. First function to process the event. The second one to traverse the tree:
async dndDropFiles(event: DragEvent): Promise<void> {
if (event.dataTransfer.types[0] !== "Files" || !event.dataTransfer.items) {
return;
}
const entries = [...(event.dataTransfer.items as any)].map(
item => item.webkitGetAsEntry()
);
const allEntries = await this.dndTraverseFileTree(entries);
const files = await Promise.all(
allEntries.map(
(entry) => new Promise((resolve, reject) => entry.file(resolve, reject))
)
);
this.files = new Set(files);
if (this.files.size > 0) {
this.progress = this.uploadService.upload(
this.files, this.currentDir, this.currentProject
);
}
}
async dndTraverseFileTree(entries: any[]): Promise<any[]> {
const dirs = entries.filter(entry => !!entry && entry.isDirectory);
const files = entries.filter(entry => !!entry && entry.isFile);
if (dirs.length) {
const childEntries = (
await Promise.all(
dirs.map(dir => new Promise(
(resolve, reject) => dir.createReader().readEntries(resolve, reject))
)
)
).flat();
return this.dndTraverseFileTree(childEntries);
}
return [ ...files ];
}

Pushing elements into the array works only inside the loop

I got some data which I'm calling from API and I am using axios for that. When data is retrieved, I dump it inside of a function called "RefractorData()" just to organize it a bit, then I push it onto existing array. The problems is, my array gets populated inside forEach and I can console.log my data there, but once I exit the loop, my array is empty.
let matches: any = new Array();
const player = new Player();
data.forEach(
async (match: any) => {
try {
const result = await API.httpRequest(
`https://APILink.com/matches/${match.id}`,
false
);
if (!result) console.log("No match info");
const refractored = player.RefractorMatch(result.data);
matches.push({ match: refractored });
console.log(matches);
} catch (err) {
throw err;
}
}
);
console.log(matches);
Now the first console.log inside forEach is displaying data properly, second one after forEach shows empty array.

Managed to do it with Promise.all() and Array.prototype.map()
.
const player = new Player();
const matches = result.data;
const promises = matches.map(async (match: any) => {
const response: any = await API.httpRequest(
`https://API/matches/${match.id}`,
false
);
let data = response.data;
return {
data: player.RefractorMatch(data)
};
});
const response: any = await Promise.all(promises);

You must understand that async functions almost always run later, because they deppend on some external input like a http response, so, the second console.log is running before the first.
There a few ways to solve this. The ugliest but easiest to figure out is to create a external promise that you will resolve once all http requests are done.
let matches = [];
let promise = new Promise((resolve) => {
let complete = 0;
data.forEach((match: any) => {
API.httpRequest(...).then((result) => {
// Your logic here
matches.push(yourLogicResult);
complete++;
if (complete === data.length) {
resolve();
}
}
}
};
console.log(matches); // still logs empty array
promise.then(() => console.log(matches)); // now logs the right array
You can solve this using other methods, for example Promise.all().
One very helpful way to solve it is using RxJs Observables. See https://www.learnrxjs.io/
Hope I helped you!

Retry nodejs http.request (post,put,delete)

What is the correct way to implement a retry on error/condition without using any third party modules in nodejs, please?
I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
Do I need to destroy/end the sockets?
I've tried looking for examples but have only found reference to third party modules and http.get samples which don't seem to work. How does one test this?
I have attempted the below without success:
async pingApi(cb) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, /ect do I reference this path?
}
};
let request = await http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', function () {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.result != 'OK') {
setTimeout(pingApi, 1000); //cant pass callback
} else {
cb(null, json.result) //works
}
}
});
})
request.end(); //does the socket need to be closed if retry is this function?
}
Any help, pointing in the right direction or criticism will be greatly appreciated as I think this is a very important learning curve for me.
Thank you in advance,

I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
I don't know for sure that everything else in your function is correct, but you can fix the recursion that you're asking about by changing this:
setTimeout(pingApi, 1000); //cant pass callback
to this:
setTimeout(() => {
this.pingApi(cb);
}, 1000);
You aren't showing the whole context here, but if pingApi() is a method, then you also need to keep track of the this value to you can call this.pingApi(db). You can preserve the value of this by using arrow function callbacks like this:
response.on('end', () => { ... });
Other things I notice that look off here:
There's no reason to use await http.request(). http.request() does not return a promise so using await with it does not do anything useful.
Without the await, there's then no reason for your function to be declared async since nobody is using a returned promise from it.
It's not clear what if (this.complete) is meant to do. Since this is inside a regular function callback, the value of this won't be your pingApi object. You should either save this higher in the scope typically with const self = this or all callbacks internally need to be arrow functions so the value of this is preserved.
You should probably put try/catch around JSON.parse() because it can throw if the input is not perfect JSON.
You should probably not retry forever. Servers really hate clients that retry forever because if something goes wrong, the client may just be bashing the server every second indefinitely. I'd suggest a certain number of max retries and then give up with an error.
Do I need to destroy/end the sockets?
No, that will happen automatically after the request ends.
How does one test this?
You have to create a test route in your server that returns the error condition for the first few requests and then returns a successful response and see if your code works with that.
Here's an attempt at a code fixup (untested):
const maxRetries = 10;
pingApi(cb, cnt = 0) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, // ect do I reference this path?
};
let request = http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', () => {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
try {
let json = JSON.parse(decoded);
if (json.result != 'OK') {
if (cnt < maxRetries)
setTimeout(() => {
this.pingApi(cb, ++cnt);
}, 1000);
} else {
cb(new Error("Exceeded maxRetries with error on pingApi()"));
}
} else {
cb(null, json.result) //works
}
} catch(e) {
// illegal JSON encountered
cb(e);
}
}
});
})
request.end();
}
Remaining open questions about this code:
What is this.complete doing and what this should it be referencing?
Why is there no request.write() to send the body of the POST request?
I know you ask for no external modules, but my preferred way of doing this would be to use promises and to use the request-promise wrapper around http.request() because it handles a lot of this code for you (checks response.status for you, parses JSON for you, uses promise interface, etc...). You can see how much cleaner the code is:
const rp = require('request-promise');
const maxRetries = 5;
pingApi(cnt = 0) {
let options = {
method: "post",
url: `http://somedomain.com/API/pingAPI?${this.auth}`,
json: true
};
return rp(options).then(result => {
if (result.result === "OK") {
return result;
} else {
throw "try again"; // advance to .catch handler
}
}).catch(err => {
if (cnt < maxRetries) {
return pingApi(++cnt);
} else {
throw new Error("pingApi failed after maxRetries")
}
});
}
And, then sample usage:
pingApi().then(result => {
console.log(result);
}).catch(err => {
console.log(err);
})

your use of async/await with core node server intrigued me and I've tried to use much as possible of this new async features.
This is what I end up with: https://runkit.com/marzelin/pified-ping
const pify = require("util").promisify;
const http = require("http");
const hostname = "jsonplaceholder.typicode.com";
const failEndpoint = "/todos/2";
const goodEndpoint = "/todos/4";
let options = {
method: "get",
path: `${failEndpoint}`,
hostname
};
async function ping(tries = 0) {
return new Promise((res) => {
const req = http.request(options, async (response) => {
let body = new Buffer(0);
response.on("data", (chunk) => {
body = Buffer.concat([body, chunk]);
})
const on = pify(response.on.bind(response));
await on("end");
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.completed) {
return res("all good");
}
if (tries < 3) {
console.log(`retrying ${tries + 1} time`);
return res(ping(tries + 1));
}
return res("failed");
})
req.on('error', (e) => {
console.error(`problem with request: ${e.message}`);
});
// write data to request body
req.end();
})
}
const status = await ping();
"status: " + status