Node/Javascript loop synchronous - javascript

I have followed multiple posts and guides about using async / await, tried multiple synchronous request libraries, I have tried promises, then blocks, callbacks, using regular loops as opposed to foreach loops, and I have run out of ideas.
This code calls the Zoom API to get a list of cloud recordings that I would like to download. Each user can have 100-200 and they are large files, so there is a limit to how many connections either my end or Zoom's end can handle without getting an understandable "RequestError: socket hang up".
Because of how the Zoom API works you can only get results for one month a t a time. So I am looping over users, then looping over months, then calling the individual URLs for stream download to a file on my workstation.
All I want this code to do is process the recordings for a single user in a single month, and WAIT until they have all downloaded, before moving onto the month and then eventually the next user.
Can anyone suggest how I might be able to accomplish that?
const fs = require("fs");
const path = require("path");
const http = require("follow-redirects").http;
const https = require("follow-redirects").https;
var syncrequest = require("sync-request");
const got = require("got");
const stream = require("stream");
const { promisify } = require("util");
(async function getFiles() {
var token = "TOKEN_HERE";
var zoomUserIDs;
var users = ["EMAILS_HERE","EMAILS_HERE"];
var dates = [
[9, 2020],
[10, 2020],
[11, 2020],
[12, 2020],
[1, 2021],
[2, 2021],
[3, 2021],
[4, 2021],
[5, 2021],
[6, 2021]
];
var path_to_save = "";
var folder_name = "";
//Loop over list of users
for (const user of users) {
console.log(user);
folder_name = "output/" + user;
if (!fs.existsSync(folder_name)) {
fs.mkdir(path.join("", folder_name), err => {
if (err) {
return console.error(err);
}
console.log(user + " Directory created successfully!");
});
}
//Loop over months
for (const date of dates) {
var url =
"https://api.zoom.us/v2/users/" +
user +
"/recordings?from=" +
date[1] +
"-" +
date[0] +
"-01&to=" +
date[1] +
"-" +
date[0] +
"-31&page_size=100";
//Call Zoom API
const res = await got.get(url, {
responseType: "json",
headers: {
Authorization: "Bearer" + token
}
});
//Get individual file url
var reponse = res.body;
if (reponse.meetings) {
for (const meeting of reponse.meetings) {
for (const recording of meeting.recording_files) {
if (
recording.recording_type == "shared_screen_with_speaker_view" ||
recording.recording_type == "shared_screen" ||
recording.recording_type == "active_speaker"
) {
var path_to_zoom_recording = recording.download_url + "?access_token=" + token;
//Dowload file
const pipeline = promisify(stream.pipeline);
path_to_save =
folder_name +
"/" +
meeting.topic.replaceAll("/", "_") +
"_" +
meeting.start_time.replaceAll(":", "-") +
".mp4";
if (!fs.existsSync(path_to_save)) {
(async () => {
console.log(
meeting.topic + " -- " + meeting.start_time + " -- START"
);
await pipeline(
got.stream(path_to_zoom_recording),
fs.createWriteStream(path_to_save)
).then(() =>
console.log(
meeting.topic + " -- " + meeting.start_time + " -- END"
)
);
})();
}
}
}
}
}
}
}
})();

I think if you get rid of the first and last line of this section you will be most of the way there
(async () => {
console.log(
meeting.topic + " -- " + meeting.start_time + " -- START"
);
await pipeline(
got.stream(path_to_zoom_recording),
fs.createWriteStream(path_to_save)
).then(() =>
console.log(
meeting.topic + " -- " + meeting.start_time + " -- END"
)
);
})();
The way you have it, your upper level isn't awaiting the pipeline.
Also you are this issue near the top:
for (const user of users) {
console.log(user);
folder_name = "output/" + user;
if (!fs.existsSync(folder_name)) {
fs.mkdir(path.join("", folder_name), err => {
if (err) {
return console.error(err);
}
console.log(user + " Directory created successfully!");
});
}
the fs.mkdir command is also not awaited. You can wrap that in promisify, and then await that too, just like you did below with another function.
It seems weird to mix it async/await with existsSync, but it probably won't hurt, since you only have one thing happening at once. It would be nice to also promisfy fs.exists, and then await that.
In newer node.js there is a pre-promisified version of all these methods. You import them from 'fs/promises' I think. https://nodejs.org/api/fs.html#fs_promise_example
also, for what it's worth, you only have to promisify a function once. This shouldn't really be in the loop: const pipeline = promisify(stream.pipeline); But node.js probably doesn't mind doing it every iteration, so that is not the issue you are looking for.

Related

Use Cloudflare HTMLRewriter to modify multiple different element types

I'd like to use Cloudflare's HTMLRewriter to modify a <p> tag with some text, and modify an <img> tag with a different src.
I've read the docs here: https://developers.cloudflare.com/workers/runtime-apis/html-rewriter but I cannot find any examples addressing re-writing multiple different elements.
I'm using Cloudflare Pages Functions so as far as I can tell I do need to do all of this in the same file.
My code currently looks like this, to just do the re-write on the <p> tag. It works, but I'd like to update a weather icon as well in the menu bar.
// ./functions/index.js
export const onRequestGet = async ({ request, next }) => {
try{
// Get the static asset response
const response = await next()
const { latitude, longitude, city, country, timezone } = request.cf
let endpoint = "https://api.openweathermap.org/data/2.5/weather?"
endpoint += "lat=" + latitude + "&lon=" + longitude + "&appid=APPID"
const init = {
headers: {
// "User-Agent" : "EMAIL",
},
}
const responseWeather = await fetch(endpoint,init)
const content = await responseWeather.json()
const currd = new Date()
var humanTime = currd.toLocaleTimeString('en-US', { timeZone: timezone })
//Get the value from the object
const currentTempC = content.main.temp
const weatherDescription = content.weather[0].description
const currentTempF = Math.round(((9/5)* (currentTempC - 273)) + 32)
var currentTempLocal
var degreesSymbol
switch(country) {
case "US":
case "BS":
case "KY":
case "LR":
case "PW":
case "FM":
case "MH":
currentTempLocal = currentTempF
degreesSymbol = "°F"
break;
default:
currentTempLocal = currentTempC
degreesSymbol = "°C"
break;
}
// US BS KY LR PW FM MH
const weatherString = "At " + humanTime + " in " + city + "there's " + weatherDescription + " and the temperature is " + currentTempLocal + degreesSymbol + "."
// var errorReport = timezone + "\n" + humanTime + "\n" + JSON.stringify(context)
// Find the placeholder in our static asset
return new HTMLRewriter().on('#weather', {
// And act on the element
element(element) {
// https://developers.cloudflare.com/workers/runtime-apis/html-rewriter#methods
element.setInnerContent(weatherString, { html: true })
}
}).transform(response)
} catch (thrown){
return new Response(thrown);
}
}
This should be as simple as chaining another .on method to your HTMLRewritter like so -
return new HTMLRewriter().on('#weather', {
element(element) {
// first handler
}).on('img', {
element(element) {
// second handler
}).transform(response)

Wait for fetch response to continue in for loop. Javascript Nodejs

I have a function that connect to a web service in SOAP. Unfortunately the web service only support a very limited connections. I have an array of items to search in the web service, if i do a for or a foreach loop, the 70% of cases complete with no error, but in the 30% the web service response a error. This occurs when the max connections is overflow. This happens because the loop is no waiting the response of the webservice and the loop cotinues creating a lot of connections.
Here's my code:
var promiseArray = [];
for (var i = 0; i < result.length; i++) {
let m = result[i].id
let xml = '<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:tem="http://tempuri.org/">' +
'<soapenv:Header/>' +
'<soapenv:Body>' +
'<tem:EjecutarConsultaXML>' +
'<!--Optional:-->' +
'<tem:pvstrxmlParametros>' +
'<![CDATA[' +
'<Consulta><NombreConexion>USERNAME</NombreConexion>' +
'<IdConsulta>QUERY</IdConsulta>' +
'<Parametros>' +
'<doc>' + m + '</doc>' +
'</Parametros>' +
'</Consulta>' +
']]>' +
'</tem:pvstrxmlParametros>' +
'</tem:EjecutarConsultaXML>' +
'</soapenv:Body>' +
'</soapenv:Envelope>';
const options = {
explicitArray: true
};
promiseArray.push(new Promise(async(resolve, reject) => {
await axios.post(url, xml, {
headers: {
'Content-Type': 'text/xml;charset=UTF-8'
}
})
.then((data) => {
xml2js.parseString(data.data, options, (err, result) => {
var temp = (result['soap:Envelope']['soap:Body'][0]['EjecutarConsultaXMLResponse'][0]['EjecutarConsultaXMLResult'][0]['diffgr:diffgram'][0]['NewDataSet'][0]['Resultado'])
resolve({
doc: m,
state: temp[0].f430_ind_estado[0]
})
});
})
.catch((err) => {
console.log(err)
});
}))
}
res.send(await Promise.all(promiseArray))
There are several issues with your code within the call to promiseArray.push().
There is no need to create a new Promise() since axios already provides one
This is actually and antipattern
There is no need for async/await in that call for the same reason.
Mixing Promises and functions that use callbacks usually doesn't turn out too well
You have no error checking in your code if the XML parser fails
The option object is not required as explicitArray: true is the default
Changes:
Removed all the extra/uneeded Promise code
Replaced xml2js.parseString with xml2js.parseStringPromise
Changed resolve to return
Since you're simply console.log() the error, removed unecessary boilerplate
Everything else is OK as written. Please let me know if I've missed something.
promiseArray.push(
axios.post(url, xml, {
headers: {
'Content-Type': 'text/xml;charset=UTF-8'
}
})
.then(data=>data.data)
.then(xml2js.parseStringPromise)
.then(result => {
var temp = result['soap:Envelope']['soap:Body'][0]['EjecutarConsultaXMLResponse'][0]['EjecutarConsultaXMLResult'][0]['diffgr:diffgram'][0]['NewDataSet'][0]['Resultado'];
return {
doc: m,
state: temp[0].f430_ind_estado[0]
};
});
.catch(console.log)
);
Just do it one by one, using async/await to do that, this means you have to use parseStringPromise instead.
var response = [];
for (var i = 0; i < result.length; i++) {
let m = result[i].id
let xml = '<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:tem="http://tempuri.org/">' +
'<soapenv:Header/>' +
'<soapenv:Body>' +
'<tem:EjecutarConsultaXML>' +
'<!--Optional:-->' +
'<tem:pvstrxmlParametros>' +
'<![CDATA[' +
'<Consulta><NombreConexion>USERNAME</NombreConexion>' +
'<IdConsulta>QUERY</IdConsulta>' +
'<Parametros>' +
'<doc>' + m + '</doc>' +
'</Parametros>' +
'</Consulta>' +
']]>' +
'</tem:pvstrxmlParametros>' +
'</tem:EjecutarConsultaXML>' +
'</soapenv:Body>' +
'</soapenv:Envelope>';
const options = {
explicitArray: true
};
try {
var { data } = await axios.post(url, xml, { // extract data from data.data
headers: {
'Content-Type': 'text/xml;charset=UTF-8'
}
})
var xmlObject = await xml2js.parseStringPromise(data)
var temp = (xmlObject['soap:Envelope']['soap:Body'][0]['EjecutarConsultaXMLResponse'][0]['EjecutarConsultaXMLResult'][0]['diffgr:diffgram'][0]['NewDataSet'][0]['Resultado'])
response.push({
doc: m,
state: temp[0].f430_ind_estado[0]
}) // push item to result array
} catch (error) {
console.log(error);
}
}
res.send(result) // send the result to client

Write javascript array elements to file

I am trying to have a node js script write some coordinates to a csv file for use in a Newman CLI script. I have the following:
const axios = require('axios');
var move_decimal = require('move-decimal-point');
var sLat = 45.029830;
var sLon = -93.400891;
var eLat = 45.069523;
var eLon = -94.286001;
var arrLatLon = []
axios.get('http://router.project-osrm.org/route/v1/driving/' + sLon + ',' + sLat + ';' + eLon + ',' + eLat + '?steps=true')
.then(function (response) {
for (let i = 0; i < response.data.routes[0].legs.length; i++) {
//console.log(response.data)
for (let ii = 0; ii < response.data.routes[0].legs[i].steps.length; ii++) {
//console.log('leg ' + i + " - step " + ii + ": " + response.data.routes[0].legs[i].steps[ii].maneuver.location[1] + "," + response.data.routes[0].legs[i].steps[ii].maneuver.location[0]);
// Declaring Latitude as 'n' & Longitude as 'nn' for decimal calculations
var n = response.data.routes[0].legs[i].steps[ii].maneuver.location[1]
var nn = response.data.routes[0].legs[i].steps[ii].maneuver.location[0]
// Latitude calculatiuons to make 'lat' values API friendly
var y = move_decimal(n, 6)
var p = Math.trunc(y);
// Longitude calculations to make 'lon' values API friendly
var yy = move_decimal(nn, 6)
var pp = Math.trunc(yy);
arrLatLon.push(p + "," + pp);
}
console.log(arrLatLon)
}
})
I have been looking through and trying numerous different tutorials/code snippets regarding writing the array elements from arrLatLon to an output file on my local machine, but none have been successful. The current code outputs the lat,lon correctly, console.log(arrLatLon) outputs:
[ '45029830,-93400894',
'44982812,-93400740',
'44977444,-93400530',
'44973116,-93410884',
'44971101,-93450400',
'45035514,-93766885',
'45035610,-93766886',
'45081631,-94286752',
'45070849,-94282026' ]
any help would be greatly appreciated. Thanks.
With nodejs you can easily write files using the fs module
const fs = require('fs');
fs.writeFile("/tmp/test", "Hey there!", function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
in your case you can simply do something like
const fs = require('fs');
// I'm converting your array in a string on which every value is
// separated by a new line character
const output = arrLatLon.join("\n");
// write the output at /tmp/test
fs.writeFile("/tmp/test", output, function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
Let me forward you to this question for more information Writing files in Node.js

Node.js: "process run out of memory"

I have the following code which leads to the error: FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory
It doesn't make any difference whether I set --max_old_space_size=4096 (or bigger numbers) or not. I have 16 GB RAM on my PC.
System: win10/node.js 6.9.2/mongodb 3.4
I have over 16 000 000 messages in the "chatModel". With a smaller amount of messages the code works.
Do you have any suggestions how to solve the problem/optimize the code?
function sortOutMessages(){
var atSymbol = "#";
var rgx = new RegExp("^\\" +atSymbol);
chatModel.find({messageContent: rgx}, function (err, doc){
var docs = doc;
var docsLength = docs.length;
for (var i =0; i<docsLength;i++) {
var directedMessagesObj = new directedMessagesModel
({
timeStamp: docs[i].timeStamp,
channelName: docs[i].channelName,
userName: docs[i].userName,
userID: docs[i].userID,
messageContent: docs[i].messageContent,
messageLength: docs[i].messageLength,
subscriber: docs[i].subscriber,
turbo: docs[i].turbo,
moderator: docs[i].moderator
});
directedMessagesObj.save({upsert:true}, function (err) {
var fs = require('fs');
if (err) {
fs.appendFile("undefinedLog.txt", "error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n")
loggerWinston.warn("error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n");
return console.log(err);
}
});
}
});
}
You are creating docs.length amount of new Promise with this code. What you should be doing instead is to limit the amount of Promise that can be running.
Since what you have been trying to write is a synchronous code, I would advise calling the next dbSave in the callback of the previous one.
If you are after a parallel system, I would create a simple semaphore system.
I advise
module.exports.asyncEach = function(iterableList, callback, done) {
var i = -1,
length = iterableList.length;
function loop() {
i++;
if (i === length) {
done();
return;
}
callback(iterableList[i], loop);
}
loop();
};
then you call asyncEach with async save model is very beter
Thank you all for your answers! I decided to go a different way and it works for now:
var cursor = chatModel.find({messageContent: rgx}).cursor();
cursor.on('data',function (doc) {
var directedMessagesObj = new directedMessagesModel
({
timeStamp: doc.timeStamp,
channelName: doc.channelName,
userName: doc.userName,
userID: doc.userID,
messageContent: doc.messageContent,
messageLength: doc.messageLength,
subscriber: doc.subscriber,
turbo: doc.turbo,
moderator: doc.moderator
});
directedMessagesObj.save({upsert:true},function (err) {
var fs = require('fs');
if (err) {
fs.appendFile("undefinedLog.txt", "error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n");
loggerWinston.warn("error at " + new Date().toLocaleString() + " directedMessagesObj.save " + "\r\n");
return console.log(err);
}
});
});

Node.js Downloading multiples files asynchronously

In trying to get a hang of node.js asynchronous coding style, I decided to write a program that would read a text file containing a bunch of URLS to download and download each file. I started out writing a function to download just one file (which works fine), but having trouble extending the logic to download multiple files.
Here's the code:
var http = require("http"),
fs = require("fs"),
input = process.argv[2],
folder = "C:/Users/Wiz/Downloads/",
regex = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/,
urls = null,
url = "",
filename = "";
fs.readFile(input, "utf8", function(e, data) {
console.log("Reading file: " + input);
if (e) console.log("Got error:" + e.message);
urls = data.split("\n");
for (var i = urls.length; i--;) {
url = urls[i];
if (!url.match(regex)) continue;
filename = folder + url.substring(url.lastIndexOf('/') + 1);
downloadQueue.addItem(url, filename);
}
});
var downloadQueue = {
queue: [],
addItem: function(p_sSrc, p_sDest) {
this.queue.push({
src: p_sSrc,
dest: p_sDest
});
if (this.queue.length === 1) {
this.getNext();
}
},
getNext: function() {
var l_oItem = this.queue[0];
http.get(l_oItem.src, function(response) {
console.log("Downloading: " + l_oItem.dest);
var file = fs.createWriteStream(l_oItem.dest);
response.on("end", function() {
file.end();
console.log("Download complete.");
downloadQueue.removeItem();
}).on("error", function(error) {
console.log("Error: " + error.message);
fs.unlink(l_oItem.dest);
});
response.pipe(file);
});
},
removeItem: function() {
this.queue.splice(0, 1);
if (this.queue.length != 0) {
this.getNext();
} else {
console.log("All items downloaded");
}
}
};
How do I structure the code so that the completion of the first download can signal the initiation of the next one. Please note that this exercise is just for learning purposes, to understand how asynchronous coding works. In practice, I'm sure there are much better tools out there to download multiple files.
Try simple at first, it look like you copy paste codes and quite don't understand what they do.
Do a simple loop, that get the url, and print something.
var http = require('http');
URL = require('url').parse('http://www.timeapi.org/utc/now?format=%25F%20%25T%20-%20%25N')
URL['headers'] = {'User-Agent': 'Hello World'}
// launch 20 queries asynchronously
for(var i = 0; i < 20; i++) {
(function(i) {
console.log('Query ' + i + ' started');
var req = http.request(URL, function(res) {
console.log('Query ' + i + ' status: ' + res.statusCode + ' - ' + res.statusMessage);
res.on('data', function(content){
console.log('Query ' + i + ' ended - ' + content);
});
});
req.on('error', function(err) {
console.log('Query ' + i + ' return error: ' + err.message);
});
req.end();
})(i);
}
All the urls will be fetched asynchronously. You can observe that the response does not arrive in order, but are still processed correctly.
The difficulty with async is not to do the things is parallel, because you just write like a single task, and execute multiple time. It becomes complicated when you need for instance to wait for all tasks to finished before continuing. And for that, have a look at promises
Here is what I started out with. Figuring that each download was invoked asynchronously, they would all be independent of each other.
var http = require("http"),
fs = require("fs"),
input = process.argv[2],
folder = "C:/Users/Wiz/Downloads/",
regex = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/,
urls = null,
url = "",
filename = "";
fs.readFile(input, "utf8",
function(e, data) {
console.log("Reading file: " + input);
if (e) console.log("Got error:" + e.message);
urls = data.split("\n");
for (var i = urls.length; i--;) {
url = urls[i];
if (!url.match(regex)) continue;
filename = folder + url.substring(url.lastIndexOf('/') + 1);
http.get(url, function(response) {
var file = fs.createWriteStream(filename);
response.on("end", function() {
file.end();
});
response.pipe(file);
})
}
});

Categories

Resources