Develop Reference

Execute block of code only after loop with requests from axios is finished

I have a script that reads an excel file and gets data from a specific column to perform a search on the Google Maps API where I use axios. For each request made, I need to save it in the newFileList variable. After completing all the requests, I must save the contents of this variable in a file. However, whenever I run my code, the file is being saved without the content of the newFileList variable. How do I wait for all requests to finish before being able to save the content in the file?
Note: the reading, writing and requesting data are working. I just need the rescue to happen only after all the loop requests are finished. I tried to solve by placing the loop inside a promisse and at the end of the execution of this loop I used resolve.
const xlsx = require("node-xlsx");
const fs = require("fs");
const coordinate = require("./coordinate");
const resourcePath = `${__dirname}/resources`;
const contentFile = xlsx.parse(`${resourcePath}/file-2.xlsx`)[0].data;
const newFile = [[...contentFile, ...["Latitude", "Longitude"]]];
for (let i = 1; i < contentFile.length; i++) {
const data = contentFile[i];
const address = data[2];
coordinate
.loadCoordinates(address)
.then((response) => {
const { lat, lng } = response.data.results[0].geometry.location;
newFile.push([...data, ...[lat.toString(), lng.toString()]]);
})
.catch((err) => {
console.log(err);
});
}
console.log(newFile);
//The code below should only be executed when the previous loop ends completely
var buffer = xlsx.build([{ name: "mySheetName", data: newFile }]); // Returns a buffer
fs.writeFile(`${resourcePath}/file-3.xlsx`, buffer, function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
The coordinate file:
const axios = require("axios");
module.exports = {
loadCoordinates(address) {
const key = "abc";
return axios
.get(`https://maps.googleapis.com/maps/api/geocode/json`, {
params: {
address,
key,
},
})
},
};

Will using an async IIFE help?
const xlsx = require("node-xlsx");
const fs = require("fs");
const coordinate = require("./coordinate");
const resourcePath = `${__dirname}/resources`;
const contentFile = xlsx.parse(`${resourcePath}/file-2.xlsx`)[0].data;
const newFile = [[...contentFile, ...["Latitude", "Longitude"]]];
(async() => {
try{
for (let i = 1; i < contentFile.length; i++) {
const data = contentFile[i];
const address = data[2];
await coordinate
.loadCoordinates(address)
.then((response) => {
const { lat, lng } = response.data.results[0].geometry.location;
newFile.push([...data, ...[lat.toString(), lng.toString()]]);
})
.catch((err) => {
console.log(err);
});
}
console.log(newFile);
//The code below should only be executed when the previous loop ends completely
var buffer = xlsx.build([{ name: "mySheetName", data: newFile }]); // Returns a buffer
fs.writeFile(`${resourcePath}/file-3.xlsx`, buffer, function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
} catch(e) {
console.log(e)
}
})();
Do note that I added await before coordinate.loadCoordinates, in order to make sure the first axios request is finished before we proceed to the next one.

You need to use Promise.all() to wait until all the promises are resolved. After that execute the writeToFile part. For more info on Promise.all(), you can refer https://www.javascripttutorial.net/es6/javascript-promise-all/
const requestPromiseArray = [];
for (let i = 1; i < contentFile.length; i++) {
const data = contentFile[i];
const address = data[2];
requestPromiseArray.push(coordinate
.loadCoordinates(address))
}
Promise.all(requestPromiseaArray).then(results=>{
// Handle "results" which contains the resolved values.
// Implement logic to write them onto a file
var buffer = xlsx.build([{ name: "mySheetName", data: results }]);
fs.writeFile(`${resourcePath}/file-3.xlsx`, buffer, function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
})

Making a web-crawler to have loop

I tried to make my web-crawler to have a loop to crawl the webpage from 1 to around 500. But the result does not include any directed one but to return an only void array.
This code is based on cheerio, jQuery, and axios. JavaScript.
const axios = require("axios");
const cheerio = require("cheerio");
const log = console.log;
const getHtml = async() => {
var i=0
while (i<493){
try {
return await axios.get("https://playentry.org/ds#!/qna?sort=created&rows=20&page="+i);
} catch (error) {
console.error(error);
}
}
};
getHtml()
.then(html => {
let ulList = [];
const $ = cheerio.load(html.data);
const $bodyList = $("div.discussContentWrapper div.discussListWrapper table.discussList").children("tr.discussRow");
$bodyList.each(function(i, elem){
ulList[i] = {
title:$(this).find('td.discussTitle div.discussTitleWrapper'),
writer:$(this).find('td.discussTitle td.discussViewCount'),
viewcount:$(this).find('td.discussTitle td.discussViewCount'),
likecount:$(this).find('td.discussTitle div.discussLikeCount'),
date:$(this).find('td.discussTitle td.discussDate'),
};
});
const data = ulList.filter(n => n.title);
return data;
})
.then(res => log(res));
The output is '''[]''' or '''[ [] ]''' with no real outputs.
Thanks for your help in advance.

how to do Api fetch and display the results

i am working on a simple app that will fetch data from an Api and display it.
a have this function
const getAUserProfile = () => {
const api = 'https://randomuser.me/api/';
// make API call here
return fetch(api)
.then(response => response.json())
.then(response => displayUserPhotoAndName(response))
notify(`requesting profile data ...`);
};
this is where i try to fetch the data and pass as a parameter to the displayUserPhotoAndName function.
Now in the displayUserPhotoAndName function i try to create a statement that de-structures the data parameter and obtains the results property from it;
i tried to Create a second statement in the next line that de-structures the results variable i just created, and obtain the first item from it (it is an Array! See https://randomuser.me/api/). the de-structured array item should be declared as profile. This represents the profile data for the user gotten from the API call that i want to display in my app.
this is the displayUserPhotoAndName function
const displayUserPhotoAndName = (data) => {
if(!data) return;
// add your code here
const {results} = data;
const {profile} = results;
document.getElementById('name').innerHTML = profile[results];
clearNotice();
};
now i am trying to display the title, first name and last name with this line of code document.getElementById('name').innerHTML = profile[0].title + profile[0].first + profile[0].last;.
this is not working when i try to run it in sapio

There is no profile property in data or in the results array. So your assignment const {profile} = results; will be undefined
You could point profile at the first item in the results array. Then use the property paths for what you want to display
const displayUserPhotoAndName = (data) => {
if (!data) return;
// add your code here
const {results} = data;
const profile = results[0];
const fullName = profile.name.first + ' ' + profile.name.last;
document.getElementById('name').innerHTML = fullName
};
const getAUserProfile = () => {
const api = 'https://randomuser.me/api/';
// make API call here
return fetch(api)
.then(response => response.json())
.then(displayUserPhotoAndName)
};
getAUserProfile()
<div id="name"></div>

You need this probably:
document.getElementById('name').innerHTML = results[0].name.title + results[0].name.first + results[0].name.last;
Because the json looks like this:
{
"results":[
{
"gender":"male",
"name":{
"title":"mr",
"first":"vidar",
"last":"sjursen"
},
"location":{
"street":"bestum tverrvei 7385",
"city":"bratsberg",
"state":"sogn og fjordane",
"postcode":"8514",
"coordinates":{
"latitude":"57.7278",
"longitude":"-95.6342"
},
"timezone":{
"offset":"+7:00",
"description":"Bangkok, Hanoi, Jakarta"
}
},
"email":"vidar.sjursen#example.com",
"login":{
"uuid":"fbc411b4-f34c-497f-acff-8294ddcf8738",
"username":"whitezebra569",
"password":"shoes",
"salt":"8prWID0j",
"md5":"02ea3e887aaa140ad312905801ae2353",
"sha1":"c9a66349e68825dc02c74618aac8572fbdd01e5b",
"sha256":"8297cc85be1127223761fb80b8f554632a6a37c35d58d435293a8f6b2dca19f3"
},
"dob":{
"date":"1952-10-19T01:20:59Z",
"age":66
},
"registered":{
"date":"2011-11-12T03:06:29Z",
"age":7
},
"phone":"88795425",
"cell":"92914506",
"id":{
"name":"FN",
"value":"19105218888"
},
"picture":{
"large":"https://randomuser.me/api/portraits/men/25.jpg",
"medium":"https://randomuser.me/api/portraits/med/men/25.jpg",
"thumbnail":"https://randomuser.me/api/portraits/thumb/men/25.jpg"
},
"nat":"NO"
}
],
"info":{
"seed":"48917bf154395ac4",
"results":1,
"page":1,
"version":"1.2"
}
}
Which means, when you do this:
const {results} = data;
Then the array will be there, and the array doesn't have profile property to get it with:
const {profile} = results;

The problem is with this line of code
const {results} = data;
--> const {profile} = results;
the "{results}" will take the results property out of your response which in turn is an array.
when you try to take {profile} from results there is no such property like profile in it.
Which makes the profile variable undefined. This was the problem.
Note: The {something} variable will look for property in your object. [this case the property is something]
Hope this one helps

this is what worked for me
const displayUserPhotoAndName = (data) => {
if(!data) return;
// add your code here
const {results} = data;
const [profile] = [...results];
const fullName = profile.name.title + ' ' + profile.name.last + ' ' + profile.name.first;
document.getElementById('name').innerHTML = fullName
document.getElementById('photo').src = profile.picture.large
displayExtraUserInfo(profile);
clearNotice();
};

const {profile} = results; here profile is undefined because results is an array with no property profile.
You need to do const profile = [...results]; and then document.getElementById('name').innerHTML = profile[0].name.title + profile[0].name.first + profile[0].name.last;
fetch('https://randomuser.me/api/')
.then(res => res.json())
.then(data => {
if (!data) return;
const {
results
} = data;
const profile = [...results];
console.log(profile)
document.getElementById('name').innerHTML = profile[0].name.title + profile[0].name.first + profile[0].name.last;
}
)
<div id="name"></div>

Web scraping and promises

I am using cheerio and node to do web scraping, but I have a problem with promises. I can scrape an article list from a page but in that list, we have more links for single pages. I need to scrape single pages as well for each item on the list.
I will show you my code for the better solution.
import rp from 'request-promise'
import cheerio from 'cheerio'
import conn from './connection'
const flexJob = `https://www.flexjobs.com`
const flexJobCategory = ['account-management', 'bilingual']
class WebScraping {
//list of article e.g for page 2
results = [] // [[title], [link for page],...]
contentPage = [] //content for each page
scrapeWeb(link) {
let fullLink = `${link}/jobs/${flexJobCategory[1]}?page=2`
const options = {
uri: fullLink,
transform(body) {
return cheerio.load(body)
}
}
rp(options)
.then(($) => {
console.log(fullLink)
$('.featured-job').each((index, value) => {
//html nodes
let shortDescription = value.children[1].children[1].children[3].children[1].children[1].children[0].data
let link = value.children[1].children[1].children[1].children[1].children[1].children[0].attribs.href
let pageLink = flexJob + '' + link
let title = value.children[1].children[1].children[1].children[1].children[1].children[0].children[0].data
let place = value.children[1].children[1].children[1].children[1].children[3].children[1].data
let jobType = value.children[1].children[1].children[1].children[1].children[3].children[0].children[0].data
this.results.push([title, '', pageLink.replace(/\s/g, ''), '', shortDescription.replace(/\n/g, ''), place, jobType, 'PageContent::: '])
})
})
.then(() => {
this.results.forEach(element => {
console.log('link: ', element[2])
this.scrapePage(element[2])
});
})
.then(() => {
console.log('print content page', this.contentPage)
})
.then(() => {
//this.insertIntoDB()
console.log('insert into db')
})
.catch((err) => {
console.log(err)
})
}
/**
* It's going to scrape all pages from list of jobs
* #param {Any} pageLink
* #param {Number} count
*/
scrapePage(pageLink) {
let $this = this
//console.log('We are in ScrapePage' + pageLink + ': number' + count)
//this.results[count].push('Hello' + count)
let content = ''
const options = {
uri: pageLink,
transform(body) {
return cheerio.load(body)
}
}
rp(options)
.then(($) => {
//this.contentPage.push('Hello' + ' : ');
console.log('Heloo')
})
.catch((err) => {
console.log(err)
})
}
/**
* This method is going to insert data into Database
*/
insertIntoDB() {
conn.connect((err) => {
var sql = "INSERT INTO contact (title, department, link, salary, short_description, location, job_type, page_detail) VALUES ?"
var values = this.results
conn.query(sql, [values], function (err) {
if (err) throw err
conn.end()
})
})
}
}
let webScraping = new WebScraping()
let scrapeList = webScraping.scrapeWeb(flexJob)
So, at 'scrapeWeb' method, at second '.then', I am calling 'scrapePage' method, however, the third promise executed before promise inside 'scrapePage' method.

You need a little more control flow at that stage. You do not want that .then()'s promise to resolve until all the calls are resolved.
You could use a Promise library like bluebird to do a Promise.each or a Promise.map for all the results you want to run.
Or use async/await to set up like .then(async () => {}) and do not use .forEach.
for(let element of this.results){
console.log('link: ', element[2])
await this.scrapePage(element[2])
}

You have a race condition problem.
The first tweak you'll need is having scrapePage returning a Promise.
scrapePage(pageLink) {
let $this = this
let content = ''
const options = {
uri: pageLink,
transform(body) {
return cheerio.load(body)
}
}
return rp(options);
}
In the second than, you need to invoke all child pages scraping eg :
.then(() => {
return Promise.all(this.results.map(childPage => this.scrapePage(childPage)));
})
This will wrap all scrapes of child pages into promises and only if all of them are resolved the code will flow.

Fetch API Download Progress Indicator?

I am trying to capture the download progress of a Fetch request and use that to change the width of a progress bar. I looked at ProgressEvent.lengthComputable as a potential solution but unsure if this can be used with the Fetch API.

without checking for errors (as in try/catch etc...)
const elStatus = document.getElementById('status');
function status(text) {
elStatus.innerHTML = text;
}
const elProgress = document.getElementById('progress');
function progress({loaded, total}) {
elProgress.innerHTML = Math.round(loaded/total*100)+'%';
}
async function main() {
status('downloading with fetch()...');
const response = await fetch('https://fetch-progress.anthum.com/30kbps/images/sunrise-baseline.jpg');
const contentLength = response.headers.get('content-length');
const total = parseInt(contentLength, 10);
let loaded = 0;
const res = new Response(new ReadableStream({
async start(controller) {
const reader = response.body.getReader();
for (;;) {
const {done, value} = await reader.read();
if (done) break;
loaded += value.byteLength;
progress({loaded, total})
controller.enqueue(value);
}
controller.close();
},
}));
const blob = await res.blob();
status('download completed')
document.getElementById('img').src = URL.createObjectURL(blob);
}
main();
<div id="status"> </div>
<h1 id="progress"> </h1>
<img id="img" />
adapted from here

Using this utility:
async function* streamAsyncIterable(stream) {
const reader = stream.getReader()
try {
while (true) {
const { done, value } = await reader.read()
if (done) return
yield value
}
} finally {
reader.releaseLock()
}
}
See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators
Then you can use for await...of loop:
const response = await fetch(url)
let responseSize = 0
for await (const chunk of streamAsyncIterable(response.body)) {
responseSize += chunk.length
}
But be aware that responseSize is response-size! Not necessarily download-size! What is the difference? There is no difference if there is no content-encoding (gzip, br, ...). But if a comperession was applied, final download-size will be the size of compressed data (the same content-length), and final response-size will be the size of uncompressed data.
See #ecthiender comment and this thread.

you can use axios instead
import axios from 'axios'
export async function uploadFile(file, cb) {
const url = `//127.0.0.1:4000/profile`
try {
let formData = new FormData()
formData.append("avatar", file)
const data = await axios.post(url, formData, {
onUploadProgress: (progressEvent) => {
console.log(progressEvent)
if (progressEvent.lengthComputable) {
let percentComplete = progressEvent.loaded / progressEvent.total;
if (cb) {
cb(percentComplete)
}
}
}
})
return data
} catch (error) {
console.error(error)
}
}