Javascript fetch multiple pages - javascript

there is url: "http:/myurl/mydata/pages/"
the task is call url+"1",url+"2",...,until there is no correspondent page, number of pages is unknown
I am not sure about design of the decision. I know I can you "async await" approach, can I avoid it? Is the next approach ok?
let url= "http:/myurl/mydata/pages/";
let finished=true;
let page=1;
let datas={};
while(!finished) {
fetch(url+page)
.then(function(data) {
collectData(page,data);
page+=1;
})
.catch(function(error) {
finished=true;
});
}
function collectData(page,data){
datas[page]=data;
}
Can I use Promises for this task?

If you want a async-await approach:
async function fetchPages(url, page) {
try{
const data = await fetch(url+page);
collectData(page+url, data);
await fetchPages(url, page + 1);
} catch(e) {
return 'Its over';
}
}
};

This looks like a job for recursion!
let url= "http:/myurl/mydata/pages/";
let page=1;
let datas={};
recurse().then(/*...*/)
function recurse() {
return new Promise(function(resolve, reject) {
fetch(url+page)
.then(function(data) {
collectData(page,data);
page+=1;
return recurse()
})
.catch(function(error) {
resolve()
});
})
}
function collectData(page,data){
datas[page]=data;
}
NOTE: I didn't test this, so might not work exactly as written. But it should set you on the right track.

Related

Async button in electron call an async function

I have an async button on my html site. When I click the button, it should call a function, which waits for a string from my Arduino and then returns the string to the other function. If the data is saved, it should display the string on the console.
Here is the code:
async function DataReceived()
{
await Port.parser.on('data',function(data){
return data;
});
}
table.send.onclick = async () =>{
let data = await DataReceived().then(()=> {
if (data != "")
{
console.log(data);
}
else
{
console.log("too late");
}
})
Normally the function should wait for DataReceived(), but it jumps directly in the if condition and sends me back too late.
What did I do wrong? I hope you can help me.
The DataReceived() function just registers the on-event, but doesn't wait for the data to occur.
You may consider moving this registration straight into the onclick function and add your logic to the function body of the Port.parser callback
Thanks to Felix King. You got me on the right way. The Port.parser.on doesn't return a promise. I had to enclose it in new Promise for it work.
a post with the answer already exist here:
async/await using serialport in node.js
My code looks now like this now:
function DataReceived()
{
return new Promise(function(resolve, reject) {
Port.parser.on('data', (data) => {
console.log (data)
resolve(data)
})
});
}
table.send.onclick = async () =>{
let data;
data = await DataReceived();
if (data != "")
{
console.log(data);
}
else
{
console.log("too late");
}
}

Promise on change event

I'm trying to understand how Promise API works.Currently, my idea is when a switch button is pressed/clicked to log some message or do some AJAX, depends on the situation.Which is a perfect example for chaining.Unfortunately, then does not work as expected and it logs the message on page load not on change event
const userID = $(".options").data('id');
let isEnabled = $("#statusToggler").data('status');
function showStatus() {
if(isEnabled) {
$('#statusToggler').bootstrapToggle('on', true);
} else {
$('#statusToggler').bootstrapToggle('off', true);
}
}
async function toggleStatus() {
return new Promise(function (resolve, reject) {
resolve(function () {
$("#statusToggler").on('change', function () {
let status = $("#statusToggler").prop('checked');
if (status === true) {
$("#statusToggler").data('status', 1);
$('#statusToggler').bootstrapToggle('on', true);
} else {
$("#statusToggler").data('status', 0);
$('#statusToggler').bootstrapToggle('off', true);
}
})
});
});
}
async function ajaxModifyStatus() {
await toggleStatus()
.then(function () {
console.log("Do something when toggle is pressed/clicked");
});
}
showStatus();
ajaxModifyStatus();
On first scan your promise syntax looks fine. The functions are both being called at the bottom of the script which is why they're running on page load.
If you want to have them run on an event you want to give the event the function like it's an argument like ajaxModifyStatus rather than ajaxModifyStatus().
Sorry I haven't used jQuery in ages so cant remember how to write it. Here's how you might do with normal js...
let element = document.getElementById('myid')
element.addEventListener('change', ajaxModifyStatus)
// Or...
element.addEventListener('change', async() => {
let result
try {
result = await ajaxModifyStatus()
} catch (err) {
console.log(err)
}
// Do something with result...
})
As the previous comments mentioned you don't necessarily need promises for this to work unless you actually want the result of the Ajax function.
Note I've actually written this as async await syntax because it's easier on my phone. Just replace that code with the promise if you want to do it that way.

Retry nodejs http.request (post,put,delete)

What is the correct way to implement a retry on error/condition without using any third party modules in nodejs, please?
I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
Do I need to destroy/end the sockets?
I've tried looking for examples but have only found reference to third party modules and http.get samples which don't seem to work. How does one test this?
I have attempted the below without success:
async pingApi(cb) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, /ect do I reference this path?
}
};
let request = await http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', function () {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.result != 'OK') {
setTimeout(pingApi, 1000); //cant pass callback
} else {
cb(null, json.result) //works
}
}
});
})
request.end(); //does the socket need to be closed if retry is this function?
}
Any help, pointing in the right direction or criticism will be greatly appreciated as I think this is a very important learning curve for me.
Thank you in advance,
I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
I don't know for sure that everything else in your function is correct, but you can fix the recursion that you're asking about by changing this:
setTimeout(pingApi, 1000); //cant pass callback
to this:
setTimeout(() => {
this.pingApi(cb);
}, 1000);
You aren't showing the whole context here, but if pingApi() is a method, then you also need to keep track of the this value to you can call this.pingApi(db). You can preserve the value of this by using arrow function callbacks like this:
response.on('end', () => { ... });
Other things I notice that look off here:
There's no reason to use await http.request(). http.request() does not return a promise so using await with it does not do anything useful.
Without the await, there's then no reason for your function to be declared async since nobody is using a returned promise from it.
It's not clear what if (this.complete) is meant to do. Since this is inside a regular function callback, the value of this won't be your pingApi object. You should either save this higher in the scope typically with const self = this or all callbacks internally need to be arrow functions so the value of this is preserved.
You should probably put try/catch around JSON.parse() because it can throw if the input is not perfect JSON.
You should probably not retry forever. Servers really hate clients that retry forever because if something goes wrong, the client may just be bashing the server every second indefinitely. I'd suggest a certain number of max retries and then give up with an error.
Do I need to destroy/end the sockets?
No, that will happen automatically after the request ends.
How does one test this?
You have to create a test route in your server that returns the error condition for the first few requests and then returns a successful response and see if your code works with that.
Here's an attempt at a code fixup (untested):
const maxRetries = 10;
pingApi(cb, cnt = 0) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, // ect do I reference this path?
};
let request = http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', () => {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
try {
let json = JSON.parse(decoded);
if (json.result != 'OK') {
if (cnt < maxRetries)
setTimeout(() => {
this.pingApi(cb, ++cnt);
}, 1000);
} else {
cb(new Error("Exceeded maxRetries with error on pingApi()"));
}
} else {
cb(null, json.result) //works
}
} catch(e) {
// illegal JSON encountered
cb(e);
}
}
});
})
request.end();
}
Remaining open questions about this code:
What is this.complete doing and what this should it be referencing?
Why is there no request.write() to send the body of the POST request?
I know you ask for no external modules, but my preferred way of doing this would be to use promises and to use the request-promise wrapper around http.request() because it handles a lot of this code for you (checks response.status for you, parses JSON for you, uses promise interface, etc...). You can see how much cleaner the code is:
const rp = require('request-promise');
const maxRetries = 5;
pingApi(cnt = 0) {
let options = {
method: "post",
url: `http://somedomain.com/API/pingAPI?${this.auth}`,
json: true
};
return rp(options).then(result => {
if (result.result === "OK") {
return result;
} else {
throw "try again"; // advance to .catch handler
}
}).catch(err => {
if (cnt < maxRetries) {
return pingApi(++cnt);
} else {
throw new Error("pingApi failed after maxRetries")
}
});
}
And, then sample usage:
pingApi().then(result => {
console.log(result);
}).catch(err => {
console.log(err);
})
your use of async/await with core node server intrigued me and I've tried to use much as possible of this new async features.
This is what I end up with: https://runkit.com/marzelin/pified-ping
const pify = require("util").promisify;
const http = require("http");
const hostname = "jsonplaceholder.typicode.com";
const failEndpoint = "/todos/2";
const goodEndpoint = "/todos/4";
let options = {
method: "get",
path: `${failEndpoint}`,
hostname
};
async function ping(tries = 0) {
return new Promise((res) => {
const req = http.request(options, async (response) => {
let body = new Buffer(0);
response.on("data", (chunk) => {
body = Buffer.concat([body, chunk]);
})
const on = pify(response.on.bind(response));
await on("end");
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.completed) {
return res("all good");
}
if (tries < 3) {
console.log(`retrying ${tries + 1} time`);
return res(ping(tries + 1));
}
return res("failed");
})
req.on('error', (e) => {
console.error(`problem with request: ${e.message}`);
});
// write data to request body
req.end();
})
}
const status = await ping();
"status: " + status

Delay the call in catch of promise

I have a method in my Db wrapper form nodejs mysql as follow:
let mysql=require('mysql')
query(sql,args,count=0){
let con=mysql.createConnection(configs[this.config])
return new Promise((resolve,reject)=>{
con.query(sql,args,(e,r)=>{
if(e){
if(e.code=='ER_CON_COUNT_ERROR'){
reject(e)
}
r=false
}
con.end(e=>{
return resolve(r)
})
})
}).catch(e=>{
//want to delay this call
if(count<5){
return this.query(sql, args,count+1)
}
return false
})
}
In case of failure I am recursively calling query but it happens instantaneously. I want to delay this call depending upon count which decide the number of retry.
This is how I am using it:
let test=async()=>{
let promises=[]
for(let i=1;i<100;i++){
promises.push(db.query('SELECT * FROM verbs'))
}
let r=await Promise.all(promises)
console.log(r[r.length-1],r.length)
}
test();
What is some good way to do that? I tried using setTimeout but that messed up promise chain and I wasn't able to exit in case of success.
Try this inside catch,
let mysql=require('mysql')
query(sql,args,count=0){
let con=mysql.createConnection(configs[this.config])
return new Promise((resolve,reject)=>{
con.query(sql,args,(e,r)=>{
if(e){
if(e.code=='ER_CON_COUNT_ERROR'){
reject(e)
}
r=false
}
con.end(e=>{
return resolve(r)
})
})
}).catch(e=>{
//want to delay this call
return new Promise((resolve,reject)=>{
setTimeout(()=>{
this.query(sql, args,count+1).then(resolve).catch(reject);
},2000);
});
})
}
Not sure why do you need infinite recursive call during exception. You have to change your design to limit retry.
Anyhow for the above given scenario, Below code can help
let mysql=require('mysql')
query(sql,args,count=0){
return new Promise((resolve,reject)=>{
let query_data = () => {
let con=mysql.createConnection(configs[this.config])
con.query(sql,args,(e,r)=>{
if(e){
if(e.code=='ER_CON_COUNT_ERROR'){
reject(e)
}
r=false
}
con.end(e=>{
return resolve(r)
})
})
}
try {
query_data();
} catch(e) {
// statements
console.log(e);
setTimeout(query_data, 3000);
}
})
}

JavaScript: Asynchronous method in while loop

I'm tackling a project that requires me to use JavaScript with an API method call. I'm a Java programmer who has never done web development before so I'm having some trouble with it.
This API method is asynchronous and it's in a while loop. If it returns an empty array, the while loop finishes. Otherwise, it loops. Code:
var done = true;
do
{
async_api_call(
"method.name",
{
// Do stuff.
},
function(result)
{
if(result.error())
{
console.error(result.error());
}
else
{
// Sets the boolean to true if the returned array is empty, or false otherwise.
done = (result.data().length === 0) ? true : false;
}
}
);
} while (!done);
This doesn't work. The loop ends before the value of "done" is updated. I've done some reading up on the subject and it appears I need to use promises or callbacks because the API call is asynchronous, but I can't understand how to apply them to the code I have above.
Help would be appreciated!
edit: see the bottom, there is the real answer.
I encourage you yo use the Promise API. Your problem can be solved using a Promise.all call:
let promises = [];
while(something){
promises.push(new Promise((r, j) => {
YourAsyncCall(() => r());
});
}
//Then this returns a promise that will resolve when ALL are so.
Promise.all(promises).then(() => {
//All operations done
});
The syntax is in es6, here is the es5 equivalent (Promise API may be included externally):
var promises = [];
while(something){
promises.push(new Promise(function(r, j){
YourAsyncCall(function(){ r(); });
});
}
//Then this returns a promise that will resolve when ALL are so.
Promise.all(promises).then(function(){
//All operations done
});
You can also make your api call return the promise and push it directly to the promise array.
If you don't want to edit the api_call_method you can always wrap your code in a new promise and call the method resolve when it finishes.
edit: I have seen now the point of your code, sorry. I've just realized that Promise.all will not solve the problem.
You shall put what you posted (excluding the while loop and the control value) inside a function, and depending on the condition calling it again.
Then, all can be wraped inside a promise in order to make the external code aware of this asynchronous execution. I'll post some sample code later with my PC.
So the good answer
You can use a promise to control the flow of your application and use recursion instead of the while loop:
function asyncOp(resolve, reject) {
//If you're using NodeJS you can use Es6 syntax:
async_api_call("method.name", {}, (result) => {
if(result.error()) {
console.error(result.error());
reject(result.error()); //You can reject the promise, this is optional.
} else {
//If your operation succeeds, resolve the promise and don't call again.
if (result.data().length === 0) {
asyncOp(resolve); //Try again
} else {
resolve(result); //Resolve the promise, pass the result.
}
}
});
}
new Promise((r, j) => {
asyncOp(r, j);
}).then((result) => {
//This will call if your algorithm succeeds!
});
/*
* Please note that "(...) => {}" equivals to "function(...){}"
*/
sigmasoldier's solution is correct, just wanted to share the ES6 version with async / await:
const asyncFunction = (t) => new Promise(resolve => setTimeout(resolve, t));
const getData = async (resolve, reject, count) => {
console.log('waiting');
await asyncFunction(3000);
console.log('finshed waiting');
count++;
if (count < 2) {
getData(resolve, reject, count);
} else {
return resolve();
}
}
const runScript = async () => {
await new Promise((r, j) => getData(r, j, 0));
console.log('finished');
};
runScript();
If you don't want to use recursion you can change your while loop into a for of loop and use a generator function for maintaining done state. Here's a simple example where the for of loop will wait for the async function until we've had 5 iterations and then done is flipped to true. You should be able to update this concept to set your done variable to true when your webservice calls have buffered all of your data rows.
let done = false;
let count = 0;
const whileGenerator = function* () {
while (!done) {
yield count;
}
};
const asyncFunction = async function(){
await new Promise(resolve => { setTimeout(resolve); });
};
const main = new Promise(async (resolve)=>{
for (let i of whileGenerator()){
console.log(i);
await asyncFunction();
count++;
if (count === 5){
done = true;
}
}
resolve();
});
main.then(()=>{
console.log('all done!');
});
Also you may try recursion solution.
function asyncCall(cb) {
// Some async operation
}
function responseHandler(result) {
if (result.error()) {
console.error(result.error());
} else if(result.data() && result.data().length) {
asyncCall(responseHandler);
}
}
asyncCall(responseHandler);
Here is a solution I came up with. Place this in an async function.
let finished = false;
const loop = async () => {
return new Promise(async (resolve, reject) => {
const inner = async () => {
if (!finished) {
//insert loop code here
if (xxx is done) { //insert this in your loop code after task is complete
finshed = true;
resolve();
} else {
return inner();
}
}
}
await inner();
})
}
await loop();
If you don't want to use Promises you can restructure your code like so:
var tasks = [];
var index = 0;
function processNextTask()
{
if(++index == tasks.length)
{
// no more tasks
return;
}
async_api_call(
"method.name",
{
// Do stuff.
},
function(result)
{
if(result.error())
{
console.error(result.error());
}
else
{
// process data
setTimeout(processNextTask);
}
}
);
}
Your loop won't work, because it is sync, your async task is async, so the loop will finish before the async task can even respond. I'd reccomend you to use Promises to manage async tasks:
//first wrapping your API into a promise
var async_api_call_promise = function(methodName, someObject){
return new Promise((resolve, reject) => {
async_api_call(methodName, someObject, function(result){
if(result.error()){
reject( result.error() )
}else{
resolve( result.data() )
}
});
})
}
now to your polling code:
//a local utility because I don't want to repeat myself
var poll = () => async_api_call_promise("method.name", {/*Do stuff.*/});
//your pulling operation
poll().then(
data => data.length === 0 || poll(), //true || tryAgain
err => {
console.error(err);
return poll();
}
).then((done) => {
//done === true
//here you put the code that has to wait for your "loop" to finish
});
Why Promises? Because they do state-management of async operations. Why implement that yourself?
let taskPool = new Promise(function(resolve, reject) {
resolve("Success!");
});
let that = this;
while (index < this.totalPieces) {
end = start + thisPartSize;
if (end > filesize) {
end = filesize;
thisPartSize = filesize - start;
}
taskPool.then(() => {
that.worker(start, end, index, thisPartSize);
});
index++;
start = end;
}

Categories

Resources