Develop Reference

Node JS function that return https get request final edited data

Hello everybody I have a problem with the Node JS function that I want it to return https get request final edited data, I know there are a lot of solutions for this async problem but I tried them all and still can't figure out what is wrong with my code?
here is my function without any other solutions editing:
function getMovie(apiKey, gen) {
const baseUrl = "https://api.themoviedb.org/3/discover/movie?api_key=" + apiKey + "&language=en-US&include_adult=false&include_video=false&page=1&with_genres=" + gen;
https.get(baseUrl, function (responce) {
console.log(responce.statusCode);
var d = "";
responce.on("data", function (data) {
d += data;
});
responce.on("end", () => {
const finalData = [];
const moviesData = JSON.parse(d);
const result = moviesData.results;
const maxx = result.length;
const rand = Math.floor(Math.random() * maxx);
const title = result[rand].title;
const rDate = result[rand].release_date;
const overview = result[rand].overview;
const imageRoot = result[rand].poster_path;
const movieId = result[rand].id;
const movieRating = result[rand].vote_average;
// here will push those variables to finalData array
// then return it
return finalData;
});
}).on('error', (e) => {
console.error(e);
});
}
and want after this finalData returns:
const finalResult = getMovie(apiKey, genre);
it always returns undefined, How can I fix this? please anyone ca help me with this problem
thanks in advance.

I solved this problem using promises using this code:
const rp = require('request-promise');
function getMovie(url) {
// returns a promise
return rp(url).then(body => {
// make the count be the resolved value of the promise
let responseJSON = JSON.parse(body);
return responseJSON.results.count;
});
}
getMovie(someURL).then(result => {
// use the result in here
console.log(`Got result = ${result}`);
}).catch(err => {
console.log('Got error from getMovie ', err);
});

How to force post-processing to wait until all fetch processes finish in node.js?

I am collecting multiple data using fetch and then use them in multiple calculations.
var data_dict = []; // objects with ids
var fetch_data = function() {
let data_history = [];
data_dict.forEach(function (item, index) {
let id = item.id;
fetch(// with id)
.then(res => res.json())
.then(data=>{// fix data})
.then(fixed_data=> data_history.push(fixed_data))
.catch(err => console.log(err))
});
return data_history;
}
var use_fetched_data = function () {
let results = [];
// use data_history in calculating results
return results;
}
var use_fetched_data2 = function () {
let results = [];
// use data_history and obtained results in calculating another results
return results;
}
// get data_history2
var data_history2 = fetch_data();
// use data_history2 to find results1
var results1 = use_fetched_data ();
// use data_history2 and results1 to find results2
var results2 = use_fetched_data2 ();
I tried to use async & await to force waiting until all data are fetched, but results still get calculated before fetch is completed.
var force_wait = async function() {
// get data_history2
var data_history2 = await fetch_data();
// use data_history2 to find results1
var results1 = use_fetched_data ();
// use data_history2 and results1 to find results2
var results2 = use_fetched_data2 ();
}
force_wait();
How to do this properly?
thanks

The problem is that your fetch_data function doesn't wait for the requests:
const fetch_data = async function () {
const data_history = [];
requests = data_dict.map(({ id }, index) => {
return fetch('https://randomuser.me/api/')
.then(res => res.json())
.then(data => data.results[0])
.then(fixed_data => data_history.push(fixed_data))
});
try {
await Promise.all(requests)
} catch (err) {
console.error(err)
}
return data_history
}
After this change, your force_wait should work as expected.

using promises and async for protobufjs loadcall

I am trying to figure out the best way to re-write the following code:
var api = function(id, contract) {
var callback = function (error, root) {
if (error)
throw error;
var by = Buffer.from(contract,'base64')
var es = root.lookupType("Contract")
var esMsg = es.decode(by)
var esBytes = es.encode(esMsg).finish()
signature = id.sign(esBytes).toString('base64')
}
return new Promise((resolve, reject) => {
protobuf.load("contract.proto", callback)
})
}
var signContract = async(privateKey, contract) => {
let signature
var id = await crypto.keys.unmarshalPrivateKey(Buffer.from(privateKey, 'base64'))
result = await api(id,contract,signature)
}
function getSessionSignature (hash, time) {
return config.id + ":" + hash + ":" + time
}
module.exports = configure(({ ky }) => {
return async function * signbatch (input, options) {
var contracts = input.Contracts
for (var i = 0 ; i < contracts.length ; i++) {
contracts[i].contract = await signContract(config.PrivKey, contracts[i].contract)
}
//add signed contracts to the searchParams
searchParams.append("arg", JSON.stringify(contracts))
let res
res = await ky.post('storage/upload/signbatch', {
searchParams
}).json()
yield JSON.stringify({})
} else {
yield JSON.stringify({error:"Private key not found"})
}
}
})
My issue is how do I write the sign async code to pass in privateKey and contract variables to api var function and return the signature back to the result variable to be assigned to contracts[i].contract ? Please note that the id.sign(..) function is Promise inside the callback function.

You need to resolve the promise in the api function, the docs suggest you could use the single argument variant here, e.g.
var root = await protobuf.load("contract.proto");
... // (The code you currently by have in 'callback'
return signature;
As the generator is async, yield will emit a Promise which you can (obviously) handle with either .then or await

Pagination in Zapier

I am trying following code to get all records from a paginated API in Zapier.
const limitPerPage = 20;
const apiUrl = "https://myurl.com/data";
var lastCursor = null;
var output = null;
const getContent = async function (cursor) {
let actualUrl = apiUrl + `?cursor=${cursor}&limit=${limitPerPage}`;
var apiResults = await fetch(actualUrl)
.then(resp => {
return resp.json;
});
}
const getEntireContentList = async function (cursor) {
const results = await getContent(cursor);
console.log("Retreiving data from API for cursor : " + cursor);
if (results.metadata.cursor !== "") {
return results.concat(await getEntireContentList(results.metadata.cursor));
} else {
return results;
}
};
(async() => {
const entireList = await getEntireContentList();
console.log(entireList);
output = entireList;
callback(null, entireList);
})();
I get error as
You did not define output! Try output = {id: 1, hello: await Promise.resolve("world")};
How can I fix this?

Your problem is that though you're awaiting in that function, the top-level carries on and execution ends before your code has had a chance to run.
The good news is, Zapier wraps your code in an async function already, so you can use await at the top level (per these docs).
Try this instead:
const limitPerPage = 20;
const apiUrl = "https://myurl.com/data";
let lastCursor = null;
// var output = null; // zapier does this for you already
const getContent = async function (cursor) {
const actualUrl = apiUrl + `?cursor=${cursor}&limit=${limitPerPage}`;
const rawResponse = await fetch(actualUrl)
return resp.json() // async function, you had it as a property
}
const getEntireContentList = async function (cursor) {
const results = await getContent(cursor);
console.log("Retreiving data from API for cursor : " + cursor);
if (results.metadata.cursor !== "") {
return results.concat(await getEntireUserList(results.metadata.cursor)); // should this be named getEntireContentList?
} else {
return results;
}
};
return {
results: await getEntireContentList()
}
I noticed this is a recursive approach. That's fine, but remember that you've got limited execution time. You also might hit memory limits (depending on how many objects you're returning), so keep an eye on that.

Updating outside value from inside a promise

I have a getValidUrls function that takes a maxId param.
Within this function it loops backwards and sends a request for the url.
Each loop decrements the maxId.
My issue is that I am trying to add these valid urls to an array, but I cannot update the array from within the .then of the promise. I have added a simple total variable to see if I could increment it and could not.
getValidUrls = (maxId) => {
return new Promise((resolve, reject) => {
let validUrls = [];
let idCounter = maxId;
let total = 0; // test to see if updated from inside (it doesn't)
// while(validUrls.length < 10 && idCounter > 0) {
for(let i = maxId; i > 0; i--){
let newsUrl = `https://hacker-news.firebaseio.com/v0/item/${i}.json?print=pretty`;
//console.log(newsUrl); // this shows all the urls & works
getJSONObject(newsUrl)
.then((story) => {
total++;
console.log(total); // this never gets shown
return getUserObject(story.by);
}).then((user) => {
if(user.karma > 5) {
validUrls.push(story);
if(validUrls.length >= 10) {
resolve(validUrls);
}
}
});
}
});
};
The following returns a json object for the url
getJSONObject = (url) => {
return new Promise((resolve, reject) => {
console.log(url); // this works and shows all urls
https.get(url, (response) => {
response.on('data', (data) => {
console.log(JSON.parse(data)); // This never gets shown
resolve(JSON.parse(data));
}, (err) => reject(err));
}, (err) => reject(err));
});
};