Fetching in order rather than all at once - javascript

I just updated the code I had with yours. I am getting an error for headers not being defined which is odd.
No need to refactor the initial twitter pulling trends code, I am just trying to get it to work. I know the getTrends() function is pulling and topNews is only grabbing the first 5.
Any idea what I am doing wrong?
let topics = [];
let topNews = [];
function getTrends() {
var myHeaders = new Headers();
myHeaders.append(
"Authorization",
"Bearer ************"
);
myHeaders.append(
"Cookie",
'personalization_id="v1_QSZs3kHuqI6knlNtIbIchQ=="; guest_id=v1%3A159630901122767291'
);
var requestOptions = {
method: "GET",
headers: myHeaders,
redirect: "follow",
};
const url =
"https://cors-anywhere-gp.herokuapp.com/https://api.twitter.com/1.1/trends/place.json?id=23424977";
fetch(url, requestOptions)
.then((response) => response.json())
.then((responseJson) => topFive(responseJson))
.catch((error) => console.log("error", error));
}
function topFive(responseJson) {
for (let i = 0; i < 5; i++) {
topics.push(responseJson[0].trends[i].name);
getNews(responseJson[0].trends[i].name.replace("#", ""), i);
}
}
function getTopicURL(topic) {
return `https://api.cognitive.microsoft.com/bing/v7.0/news/search?q=${topic}&count=5`;
}
function getHeaders() {
var headers = new Headers();
headers.append('Ocp-Apim-Subscription-Key', '******');
return headers;
}
function getOptions() {
return {
headers: getHeaders(),
method: 'GET',
redirect: 'follow'
};
}
function fetchAsJSON(url, options) {
return fetch(url, options).then(response => response.json());
}
function toThunk(fn, ...args) {
return () => fn(...args);
}
function delay(ms, fn) {
return new Promise((resolve, reject) => {
setTimeout(function () {
fn().then(resolve).catch(reject);
}, ms);
});
}
function getNews(topic, index) {
return delay(
index * 1000,
toThunk(
fetchAsJSON,
getTopicURL(topic),
getOptions()
)
);
}
Promise.
all(topics.map(getNews)).
then(topicsArray => {
topicsArray.forEach((topic, index) => {
topNews[index] = topic.value;
});
}).
catch(exception => {
console.log('error:', exception);
});
getTrends();
getNews();
console.log(topNews);

You could use Promises and setTimeout. Below is some example code that you can try. I've added several comments as explanation.
I refactored your code from inline to separate utility functions to make it a bit more modular.
// --- FETCH UTILITY FUNCTIONS ---
function getTopicURL(topic) {
return `https://api.cognitive.microsoft.com/bing/v7.0/news/search?q=${topic}&count=5`;
}
function getHeaders() {
var headers = new Headers();
headers.append('Ocp-Apim-Subscription-Key', '******');
return headers;
}
function getOptions() {
return {
headers: getHeaders(),
method: 'GET',
redirect: 'follow'
};
}
function fetchAsJSON(url, options) {
return fetch(url, options).then(response => response.json());
}
// --- "LAZY" UTILITIES ---
function toThunk(fn, ...args) {
// creates a `thunk`
return () => fn(...args);
}
function delay(ms, fn) {
// returns a promise that wraps a call to setTimeout. note that
// the `fn` parameter has to be a thunk!
return new Promise((resolve, reject) => {
setTimeout(function () {
fn().then(resolve).catch(reject);
}, ms);
});
}
// --- PROG ---
function getNews(topic, index) {
// grabs 5 news for 1 topic, defers fetching by 1 second per item
// (first item = 0 ms, second item = 1000 ms, third item = 2000 ms, etc.)
return delay(
index * 1000,
toThunk(
fetchAsJSON,
getTopicURL(topic),
getOptions()
)
);
}
let topics = ['topic-a', 'topic-b', 'topic-c', 'topic-d', 'topic-e'];
Promise.
all(topics.map(getNews)).
then(topicsArray => {
// when all topics return, iterate over them and store all news per topic
topicsArray.forEach((topic, index) => {
topNews[index] = topic.value;
});
checkNewsDone(); // <-- not sure if needed or not since I don't know what it does
}).
catch(exception => {
console.log('error:', exception);
});

Related

Share variable between class methods

I'm not a master in JavaScript. I'm starting use the OOP syntax and I have a small class like this one:
class Scraper {
constructor() {}
question(inputText) {
rl.setPrompt(inputText);
rl.prompt();
return new Promise( (resolve, reject) => {
let answer;
rl.on('line', (input) => {
answer = input;
rl.close();
});
rl.on('close', () => {
resolve(answer);
});
})
}
startFetch(username) {
this.username = String(username);
return axios({
url: `https://www.instagram.com/${this.usernamee}/?__a=1`
}).then( (response) => {
//response.data.graphql.user);
this.user_id = response.data.graphql.user.id;
//totalMedia = response.data.graphql.user.edge_owner_to_timeline_media.count
this.has_next_page = response.data.graphql.user.edge_owner_to_timeline_media.page_info.has_next_page;
response.data.graphql.user.edge_owner_to_timeline_media.edges.map( (item, index) => {
this.processLink(item.node.display_url, index);
});
if( this.has_next_page ){
this.currCursor = response.data.graphql.user.edge_owner_to_timeline_media.page_info.end_cursor;
this.fetchNextPage(this.user_id);
} else {
console.log('Completed');
}
//let nodes = response.data.graphql.user.edge_owner_to_timeline_media.edges.length;
});
}
fetchNextPage(id) {
return axios({
method: 'GET',
baseURL: 'https://www.instagram.com/graphql/query/',
params: {
query_hash: '42323d64886122307be10013ad2dcc44',
variables: {
id: id,
first: "12",
after: this.currCursor
}
}
}).then( (response) => {
//console.log(response.data.data.user.edge_owner_to_timeline_media.edges[0].node)
//totalMedia = response.data.data.user.edge_owner_to_timeline_media.count
response.data.data.user.edge_owner_to_timeline_media.edges.map( (item, index) => {
index++;
this.processLink(item.node.display_url, index);
});
if( response.data.data.user.edge_owner_to_timeline_media.page_info.has_next_page ){
this.currCursor = response.data.graphql.user.edge_owner_to_timeline_media.page_info.end_cursor;
}
});
}
processLink(imageURI, n) {
let filename = path.format({dir: destinationPath, base: `${n}.jpg`});
let file = fs.createWriteStream(filename);
https.get(imageURI, (res) => {
res.pipe(file);
});
}
}
How I can set a variable inside a method and then share it and overwrite if needed inside another method of the same class? The class I'm creating is responsible to scrape an Instagram public profile. After the first request, as suggested to me here I'm calling the fetchNextPage method. To make it works as expected I need to set the currCursor variable and maintain it updated with the new cursors after each request. will the this.user_id be set and called from the method, and the this.currCursor will be updated with new value?

problem awaiting the callback given to a function

I have a function who send a message to the server to get the answer and if the answer is true I want my application to send an error to the user. The problem is that I can't manage to await the callback in the Fetch function I wrote.
This is the function who send the question to the server.
async donglePaired(){
if (Platform.OS !=='ios'){
var pairedDevices = await BluetoothScanner.getPairedDevices();
console.log('Sending........');
let data={
data:pairedDevices,
};
new Api().fetch("bluetoothCheck",{devices:JSON.stringify(data),userid:this.state.probe.UID},(result) => {
if (!result.err) return false;
console.log("Dongle already paired");
return true;
//logNetworkState
});
}
}
This is the Api.fetch function i wrote
fetch(action,data,cb){
let url=Config.URL+"?version="+Config.VERSION+"&action="+action;
let post="";
let formData=new FormData();
for(let k in data) formData.append(k,data[k]);
for(let k in data) post+="&"+k+"="+encodeURIComponent(data[k]).replace(/%20/g,'+');
console.log(url+post);
console.log(url);
if (data.batch) console.log(data.batch);
let sending=true;
fetch(url,{
method: 'post',
body: formData
})
.then(function(response){
if (true) return response.json();
let txt=response.text();
console.log(txt);
return JSON.parse(txt);
})
.then(
(result)=>{
if (!sending) return;
sending=false;
console.log(JSON.stringify(result));
if (cb) cb(result);
},
(error)=>{
if (!sending) return;
sending=false;
console.log("fetch error");
console.log(error);
if (cb) cb();
}
);
setTimeout(()=>{
console.log("http timeout")
if (!sending) return console.log("nothing to abort");
if (cb) cb();
},Config.HTTP_TIMEOUT*1000)
}
}
And this is my main code where I wait for the first function donglePaired, and if donglePaired return true I send an error to the user.
let donglePaired = await this.props.app.donglePaired();
if (donglePaired) return this.props.app.setError("ERR_DONGLE");
The problem is that the program doesnt wait for donglePaired, despite of the await
your code here is inappropriate
let donglePaired = await this.props.app.donglePaired();
if (donglePaired) return this.props.app.setError("ERR_DONGLE");
Async function cannot return value normally unless it is a Promise
See my simple demo below!
async function test() {
const result = await asyncRequest()
return result
}
function asyncRequest() {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve('success')
}, 2000)
})
}
test().then((data) => {
console.log(data)
})
The snippets should give you an idea how to await the callback
Sending to the API
async function remove_configuration(filename) {
const data = { filename };
const options = {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(data)
};
await fetch('/delete', options);
}
Just Retrieving Data
async function display() {
let response = await fetch('/get-available-configurations')
let data = await response.json(); // JSON or Text what do you prefer
// do something with data
}
You could return an Promise.race() with your timeout function.
fetch(action, data, cb) {
let url = Config.URL + "?version=" + Config.VERSION + "&action=" + action;
let post = "";
let formData = new FormData();
for (let k in data) formData.append(k, data[k]);
for (let k in data)
post += "&" + k + "=" + encodeURIComponent(data[k]).replace(/%20/g, "+");
console.log(url + post);
console.log(url);
if (data.batch) console.log(data.batch);
let sending = true;
return Promise.race([
fetch(url, {
method: "post",
body: formData
})
.then(res => res.json())
.then(result => {
if (!sending) return;
sending = false;
return result;
}),
sleep(Config.HTTP_TIMEOUT * 1000)
]);
}
const sleep = ms => new Promise((_, rej) => setTimeout(rej("TIMEOUT"), ms));
It either returns you the value or it rejects with TIMEOUT or it rejects with an error from fetch
And donglePaired looks like this then. I have wrapped it with an try / catch
async donglePaired() {
if (Platform.OS !== "ios") {
var pairedDevices = await BluetoothScanner.getPairedDevices();
console.log("Sending........");
let data = {
data: pairedDevices
};
try {
let result = await new Api().fetch("bluetoothCheck", {
devices: JSON.stringify(data),
userid: this.state.probe.UID
});
if (!result.err) return false;
console.log("Dongle already paired");
return true;
//logNetworkState
} catch (err) {
console.log(err);
}
}
}
One possibility is to drop the async and change it to this:
donglePaired() {
return new Promise( function(resolve, reject) {
if (Platform.OS !=='ios'){
var pairedDevices = await BluetoothScanner.getPairedDevices();
console.log('Sending........');
let data={
data:pairedDevices,
};
new Api().fetch("bluetoothCheck",{devices:JSON.stringify(data),userid:this.state.probe.UID},(result) => {
if (!result.err) reject(false);
console.log("Dongle already paired");
resolve(true);
//logNetworkState
});
}
reject(false);
});
}
And:
this.props.app.donglePaired().then( (response) => {
// do something here, this will only run if the response is true
});

I want to dispatch 4 different value from Get service with different action. want to make one call

I am trying to show 4 different array's data. I am calling get service but calling it 4 times. instead i want to make one call. with same link but want to dispatch 4 different actions for different data. as you can see there are 4 const which i want to dispatch and i have to make 4 calls right now. i am using initialload() as to reach to my view in Redux.
export function getcoCodeFilter() {
return new Promise((resolve, reject) => {
fetch(getServiceContext() + 'getfilteroptions', {
method: 'GET',
credentials: 'same-origin'
})
.then((response) => {
if (response.ok) {
response
.json()
.then((json) => {
const filterDisplay = json.data;
const companyList = filterDisplay.companyCodes;
const formtypeList = filterDisplay.formTypes;
const yearList = filterDisplay.yearList;
const qtrList = filterDisplay.quarterList;
resolve(companyList);
});
}
else {
response
.json()
.then((json) => {
const errors = json;
reject(errors ? errors.exceptionMessages : []);
});
}
});
});
}
// get filter formtypes
export function getFormTypesFilter() {
return new Promise((resolve, reject) => {
fetch(getServiceContext() + 'getfilteroptions', {
method: 'GET',
credentials: 'same-origin'
})
.then((response) => {
if (response.ok) {
response
.json()
.then((json) => {
const coTypesList = json.data;
resolve(coTypesList.formTypes);
});
}
else {
response
.json()
.then((json) => {
const errors = json;
reject(errors ? errors.exceptionMessages : []);
});
}
});
});
}
// get year for Filters
export function getYearFilter() {
return new Promise((resolve, reject) => {
fetch(getServiceContext() + 'getfilteroptions', {
method: 'GET',
credentials: 'same-origin'
})
.then((response) => {
if (response.ok) {
response
.json()
.then((json) => {
const coTypesList = json.data;
resolve(coTypesList.yearList);
});
}
else {
response
.json()
.then((json) => {
const errors = json;
reject(errors ? errors.exceptionMessages : []);
});
}
});
});
}
// get quarters
export function getQTRFilter() {
return new Promise((resolve, reject) => {
fetch(getServiceContext() + 'getfilteroptions', {
method: 'GET',
credentials: 'same-origin'
})
.then((response) => {
if (response.ok) {
response
.json()
.then((json) => {
const coTypesList = json.data;
resolve(coTypesList.quarterList);
});
}
else {
response
.json()
.then((json) => {
const errors = json;
reject(errors ? errors.exceptionMessages : []);
});
}
});
});
}
export const getInitialLoad = (dispatch) => {
return new Promise((resolve) => {
getcoCodeFilter().then((companyList) => {
dispatch({
type: 'COCODE_FILTER_DISPLAY',
value: companyList
});
resolve();
});
getFormTypesFilter().then((formtypeList) => {
dispatch({
type: 'FORMTYPES_FILTER_DISPLAY',
value: formtypeList
});
resolve();
});
getYearFilter().then((yearList) => {
dispatch({
type: 'YEAR_FILTER_DISPLAY',
value: yearList
});
resolve();
});
getQTRFilter().then((qtrList) => {
dispatch({
type: 'QTR_FILTER_DISPLAY',
value: qtrList
});
resolve();
});
});
};
What I often do is store all information in an object and dispatch an action with the object. The action will get picked up by one more many reducers.
something similar to this.
export const getInitialLoad = (dispatch) => {
const ResponseData = {}
return new Promise((resolve) => {
getcoCodeFilter().then((companyList) => {
ResponseData["companyList"] = companyList;
resolve();
});
getFormTypesFilter().then((formtypeList) => {
ResponseData["formtypeList"] = formtypeList;
resolve();
});
getYearFilter().then((yearList) => {
ResponseData["yearList"] = yearList;
resolve();
});
getQTRFilter().then((qtrList) => {
ResponseData["qtrList"] = qtrList;
dispatch({
type: 'INITIAL_LOAD_ACTION',
value: ResponseData
});
resolve();
});
});
};
INITIAL_LOAD_ACTION can be called anything and used in any number of reducers. all you have to do is set the sate using something along the lines of
action.payload.value.ResponseData where ResponseData is one of the 4 keys you set above.
EDIT:
export const getInitialLoad = async (dispatch) => {
const ResponseData = {}
ResponseData["companyList"] = await getcoCodeFilter();
ResponseData["formtypeList"] = await getFormTypesFilter();
ResponseData["yearList"] = await getYearFilter();
ResponseData["qtrList"] = await getQTRFilter();
dispatch({
type: 'INITIAL_LOAD_ACTION',
value: ResponseData
});
};
OR you could do something like
export const getInitialLoad = async (dispatch) => {
const ResponseData = await Promise.all([getcoCodeFilter, getFormTypesFilter, getYearFilter, getQTRFilter])
dispatch({
type: 'INITIAL_LOAD_ACTION',
value: ResponseData
});
};
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all

Dispatch async action

By changing my action to async I am not able to dispatch it. Whenever I make the dispatch it enters the cath with the value false. Here is my page where I dispatch the action from mounted hook (I tried created too)
mounted () {
this.$store.dispatch('productById', this.$route.params['id']).then((response) => {
this.product = response
})
.catch(err => {
console.log(err)
})
}
And this is my action
async productById ({commit}, payload) {
const AuthStr = await getAdminOrRespondentAuth()
return new Promise((resolve, reject) => {
commit(PRODUCT_BY_ID)
axios.get(`${API_BASE}/products/${payload}`, {
params: {
origin: '1'
},
transformRequest: [function (data, headers) {
delete headers.common.Authorization
headers.Authorization = AuthStr
return data
}],
paramsSerializer: params => parseParams(params)
}).then(response => {
if (response.status === 200) {
commit(PRODUCT_BY_ID_SUCCESS, response.data)
resolve(response.data)
} else {
reject(response)
}
})
.catch(err => {
if (err.response.data.idStatus === 1) {
commit(PRODUCT_BY_ID_SUCCESS, err.response.data.data)
reject(err)
}
})
})
}
When the Vue enters mounted hook it "dispatch the action" and goes straight into the catch block not calling my action. My action is not executed.
If I change my action to sync, everything works normally. I need this action to be async because getAdminOrRespondentAuth function thats call oidc async method to recover user.
What I'm doing wrong?
#Samurai8 is right. Thanks for the help. My getAdminOrRespondentAuth was not returning a promise correctly. After correcting the error of the function everything came back to work. This is the function that generate error discribed:
async function getAdminOrRespondentAuth () {
let mgr = new Mgr()
var adminToken = await mgr.getToken()
if (adminToken !== false) {
return 'Bearer '.concat(adminToken)
} else {
let usrToken = localStorage.getItem('user-token')
return 'Bearer '.concat(usrToken)
}
}
Here is the function that works:
async function getAdminOrRespondentAuth () {
var adminToken = ''
return new Promise(async (resolve, reject) => {
let mgr = new Mgr()
try {
adminToken = await mgr.getToken()
} catch (error) {
adminToken = error
}
if (adminToken !== false) {
resolve('Bearer '.concat(adminToken))
} else {
let usrToken = localStorage.getItem('user-token')
if (usrToken !== null) {
resolve('Bearer '.concat(usrToken))
} else {
resolve('')
}
}
})
}

async Iterate over API results

I am calling an API which returns results in pages and I am trying to find an 'elegant' way of retrieving them.
Ideally I want to consume them like this:
let results = api.get();
for await (const page of results) {
// do stuff with page
}
I can active close to this using a generator function like this:
class Results {
constructor(url, token) {
this.url = url;
this.token = token;
}
async *page() {
let url = this.url;
while (true) {
const response = await fetch(url, {
headers: { 'Authorization': 'Bearer ' + this.token }
});
const data = await response.json();
yield data.values;
if (!data.next) return;
url = data.next;
}
}
}
And calling it like:
for await (const page of results.page()) {
// do stuff with page
}
I have tried to do it with a [Symbol.iterator] like this, but cannot get it to work:
[Symbol.iterator]() {
let that = this;
return {
next: async function() {
if (!that.page) {
that.page = that.url;
return {done: true};
}
const response = await fetch(that.page, {
headers: { 'Authorization': 'Bearer ' + that.token }
});
const data = await response.json();
that.page = data.data.next;
return {
value: data,
done: false
}
}
}
}
This issue is I need to get the link to the next page from the current page to determine if there is a next page, but as its a promise i cannot access it in the function.
Any ideas how to get the iterator working?
Following advice here is a working function. [Symbol.asyncIterator] made all the difference. Thanks:
[Symbol.asyncIterator]() {
let that = this;
return {
page: that.url,
token: that.token,
next: async function() {
if (!this.page) {
this.page = that.url;
return {done: true};
}
const response = await fetch(this.page, {
headers: { 'Authorization': 'Bearer ' + this.token }
});
const data = await response.json();
this.page = data.next;
return {
value: data,
done: false
}
}
}
Now that its working ideally I just want to be able to iterate through all results and not know about pages so here is a working solution to this for info:
[Symbol.asyncIterator]() {
let that = this;
return {
page: that.url,
token: that.token,
values: [],
next: async function() {
if (!this.page && this.values.length === 0) {
this.page = that.url;
return {done: true};
}
if (this.values.length > 0) {
return {
value: this.values.pop(),
done: false
}
}
const response = await fetch(this.page, {
headers: { 'Authorization': 'Bearer ' + this.token }
});
const data = await response.json();
this.page = data.next;
this.values = data.values;
if (this.values.length === 0) {
return { done: true }
}
return {
value: this.values.pop(),
done: false
}
}
}
This code can be simplified by using an async generator function like so:
async *[Symbol.asyncIterator]() {
let url = this.url;
const getPage = url =>
fetch(url, this.header)
.then(response => response.json())
.then(data => ({
next: data.next,
values: data.values
}));
while(url) {
const page = await getPage(url);
for (const value of page.values) {
yield value;
}
url = page.next;
}
}
So the full class looks like this:
class Response {
constructor(url, token) {
this.url = url;
this.header = {
headers: {
Authorization: `Bearer ${token}`
}
};
}
async* [Symbol.asyncIterator]() {
let url = this.url;
const getPage = url =>
fetch(url, this.header)
.then(response => response.json())
.then(data => ({
next: data.next,
values: data.values
}));
while (url) {
const page = await getPage(url);
for (const value of page.values) {
yield value;
}
url = page.next;
}
}
}
Which allows you to easily loop through paged API results like this:
for await (const item of response) {
...
}

Categories

Resources