I am using Ionic2/Typescript.
I have 2 Promise's that I want to have complete, before I continue (i.e synchronous). So I put the call to the 2 functions in a Promise.all(...), expecting them to complete before resolve is called.
I have the following code:
public openDatabase(): Promise<Array<Message>> {
let promise: Promise<Array<Message>> = new Promise<Array<Message>>(resolve => {
if (false && this.database && this.database != null) {
Promise.all([this.refreshChats(this.database), this.refreshMessages(this.database)]).then(() => {
console.log('openDatabase1: resolve');
resolve(this.messages);
});
} else {
this.database = new SQLite();
this.database.openDatabase({
name: "data.db",
location: "default"
}).then(() => {
Promise.all([this.refreshChats(this.database), this.refreshMessages(this.database)]).then(() => {
console.log('openDatabase2: resolve');
resolve(this.messages);
});
}, (error) => {
console.log("OPEN ERROR: ", error);
});
}
});
return promise;
}
public refreshChats(db: any): Promise<any> {
console.log('refreshChats ');
return db.executeSql("SELECT * FROM chats", [])
.then((chatData) => {
let promises: Array<any> = [];
this.chats = [];
if (chatData.rows.length > 0) {
for (var i = 0; i < chatData.rows.length; i++) {
promises.push(this.populateChat(db, chatData.rows.item(i)));
}
}
Promise.all(promises).then(() => {
console.log('refreshChats return this.chats.length = ' + this.chats.length);
return this.chats;
});
})
.catch(error => {
console.log("ERROR REFRESHING CHATS: " + JSON.stringify(error));
console.log(error);
});
}
From my console output, you can see that resolve is called before the chats are finished refreshing:
refreshChats
populateChat Object {...}
openDatabase2: resolve
refreshChats return this.chats.length = 1
Any advise on how I should structure my Promises is appreciated.
You need to return new promises from inner promise callbacks:
public openDatabase(): Promise<Array<Message>> {
let promise: Promise<Array<Message>> = new Promise<Array<Message>>(resolve => {
if (false && this.database && this.database != null) {
return Promise.all([this.refreshChats(this.database), this.refreshMessages(this.database)]).then(() => {
console.log('openDatabase1: resolve');
resolve(this.messages);
});
} else {
this.database = new SQLite();
return this.database.openDatabase({
name: "data.db",
location: "default"
}).then(() => {
return Promise.all([this.refreshChats(this.database), this.refreshMessages(this.database)]).then(() => {
console.log('openDatabase2: resolve');
resolve(this.messages);
});
}, (error) => {
console.log("OPEN ERROR: ", error);
});
}
});
return promise;
}
public refreshChats(db: any): Promise<any> {
console.log('refreshChats ');
return db.executeSql("SELECT * FROM chats", [])
.then((chatData) => {
let promises: Array<any> = [];
this.chats = [];
if (chatData.rows.length > 0) {
for (var i = 0; i < chatData.rows.length; i++) {
promises.push(this.populateChat(db, chatData.rows.item(i)));
}
}
return Promise.all(promises).then(() => {
console.log('refreshChats return this.chats.length = ' + this.chats.length);
return this.chats;
});
})
.catch(error => {
console.log("ERROR REFRESHING CHATS: " + JSON.stringify(error));
console.log(error);
});
}
Note, return Promise.all and return this.database.openDatabase.
Related
I am new to Scraping.
This is my PDF downloading code.
I want to use Async Await in this code.
I don't know where I have to use async await in my code.
function scrapPdf(config, search_url, message) {
console.log('PDF downloading');
got(search_url).then(response => {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}).catch(err => {
console.log(err);
});
}
Here's one possibility assuming you only want to use async/await on the Promise returned from the .then:
async function scrapPdf(config, search_url, message) {
let response = null
console.log('PDF downloading');
try {
response = await got(search_url);
} catch (err) {
console.log(err);
}
if (response) {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}
}
I want to get the result of an async function - depending on the result I either want to execute another function or continue the for loop. Here is my Cloud Function:
return query.get().then(geoSnapshot => {
// If GeoquerySnapshot returns 0 docs, createDoc()
let docs = geoSnapshot.docs;
if (docs.length === 0) return createDoc(data);
for(let [index, doc] of docs.entries()){
// If age preferences are valid, joinDoc
let currentDoc = docs[index].data();
if (validAge(currentDoc, data)){
let matchedBefore = await(matchedBefore(currentDoc, data))
if (!matchedBefore){
return joinDoc(docs[index], data);
} else if (index === (docs.length - 1)) {
return createDoc(data);
}
}
return
}
}).catch( error => {
console.log("error query: " + error);
return { error: error };
})
async function matchedBefore(currentDoc, data){
return db.collection('users').doc(data.uid).get().then( doc => {
if ( !doc.exists ) return false;
// If user1 in matchedUsers
let matchedUsers = doc.get('matchedUsers');
if (matchedUsers === undefined) return true
let matchedBefore = matchedUsers.includes(currentDoc.user1);
console.log('matchedBefore: ' + matchedBefore);
if (matchedBefore) {
return false
} else {
return true
}
})
}
I'm getting this following error on let matchedBefore = await(matchedBefore(currentDoc, data)):
Each then() should return a value or throw
How can I ensure the function matchedBefore() finishes before the for loop continues?
I think you are too confused with the implementation of async-await. The correct way would be:
return query.get().then(async geoSnapshot => {
// If GeoquerySnapshot returns 0 docs, createDoc()
let docs = geoSnapshot.docs;
if (docs.length === 0) return createDoc(data);
/* eslint-disable no-await-in-loop */
for(let [index, doc] of docs.entries()) {
// If age preferences are valid, joinDoc
let currentDoc = docs[index].data();
if (validAge(currentDoc, data)){
let matchedBefore = await(matchedBefore(currentDoc, data))
if (!matchedBefore){
return joinDoc(docs[index], data);
} else if (index === (docs.length - 1)) {
return createDoc(data);
}
}
return
}
/* eslint-enable no-await-in-loop */
}).catch( error => {
console.log("error query: " + error);
return { error: error };
})
async function matchedBefore(currentDoc, data){
let doc = await db.collection('users').doc(data.uid).get()
if ( !doc.exists ) return false;
// If user1 in matchedUsers
let matchedUsers = doc.get('matchedUsers');
if (matchedUsers === undefined) return true
let matchedBefore = matchedUsers.includes(currentDoc.user1);
console.log('matchedBefore: ' + matchedBefore);
return !matchedBefore
}
I'm using a snippet example for Amazon Athena just to test inserting some data. I can't tell why it isn't working and CloudWatch logs does not show any output when the statement execution is completed. Even when I change it to a simple select statement I can't see any output. I know the query, database and table is fine, because when I test it using the Athena query editor it executes without a problem.
module.exports.dlr = async event => {
let awsFileCreds = {
accessKeyId: "XXX",
secretAccessKey: "XXX"
};
let creds = new AWS.Credentials(awsFileCreds);
AWS.config.credentials = creds;
let client = new AWS.Athena({ region: "eu-west-1" });
let q = Queue((id, cb) => {
startPolling(id)
.then(data => {
return cb(null, data);
})
.catch(err => {
console.log("Failed to poll query: ", err);
return cb(err);
});
}, 5);
const sql = "INSERT INTO delivery_receipts (status, eventid, mcc, mnc, msgcount, msisdn, received, userreference) VALUES ('TestDLR', 345345, 4353, '5345435', 234, '345754', 234, '8833')"
makeQuery(sql)
.then(data => {
console.log("Row Count: ", data.length);
console.log("DATA: ", data);
})
.catch(e => {
console.log("ERROR: ", e);
});
function makeQuery(sql) {
return new Promise((resolve, reject) => {
let params = {
QueryString: sql,
ResultConfiguration: { OutputLocation: ATHENA_OUTPUT_LOCATION },
QueryExecutionContext: { Database: ATHENA_DB }
};
client.startQueryExecution(params, (err, results) => {
if (err) return reject(err);
q.push(results.QueryExecutionId, (err, qid) => {
if (err) return reject(err);
return buildResults(qid)
.then(data => {
return resolve(data);
})
.catch(err => {
return reject(err);
});
});
});
});
}
function buildResults(query_id, max, page) {
let max_num_results = max ? max : RESULT_SIZE;
let page_token = page ? page : undefined;
return new Promise((resolve, reject) => {
let params = {
QueryExecutionId: query_id,
MaxResults: max_num_results,
NextToken: page_token
};
let dataBlob = [];
go(params);
function go(param) {
getResults(param)
.then(res => {
dataBlob = _.concat(dataBlob, res.list);
if (res.next) {
param.NextToken = res.next;
return go(param);
} else return resolve(dataBlob);
})
.catch(err => {
return reject(err);
});
}
function getResults() {
return new Promise((resolve, reject) => {
client.getQueryResults(params, (err, data) => {
if (err) return reject(err);
var list = [];
let header = buildHeader(
data.ResultSet.ResultSetMetadata.ColumnInfo
);
let top_row = _.map(_.head(data.ResultSet.Rows).Data, n => {
return n.VarCharValue;
});
let resultSet =
_.difference(header, top_row).length > 0
? data.ResultSet.Rows
: _.drop(data.ResultSet.Rows);
resultSet.forEach(item => {
list.push(
_.zipObject(
header,
_.map(item.Data, n => {
return n.VarCharValue;
})
)
);
});
return resolve({
next: "NextToken" in data ? data.NextToken : undefined,
list: list
});
});
});
}
});
}
function startPolling(id) {
return new Promise((resolve, reject) => {
function poll(id) {
client.getQueryExecution({ QueryExecutionId: id }, (err, data) => {
if (err) return reject(err);
if (data.QueryExecution.Status.State === "SUCCEEDED")
return resolve(id);
else if (
["FAILED", "CANCELLED"].includes(data.QueryExecution.Status.State)
)
return reject(
new Error(`Query ${data.QueryExecution.Status.State}`)
);
else {
setTimeout(poll, POLL_INTERVAL, id);
}
});
}
poll(id);
});
}
function buildHeader(columns) {
return _.map(columns, i => {
return i.Name;
});
}
return { message: 'Go Serverless v1.0! Your function executed successfully!', event };
};
Figured it out. Using aws lambda events with athena is easy using the athena-express package. You can specify your configuration and query the athena database like you normally would with significantly less code than what's provided in the amazon athena nodejs example.
This is the code I used to achieve a result:
"use strict";
const AthenaExpress = require("athena-express"),
aws = require("aws-sdk");
const athenaExpressConfig = {
aws,
db: "messaging",
getStats: true
};
const athenaExpress = new AthenaExpress(athenaExpressConfig);
exports.handler = async event => {
const sqlQuery = "SELECT * FROM delivery_receipts LIMIT 3";
try {
let results = await athenaExpress.query(sqlQuery);
return results;
} catch (error) {
return error;
}
};
I am trying to test the following snippet using the Chrome Dev Tool, but I get this error on the last line of the snippet ... cannot find where it's wrong ...
function myPromise1(time1, ok1){
return new Promise((resolve, reject) => {
// do stuff 1 async
setTimeout(() => {
if ( ok1 ) {
resolve('stuff-1 worked')
} else {
reject(Error('stuff-1 failed'))
}
}, time1)
});
function myPromise2(time2, ok2){
return new Promise((resolve, reject) => {
// do stuff 2 async
setTimeout(() => {
if ( ok2 ) {
resolve('stuff-2 worked')
} else {
reject(Error('stuff-2 failed'))
}
}, time2)
});
function myPromise3(time3, ok3){
return new Promise((resolve, reject) => {
// do stuff 3 async
setTimeout(() => {
if ( ok3 ) {
resolve('stuff-3 worked')
} else {
reject(Error('stuff-3 failed'))
}
}, time3)
});
const c1 = '';
const c2 = '';
const c3 = '';
const promise1 = (val11, val12) => myPromise1(val11, val12)
.then(p1Result => {
c1 = p1Result;
return;
}, function(err1) {
console.log('Error: ', err1);
return;
});
const promise2 = (val21, val22) => myPromise2(val21, val22)
.then((p2Result) => {
c2 = p2Result;
return;
}, function(err2) {
console.log('Error: ', err2);
return;
});
const promise3 = (val31, val32) => myPromise2(val31, val32)
.then((p3Result) => {
c3 = p3Result;
return;
}, function(err3) {
console.log('Error: ', err3);
return;
});
const conditionalPromiseFlow = (...fns) => {
if(fns.length === 0) return Promise.resolve()
const [next] = fns;
return next()
.then(result => {
if(result) {
return conditional(...fns.slice(1));
}
return result;
})
}
conditionalPromiseFlow(() => promise1(1000, true), () => promise2(2000, true), () => promise3(3000, true))
.then(() => {
console.log('Status 200 - ALL DONE');
return;
})
.catch((error) => {
console.log('Status ', error.status, ' - ', error.message);
return;
}) <== Error stated here
this is the corrected snippets to test promises flow...
I can play with the main() line
conditionalPromiseFlow(() => promise1(1000, true), () => promise2(2000, true), () => promise3(3000, true))
the flow stops when one promise is rejected
function myPromise1(time1, ok1){
console.log('Promise1', time1, ok1);
return new Promise((resolve, reject) => {
setTimeout(() => {
if ( ok1 ) {
resolve('stuff-1 worked');
} else {
reject(Error('stuff-1 failed'));
}
}, time1);
});
}
function myPromise2(time2, ok2){
console.log('Promise2', time2, ok2);
return new Promise((resolve, reject) => {
setTimeout(() => {
if ( ok2 ) {
resolve('stuff-2 worked');
} else {
reject(Error('stuff-2 failed'));
}
}, time2);
});
}
function myPromise3(time3, ok3){
console.log('Promise3', time3, ok3);
return new Promise((resolve, reject) => {
setTimeout(() => {
if ( ok3 ) {
resolve('stuff-3 worked');
} else {
reject(Error('stuff-3 failed'));
}
}, time3);
});
}
let c1 = '';
let c2 = '';
let c3 = '';
const promise1 = (val11, val12) => myPromise1(val11, val12)
.then(p1Result => {
c1 = p1Result;
return;
}, function(err1) {
err1.status = 300;
throw err1;
});
const promise2 = (val21, val22) => myPromise2(val21, val22)
.then((p2Result) => {
c2 = p2Result;
return;
}, function(err2) {
err2.status = 300;
throw err2;
});
const promise3 = (val31, val32) => myPromise3(val31, val32)
.then((p3Result) => {
c3 = p3Result;
return;
}, function(err3) {
err3.status = 300;
throw err3;
});
const conditionalPromiseFlow = (...fns) => {
if(fns.length === 0) return Promise.resolve();
const [next] = fns;
return next()
.then(result => {
if(!result) {
return conditionalPromiseFlow(...fns.slice(1));
}
return result;
});
};
conditionalPromiseFlow(() => promise1(1000, true), () => promise2(2000, true), () => promise3(3000, true))
.then(() => {
console.log('Status 200 - ALL DONE');
return;
})
.catch((error) => {
console.log('Status ', error.status, ' - ', error.message);
return;
});
I am using Ionic 2 with SQLite. I am getting an error, and I suspect it is due to me not using Promises correctly.
I get the following error:
ERROR REFRESHING CHATS: {}
TypeError {stack: (...), message: "Cannot read property 'executeSql'
of undefined"}
ts
private database = new SQLite();
public openDatabase(): Promise<Array<Message>> {
let promise: Promise<Array<Message>> = new Promise<Array<Message>>(resolve => {
if (this.database) {
let promiseChats: Promise<any> = this.refreshChats();
let promiseMessages: Promise<any> = this.refreshMessages();
Promise.all([promiseChats, promiseMessages]).then(() => { resolve(this.messages) });
} else {
this.database.openDatabase({
name: "data.db",
location: "default"
}).then(() => {
let promiseChats: Promise<any> = this.refreshChats();
let promiseMessages: Promise<any> = this.refreshMessages();
Promise.all([promiseChats, promiseMessages]).then(() => { resolve(this.messages) });
}, (error) => {
console.log("OPEN ERROR: ", error);
});
}
});
return promise;
}
public refreshChats(): Promise<any> {
return this.database.executeSql("SELECT * FROM chats", [])
.then((chatData) => {
this.chats = [];
if (chatData.rows.length > 0) {
for (var i = 0; i < chatData.rows.length; i++) {
this.populateChat(chatData, i);
}
}
return this.chats;
})
.catch(error => {
console.log("ERROR REFRESHING CHATS: " + JSON.stringify(error));
console.log(error);
});
}
When I debug the code, and break on this.database.executeSql, this.database is not undefined. Also, you can see there is a check on this.database to see if it's undefined before too. Yet, the debugger steps out the Promise into the error, reporting the error shown.
Also, after the Promise finishes, the this.chats are populated. So I am very confused.
If anyone can suggest what I am doing incorrectly, I would appreciate it.
UPDATE
I have updated the code as follows, but still get the same error:
public openDatabase(): Promise<Array<Message>> {
let promise: Promise<Array<Message>> = new Promise<Array<Message>>(resolve => {
if (this.database) {
Promise.all([this.refreshChats(), this.refreshMessages()]).then(() => { resolve(this.messages) });
} else {
this.database.openDatabase({
name: "data.db",
location: "default"
}).then(() => {
Promise.all([this.refreshChats(), this.refreshMessages()]).then(() => { resolve(this.messages) });
}, (error) => {
console.log("OPEN ERROR: ", error);
});
}
});
return promise;
}
public refreshChats(): Promise<any> {
return this.database.executeSql("SELECT * FROM chats", [])
.then((chatData) => {
let promises: Array<any> = [];
this.chats = [];
if (chatData.rows.length > 0) {
for (var i = 0; i < chatData.rows.length; i++) {
promises.push(this.populateChat(chatData.rows.item(i)));
}
}
Promise.all(promises).then(() => {
return this.chats;
});
})
.catch(error => {
console.log("ERROR REFRESHING CHATS: " + JSON.stringify(error));
console.log(error);
});
}
You are employing a bit of a Promise antipattern which I will describe to you. Although I can't see a reason it would cause your problem, it may help with debugging.
It's known as the explicit-construction anti-pattern and essentially it is when you create a new Promise unnecessarily.
For instance in your code, both of your functions create a new Promise at the start, where they could instead just return the promise used later;
public refreshChats(): Promise<Array<Chat>> {
if (!this.database || this.database === null) {
console.log('ERROR refreshing chats: database = ' + this.database);
}
return this.database.executeSql("SELECT * FROM chats", [])
.then((chatData) => {
this.chats = [];
if (chatData.rows.length > 0) {
for (var i = 0; i < chatData.rows.length; i++) {
this.populateChat(chatData, i);
}
}
return this.chats;
})
.catch(error => {
console.log("ERROR REFRESHING CHATS: " + JSON.stringify(error));
console.log(error);
});
}
This may help to break up your error so that you can see where it originates. If you produce any further information, ping me and I will update my answer.