I am trying to read some data from a file and store it in a database.
This is part of a larger transaction and I need the returned ids for further steps.
async parseHeaders(mysqlCon, ghID, csv) {
var self = this;
var hIDs = [];
var skip = true;
var idx = 0;
console.log("Parsing headers");
return new Promise(async function(resolve, reject) {
try {
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream(csv)
});
await lineReader.on('close', async function () {
console.log("done: ", JSON.stringify(hIDs));
resolve(hIDs);
});
await lineReader.on('line', async function (line) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx++] = await self.getParamID(mysqlCon, ghID, v, c, data);//, function(hID,sidx) { //add data in case the parameter is not in DB, yet.
}
});
} catch(e) {
console.log(JSON.stringify(e));
reject("some error occured: " + e);
}
});
}
async getParamID(mysqlCon,ghID,variable,category,data) {
return new Promise(function(resolve, reject) {
var sql = "SELECT ID FROM Parameter WHERE GreenHouseID="+ghID+" AND Variable = '" + variable + "' AND Category='" + category + "'";
mysqlCon.query(sql, function (err, result, fields) {
if(result.length === 0 || err) { //apparently not in DB, yet ... add it (Acronym and Machine need to be set manually).
sql = "INSERT INTO Parameter (GreenHouseID,Variable,Category,Control) VALUES ("+ghID+",'"+variable+"','"+category+"','"+data[3]+"')";
mysqlCon.query(sql, function (err, result) {
if(err) {
console.log(result,err,this.sql);
reject(err);
} else {
console.log("Inserting ",variable," into DB: ",JSON.stringify(result));
resolve(result.insertId); //added, return generated ID.
}
});
} else {
resolve(result[0].ID); //found in DB .. return ID.
}
});
});
}
The functions above are in the base class and called by the following code:
let headerIDs = await self.parseHeaders(mysqlCon, ghID, filePath);
console.log("headers:",JSON.stringify(headerIDs));
The sequence of events is that everything in parseHeaders completes except for the call to self.getParamID and control returns to the calling function which prints an empty array for headerIDs.
The console.log statements in self.getParamID are then printed afterward.
What am I missing?
Thank you
As you want to execute an asynchronous action for every line we could define a handler to do right that:
const once = (target, evt) => new Promise(res => target.on(evt, res));
function mapLines(reader, action) {
const results = [];
let index = 0;
reader.on("line", line => results.push(action(line, index++)));
return once(reader, "close").then(() => Promise.all(results));
}
So now you can solve that easily:
let skip = false;
const hIDs = [];
await mapLines(lineReader, async function (line, idx) {
line = line.replace(/\"/g, '');
if (line.startsWith("Variable")) { //Variable,Statistics,Category,Control
console.log("found variables");
skip = false; //Ignore all data and skip to the parameter description.
return; //Skip also the header line.
}
if (!skip) {
var data = line.split(",");
if (data.length < 2) { //Variable section done return results.
console.log("Found sub?",line);
return lineReader.close();
}
var v = data[0];
var bidx = data[0].indexOf(" [");
if (bidx > 0)
v = data[0].substring(0, bidx); //[] are disturbing mysql (E.g.; Air temperature [�C])
var c = data[2];
hIDs[idx] = await self.getParamID(mysqlCon, ghID, v, c, data);
}
});
Related
I am trying to commit to batch but I keep getting this error but cant figure out where i am going wrong, i tried moving the batch outside certain functions but still not progress, here is my code,i am not sure what I'm missing , i keep getting this error :
scheduledFunction
Error: Cannot modify a WriteBatch that has been committed.
at WriteBatch.verifyNotCommitted (/workspace/node_modules/#google-cloud/firestore/build/src/write-batch.js:117:19)
at WriteBatch.update (/workspace/node_modules/#google-cloud/firestore/build/src/write-batch.js:313:14)
at /workspace/index.js:152:48
at QuerySnapshot.forEach (/workspace/node_modules/#google-cloud/firestore/build/src/reference.js:748:22)
at step2 (/workspace/index.js:91:39)
at runMicrotasks (<anonymous>)
at processTicksAndRejections (internal/process/task_queues.js:97:5)
here is my code
async function updateBets() {
var marketRef = db.collection('matches');
var snapshot = await marketRef.where('matchStatus', '==', 'FINISHED').get();
if (snapshot.empty) {
console.log('No matching documents.');
return;
}
console.log('I found documents');
snapshot.forEach(doc => {
if (doc.data().matchId == null) {
//console.log('document with error" ' + doc.id);
}
step2();
async function step2() {
if (doc.data().matchId != null) {
var query = doc.data().matchId.toString();
// console.log('here is match ID' + query);
var marketRef2 = db.collection('markets');
var snapshot2 = await marketRef2.where('marketId', '==', query).get();
snapshot2.forEach(doc2 => {
if (doc2.data().marketTitleId == 'FULL_TIME_RESULT') {
var a = doc.data().homeTeamScore;
var b = doc.data().awayTeamScore;
var winnerIndex;
if (a > b) {
winnerIndex = 0;
var resultIndex = ['WINNER', 'LOSER', 'LOSER'];
var docName = `${doc.data().matchId}` + '000' + '1';
var sfRef = db.collection('markets').doc(docName);
batch5.update(sfRef, {
results: resultIndex
});
} else if (a == b) {
winnerIndex = 1;
var docName = `${doc.data().matchId}` + '000' + '1';
var resultIndex = ['LOSER', 'WINNER', 'LOSER'];
var sfRef = db.collection('markets').doc(docName);
batch5.update(sfRef, {
results: resultIndex
});
} else if (a < b) {
winnerIndex = 2;
var docName = `${doc.data().matchId}` + '000' + '1';
var resultIndex = ['LOSER', 'LOSER', 'WINNER'];
var sfRef = db.collection('markets').doc(docName);
batch5.update(sfRef, {
results: resultIndex
});
}
}
})
}
}
});
try {
await batch5.commit();
console.log("im done with results");
} catch (err) {
console.log('Mac! there was an error with results: ', err);
}
}
.....................................................................................................
Your main function is an async which will continue to run all methods asynchronously, ignoring any internal functions. You must make sure that all jobs being applied to the batch are added before attempting to commit, at the core this is a race condition.
You can handle these multiple ways, such as wrapping it with a promise chain or creating a callback.
For a callback, add the following before your forEach
var itemsProcessed = 0;
Insert this at the end of the forEach
itemsProcessed++;
if(itemsProcessed === array.length) {
CommitBatch();
}
Then put your batch.commit() method inside this callback function
function CommitBatch(){
try {
await batch5.commit();
console.log("im done with results");
} catch (err) {
console.log('Mac! there was an error with results: ', err);
}
}
I created a function in file and I want to return value from function and then use the value in if else condition but i get undefined here is my code for importable file:
export const check_student_in_db = (st_idd) => {
//check student is in attendence list
var today = new Date();
let date = today.getDate() + "/" + parseInt(today.getMonth() + 1) + "/" + today.getFullYear();
let slash_removed = date.replace(/\//g, "")
AsyncStorage.getItem('student_status', (err, result) => {
if (!err && result != null) {
//console.warn('note a null');
let db_data = JSON.parse(result); // store db data of attendence
db_dataa = JSON.parse(result);
// concat db data with new data
let ext = db_data[slash_removed]; //check data exist with today date
//console.warn('get a test ' + ext);
if (ext) {
//search for student in dabase json data using student id
ext.map(function (item) {
if(item.st_id == st_idd){
//console.warn(st_idd);
return st_idd;
}else {
return false;
}
});
} else {
return false;
}
}
else {
return false;
}
});
}
and here I my code to use this function
import {insert_st_attendence, check_student_in_db} from './functions/functions';
load_memory = async () => {
const status = await check_student_in_db('18ds1');
console.warn(status);
if (status) {
console.warn(status);
}
return true;
}
My function is simple, this is save data:
export async function saveData(key, value) {
try {
await AsyncStorage.setItem(key, value);
} catch (error) {
console.log('Error when saving data: ' + error.toString())
}
}
function getData
export async function getData(key, callback) {
let data = await AsyncStorage.getItem(key);
let result = JSON.parse(data);
callback(result);
}
and use this anywhere in your project
import {getData} from "...link to file/";
getData("Your Key", function (result) {
// Your code here
})
hope this help
First of all the await expression causes async function execution to pause until a Promise is resolved, that is fulfilled or rejected.
So in your check_student_in_db function you need to return Promise
export const check_student_in_db = (st_idd) => {
//check student is in attendence list
return new Promise((resolve, reject) => {
var today = new Date();
let date = today.getDate() + "/" + parseInt(today.getMonth() + 1) + "/" + today.getFullYear();
let slash_removed = date.replace(/\//g, "")
AsyncStorage.getItem('student_status', (err, result) => {
if (!err && result != null) {
//console.warn('note a null');
let db_data = JSON.parse(result); // store db data of attendence
db_dataa = JSON.parse(result);
// concat db data with new data
let ext = db_data[slash_removed]; //check data exist with today date
//console.warn('get a test ' + ext);
if (ext) {
//search for student in dabase json data using student id
ext.map(function (item) {
if(item.st_id == st_idd){
//console.warn(st_idd);
resolve(st_idd);
}else {
reject(false);
}
});
} else {
reject(false)
}
}
else {
reject(false)
}
});
}
I need to perform a rather complicated chain of promise resolutions to get and save data that is located in PDFs that are uploaded by a user. Everything works fine for single PDFs, but it breaks as soon as I try to upload multiple PDFs because the order in which the resolution proceeds isn't as I expect.
I call the function saveDoc on each file in an array:
saveDoc: function(){
var files = this.$refs.upload.uploadFiles
var self=this;
var promises = []
for (var i=0; i<files.length; i++){
(function(){
var file = files[i]
var name = file['name']
if (!(/\.pdf/i.test(name))){
name+='.pdf'
}
var type = mime.lookup(name)
var file = file['raw'];
var beforeUrl = self.selectedCategories.join('&&&')
console.log('going to save: ' + name) // Because of the IIFE, I'm not expecting this to log only after all steps 1 - 20 have been completed for each file
Store.getFileData(file).then(function(data){
console.log(2)
return Store.saveDoc(name, type, data, beforeUrl).then(url => {
console.log(8)
return Text.getText(url).then(text => {
console.log(12)
return Text.getMetadata(text, url).then(metadata => {
console.log(20)
if (metadata.length){
return Store.saveMetadata(beforeUrl, metadata, name)
}
return Store.createCategory(name, self.selectedCategories, '')
})
})
})
})
})()
}
},
I'm sure the promises could use some work, but what seems to be the problem is that the line console.log('going to save: ' + name) is called twice before the entire sequence 1-20 is carried out for one file (see error message of number sequence at bottom of this post). I tried to prevent this using an IIFE, but I guess I didn't do this right.
store.js
getData: function(url){
return new Promise(function(accept, reject){
console.log(4)
documentation.get(url, function(err, body) {
console.log(5)
if (!err){
console.log(body);
accept(body)
}
});
})
},
getFileData: function(file){
var reader = new FileReader();
return new Promise(function(accept, reject){
console.log(1)
reader.onload = (e) => {
// var data = e.target.result.replace(/^data:[A-Za-z]+\/[A-Za-z]+;base64,/, '')
// console.log('base 64: ' + data)
var res = new Uint8Array(e.target.result)
accept(res)
};
reader.readAsArrayBuffer(file);
})
},
saveDoc: function(name, type, filedata, url){
console.log(3)
var self=this
return new Promise(function(accept, reject){
return self.getData(url).then(data => {
console.log(6)
var rev = data['_rev']
return documentation.attachment.insert(url, name, filedata, type,
{ rev: rev }, function(err, body) {
if (!err){
console.log(7)
var fullUrl = 'http://dev04/documentation/'+url+'/'+name
accept(fullUrl)
}
else {
console.log(err)
}
})
}).catch(err => {
console.log(err)
})
})
},
saveMetadata: function(url, metadata, name){
var fileName = path.basename(name)
var self=this
return new Promise(function(accept, reject){
self.getData(url).then(data => {
var meta
var rev = data['_rev']
if(!data['metadata']){
data['metadata'] = {}
}
data['metadata'][fileName] = metadata
var datastring = JSON.stringify(data)
documentation.insert(data, url, function(err, body, header) {
if (err) {
console.log(err.message);
return;
}
});
}).catch(err => {
console.log(err)
})
})
},
text.js
export default {
getText: function(url){
console.log(9)
var result = []
return new Promise(function(accept, reject){
console.log(10)
return pdfjs.getDocument(url).then(pdf => {
console.log(11)
var pdf = pdfjs.getDocument(url);
return pdf.then(function(pdf) { // get all pages text
var maxPages = pdf.pdfInfo.numPages;
var countPromises = []; // collecting all page promises
for (var j = 1; j <= maxPages; j++) {
var page = pdf.getPage(j);
var txt = "";
countPromises.push(page.then(function(page) { // add page promise
var textContent = page.getTextContent();
// console.log('the content is ' + textContent)
return textContent.then(function(text){ // return content promise
var val = text.items.map(function (s) { return s.str; }).join('&&&')+'&&&'; // value page text
result.push(val)
// console.log(val + ' should be one page of text')
return val
});
}));
}
// Wait for all pages and join text
return Promise.all(countPromises).then(function (texts) {
accept(texts.join(''))
});
});
});
})
},
getMetadata: function(text, url){
console.log(13)
var result = []
var self = this
return new Promise(function(accept, reject){
self.getpageno(url).then(function(pagecount){
console.log(19)
try {
var dataMatch = rx.exec(text)
var produktDaten = dataMatch[1].split("&&&").filter(Boolean);
console.log(produktDaten)
var dokuArr = dataMatch[2].split("&&&").filter(Boolean);
for (var i=0; i<produktDaten.length; i+=4){
var entry = {}
entry.pagecount = pagecount
entry.kks = {}
entry.kks.pages = {}
var kksNummer = produktDaten[i];
entry.kks.nummer = kksNummer;
for(var j=0; j<dokuArr.length; j+=3){
var nummer = dokuArr[j];
var beschreibung = dokuArr[j+1];
var seite = dokuArr[j+2];
// make sure seite is a digit
if (!(/^\d+$/.test(seite))){
console.log(seite + ' was not a valid page number')
throw err
}
if (/(A|a)lle?/i.test(nummer)){
entry.kks.pages[beschreibung] = seite;
// self.tableEntry.kks.url = url;
// self.tableEntry.fileName = name;
///// kksNummern.forEach(function(kks){
// self.tableEntry.kks;
// })
}
else if (nummer === kksNummer) {
entry.kks.pages[beschreibung] = seite;
// entry.kks.url = url;
// entry.fileName = name
}
}
entry.hersteller = produktDaten[i+1]
entry.typ = produktDaten[i+2]
entry.artikelNummer = produktDaten[i+3]
result.push(entry)
}
}
catch(e){
return accept(result)
}
return accept(result)
/* if (result.length>0){
console.log('accepting the result')
}
reject()*/
}).catch(err => {
console.log(err)
})
})
},
getpageno: function(url){
console.log(14)
var self=this
var pdf = pdfjs.getDocument(url);
return pdf.then(function(pdf){
console.log(15)
var maxPages = pdf.pdfInfo.numPages;
var countPromises = []; // collecting all page promises
for (var j = 1; j <= maxPages; j++) {
try {
var page = pdf.getPage(j);
var txt = "";
countPromises.push(page.then(function(page) { // add page promise
var textContent = page.getTextContent();
return textContent.then(function(text){ // return content promise
console.log(16)
return text.items.map(function (s) { return s.str; }).join('&&&'); // value page text
});
}));
}
catch(e){
console.log(e)
}
}
// Wait for all pages and join text
return Promise.all(countPromises).then(function (texts) {
// since doumentation pages do not add the final '&&&', must add one manually (only after rx has been found)
console.log(17)
var fulltext = texts.reduce(function(full, text){
if (rx.test(full)){
var next = '&&&'+text
return full+=next
}
return full+=text
}, '')
return [fulltext, texts]
});
}).then(function(textarr){
console.log(18)
var fulltext = textarr[0]
self.fulltext = fulltext;
var texts = textarr[1]
try {
var partialmatch = rx.exec(fulltext)[0]
var count = texts.reduce(function(pageno, text){
var tomatch = text.replace(/.*Typ&&&/, '')
if (tomatch.length>0 && partialmatch.indexOf(tomatch) > -1){
pageno++
}
return pageno;
}, 0)
}
catch(e){
console.log(e)
}
return count;
}).catch(err => {console.log(err)})
}
}
I use the console to log the order I am expecting for the functions. I am expecting the numbers 1 - 20 in order, but I get the following.
going to save: 03_.pdf selector.js:211:6
1 store.js:227:4
going to save: 2017.07.05_0016 E161206.pdf selector.js:211:6
1 store.js:227:4
2 selector.js:213:7
3 store.js:239:3
4 store.js:213:4
2 selector.js:213:7
3 store.js:239:3
4 store.js:213:4
5 store.js:215:5
Object { _id: "Test", _rev: "36-85a08c0852ccab78c0b4c10369e83fb2", rank: 7, icon: "wrench", metadata: Object, _attachments: Object } store.js:217:6
6 store.js:243:5
5 store.js:215:5
Object { _id: "Test", _rev: "36-85a08c0852ccab78c0b4c10369e83fb2", rank: 7, icon: "wrench", metadata: Object, _attachments: Object } store.js:217:6
6 store.js:243:5
7 store.js:247:7
8 selector.js:215:8
9 text.js:12:3
10
11 text.js:17:5
12 selector.js:217:9
13 text.js:50:3
14 text.js:112:3
15 text.js:116:4
16 text.js:129:8
17 text.js:142:5
18 text.js:153:4
19 text.js:57:5
20
Can anyone offer any advice on how to get this order correct? Thank you.
I have a promise that is meant to collect metadata on a file and then resolve it with the metadata that was collected. Here is how I am trying to get its result:
getMetadata: function(text, url){
return this.getpageno(url).then(function(pagecount){
return new Promise(function(accept, reject){
var result = []
var dataMatch = rx.exec(text)
var produktDaten = dataMatch[1].split("&&&").filter(Boolean);
var dokuArr = dataMatch[2].split("&&&").filter(Boolean);
console.log('the produktdaten are ' + produktDaten)
for (var i=0; i<produktDaten.length; i+=4){
var entry = {}
for(var j=0; j<dokuArr.length; j+=3){
var seite = dokuArr[j+2];
// make sure seite is a digit
if (!(/^\d+$/.test(seite))){
console.log(seite + ' was not a valid page number')
throw err
}
if (/(A|a)lle?/i.test(nummer)){
entry.kks.pages[beschreibung] = seite;
// })
}
else if (nummer === kksNummer) {
entry.kks.pages[beschreibung] = seite;
}
}
entry.hersteller = produktDaten[i+1]
entry.typ = produktDaten[i+2]
entry.artikelNummer = produktDaten[i+3]
result.push(entry)
}
if (result.length>0){
return accept(result)
}
return reject()
})
})
},
getpageno: function(url){
var self=this
var pdf = pdfjs.getDocument(url);
return pdf.then(function(pdf){
var maxPages = pdf.pdfInfo.numPages;
var countPromises = []; // collecting all page promises
for (var j = 1; j <= maxPages; j++) {
try {
var page = pdf.getPage(j);
var txt = "";
countPromises.push(page.then(function(page) { // add page promise
var textContent = page.getTextContent();
return textContent.then(function(text){ // return content promise
return text.items.map(function (s) { return s.str; }).join('&&&'); // value page text
});
}));
}
catch(e){
console.log(e)
}
}
// Wait for all pages and join text
return Promise.all(countPromises).then(function (texts) {
// since doumentation pages do not add the final '&&&', must add one manually (only after rx has been found)
var fulltext = texts.reduce(function(full, text){
if (rx.test(full)){
var next = '&&&'+text
return full+=next
}
return full+=text
}, '')
return [fulltext, texts]
});
}).then(function(textarr){
var fulltext = textarr[0]
self.fulltext = fulltext;
var texts = textarr[1]
try {
var partialmatch = rx.exec(fulltext)[0]
var count = texts.reduce(function(pageno, text){
var tomatch = text.replace(/.*Typ&&&/, '')
if (tomatch.length>0 && partialmatch.indexOf(tomatch) > -1){
pageno++
}
return pageno;
}, 0)
}
catch(e){
console.log(e)
}
return count;
})
}
Edited the entry to show that I'm now returning the value of the gepageno function.
The data that I am expecting is logged but not available as a result of accept(). Can anyone tell what could be going wrong?
I have tried to simulate and give a solution of your promise. I have posted pseudo code.
Promise flow
var testFunction = ((result) => {
return new Promise(function (resolve, reject) {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("2. testFunction resolved");
resolve("Final response");
}
});
});
var getpageno = ((url) => {
return new Promise((resolve, reject) => {
var err = null;
if (!!err) {
reject(err);
} else {
console.log("1. getpageno resolved");
resolve(45);
}
});
});
var getMetadata = ((text, url) => {
console.log("0. getmetadata resolved");
var self = this;
getpageno(url)
.then((pageCount) => {
console.log("> pageCount :", pageCount);
return testFunction(pageCount);
})
.then((data) => {
console.log(">", data);
});
});
getMetadata("hell", "https://www.google.com");
output
0. getmetadata resolved
1. getpageno resolved
> pageCount : 452. testFunction resolved
> Final response
you need to return the promise on the third line
return self.getpageno(url).then
to access its eventual fulfillment value.
I am trying to query sqlite3 from node function. Depending on result it should return TRUE or FALSE back to the calling code. This is what I have done, but always get 'undefined':
var sqlite3 = require("sqlite3").verbose();
var dbOpen = function dbOpen() {
sqlite3 = require('sqlite3').verbose();
db = new sqlite3.Database('./internaldb/OPMSE.db');
}
module.exports.dbOpen = dbOpen;
dbOpen();
mKey = 'AIzaSyCrIYuHJ_jiZqmGYoAs7mxiloB3-OBx5WI';
async function db1() {
function db2() {
db.each("SELECT STATUS status FROM API_Keys WHERE KEY = ? AND STATUS = ?", [mKey, '1'], (err, row, x) => {
var keyValid = false;
if (err) {
console.log('There is an error in API Validation')
} else {
console.log(keyValid + '<--This is from upper fun')
keyValid = true;
return keyValid;
}
db.close();
return keyValid;
});
} // db2
d = await db2();
console.log(d + '<--Supposed to return TRUE or FALSE')
} //db1
x = db1();
console.log(x)
/*Returns:
Promise { <pending> }
undefined<--Supposed to return TRUE or FALSE
false<--This is from upper fun
*/
If you let db2 return a promise it will work, if you want to know if the key is in the database it would be better to select count it'll return one record and you don't need db.each (not sure what the method is to get one record).
Here is the code with some comments, this should work:
var sqlite3 = require("sqlite3").verbose();
var dbOpen = function dbOpen() {
sqlite3 = require('sqlite3').verbose();
db = new sqlite3.Database('./internaldb/OPMSE.db');
}
module.exports.dbOpen = dbOpen;
dbOpen();
mKey = 'AIzaSyCrIYuHJ_jiZqmGYoAs7mxiloB3-OBx5WI';
async function db1() {
function db2() {
//return a promise
return new Promise(
(resolve,reject)=>
db.serialize(
function() {
var keyValid = false;
db.each(//maybe not use each but just get one record and select count(*)
"SELECT STATUS status FROM API_Keys WHERE KEY = ? AND STATUS = ?",
[mKey, '1'], (err, row, x) => {
if (err) {
reject(err);//reject on error
} else {
console.log(keyValid + '<--This is from upper fun');
keyValid = true;
}
}
);
db.colose();
}
)
)
} // db2
d = await db2();
console.log(d + '<--Supposed to return TRUE or FALSE')
} //db1
x = db1();
console.log(x)