I am new to the parse platform and i'm trying to insert 81000 rows of data in to the Parse DB, here the code
const uri = "/the.json"
const res = await axios.get(uri)
const dataresult = Object.keys(res.data)
if (dataresult.length > 0) {
res.data.forEach(function (datakp) {
var kp = new Parse.Object("theClass");
kp.save(datakp)
.then((res) => {
console.log('oke ' + res.id)
}),
(error) => {
console.log('err : '+ error.message)
}
})
}
There is no error in console log, and no data is saved in Parse DB, but if I only insert 1000 rows, it will save to the database.
EG:
if (dataresult.length > 0) {
res.data.forEach(function (datakp, index) {
if (index < 1000) {
var kp = new Parse.Object("theClass");
kp.save(datakp)
.then((res) => {
console.log('oke ' + res.id)
}),
(error) => {
console.log('err : '+ error.message)
}
})
}
}
Thank You
UPDATE
I fix this case based on answer #davi-macêdo
here a complete code
const uri = "/the.json"
const res = await axios.get(uri)
const dataresult = Object.keys(res.data)
const objs = [];
const theKP = Parse.Object.extend("theClass")
if (dataresult.length > 0) {
res.data.forEach(function (datakp) {
var thekp = new theKP()
thekp.set(datakp)
objs.push(thekp);
})
}
Parse.Object.saveAll(objs)
.then((res) => {
console.log('oke updated ' + dataresult.length)
}),
(error) => {
console.log('err : '+ error.message)
}
The most efficient way is using Parse.Object.saveAll function. Something like this:
const uri = "/the.json"
const res = await axios.get(uri)
const dataresult = Object.keys(res.data)
const objs = [];
if (dataresult.length > 0) {
res.data.forEach(function (datakp) {
objs.push(new Parse.Object("theClass", datakp));
})
}
Parse.Object.saveAll(objs)
.then((res) => {
console.log('oke ' + res.id)
}),
(error) => {
console.log('err : '+ error.message)
}
Anyways, since you have no error and no data currently being saved, you might be kitting some memory limit. So that's something you also need to be aware about.
You're probably hitting rate limits, I can't imagine saving 81,000 records in one shot is normal behaviour for many applications.
I looked through the documentation and couldn't find anything that might mention a save limit, however sending 1000 requests would trigger most rate limit protection
Related
hi i wonder why i cannot retrieve an objectId from this json object even i can printout the stringify on console.
I can retrieve all other column with no problem but not objectId. It happen to all table in my back4app Parse server.
i need the objectId in order to update certain column in my program
below is my code
1)
const parseQuery = new Parse.Query("User");
parseQuery.equalTo("username", "Azha");
let queryResult = await parseQuery
.find()
.then((results) => {
results.forEach((prod) => {
//if i change below to prod.get("objectId") error undefined appear
console.log("Product ID Available : " + prod.get("username"));
});
})
.catch((error) => {
console.log(error);
});
const parseQuery = new Parse.Query("User");
parseQuery.equalTo("username", "Azha");
try {
let todos = await parseQuery.find();
if (todos.length > 0) {
//if i change below to todos[0].get("objectId") error undefined appear
console.log("yes Approval : " + todos[0].get("companyname"));
} else {
console.log("No Approval");
}
console.log("- value is : " + JSON.stringify(todos));
console.log("----------------------");
} catch (error) {
Alert.alert("Error!", error.message);
}
below is the json printout
[{"sessionToken":"r:d9166aa9d7143463c46725d095b53946","username":"Azha","createdAt":"2021-09-21T15:27:01.088Z","updatedAt":"2021-10-10T13:01:27.126Z","companyname":"XXX","fullname":"XXX","email":"azha#abc.com.my","emailVerified":true,"accesslevel":"Maintenence","companydivision":"Maintenence","position":"Technician","phonenumber":"999","userteam":"B","useremail":"azha#abc.com.my","ACL":{"*":{"read":true},"IuBGmCtxyu":{"read":true,"write":true}},"objectId":"IuBGmCtxyu"}]
Yes i just found my solution. Using object1 below:
const parseQuery = new Parse.Query("User");
parseQuery.equalTo("username", "Azha");
try {
let todos = await parseQuery.find();
var object1 = JSON.parse(JSON.stringify(todos));
console.log("2- value is : " + object1[0].objectId);
} catch (error) {
Alert.alert("Error!", error.message);
}
I'm working on a Bus Stop google assistant script in node.js
I based it on the weather API example by Google. Given the right API key, the weather function will work and return the weather for a place on a date.
The Bus Stop API will return the correct output in the console.log, but the output does not get passed on to the else if statement where the function is called.
I get 2 errors:
"Unhandled rejection" Which can be alleviated by commenting out the reject code in the callBusApi.
"TypeError: Cannot read property 'json' of undefined
at callBusApi.then.catch (/user_code/index.js:45:9)
at process._tickDomainCallback (internal/process/next_tick.js:135:7)" This is where it breaks. I think because it doesn't get the output from the function.
My script looks as follows:
'use strict';
const http = require('http');
const host = 'api.worldweatheronline.com';
const wwoApiKey = 'enter a working key';
exports.weatherWebhook = (req, res, re) => {
if(req.body.queryResult.intent['displayName'] == 'weather'){
// Get the city and date from the request
let city = req.body.queryResult.parameters['geo-city']; // city is a required param
// Get the date for the weather forecast (if present)
let date = '';
if (req.body.queryResult.parameters['date']) {
date = req.body.queryResult.parameters['date'];
console.log('Date: ' + date);
}
// Call the weather API
callWeatherApi(city, date).then((output) => {
res.json({ 'fulfillmentText': output }); // Return the results of the weather API to Dialogflow
}).catch(() => {
res.json({ 'fulfillmentText': `I don't know the weather but I hope it's good!` });
});
}
else if (req.body.queryResult.intent['displayName'] == 'mytestintent'){
callBusApi().then((output) => {
re.json({ 'fulfillmentText': output }); // Return the results of the bus stop API to Dialogflow
}).catch(() => {
re.json({ 'fulfillmentText': `I do not know when the bus goes.` });
});
}
};
function callBusApi () {
return new Promise((resolve, reject) => {
http.get({host: 'v0.ovapi.nl', path: '/stopareacode/beunav/departures/'}, (re) => {
let boy = '';
re.on('data', (d) => {boy+=d});
re.on('end',() => {
let response = JSON.parse(boy)
var firstKey = Object.keys(response['beunav']['61120250']['Passes'])[0];
var timeKey = Object.keys(response['beunav']['61120250']['Passes'][firstKey])[19];
var destKey = Object.keys(response['beunav']['61120250']['Passes'][firstKey])[1];
let destination = response['beunav']['61120250']['Passes'][firstKey][destKey];
let datetime = response['beunav']['61120250']['Passes'][firstKey][timeKey];
let fields = datetime.split('T');
let time = fields[1];
let output = `Next bus to ${destination} departs at ${time} .`;
console.log(output)
resolve(output);
});
re.on('error', (error) => {
console.log(`Error talking to the busstop: ${error}`)
reject();
});
});
});
};
function callWeatherApi (city, date) {
return new Promise((resolve, reject) => {
let path = '/premium/v1/weather.ashx?format=json&num_of_days=1' +
'&q=' + encodeURIComponent(city) + '&key=' + wwoApiKey + '&date=' + date;
console.log('API Request: ' + host + path);
http.get({host: host, path: path}, (res) => {
let body = '';
res.on('data', (d) => { body += d; });
res.on('end', () => {
let response = JSON.parse(body);
let forecast = response['data']['weather'][0];
let location = response['data']['request'][0];
let conditions = response['data']['current_condition'][0];
let currentConditions = conditions['weatherDesc'][0]['value'];
let output = `Current conditions in the ${location['type']}
${location['query']} are ${currentConditions} with a projected high of
${forecast['maxtempC']}°C or ${forecast['maxtempF']}°F and a low of
${forecast['mintempC']}°C or ${forecast['mintempF']}°F on
${forecast['date']}.`;
console.log(output);
resolve(output);
});
res.on('error', (error) => {
console.log(`Error calling the weather API: ${error}`)
reject();
});
});
});
}
It appears that your method has a parameter too much
exports.weatherWebhook = (req, res, re) => {
should be:
exports.weatherWebhook = (req, res) => {
And as well on the variable 're' used in the handling of the 'mytestintent' inside the webhook.
This explains the 'not defined' error when trying to set a json value on it.
Regarding your 2 question: It usually comes when the value of the variable is not defined.
First check wheather you have defined the JSON variable in your .js file.
Or its in some other format.
I am using cheerio and node to do web scraping, but I have a problem with promises. I can scrape an article list from a page but in that list, we have more links for single pages. I need to scrape single pages as well for each item on the list.
I will show you my code for the better solution.
import rp from 'request-promise'
import cheerio from 'cheerio'
import conn from './connection'
const flexJob = `https://www.flexjobs.com`
const flexJobCategory = ['account-management', 'bilingual']
class WebScraping {
//list of article e.g for page 2
results = [] // [[title], [link for page],...]
contentPage = [] //content for each page
scrapeWeb(link) {
let fullLink = `${link}/jobs/${flexJobCategory[1]}?page=2`
const options = {
uri: fullLink,
transform(body) {
return cheerio.load(body)
}
}
rp(options)
.then(($) => {
console.log(fullLink)
$('.featured-job').each((index, value) => {
//html nodes
let shortDescription = value.children[1].children[1].children[3].children[1].children[1].children[0].data
let link = value.children[1].children[1].children[1].children[1].children[1].children[0].attribs.href
let pageLink = flexJob + '' + link
let title = value.children[1].children[1].children[1].children[1].children[1].children[0].children[0].data
let place = value.children[1].children[1].children[1].children[1].children[3].children[1].data
let jobType = value.children[1].children[1].children[1].children[1].children[3].children[0].children[0].data
this.results.push([title, '', pageLink.replace(/\s/g, ''), '', shortDescription.replace(/\n/g, ''), place, jobType, 'PageContent::: '])
})
})
.then(() => {
this.results.forEach(element => {
console.log('link: ', element[2])
this.scrapePage(element[2])
});
})
.then(() => {
console.log('print content page', this.contentPage)
})
.then(() => {
//this.insertIntoDB()
console.log('insert into db')
})
.catch((err) => {
console.log(err)
})
}
/**
* It's going to scrape all pages from list of jobs
* #param {Any} pageLink
* #param {Number} count
*/
scrapePage(pageLink) {
let $this = this
//console.log('We are in ScrapePage' + pageLink + ': number' + count)
//this.results[count].push('Hello' + count)
let content = ''
const options = {
uri: pageLink,
transform(body) {
return cheerio.load(body)
}
}
rp(options)
.then(($) => {
//this.contentPage.push('Hello' + ' : ');
console.log('Heloo')
})
.catch((err) => {
console.log(err)
})
}
/**
* This method is going to insert data into Database
*/
insertIntoDB() {
conn.connect((err) => {
var sql = "INSERT INTO contact (title, department, link, salary, short_description, location, job_type, page_detail) VALUES ?"
var values = this.results
conn.query(sql, [values], function (err) {
if (err) throw err
conn.end()
})
})
}
}
let webScraping = new WebScraping()
let scrapeList = webScraping.scrapeWeb(flexJob)
So, at 'scrapeWeb' method, at second '.then', I am calling 'scrapePage' method, however, the third promise executed before promise inside 'scrapePage' method.
You need a little more control flow at that stage. You do not want that .then()'s promise to resolve until all the calls are resolved.
You could use a Promise library like bluebird to do a Promise.each or a Promise.map for all the results you want to run.
Or use async/await to set up like .then(async () => {}) and do not use .forEach.
for(let element of this.results){
console.log('link: ', element[2])
await this.scrapePage(element[2])
}
You have a race condition problem.
The first tweak you'll need is having scrapePage returning a Promise.
scrapePage(pageLink) {
let $this = this
let content = ''
const options = {
uri: pageLink,
transform(body) {
return cheerio.load(body)
}
}
return rp(options);
}
In the second than, you need to invoke all child pages scraping eg :
.then(() => {
return Promise.all(this.results.map(childPage => this.scrapePage(childPage)));
})
This will wrap all scrapes of child pages into promises and only if all of them are resolved the code will flow.
I am doing a Progressive Web App and I want to access the data I have stored on IndexedDB when the ajax reponse fails. The data gets filled when the query is successful. Below is one of the attempts. In comments is the original format before I inserted the offline part (I already had base code).
getAudits: (startOfMonth, endOfMonth) => {
return new Promise((resolve, reject) => {
request
.get(url + "api/?method=Audit.data")
.query({
pageLength: "all",
format: "json",
where: startOfMonth
? Utils.hashQuery(
"a.date",
"between",
[startOfMonth, endOfMonth].toString()
)
: "",
ts: new Date().getTime()
})
.end((error, response) => {
var res;
if (error) {
res = getCacheAudits(startOfMonth, endOfMonth);
//reject(error);
} else {
let _audits = [];
const audits = Utils.toArray(JSON.parse(response.text).rows);
audits.forEach(audit => {
_audits.push(audit);
//console.log(audit);
});
res = _audits;
getServerAudits(_audits);
//resolve(res);
}
resolve(res);
});
});
}
Does anyone have any idea about how to handle this correctly? Thanks in advance.
Not sure why you copy Utils.toArray(JSON.parse(response.text).rows to audits, _audits and res, they are all the same.
Also not sure what getServerAudits is supposed to do because it doesn't return anything and you're not doing anything with it's return value (usually a bad design sign to call functions and ignore their return)
You could probably do something like this:
getAudits: (startOfMonth, endOfMonth) => {
return new Promise((resolve, reject) => {
request
.get(url + "api/?method=Audit.data")
.query({
pageLength: "all",
format: "json",
where: startOfMonth
? Utils.hashQuery(
"a.date",
"between",
[startOfMonth, endOfMonth].toString()
)
: "",
ts: new Date().getTime()
})
.end((error, response) =>
(error)
? reject([error,startOfMonth, endOfMonth])
: resolve(Utils.toArray(JSON.parse(response.text).rows))
)
})
.catch(
([err,startOfMonth, endOfMonth]) =>
getCacheAudits(startOfMonth, endOfMonth)
)
.then(
audits => {
//not sure what this does, you don't seem to care about it's
// result because your code doesn't do anything with it
getServerAudits(audits);
//probably want to save audits to local storage here
return Promise.all([
audits,
saveCacheAudits(startOfMonth,endOfMonth,audits)
]);
}
)
.then(
//resolve audits
([audits])=>audits
)
}
Lets make this easy with indexedb library jsstore -
var con= new JsStore.Instance(<db_name>);
getCacheAudits: (startOfMonth, endOfMonth)=>{
return con.select({
From: "Table_Name",
Where:
{
startOfMonth:startOfMonth,
endOfMonth: endOfMonth
}
})
}
getAudits: (startOfMonth, endOfMonth) => {
return new Promise((resolve, reject) => {
request
.get(url + "api/?method=Audit.data")
.query({
pageLength: "all",
format: "json",
where: startOfMonth
? Utils.hashQuery(
"a.date",
"between",
[startOfMonth, endOfMonth].toString()
)
: "",
ts: new Date().getTime()
})
.end((error, response) => {
var res;
if (error) {
//get the data from indexeddb
getCacheAudits.
then(function (results){
resolve(results);
})
.catch(function (error) {
reject(error._message);
});
} else {
let _audits = [];
const audits = Utils.toArray(JSON.parse(response.text).rows);
audits.forEach(audit => {
_audits.push(audit);
//console.log(audit);
});
res = _audits;
getServerAudits(_audits);
resolve(res);
}
//resolve(res);
});
});
}
You can also use where (http://jsstore.net/tutorial/where) and all others options to query the data.
If you are using angular to create the app, check out this article - http://ujjwalguptaofficial.blogspot.in/2017/10/angular4-crud-operation-in-indexeddb.html
Hope this will help you.
Just solved the problem. It wasn't actually just in this part, but in the getCacheAudits function too.
The getCacheAudits call became:
getCacheAudits(startOfMonth, endOfMonth, function (res) {
resolve(res);
})
And in the getCacheAudits function was added a callback
function getCacheAudits(startOfMonth, endOfMonth, callback){
let initOpenReq = indexedDB.open("AuditsDB");
let agregate = [];
initOpenReq.onsuccess = function() {
var db = initOpenReq.result;
var transaction = db.transaction("audits","readonly");
var obj_store = transaction.objectStore("audits");
obj_store.openCursor().onsuccess = function (event){
let cursor = event.target.result;
if(cursor){
agregate.push(cursor.value);
cursor.continue();
}else{
callback(agregate);
}
};
}
There are a few more bells a whistles in the function, but this made it work properly.
Sorry for not including the function right from the beginning.
I am trying to search my local db for a user by email, but when I try to reference a function that does that from a different js file, via an import, I get undefined results. I have searched a bit on Stack about this issue I am having, and heard of something referred to as a callback, is this something that I would need to implement? If so could you point me to an example ?
Thanks in advance!
Here is my code that is exported (db.js file) :
var neo4j = require('neo4j-driver').v1;
var driver = neo4j.driver("bolt://localhost:7687", neo4j.auth.basic("neo4j", "neo4j"));
var session = driver.session();
exports.findEmail = (email) => {
// console.log('hi');
session
.run("MATCH (a:Person) WHERE a.email = {email} RETURN a.name AS name, a.email AS email, a.location AS location", {
email: email
})
.then((result) => {
let result_string = '';
result.records.forEach((record) => {
console.log(record._fields);
result_string += record._fields + ' ';
});
return result_string;
})
.catch((e) => {
return ('error : ' + JSON.stringify(e));
})
}
Here is my code calling the export : (test.js)
var tester = require('./db.js');
let temp = tester.findEmail("testemail#yahoo.com");
console.log(temp);
The thing is that JS is asynchronous, and you using it as it is synchronous code.
Can you try this one, should work:
var neo4j = require('neo4j-driver').v1;
var driver = neo4j.driver("bolt://localhost:7687", neo4j.auth.basic("neo4j", "neo4j"));
var session = driver.session();
const findEmail = (email, callback) => {
console.log('hi :', email);
session
.run("MATCH (a:Person) WHERE a.email = {email} RETURN a.name AS name, a.email AS email, a.location AS location", {
email: email
})
.then((result) => {
let result_string = '';
result.records.forEach((record) => {
console.log(record._fields);
result_string += record._fields + ' ';
});
return callback(null, result_string);
})
.catch((e) => {
return callback('error : ' + JSON.stringify(e)));
})
}
module.exports = {
findEmail
};
Then in test.js:
var tester = require('./db');
tester.findEmail("testemail#yahoo.com", (err, temp) => {
if (err) return console.error(err);
console.log(temp);
} );
The idea behind this is that all the flow in db file is asynchronous.
So in order to catch result you need to pass the function, callback, which will be triggered when the ansynchronous flow is done.