I am implementing Vitaly's pg-promise performance patterns, as advised here and there.
Here is my code :
for (var i=0;i<chunkedData.length;i++){
var insertData = chunkedData[i].map(function (d) {
return {
application_id: d.application_id,
country_id: d.country_id,
collection_id: collectionId
};
});
// Would need to make a loop here, and thus turning the result into an array
var updateData = {
application_id: chunkedData[i][j].application_id,
country_id: chunkedData[i][j].country_id,
collection_id: collectionId
};
var query = h.insert(insertData, cs) +
" ON CONFLICT ON CONSTRAINT application_average_ranking_application_id_country_id_colle_key DO UPDATE SET " +
h.sets(updateData, cs);
db.none(query)
.then(data => {
console.log('success');
})
.catch(error=> {
console.log('insert error : ' + error);
});
}
My problem is that insertData is an Array of Objects, and the library's insert helper builds an insert request using that Array, as specified in pg-promise API. Whereas updateData must be a simple Object.
I would like that when :
ON CONFLICT ON CONSTRAINT constraintName DO UPDATE
is triggered, the update values match the corresponding object in 'insertData' array.
How can I work around that problem ?
I've tried to put everything in a loop, but it leaks memory like crazy, and well, I lose the benefits of the pattern...
EDIT :
I want my query to be the equivalent of :
var inserts = data.map(entry => {
return t.none(" INSERT INTO application_average_ranking (application_id,country_id,collection_id) VALUES ($1,$2,$3)" +
" ON CONFLICT ON CONSTRAINT application_average_ranking_application_id_country_id_colle_key" +
" DO UPDATE SET country_id=$2,collection_id=$3",
[entry.application_id,entry.country_id,collectionId]
);
});
In that case when Update is called, the parameters refer to values originally proposed for insertion.
Your task requires a static SQL to implement that kind of logic, by using EXCLUDED as the table reference with rows excluded due to the conflict:
var sqlConflict = " ON CONFLICT ON CONSTRAINT" +
" application_average_ranking_application_id_country_id_colle_key" +
" DO UPDATE SET application_id = excluded.application_id" +
" country_id = excluded.country_id, collection_id = excluded.collection_id";
var insertData = chunkedData.map(function (d) {
return {
application_id: d.application_id,
country_id: d.country_id,
collection_id: collectionId
};
});
var query = h.insert(insertData, cs) + sqlConflict;
db.none(query)
.then(data => {
console.log('success');
})
.catch(error=> {
console.log('insert error : ' + error);
});
UPDATE
And in case your static list of excluded fields is too long and you want to simplify it, you can can always rely on flexibility of the helpers methods:
// or pull them from an object using `Object.keys(obj)`:
var cols = ['application_id', 'country_id', 'collection_id'];
var sets = pgp.helpers.sets({}, cols.map(c=> ({
name: c, mod: '^', def: 'excluded.' + pgp.as.name(c)
})));
console.log(sets);
//=> "application_id"=excluded."application_id","country_id"=excluded."country_id",
// "collection_id"=excluded."collection_id"
// or its simple JavaScript equivalent:
var sets = cols.map(c=> {
var name = pgp.as.name(c);
return name + '=excluded.' + name;
}).join();
UPDATE
With version 7.3.0 of the library and later, you should use method assignColumns to generate all of the excluded sets, like this:
cs.assignColumns({from: 'EXCLUDED'})
//=> "application_id"=EXCLUDED."application_id","country_id"=EXCLUDED."country_id","collection_id"=EXCLUDED."collection_id"
or, if you want to skip application_id, then you can do:
cs.assignColumns({from: 'EXCLUDED', skip: 'application_id'})
//=> "country_id"=EXCLUDED."country_id","collection_id"=EXCLUDED."collection_id"
See ColumnSet.assignColumns
Don't use h.sets(). Just write the conflict_action yourself. Handbook says
The SET and WHERE clauses in ON CONFLICT DO UPDATE have access to the existing row using the table's name (or an alias), and to rows proposed for insertion using the special excluded table.
Postgres - Insert
Related
So I'm building this web forecast app using OpenWeatherMap API, and so far I'm being able to fetch the data from the first iteration of the list's output, however I need to obtain the data of other specific fields aswell. Here's a bit of my code:
ajaxGet("https://api.openweathermap.org/data/2.5/onecall?lat=4.6097&lon=-74.0817&exclude=current,minutely,hourly,alerts&appid=APPID&units=metric", function (response) {
var data = JSON.parse(response);
console.log(data);
var temperature = document.createElement("h6");
temperature.textContent = data.daily[0].temp.max + "°" + " / " + data.daily[0].temp.min + "°";
document.getElementById("temperaturaBogVier").appendChild(temperature);
});
And here's an example of what the API's output looks like (I'm only showing the first iteration in here, but there are at least 6 in total, https://api.openweathermap.org/data/2.5/onecall?lat=4.6097&lon=-74.0817&exclude=current,minutely,hourly,alerts&appid=APPID&units=metric):
{"lat":4.61,"lon":-74.08,"timezone":"America/Bogota","timezone_offset":-18000,"daily":
[
{"dt":1600876800,
"sunrise":1600857917,
"sunset":1600901504,
"temp":{"day":18.14,"min":8.99,"max":18.14,"night":12.08,"eve":15.45,"morn":8.99},
"feels_like":{"day":17,"night":11.02,"eve":14.6,"morn":7.58},
"pressure":1017,"humidity":54,
"dew_point":8.69,
"wind_speed":1.2,
"wind_deg":164,
"weather":[{"id":501,"main":"Rain","description":"moderate rain","icon":"10d"}],
"clouds":82,
"pop":0.94,
"rain":5.85,
"uvi":15.14}
]
}
So as you can see, I'm being able to print into my HTML the data contained into "data.daily[0].temp.", but it only works for the first set of fields and I got no clue how to select a specific iteration. I'm sure I'm missing something into the concat, but nothing I've tried has worked so far.
Any help would be greatly appreciated, and rewarded with an imaginary waffle. THX :D
The temperatures for each day data.daily are defined as an JavaScript array of objects. You can simply access them by their index, which indicates their position in the array.
data.daily[0] // First element
data.daily[1] // Second element
data.daily[2] // Third element
Once you have selected an object within the array, you can then access certain values like data.daily[2].temp.max.
The cool thing about arrays is that you can iterate them with a loop. This will save you a lot of writing, if you want to print out each temperatures for every day:
ajaxGet("https://api.openweathermap.org/data/2.5/onecall?lat=4.6097&lon=-74.0817&exclude=current,minutely,hourly,alerts&appid=YOUR_API_KEY_HERE&units=metric", function (response) {
var data = JSON.parse(response);
console.log(data);
data.daily.forEach(function (date) {
var temperature = document.createElement("h6");
temperature.textContent = date.temp.max + "°" + " / " + date.temp.min + "°";
document.getElementById("temperaturaBogVier").appendChild(temperature);
})
});
Please note: I've removed the appid=XXXXX part of the request URL, because it contains your personal API key for OpenWeather, which you should not share publicly.
If I understand the question correctly, you want to take all daily max/min-values and put them into elements that you want to append to another element.
Here is a way to do that
ajaxGet("https://api.openweathermap.org/data/2.5/onecall?lat=4.6097&lon=-74.0817&exclude=current,minutely,hourly,alerts&units=metric", function (response) {
var data = JSON.parse(response);
console.log(data);
data.daily
.map(datapoint => datapoint.temp) //get the temp value/object from each datapoint
.map(temp => { //create an element and add each max/min value to that element's textContent
const temperature = document.createElement("h6");
temperature.textContent = temp.max + "° / " + temp.min + "°";
return temperature;
})
.forEach(element => { //add each of the elements created in the previous map to the desired element
document.getElementById("temperaturaBogVier").appendChild(element);
});
});
As pointed out in the other answer, I've also removed the app-id query parameter
I was having some trouble when trying to retrieve from firebase. Here is my firebase structure:
What I tried to do is firstly, I wanted to get list of receiptItemID in the first picture. Then, after I get the IDs, for each ID, I wanted to get its quantity and type. After that, I will store them into array and perform some sorting.
Here is my code:
var query = firebase.database().ref('');
query.once( 'value', data => {
data.forEach(subtypeSnapshot => {
var itemKey = subtypeSnapshot.key;
var query = firebase.database().ref('').child(itemKey);
});
});
});
I managed to get the itemKey. However, when I tried to get the details of each receiptItem by the console.log that part, it prints out undefined for both. Any ideas on how to retrieve the data?
You don't need the forEach cycle, it's one level too deep. Instead, use the 'data' argument directly. This new callback should work:
var itemDetail = data.val();
var subtype = itemDetail.type;
var quantity = itemDetail.quantity;
console.log(subtype + ' ' + quantity);
In the first iteration of the forEach of your sample code, itemDetail will be equal to "Farmland" instead of the whole object; thus, subtype and quantity are null. In the new callback, itemDetail will be equal to the whole object, so subtype and quantity can be successfully declared.
var query = firebase.database().ref('receiptItems').child(itemKey);
query.once( 'value', data => {
var itemDetail = data.val();
var subtype = data.type;
// you may try data.toJSON().type as well
var quantity = data.quantity;
// try data.toJSON().quantity
console.log(subtype + ' ' + quantity);
});
In second retrieval you already have access to receiptItems/itemKey .This is a particular entry in receiptItems , not the whole receiptItems array.
So no need to apply data.forEach() once more as there is only one record. We apply data.forEach() to fetch an array of records/object. In your case it is just one entry.
I'm using vuejs for this project, but this problem is not necessarily connected - but if there is a vue-way, I would prefer that.
I'm building a table, that enables the user to have per-column-filters (in this case simple inputs). The columns are dynamic, so is the amount of data (thousands of rows, but less than 100.000 entries).
// example data
var columns = ['id', 'title', 'date', 'colour']
var data = [{ id: 1, title: 'Testentry 1', date: '2017-02-21T07:10:55.124Z', colour: 'green'}]
Here is the problem: I'm iterating over the columns, checking if a search-input exists, and if so, I try to filter the data based on the searchquery. In case of the ID, the time complexity is O(n). If I know search for a title additionally, I can reuse the result of the first searchquery, dramatically reducing the amount of data has to be looked at.
The searchqueries are stored in an object search, and the filtered data is a computed property, that gets updated whenever search changes. The way how that works though is, that if I change the searchquery for title, it would re-evaluate the searchquery even for the ID, although the searchquery for that didn't change.
This would require some kind of caching of data filtered for each column. And only the proceeding columns need to be queried upon.
edit: added code for the filtering:
filteredRows () {
var rows = this.data
for (var i = 0; i < this.columns.length; i++) {
var column = this.columns[i].name
var search = this.tSearch[column]
if (!search && search.length === 0) continue
console.log(column + ': ' + ' (' + search + ') -> ' + rows.length)
rows = _.filter(rows, (row) => {
var value = '' + row[column]
value.search(search) > -1
})
}
return rows
}
Just a suggestion, but did you try to use a watcher to get old and new value of input.
data: function() {
return {
propertyToWatch: 'something'
}
},
computed: {
...
},
watch: {
'propertyToWatch': function (val, oldVal) {
console.log(oldVal); // logs old value
console.log(val); // logs current value
// here you can call a function and send both of these args and detect diff
}
},
....
I found this method, with which I have ordered the hours what I store in my records, I wanted them to be sorted from highest to lowest, but when I test the code, I notice that only two values of the array are compared, which are the first registers.
I've seen other methods of comparison and the logic is the same, what am I doing wrong? I group the messages per user, using the id of the user as key of the array , then I save the rest of data. I do this for retrieve the current messages, since I want show a list of the last currently messages sent.
This is the code:
var ref = new Firebase('https://chatfbexample.firebaseio.com/all-messages');
ref.on("value", function (snapshot) {
var Messages = [];
var value = 0;
snapshot.forEach(function (snap) {
value = snap.val().id;
fecha = snap.val().id;
Messages[value] = [];
Messages[value]['fecha'] = snap.val().fechahora; //I receive a date with the format HH:MM:SS
Messages[value]['texto'] = snap.val().texto;
});
function compare(a, b) {
var time1 = a['fecha'].replace(/:/gi, '1');
var time2 = b['fecha'].replace(/:/gi, '1');
var data1 = parseInt(time1);
var data2 = parseInt(time2);
// alert(time1);
if (data1 > data2)
return -1;
if (data1 < data2)
return 1;
return 0;
}
Messages.sort(compare);
for (var i in Messages) {
console.log("Hour: " + Messages[i]['fecha'] + ' ' + ' Message: ' + Messages[i]['texto']);
}
});
the result is something like this
Hour: 12:11:13 Message: whats'up?
Hour: 11:38:44 Message: p
Hour: 11:49:01 Message: hey?
the second and the third messages are not being compared
an image of my Firebase database
First off it probably would be better for you to save a timestamp from Firebase instead of an actual time, which was probably created on the client side, inside your database. This way you could easily let Firebase handle the sorting.
You would need to use firebase.database.ServerValue.TIMESTAMP to save the exact time at which Firebase received your message inside fechahora.
Following you could simply query all of your messages ordered by their child fechahora.
ref.orderByChild("fechahora").on("value", function (snapshot) {
//Do stuff with your already sorted data
}
If you want your query to have a better performance set the .indexOn property of your node containing all the messages inside your Firebase Database rules to fechahora.
Bonus: If you want your data to be ordered newest to oldest instead of oldest to newest, you just need to use the negative value of the timestamp created by Firebase.
Instead of ordering client-side, you'll be better off asking the server to sort the messages:
ref.orderByChild("fechahora").on(...
Make sure that you define the proper index in your Firebase security rules:
{
"rules": {
"all-messages": {
".indexOn": "fechahora"
}
}
}
Without that the sorting will still work, but will happen on the client instead of the server.
In parse I have a class named "TestScore". Each object has a key named "quizName".
I need to get an array of unique "quizName" values. I wrote the below which queries the "TestClass" and loops through the results looking for unique "quizName" values.
At first seemed to do the job. But then I realized that the maximum number of returned objects is 1000. Soon there will be more than 1000 objects stored which means that this method will not guarantee that I end up will all values.
function loadTests(){
//create an array to hold each unique test name as we find them
var uniqueEntries = [];
//query parse to return TestScore objects
var TestScore = Parse.Object.extend("TestScore");
var query = new Parse.Query(TestScore);
query.limit(1000) //added this after realizing that the default query limit is only 100
query.find({
success: function(testScore) {
$(testScore).each(function(index, score) {
//here I loop though all of the returned objects looking at the "quizName" for each
if($.inArray(score.get("quizName"), uniqueEntries) === -1) {
//if the quiz name is not already in the "uniqueEntries" array, I add it to the array
uniqueEntries.push(score.get("quizName"));
}
});
//do stuff with quiznames here...., add them as options in select boxes mostly
}
});
}
I looked at {Parse.Query} notContainedIn(key, values) which looks promising but cant figure out if I can add values to the array as I find them. It seems like I would have to have an array to start with (defeating the whole point.)
This part of the docs "{Parse.Query} Returns the query, so you can chain this call." makes me think I might be able to chain queries together to get what I need, but that doesn't seem very efficient.
How can I retrieve unique values for key "quizName" when my class has > 1000 objects?
I'm sure you're long past this by now, but only way I know of to do it is to use one query after another by using a .skip(#) value for each query. So get 1000, then query again with .skip(1000), concatenate the items from the first list and second, then query again with .skip(2000), etc...
Be aware that I think there's a limit on skip values of 10,000. Don't take my word on that, just pointing you to something that I think is right that you should confirm if you think it applies to your situation.
I eventually found a tutorial online that I was able to modify and came up with the below. This effectively sets the return limit to 10,000 instead of 1,000 and allows setting several different parameters for the query.
My changes could surely be written better, maybe as an options object or similar but it works for my needs.
You can see a working demo here
function getStuff(){
// here we will setup and call our helper functions with callbacks to handle the results
var scheme =['SOTest',true]; // return all objects with value `true` in the `SOTest` column
// var scheme =['descending','createdAt']; // return all objects with sort order applied
// var scheme =''; // or just return all objects
// see `findChunk()` below for more info
var Remark = Parse.Object.extend("Remark");
schemePromise(Remark, scheme).done(function (all) {
console.log('Found ' + all.length+' Remarks');
$.each( all, function(i, obj){
$('#test').append(obj.get('Remark') +'<br>');
});
})
.fail(function (error) {
console.log("error: " + JSON.stringify(error));
});
}
getStuff(); // call our function
// helper functions used to get around parse's 1000 query limit
// raises the limit to 10,000 by using promises
function findChunk(model, scheme, allData) {
// if `scheme` was an empty string, convert to an array
// this is the default and returns all objects in the called class
if(scheme==''){ ['scheme',''] };
// will return a promise
var limit = 1000;
var skip = allData.length;
var findPromise = $.Deferred();
var query = new Parse.Query(model);
// to get all objects from the queried Class then sort them by some column
// pass `scheme` as an array like [ sort method, column to sort ]
if (scheme[0]=='descending') query.descending(scheme[1]);
else if (scheme[0]=='ascending') query.ascending(scheme[1]);
// to limt results to objects that have a certain value in a specific column
// pass `scheme` as an array like [ column name, value ]
else query.equalTo(scheme[0], scheme[1]);
// more options can easily be built in here using `scheme`
query
.limit(limit)
.skip(skip)
.find()
.then(function (results) {
findPromise.resolve(allData.concat(results), !results.length);
}, function (results) {
findPromise.reject(error);
});
return findPromise.promise();
}
function schemePromise(model, scheme, allResults, allPromise) {
// find a scheme at a time
var promise = allPromise || $.Deferred();
findChunk(model, scheme, allResults || [])
.done(function (results, allOver) {
if (allOver) {
// we are done
promise.resolve(results);
} else {
// may be more
schemePromise(model, scheme, results, promise);
}
})
.fail(function (error) {
promise.reject(error);
});
return promise.promise();
}