I am working on a performance issue of a function, that takes 15sec to response, which makes a request to firebase for all documents that are
"ErrorID" "==" "0"
The problem is that there are many documents and they are kind of very large objects, and I only need TWO FIELDS (Order and Amount) of each document, there are any way to request only those two fields that accomplish the condition?
Something like :
firestore.collection("purchases").where("ErrorID", "==", "0").get(Order, Amount);
The function that im talking about:
const totalEarn = async (req, res, next) => {
const DAY = 86400000;
const WEEK = 604800016;
const MONTH = 2629800000;
try {
let snap = await firestore.collection("purchases").where("ErrorID", "==", "0").get(); // CONDITION
let totalToday = 0;
let totalYesterday = 0;
let totalLastWeek = 0;
let totalLastMonth = 0;
let now = Date.now();
let Yesterday = now - 86400000;
await snap.forEach((doc) => { // THIS FOR EACH TAKES TOO MUCH TIME
let info = doc.data();
let time = info.Order.split("-")[2]; // FIRESTORE FIELD -> ORDER
let amount = info.AmountEur * 1; // FIRESTORE FIELD -> AMOUNT
if (time > now - DAY) {
totalToday = totalToday + amount;
}
if (time < now - DAY && time > Yesterday - DAY) {
totalYesterday = totalYesterday + amount;
}
if (time > now - WEEK) {
totalLastWeek = totalLastWeek + amount;
}
if (time > now - MONTH) {
totalLastMonth = totalLastMonth + amount;
}
});
res.send({
status: true,
data: {
totalToday: totalToday.toFixed(2),
totalYesterday: totalYesterday.toFixed(2),
totalLastWeek: totalLastWeek.toFixed(2),
totalLastMonth: totalLastMonth.toFixed(2),
},
});
} catch (error) {
res.status(410).send({
status: false,
error: "some error occured counting the numbers",
e: error.message,
});
}
};
The document im talking about
If you use Firestore Node.JS client or Firebase Admin SDK, then you can use select() to select fields:
import { Firestore } from "#google-cloud/firestore";
const firestore = new Firestore();
const snap = firestore
.collection("purchases")
.where("ErrorID", "==", "0")
.select("Order", "Amount")
.get();
How would one fetch messages from text channel beginning with the latest/newest message until a specific date. For example until the date two days ago.
The desired result is having a function that will do the job and return an array of messages dating in a range: now -> end date specified as the function's argument.
This would be my approach, feel free to post your own better answers :3
async function fetchMessagesUntil(channel, endDate, lastID) {
let messages = (await channel.messages.fetch({ limit: 100, before: lastID })).array();
if (messages.length == 0) return messages;
for (let i = 0; i < messages.length; i++) {
if (messages[i].createdAt.getTime() < endDate.getTime()) {
return messages.slice(0, i);
}
}
return messages.concat(
await fetchMessagesUntil(channel, endDate, messages[messages.length - 1].id)
);
}
Example usage
let end = new Date();
end.setDate(end.getDate() - 2); // Subtract two days from now
(await fetchMessagesUntil(message.channel, end)).forEach(x => console.log(x.content));
I have this function:
function getInfoSchoolTime() {
var date = new Date();
var schoolBellTime = ["8:10","9:02","9:54","9:59","10:51","11:43","11:58","12:48","13:35","13:40","14:10","15:02","15:54"];
var remainingTime, currentHour;
for (var i = 0; i < schoolBellTime.length-1; i++) {
var startTime = schoolBellTime[i].split(":");
var endTime = schoolBellTime[i+1].split(":");
if (parseInt(startTime[0]) >= date.getHours() && parseInt(startTime[1]) >= date.getMinutes())
if (parseInt(endTime[0]) <= date.getHours() && parseInt(endTime[1]) <= date.getMinutes()) {
currentHour = i;
remainingTime=(parseInt(endTime[1])-date.getMinutes()+60)%60;
break;
}
}
if (currentHour == undefined)
return {current: -1, remaining: "not available"};
return {current: currentHour, remaining: remainingTime};
}
var info = getInfoSchoolTime();
console.log(info.current, info.remaining);
I have the schoolBellTime array that contains the timestamps of my school bell (I know, my school has strange bell times, these timestamps includes playtimes and lunchtime), this function is meant to return the 1st hour/2nd hour/3rd hour ... and the minutes that remains to the next hour/breaktime.
I checked all the code and can't find the error, it keeps returning {current: -1, remaining: "not available"}
The function at the top: setDateTime() takes a date and a time, and constructs a date object for that time.
Then I updated your function, I convert start and end to times on the current day, and then check if date.getTime() occurs between them. Then I simply subtract date.getTime() from end, and convert the result to minutes from milliseconds.
var setDateTime = function(date, str) {
var sp = str.split(':');
date.setHours(parseInt(sp[0], 10));
date.setMinutes(parseInt(sp[1], 10));
return date;
}
function getInfoSchoolTime() {
var date = new Date();
var schoolBellTime = ["8:10", "9:02", "9:54", "9:59", "10:51", "11:43", "11:58", "12:48", "13:35", "13:40", "14:10", "14:10", "15:02", "15:54"];
var remainingTime, currentHour, currentPeriod;
for (var i = 0; i < schoolBellTime.length - 1; i++) {
start = setDateTime(new Date(), schoolBellTime[i])
end = setDateTime(new Date(), schoolBellTime[i + 1])
if (date.getTime() > start.getTime() && date.getTime() < end.getTime()) {
currentHour = i
remainingTime = end.getTime() - date.getTime()
currentPeriod = ([schoolBellTime[i], schoolBellTime[i+1]]).join('-')
}
}
return {current: currentHour, currentPeriod: currentPeriod, remaining: Math.round(remainingTime * 0.0000166667)}
}
console.log(getInfoSchoolTime())
Here's a somewhat different approach, both to the code and the API. It uses two helper functions. Each should be obvious with a single example: pad(7) //=> "07" and pairs(['foo', 'bar', 'baz', 'qux']) //=> [['foo', 'bar'], ['bar', 'baz'], ['baz', 'qux']].
The main function takes a list of bell times and returns a function which itself accepts a date object and returns the sort of output you're looking for (period, remaining time in period.) This API makes it much easier to test.
const pad = nbr => ('00' + nbr).slice(-2)
const pairs = vals => vals.reduce((res, val, idx) => idx < 1 ? res : res.concat([[vals[idx - 1], val]]), [])
const schoolPeriods = (schoolBellTime) => {
const subtractTimes = (t1, t2) => 60 * t1.hour + t1.minute - (60 * t2.hour + t2.minute)
const periods = pairs(schoolBellTime.map(time => ({hour: time.split(':')[0], minute: +time.split(':')[1]})))
return date => {
const current = {hour: date.getHours(), minute: date.getMinutes()}
if (subtractTimes(current, periods[0][0]) < 0) {
return {message: 'before school day'}
}
if (subtractTimes(current, periods[periods.length - 1][1]) > 0) {
return {message: 'after school day'}
}
const idx = periods.findIndex(period => subtractTimes(current, period[0]) >= 0 && subtractTimes(period[1], current) > 0)
const period = periods[idx]
return {
current: idx + 1,
currentPeriod: `${period[0].hour}:${pad(period[0].minute)} - ${period[1].hour}:${pad(period[1].minute)}`,
remaining: subtractTimes(period[1], current)
}
}
}
const getPeriod = schoolPeriods(["8:10","9:02","9:54","9:59","10:51","11:43","11:58","12:48","13:35","13:40","14:10","14:10","15:02","15:54"])
console.log("Using current time")
console.log(getPeriod(new Date()))
console.log("Using a fixed time")
console.log(getPeriod(new Date(2017, 11, 22, 14, 27))) // will Christmas break ever come?!
I made a random guess at the behavior you would want if the date is outside the period range.
Internally, it creates a list of period objects that look like
[{hour:9, minute: 59}, {hour: 10, minute: 51}]
Perhaps it would be cleaner if instead of a two-element array it was an object with start and end properties. That would be an easy change.
Do note that for this to make sense, the bells need to be listed in order. We could fix this with a sort call, but I don't see a good reason to do so.
Here is an ES6 example using deconstruct (const [a,b]=[1,2]), array map, array reduce, partial application (closure) and fat arrow function syntax.
This may not work in older browsers.
//pass date and bellTimes to function so you can test it more easily
// you can partially apply bellTimes
const getInfoSchoolTime = bellTimes => {
//convert hour and minute to a number
const convertedBellTimes = bellTimes
.map(bellTime=>bellTime.split(":"))//split hour and minute
.map(([hour,minute])=>[new Number(hour),new Number(minute)])//convert to number
.map(([hour,minute])=>(hour*60)+minute)//create single number (hour*60)+minutes
.reduce(//zip with next
(ret,item,index,all)=>
(index!==all.length-1)//do not do last one, create [1,2][2,3][3,4]...
? ret.concat([[item,all[index+1]]])
: ret,
[]
);
return date =>{
//convert passed in date to a number (hour*60)+minutes
const passedInTime = (date.getHours()*60)+date.getMinutes();
return convertedBellTimes.reduce(
([ret,goOn],[low,high],index,all)=>
//if goOn is true and passedInTime between current and next bell item
(goOn && passedInTime<high && passedInTime>=low)
? [//found the item, return object and set goOn to false
{
current: index+1,
currentPeriod: bellTimes[index]+"-"+bellTimes[index+1],
remaining: high-passedInTime
},
false//set goOn to false, do not continue checking
]
: [ret,goOn],//continue looking or continue skipping (if goOn is false)
[
{current: 0, currentPeriod: "School is out", remaining: 0},//default value
true//initial value for goOn
]
)[0];//reduced to multiple values (value, go on) only need value
}
};
//some tests
const date = new Date();
//partially apply with some bell times
const schoolTime = getInfoSchoolTime(
[
"8:10", "9:02", "9:54", "9:59", "10:51",
"11:43", "11:58", "12:48", "13:35", "13:40",
"14:10", "14:10", "15:02", "15:54"
]
);
//helper to log time from a date
const formatTime = date =>
("0"+date.getHours()).slice(-2)+":"+("0"+date.getMinutes()).slice(-2);
date.setHours(11);
date.setMinutes(1);
console.log(formatTime(date),schoolTime(date));//11:01
date.setHours(15);
date.setMinutes(53);
console.log(formatTime(date),schoolTime(date));//15:53
date.setHours(23);
date.setMinutes(1);
console.log(formatTime(date),schoolTime(date));//23:01
I am trying to parse and add into mongodb database some data that am getting from an API. I want to get all the data for every user from specific time till today.
So what I am doing is, I am retrieving for each user data for 5 days for each iteration, so its like 2-3 month data seperating into 5 days.
For some reason I am getting this error with the Allocation Failer - Process out of memory.
Seems like I get this error at the time when I arrive at a particular user, cz he seems having more data than others.
I did tried this command when running the script: node --max-old-space-size=4028 worksnap.js.
My code looks like this:
var currentMonth = new Date();
var startDate = new Date("February 1, 2016 00:00:00"); //Start from February
var counter = 1;
while (startDate.getMonth() <= currentMonth.getMonth()) {
//todo:: look if u have to increaze the start time, due the previous end time becomes start time it can take the same time time entries (have to be reviewd and make sure)....
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 5);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
}
function getTimeEntriesFromWorksnap(error, response, body) {
//console.log(response.statusCode);
if (!error && response.statusCode == 200) {
parser.parseString(body, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
_.forEach(timeEntries, function (timeEntry) {
_.forEach(timeEntry, function (item) {
saveTimeEntry(item);
});
});
});
}
}
function saveTimeEntry(item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
throw err;
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('item inserted...');
}
});
});
}
function iterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
_.forEach(students, function (student, i) {
if (student.worksnap.user != null) {
setTimeout(function () {
var options = {
url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic bGhNSVwJkVUFasSxx2loOFVyZkFyOENEZEsdxxxCdUlHdElWMHo0czo='
}
};
request(options, getTimeEntriesFromWorksnap);
}, 5000 * i);
}
});
});
}
Anyone knows what I am doing wrong here?
This is more a comment, as it does not contain a solution.
There are two things that looks fishy:
One problem is with:
while (startDate.getMonth() <= currentMonth.getMonth()) {
//todo:: look if u have to increaze the start time, due the previous end time becomes start time it can take the same time time entries (have to be reviewd and make sure)....
var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 5);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
}
You don't wait until you process the data of one student, but you request the data of all students in parallel.
A similar problem is the setTimeout, because depending on the execution time your code needs to hold the data of multiple requests in memory.
You should use something like async or Promise to solve asynchrone loops.
I have a Keystone.js blog and I want to add blog archives similar to Wordpress /archive/year/month. I added some extra date fields to the post object but I feel there is a way to do this using the published date.
Right now archive year is just '2014' and archive month is '06', while the '-publishedDate' value would be something like "publishedDate" : Date( 1355644800000 ). Is there a way to write a function in the query to parse the date as a JS date object then match the values?
// Load the posts
view.on('init', function(next) {
var q = keystone.list('Post').paginate({
page: req.query.page || 1,
perPage: 10,
maxPages: 10
})
.where('state', 'published')
.sort('-publishedDate')
.populate('author categories');
if (locals.data.category) {
q.where('categories').in([locals.data.category]);
}
// If archive section, filter by year and month
if (locals.data.archiveYear && locals.data.archiveMonth) {
q.where('-publishedDate',locals.data.archiveYear);
q.where('-publishedDate',locals.data.archiveMonth);
}
q.exec(function(err, results) {
locals.data.posts = results;
next(err);
});
});
Using moment.js similar code than user1572796
if (locals.filters.year) {
var start = moment().year(locals.filters.year).month(locals.filters.month).startOf('month');
var end = moment().year(locals.filters.year).month(locals.filters.month).endOf('month');
q.where('publishedDate', { $gt: start, $lt: end });
}
This seems to work:
// Load the posts
view.on('init', function(next) {
var q = keystone.list('Post').paginate({
page: req.query.page || 1,
perPage: 10,
maxPages: 10
})
.where('state', 'published')
.sort('-publishedDate')
.populate('author categories');
if (locals.data.category) {
q.where('categories').in([locals.data.category]);
}
function daysInMonth(month,year) {
return new Date(year, month, 0).getDate();
}
if (locals.filters.year && locals.filters.month) {
var postMonth = locals.filters.month - 1;
var start = new Date(locals.filters.year, postMonth, 1);
var end = new Date(locals.filters.year, postMonth, daysInMonth(locals.filters.month, locals.filters.year));
q.find({publishedDate: { $gte: start, $lt: end }});
}
q.exec(function(err, results) {
locals.data.posts = results;
next(err);
});