I have been using firebase-queue for years to invoke jobs for years, but recently it started failing with transaction errors competing on resources, and had serious scaling issue.
So I migrated firebase-queue to cloud function and it seemed to be fast and very scalable initially. I wrote a simple script that is hooked up by onWrite on queue/task/{taskId}. But it seems like sometimes onWrite is never triggered randomly. I can not see the specific job triggered in the logs at all.
We are using blaze plan, promise is returned on the function, timelimit is set as maximum, 99.9% of the jobs are taking less than 5000ms.
We have like 50K invocations per day, and around 100 jobs are not triggered. This is so intermittent and have no clue on that. One strange thing is I don't see any errors on log viewer, but on the invocations graph, I can see some load error and connection error, and still no idea what are they. Any insight to resolve the issue would be appreciated.
const functions = require(`firebase-functions`);
const admin = require(`firebase-admin`);
const _ = require(`lodash`);
const specs = require(`./specs.json`);
const queueSpecs = require(`./queue-specs.js`);
const SERVER_TIMESTAMP = { ".sv": `timestamp` };
admin.initializeApp();
const workers = {};
queueSpecs.forEach((spec) => {
let specString = spec;
if (typeof spec === `object`) {
specString = spec.id;
}
workers[specString] = require(`./lib/workers/${specString}`);
});
const taskHandler = (change, context) => {
const data = change.after.val();
const ref = change.after.ref;
const { taskID } = context.params;
if (!data) {
return null;
}
const worker = workers[data._state];
if (!worker) {
console.error(`Worker not found for task ${taskID} - ${data._state}`);
return null;
}
const specID = data._state.replace(`/`, `_`);
const spec = specs[specID];
if (!spec) {
console.error(`Spec not found for task ${taskID} - ${specID}`);
return null;
}
console.log(
`Running ${taskID} - ${data._state} - ${JSON.stringify(data, null, 2)}`
);
return new Promise((resolve, reject) => {
const taskResolve = newTask => {
const updates = _.clone(newTask);
let newState = _.get(newTask, `_new_state`, ``);
newState = newState || spec.finished_state;
if (!updates || !newState) {
console.log(`Exiting task - ${taskID}`);
return ref.set(null).then(resolve, reject);
}
updates._state = newState || spec.finished_state;
updates._state_changed = SERVER_TIMESTAMP;
updates._owner = null;
updates._progress = null;
updates._error_details = null;
console.log(`Resolving`);
return ref.update(updates).then(resolve, reject);
};
const taskReject = error => {
const updates = {};
let errorString;
if (_.isError(error)) {
errorString = error.message;
} else if (_.isString(error)) {
errorString = error;
} else if (!_.isUndefined(error) && !_.isNull(error)) {
errorString = error.toString();
}
if (updates._state === `error`) {
console.log(`Exiting task on reject - ${taskID}`);
return ref.set(null).then(resolve, reject);
}
updates._state = spec.error_state || `error`;
updates._state_changed = SERVER_TIMESTAMP;
updates._owner = null;
updates._progress = null;
updates._error_details = {
previous_state: spec.in_progress_state,
error: errorString,
errorStack: _.get(error, `stack`, null),
attempts: 1
};
console.log(`Rejecting`);
return ref.update(updates).then(resolve, reject);
};
const taskProgress = () => {
// eslint-disable-line
// progress is never used, thus just resolving
const updates = {};
console.log(`Progress ????`);
return ref.update(updates).then(resolve, reject);
};
worker(data, taskProgress, taskResolve, taskReject);
});
};
exports.taskRunner = functions.database
.ref(`/queue/tasks/{taskID}`)
.onWrite(taskHandler);
Related
I am making a function for firebase cloud functions, I want a function to be called every time a new document is created in "posts". I want this function to perform the tasks that I put inside the "onCeatePost" function.
The problem I have is that I'm not sure if this is the correct way to structure such a function.
In several firebase examples I have seen that it is always called return _; or return null; at the end of a task, but I don't know how to structure the function so that all the tasks are carried out, could someone help me to restructure my function or tell me what is wrong please.
There are several if statements in the function, if the created publication does not comply with them, I would like it to skip them but continue with the other tasks that I put inside the function.
I don't know if it's too much to ask, but I'm new to this language and I haven't been able to find the answer I'm looking for. Thank you!
exports.onPostCreate = functions.firestore.document("/posts/{postId}").onCreate(async (snap) => {
const post = snap.data();
if (post) {
try {
const topic = post.topic;
const contentForFeed = post.contentForFeed;
const uid = post.uid;
const previous = post.prev;
await db.collection("users").doc(uid).update({"stats.posts": admin.firestore.FieldValue.increment(1)});
if (topic) {
await db.collection("topics").doc(topic.id).collection("user-authors").doc(uid).set({"date": snap.createTime});
}
if (contentForFeed == true) {
const userPath = db.collection("users").doc(uid);
await userPath.update({"stats.lastUpdate": snap.createTime});
}
if (previous) {
const previousId = previous.id;
const previousUid = previous.uid;
const refPrev = db.collection("posts").doc(previousId);
await db.runTransaction(async (t) => {
const doc = await t.get(refPrev);
const priority = doc.data().stats.date;
const newDate = new admin.firestore.Timestamp(priority.seconds + 120, priority.nanoseconds);
await db.collection("posts").doc(previousId).update({"newDate": newDate});
});
if (previousUid != uid) {
const path = db.collection("users").doc(uid).collection("user-posts");
const dataToSet = {"timestamp": snap.createTime, "uid": uid, "postId": onReplyToPostId};
await path(dataToSet);
}
}
} catch (err) {
functions.logger.log(err);
}
} else {
return null;
}
});
You'll find below the adapted code (untested) with 4 corrections.
Here are explanations for the two most important ones:
(Correction 2) In a transaction you need to use the transaction's update() method and not the "standard one"
(Correction 4) When all the asynchronous work is complete you need to return a value or a Promise. See this documntation page for more details.
exports.onPostCreate = functions.firestore
.document('/posts/{postId}')
.onCreate(async (snap) => {
const post = snap.data();
if (post) {
try {
const topic = post.topic;
const contentForFeed = post.contentForFeed;
const uid = post.uid;
const previous = post.prev;
await db
.collection('users')
.doc(uid)
.update({
'stats.posts': admin.firestore.FieldValue.increment(1),
});
if (topic) {
await db
.collection('topics')
.doc(topic.id)
.collection('user-authors')
.doc(uid)
.set({ date: snap.createTime });
}
if (contentForFeed == true) {
const userPath = db.collection('users').doc(uid);
await userPath.update({ 'stats.lastUpdate': snap.createTime });
}
let previousUid; // <= Correction 1
if (previous) {
const previousId = previous.id;
previousUid = previous.uid; // <= Correction 1
const refPrev = db.collection('posts').doc(previousId);
await db.runTransaction(async (t) => {
const doc = await t.get(refPrev);
const priority = doc.data().stats.date;
const newDate = new admin.firestore.Timestamp(
priority.seconds + 120,
priority.nanoseconds
);
t.update(refPrev, { newDate: newDate }); // <= Correction 2
});
if (previousUid != uid) {
const path = db
.collection('users')
.doc(uid)
.collection('user-posts');
const dataToSet = {
timestamp: snap.createTime,
uid: uid,
postId: onReplyToPostId,
};
await path.add(dataToSet); // <= Correction 3
}
}
return null; // <= Correction 4
} catch (err) {
functions.logger.log(err);
}
} else {
return null;
}
});
First time using stackoverflow. It is a bot made to post result whenever new episode of show in search list gets added on nyaa.si. I want bot to post result only once for every episode but bot post same episode multiple time in different time frames. It gets fixed for while after I restart the bot.
The code to add show to search list.
async addShow(msg) {
const regex = /"(.+?)" "(.+?)"(?: "(.+?)")?/g;
const found = regex.exec(msg.content);
if (found === null) {
await msg.channel.send(`Invalid new syntax:\n${COMMAND_CHARACTER} new \"show search phrase\" \"MALURL\" \"attribute regex\" (optional last)`);
return;
}
let [f, search, url, reg] = found;
let count = await this.db.get('search').last().value();
if (_.isUndefined(count)) {
count = 0;
}
else {
count = count.id;
}
await this.db.get('search').push({id: count + 1, search, url, regex: reg}).write();
logger.info(`New show has been added to the searchlist - ${search} - ${url} for server ${this.guildID}`);
await msg.channel.send("Saved!");
}
The code to search
async searchShow(id, query, channel = null, OG = null) {
const results = await this.nyaa.getResults(query);
if (!results.length) {
return;
}
logger.info(`Results found for ${query}: ${results.length}`);
const embedFunction = this.getRichEmbed.bind(this);
for (let i of results) {
const item = await this.db.get('rss').find({guid: i.guid}).value();
if (!_.isUndefined(item)) {
continue;
}
if (await this.postShow(embedFunction, i, channel, OG)) {
await this.db.get('rss').push({...i, searchID: id}).write();
}
}
}
Code to post result when new episode comes.
async postShow(embedFunc, item, channel = null, og = null, channelType = NYAA_UPDATES) {
if (channel === null) {
channel = await this.getGuildChannel(channelType);
if (!channel) {
return false;
}
}
return new Promise(async (resolve) => {
const title = (og !== null ? og.title ?? item.title : item.title);
const embed = await embedFunc(item, title);
if (og !== null) {
const img = og.image ?? null;
if (img) {
embed.setThumbnail(img);
}
const url = og.url ?? null;
if (url) {
embed.setURL(url);
}
}
let retryCounter = 0;
logger.info(`Posting new result for ${title} with guid ${item.guid} for server ${this.guildID}`);
while (true) {
try {
await channel.send(embed);
setTimeout(() => {
resolve(true);
}, 2000);
break;
}
catch (e) {
logger.warn(`An error has occured while posting: ${e.toString()}, retrying (${++retryCounter} in 5 seconds`);
await new Promise((res) => {
setTimeout(() => {
res();
}, 5000);
});
if (retryCounter > 10) {
resolve(false);
}
}
}
});
}
Also one who wrote most of code was different person and I only added few additional function here and there which doesn't affect the code much. One who wrote most of core code had to leave discord so I was left to host the bot which I am hosting at repl.it. It will be great help to know whether the problem is with the code or not.
As Julian Kleine mentioned above, the most common reason a bot posts multiple times is if you are running multiple instances of the host. Close all instances of command prompt, and check task manager to see if any other hidden instances are running in the background.
There is a custom hook
const useStompSubscription = () => {
const dispatch = useDispatch();
const routerLocation = useLocation();
const namespace = useSelector(bootstrapNamespaceSelector);
console.log("routerLocation", routerLocation); // logs correctly when routerLocation changes
useEffect(() => {
console.log("useEffect", routerLocation); // logs correctly when routerLocation changes
if (!namespace) {
return;
}
const stompClient = getSTOMPClient(
namespace,
function(message) {
console.log("onReceiveQueueMessage", routerLocation); // logs only the first route where initialization happened
if (
routerLocation.pathname !== QUEUE_PATH &&
routerLocation.pathname !== WORKLOAD_PATH
)
return;
let payload = JSON.parse(message.body);
if (payload.status === PRINTED || payload.status === CANCELLED) {
dispatch(onDeletePrintingQueueAction(payload));
} else {
dispatch(onCreateOrUpdatePrintingQueueAction(payload));
}
},
function(message) {
console.log("onReceiveManifestMessage", routerLocation); // logs only the first route where initialization happened
if (routerLocation.pathname !== DELIVERY_MANIFEST_PATH) return;
let payload = JSON.parse(message.body);
dispatch(onCreateOrUpdateDeliveryPackageAction(payload));
},
);
stompClient.activate();
return () => {
stompClient.deactivate();
};
}, [dispatch, namespace, routerLocation, routerLocation.pathname]);
};
It calls getSTOMPClient function, passing 2 callbacks - onReceiveQueueMessage and onReceiveManifestMessage, which access the routerLocation from the parent scope.
let instance;
export const getSTOMPClient = (
namespace,
onReceiveQueueMessage,
onReceiveManifestMessage,
) => {
if (isNil(instance)) {
let host;
let url;
if (window.location.port === "") {
host = `${window.location.hostname}`;
} else {
host = `${window.location.hostname}:${window.location.port}`;
}
if (window.location.protocol === "https:") {
url = `wss://${host}/ws`;
} else {
url = `ws://${host}/ws`;
}
instance = new Client({
brokerURL: url,
onConnect() {
const queueTopic = `/ui/${namespace}/queue`;
this.queueSubscription = this.subscribe(
queueTopic,
onReceiveQueueMessage,
);
const dmTopic = `/ui/${namespace}/dm`;
this.dmSubscription = this.subscribe(dmTopic, onReceiveManifestMessage);
},
onDisconnect() {
const defaultSub = { unsubscribe: noop };
const queueSubscription = defaultTo(this.queueSubscription, defaultSub);
const dmSubscription = defaultTo(this.dmSubscription, defaultSub);
queueSubscription.unsubscribe();
dmSubscription.unsubscribe();
},
// Helps during debugging, remove in production
debug: str => {
console.log(new Date(), str);
},
});
}
return instance;
};
What I expect is that whenever the callbacks are fired they will access the routerLocation from useStompSubscription scope.
The console.log("routerLocation", routerLocation); and console.log("useEffect", routerLocation); log correctly the changes.
However in the callbacks always the first routerLocation is used, the changes doesn't seem to affect it.
This seems to be simple closure.
What am I missing in here?
I wrote an Async/Await function to return promises for drivers report and analysis.
I have three different promise API files I extracted details from to do my analysis. However running test ith jest I get the error
Timeout - Async callback was not invoked within the 5000ms timeout specified by jest.setTimeout.Error:
I have refactored my code more than three times in two days but the error returns.
I will like to know how to manage my promises, perhaps there is something am not doing well and I am keen on this for optimization.
Is there a way to manage the promises in the code below to bypass the jest error?
any other suggestion will be highly appreciated.
NB: sorry I have post all the code for better insight.
code
const { getTrips } = require('api');
const { getDriver } = require('api')
const { getVehicle } = require('api')
/**
* This function should return the data for drivers in the specified format
*
* Question 4
*
* #returns {any} Driver report data
*/
async function driverReport() {
// Your code goes here
let trip = await getTrips()
trip = trip.map(item => {
item.billedAmount = parseFloat(item.billedAmount.toString().replace(',', '')).toFixed(2);
return item;
})
let getId = trip.reduce((user, cur) => {
user[cur.driverID] ? user[cur.driverID] = user[cur.driverID] + 1 : user[cur.driverID] = 1
return user
}, {})
// console.log(getId)
let mapId = Object.keys(getId)
// console.log(mapId)
let eachTripSummary = mapId.reduce((acc, cur) => {
let singleTrip = trip.filter(item => item.driverID == cur)
acc.push(singleTrip)
return acc
}, [])
// eachTripSummary = eachTripSummary[0]
// console.log(eachTripSummary)
// console.log(trip)
let reducedReport = eachTripSummary.reduce(async(acc, cur) =>{
acc = await acc
// console.log(acc)
let user = {}
let cash = cur.filter(item => item.isCash == true)
// console.log(cash.length)
let nonCash = cur.filter(item => item.isCash == false)
let driverSummary = await getDriverSummary(cur[0]['driverID'])
let trips = []
let customer = {}
cur[0].user ? (customer['user'] = cur[0]['user']['name'], customer['created'] = cur[0]['created'], customer['pickup'] = cur[0]['pickup']['address'],
customer['destination'] = cur[0]['destination']['address'], customer['billed'] = cur[0]['billedAmount'], customer['isCash'] = cur[0]['isCash']) : false
trips.push(customer)
let vehicles = []
if(driverSummary == undefined){
// console.log(cur)
user = {
id: cur[0]['driverID'],
vehicles: vehicles,
noOfCashTrips: cash.length,
noOfNonCashTrips: nonCash.length,
noOfTrips: cur.length,
trips: trips
}
acc.push(user)
// console.log(user)
return acc
}
let driverInfo = driverSummary[0]
let vehicleInfo = driverSummary[1]
let { name, phone } = driverInfo
let { plate, manufacturer } = vehicleInfo[0]
// console.log(plate)
let vpm = {
plate,
manufacturer
}
vehicles.push(vpm)
// console.log(cash.length)
user ={
fulName: name,
phone,
id: cur[0]['driverID'],
vehicles: vehicles,
noOfCashTrips: cash.length,
noOfNonCashTrips: nonCash.length,
noOfTrips: cur.length,
trips: trips
}
acc.push(user)
// console.log(acc)
return acc
}, [])
// reducedReport.then(data =>{console.log(data)})
return reducedReport
}
async function getDriverSummary(param) {
let driverDetails = await getDriver(param)
.then(data => {return data}).catch(err => {return err})
// console.log(driverDetails)
let vehicleDetails;
let { vehicleID } = driverDetails
if(driverDetails != "Error" & vehicleID != undefined){
// console.log(vehicleID)
vehicleDetails = vehicleID.map(async item => {
let vehicleSummary = getVehicle(item)
return vehicleSummary
})
// console.log(await vehicleDetails)
return await Promise.all([driverDetails, vehicleDetails])
}
}
driverReport().then(data => {
console.log(data)
})
module.exports = driverReport;
Use jest.setTimeout(30000); to increase the timeout. It will increase the timeout globally.
// jest.config.js
module.exports = {
setupTestFrameworkScriptFile: './jest.setup.js'
}
// jest.setup.js
jest.setTimeout(30000)
Or you can use user test example like this
describe("...", () => {
test(`...`, async () => {
...
}, 30000);
});
I am trying to get the new wit.ai Bot Engine connected to hubot with Javascript.
Unfortunately I am not a JS Developer so I'm struggling.
Here's the code I have:
'use strict';
const Wit = require('../../../node-wit').Wit;
const firstEntityValue = (entities, entity) => {
const val = entities && entities[entity] &&
Array.isArray(entities[entity]) &&
entities[entity].length > 0 &&
entities[entity][0].value
;
if (!val) {
return null;
}
return typeof val === 'object' ? val.value : val;
};
const actions = {
say: (sessionId, msg, cb) => {
console.log(msg);
cb();
},
merge: (context, entities, cb) => {
const loc = firstEntityValue(entities, "location");
if (loc) context.loc = loc;
cb(context);
},
error: (sessionId, msg) => {
console.log('Oops, I don\'t know what to do.');
},
'fetch-weather': (context, cb) => {
// Here should go the api call, e.g.:
// context.forecast = apiCall(context.loc)
context.forecast = 'sunny';
cb(context);
},
};
const client = new Wit('MY_TOKEN_HERE', actions);
client.interactive();
module.exports = function(robot) {
robot.respond(/hey\s+(.+$)/i, function(msg){
var match = msg.match[1];
msg.send("I've heared: " + match);
console.log(match)
process.stdout.write(match);
});
}
The script listens to "hey botname" and outputs what was written after this. My Problem is I have no clue how to get this input sent to the wit client. Using this script in bash without the hubot stuff works fine towards wit as this is based on the quick start example from wit.ai.
The other issue I'm facing is that I would like to have hubot listen in a private channel with every user and have it respond to every message without prefix. Just as the node example does in the console.
Help is highly apprechiated!
Ok, after fiddling a while I made this work.
Here's what my hubot script looks like now:
'use strict';
const Wit = require('../../../node-wit').Wit;
var room;
const firstEntityValue = (entities, entity) => {
const val = entities && entities[entity] &&
Array.isArray(entities[entity]) &&
entities[entity].length > 0 &&
entities[entity][0].value
;
if (!val) {
return null;
}
return typeof val === 'object' ? val.value : val;
};
const actions = {
say: (sessionId, msg, cb) => {
console.log(msg);
room.send(msg)
cb();
},
merge: (context, entities, cb) => {
const loc = firstEntityValue(entities, "location");
if (loc) context.loc = loc;
cb(context);
},
error: (sessionId, msg) => {
console.log('Oops, I don\'t know what to do.');
room.send('Oops, I don\'t know what to do.')
},
};
const client = new Wit('MY_TOKEN_HERE', actions);
//client.interactive();
module.exports = function(robot) {
robot.listen(function(msg) {
var userID = msg.user.id;
var roomID = msg.user.room;
// is this a direct chat(private room)?
if(roomID.indexOf(userID) >= 0){
if(typeof msg.text == "string"){
client.pxMessage(msg.text);
}
}
return true;
},
function(response){
// save room for replys
room = response;
});
}
additionally I made an awful hack to wit.js to get this work. I added the following function as I was not able to use the available methods to get this working. Basically callbacks and session were holding me back:
this.pxMessage = (message) => {
const sessionId = uuid.v1();
const context = {};
const steps = 5;
this.runActions(
sessionId,
message,
context,
(error, context) => {
if (error) {
l.error(error);
}
rl.prompt();
},
steps
);
}
If someone would take this further and implement it properly I would love to see the result. This hack works and we now have a really smart bot in our rocket.chat who understands natural language and learns every day.
You can directly use this module, it seems to be working well: https://www.npmjs.com/package/hubot-wit
I have just now finished integration. You just need to provide the environment variable WIT_TOKEN and it works beautifully!