Develop Reference

Firestore : why using serverTimestamp gives different results?

I am having a hard time understanding serverTimestamp in firestore.
When I save a document in database in a firebase function using Fieldvalue.serverTimestamp() or in a javascript client code using serverTimestamp() it sometimes doesn't save the same thing in the database.
See screenshots below :
Sometime I get an object with {nanoseconds: xxx, seconds: xxx} and sometimes I get a timestamp formatted date...
The problem is when I try to query my orders using query(collectionRefOrders, orderBy('createdAt', 'desc'), limit(10)).
The orders with the object appears before the others ones even if they are created after...
Any clue why this happens ? What am I doing wrong ?
Thanks a lot.
EDIT :
Here is the code I use to add documents in the my firebase function (it is a request function I call in a website) :
const { getFirestore, FieldValue } = require('firebase-admin/firestore');
const firebaseDB = getFirestore();
exports.createOrderFromTunnel = functions.region('europe-west3')
.runWith({
timeoutSeconds: 10,
memory: "4GB",
})
.https
.onRequest(async (req, res) => {
cors(req, res, async () => {
try {
const { apiKey } = req.body;
const project = await getProjectFromApiKey(apiKey);
if (!project) {
return res.json({
success: false,
error: 'Unauthorized: invalid or missing api key'
});
}
const contactData = {
address: {},
createdAt: FieldValue.serverTimestamp()
};
const orderData = {
accounting: {
totalHT: 0,
totalTTC: 0,
totalTVA: 0,
},
createdAt: FieldValue.serverTimestamp(),
status: 'NEW',
};
const refProject = firebaseDB
.collection('projects')
.doc(project.id);
const colOrder = firebaseDB.collection(`projects/${project.id}/orders`)
const refOrder = colOrder.doc();
const colContact = firebaseDB.collection(`projects/${project.id}/contacts`)
const refContact = colContact.doc();
await firebaseDB.runTransaction(async transaction => {
const snapProject = await transaction.get(refProject);
const dataProject = snapProject.data();
const sequenceContact = dataProject.sequenceContact;
const sequenceOrder = dataProject.sequenceOrder;
contactData.sequence = sequenceContact;
orderData.sequenceNumber = sequenceOrder;
await transaction.set(refContact, contactData);
orderData.customer.id = refContact.id;
orderData.customer.sequence = sequenceContact;
await transaction.set(refOrder, orderData);
await transaction.update(refProject, {
sequenceContact: sequenceContact + 1,
sequenceOrder: sequenceOrder + 1,
totalContacts: dataProject.totalContacts + 1,
totalOrders: dataProject.totalOrders + 1,
});
return refOrder.id;
});
return res.json({
success: true
});
} catch (err) {
functions.logger.error(err);
return res.json({
success: false,
err
});
}
});
});
Here is the code I use to add documents in my client code (it is a web app in javascript) :
const createOrder = async (projectId) => {
try {
const orderData = {
accounting: {
totalHT: 0,
totalTTC: 0,
totalTVA: 0,
},
createdAt: serverTimestamp(),
status: 'NEW',
surface: 0,
};
const refProject = doc(firebaseDB, 'projects', projectId);
const colOrder = collection(firebaseDB, `projects/${projectId}/orders`)
const refOrder = doc(colOrder);
return await runTransaction(firebaseDB, async (transaction) => {
const snapProject = await transaction.get(refProject);
if (!snapProject.exists()) {
throw "Document does not exist!";
}
const dataProject = snapProject.data();
const sequence = dataProject.sequenceOrder;
orderData.sequenceNumber = sequence;
transaction.set(refOrder, orderData);
transaction.update(refProject, { sequenceOrder: sequence + 1, totalOrders: dataProject.totalOrders + 1 });
return refOrder.id;
});
} catch (e) {
console.error(e);
return null;
}
};

Node function not working, await part is not getting executed in the given function

While hitting API I'm getting function from my services/ElasticSearch.js and for some reason function there is not working after the axios part.
In the below file I've called function elasticService.updateDocument this function has been brought from another file.
'''
class ProductController {
constructor() { }
async saveProduct(req, res) {
console.log('ITs coming here')
let { _id, Product } = req.body;
if (_id) delete req.body._id;
let elasticResult;
try {
if (Product && Product.Category) {
req.body.Category = Product.Category
delete Product.Category
}
if (Product && Product.URL) {
const exists = await ProductService.checkProductByUrl(Product.URL);
_id = exists._id
}
const result = await ProductService.saveProduct(req.body, _id);
if (result) {
if (_id) {
console.log('Here.... UPDATE')
const savedProduct = await ProductModel.createPayload(req.body);
console.log(savedProduct,'saved_product')
let elaticDoc = await this.createElasticDocData(savedProduct);
console.log(elaticDoc.id,'elasticResult')
elaticDoc.id = result._id;
elaticDoc = new Elastic(elaticDoc);
console.log(elaticDoc,'<----------elaticdoc-------------->')
elasticResult = await elasticService.updateDocument(JSON.stringify(elaticDoc), req.body.Category)
console.log(elasticResult,'elasticResult')
}
else {
console.log('Here.... ADD')
const savedProduct = await ProductModel.createPayload(result);
let elaticDoc = await this.createElasticDocData(savedProduct);
elaticDoc.id = result._id;
elaticDoc = new Elastic(elaticDoc);
elasticResult = await elasticService.createDocument(JSON.stringify(elaticDoc), req.body.Category)
}
const response = new Response(1, "Product is saved successfully", "", "", { product: result, elasticResult: elasticResult });
return res.status(200).send(response);
}
const response = new Response(0, "Error in saving Product", 0, "Product not saved", {});
return res.status(200).send(response);
} catch (error) {
const response = new Response(0, "Unexpected Error", 0, error, {});
return res.status(400).send(response);
}
}
'''
This is the elasticappsearch file where above mentioned is coming from and for some reason it's not working after axios.patch part.
'''
const private_key = process.env.elastic_private_key
const search_key = process.env.elastic_search_key
const axios = require("axios")
class ElasticAppSearch {
async updateDocument(body, engine) {
console.log('Its coming in updateDOCS here')
const response = await axios.patch(`${process.env.elastic_url}/${engine}/documents`, body, {
headers: {
Authorization: `Bearer ${private_key}`,
},
});
console.log(response,'<--===-=-=-=-=-=-=-=-=-=-=-=-response')
return response.data
}
'''

Asynchronous code works independently, but not together. Not getting any exceptions, just timing out

I am writing a lambda function to add hosts to a SQS queue for a rolling restart. The code I have written works individually, but not together. Even when I hard code values in the constructor. This doesn't appear to be a memory/CPU. I tried running the function with 1GB of memory, even though it only uses about 80MB. The average execution time for the individual functions is about 0.5 seconds (shouldn't take more than about 1.5 seconds to execute in total). I did trying running this function with a 30 second timeout, but it still timed out.
I work behind a corporate proxy, and have to hand jam the code. I don't have an IDE or intellisense on my internet facing network. There may be typos here, but not in the actual code. I have omitted my module imports and variable declarations to save time. It isn't relevant to the issue at hand.
EDIT: I added the module imports and variable declarations to the first example to hopefully alleviate some confusion.
Here are just a few things I have tried. This does not work (timing out):
// Custom lambda layer
const { marklogic, aws } = require('nodejs-layer-lib');
const { HOSTS, DOMAIN, PORT, USERNAME, PASSWORD, RESTART_QUEUE_NAME } = process.env;
const params = [
'format=json'
];
const options = {
port: PORT,
params: params,
httpOptions: {
headers: {
'Authorization': `Basic ${Buffer.from(`${USERNAME}:${PASSWORD}`).toString('base64')}`
},
method: 'GET'
}
};
const taskServers = (HOSTS.split(',') || []).map(host => {
const _host = host.split(':');
return {
id: _host[0],
name: `http://${_host[1].toLowerCase()}.${DOMAIN}`
};
});
exports.handler = async () => {
let hosts, queueUrl, addToQueueResults;
try {
hosts = (await marklogic.hosts.getHosts(taskServers, options) || []);
} catch (e) { console.error('hosts', e); }
try {
queueUrl = await aws.sqs.getQueueUrlByName(RESTART_QUEUE_NAME);
} catch (e) { console.error('queueUrl ', e); }
try {
addToQueueResults = await aws.sqs.addMessages(queueURL, hosts);
} catch (e) { console.error('addToQueueResults ', e); }
return {
status: 200,
body: addToQueueResults
};
}
This does not work (timing out):
// Modules imports and variable declarations here...
exports.handler = async () => {
const hosts = (await marklogic.hosts.getHosts(taskServers, options) || []);
const queueUrl = await aws.sqs.getQueueUrlByName(RESTART_QUEUE_NAME);
const addToQueueResults = await aws.sqs.addMessages(queueURL, hosts);
return {
status: 200,
body: addToQueueResults
};
}
This does not work (timing out):
// Modules imports and variable declarations here...
exports.handler = async () => {
const hosts = (await marklogic.hosts.getHosts(taskServers, options) || []);
const queueUrl = await aws.sqs.getQueueUrlByName('my-queue-name');
const addToQueueResults = await aws.sqs.addMessages('http://queueurl.com', ['anything', 'in', 'here']); // Doesn't even need the queueUrl or hosts anymore
return {
status: 200,
body: addToQueueResults
};
}
This works. It will return the host objects I am expecting in the response:
// Modules imports and variable declarations here...
exports.handler = async () => {
const hosts = (await marklogic.hosts.getHosts(taskServers, options) || []);
return {
status: 200,
body: hosts
};
}
This works. It will get the queue url, then add messages to my SQS queue and return the SQS response:
// Modules imports and variable declarations here...
exports.handler = async () => {
const queueUrl = await aws.sqs.getQueueUrlByName(RESTART_QUEUE_NAME);
const addToQueueResults = await aws.sqs.addMessages(queueUrl , ['anything', 'in', 'here']);
return {
status: 200,
body: addToQueueResults
};
}
I tried implementing the Async handler in AWS Lambda function handler in Node.js and reviewed many AWS Lambda execution troubleshooting documents. The marklogic management API runs on port 8002 by default and I think the aws-sdk module uses http/https (80/443), so I don't think the ports are getting tied up.
What am I missing here?
EDIT 2: This has something to do with how promises are handled with AWS Lambda. I cannot find much information about this. Even following the instructions in AWS Lambda function handler in Node.js for "Async Handlers" I cannot get this to work. It works perfectly fine locally with or without my custom lambda layer.
Node.js runtime: 12.x (I didn't mention this before)
This also doesn't work (timing out):
// Modules imports and variable declarations here...
exports.handler = async function (event) {
const promise = function () {
return new Promise(async function (resolve, reject) {
try {
const hosts = await marklogic.hosts.getHosts(taskServers, options) || [];
const queueUrl = await aws.sqs.getQueueUrlByName(RESTART_QUEUE_NAME);
const addToQueueResults = await aws.sqs.addMessages(queueUrl, hosts);
resolve({
status: 200,
body: addToQueueResults
});
} catch (error) {
reject({
status: 500,
error: error
});
}
});
};
return promise(); // Throws error without constructor despite the AWS doc example
}
Unless someone AWS Lambda genius has ran into a similar issue before with Node.js, I am just going to convert it into 2 lambda functions and use Step Functions to process them.

There was a typo in queueUrl (I imagine not that, but worth a try!)
Please run:
// Custom lambda layer
const { marklogic, aws } = require('nodejs-layer-lib');
const { HOSTS, DOMAIN, PORT, USERNAME, PASSWORD, RESTART_QUEUE_NAME } = process.env;
const params = [
'format=json'
];
const options = {
port: PORT,
params,
httpOptions: {
headers: {
Authorization: `Basic ${Buffer.from(`${USERNAME}:${PASSWORD}`).toString('base64')}`
},
method: 'GET'
}
};
const taskServers = (HOSTS.split(',') || []).map(host => {
const _host = host.split(':');
return {
id: _host[0],
name: `http://${_host[1].toLowerCase()}.${DOMAIN}`
};
});
exports.handler = async () => {
let hosts, queueUrl, addToQueueResults;
try {
hosts = (await marklogic.hosts.getHosts(taskServers, options) || []);
} catch (e) { console.error('hosts', e); }
try {
queueUrl = await aws.sqs.getQueueUrlByName(RESTART_QUEUE_NAME);
} catch (e) { console.error('queueUrl ', e); }
try {
addToQueueResults = await aws.sqs.addMessages(queueUrl, hosts);
} catch (e) { console.error('addToQueueResults ', e); }
return {
status: 200,
body: JSON.stringify(addToQueueResults)
};
};
// keeping the same format.. ^^
If no luck - what is on my mind, is as aws-sdk is included out of the box in lambda.. it's not customary to require it extraneously via a layer and although it may not look to be imported at top level by marklogic, it may be bundled deep within marklogic, then when you import AWS and change config (in the layer) it overwrites it
Let's find out..:
Step 1:
So, this you say should work.. if we ignore the AWS import, and just import marklogic?
// Custom lambda layer
// const { marklogic, aws } = require('nodejs-layer-lib'); // ignoring AWS for now
const { marklogic } = require('nodejs-layer-lib');
const { HOSTS, DOMAIN, PORT, USERNAME, PASSWORD, RESTART_QUEUE_NAME } = process.env;
const params = [
'format=json'
];
const options = {
port: PORT,
params,
httpOptions: {
headers: {
Authorization: `Basic ${Buffer.from(`${USERNAME}:${PASSWORD}`).toString('base64')}`
},
method: 'GET'
}
};
const taskServers = (HOSTS.split(',') || []).map(host => {
const _host = host.split(':');
return {
id: _host[0],
name: `http://${_host[1].toLowerCase()}.${DOMAIN}`
};
});
exports.handler = async () => {
// let hosts, queueUrl, addToQueueResults;
let hosts;
try {
hosts = (await marklogic.hosts.getHosts(taskServers, options) || []);
console.log('hosts => ', hosts);
// queueUrl = await aws.sqs.getQueueUrlByName(RESTART_QUEUE_NAME);
// addToQueueResults = await aws.sqs.addMessages(queueUrl, hosts);
return {
status: 200,
body: JSON.stringify(hosts)
};
} catch (error) {
console.log('error => ', error);
throw error;
}
};
Ok, so if that works..:
Step 2 (Please set the region for SQS and also hard code in the queueUrl):
// Custom lambda layer
// const { marklogic, aws } = require('nodejs-layer-lib');
const { marklogic } = require('nodejs-layer-lib');
const AWS = require('aws-sdk');
AWS.config.update({ region: 'eu-west-1' }); // Please set region accordingly
const sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
const { HOSTS, DOMAIN, PORT, USERNAME, PASSWORD, RESTART_QUEUE_NAME } = process.env;
const params = [
'format=json'
];
const options = {
port: PORT,
params,
httpOptions: {
headers: {
Authorization: `Basic ${Buffer.from(`${USERNAME}:${PASSWORD}`).toString('base64')}`
},
method: 'GET'
}
};
const taskServers = (HOSTS.split(',') || []).map(host => {
const _host = host.split(':');
return {
id: _host[0],
name: `http://${_host[1].toLowerCase()}.${DOMAIN}`
};
});
exports.handler = async () => {
let hosts, addToQueueResults;
try {
hosts = (await marklogic.hosts.getHosts(taskServers, options) || []);
console.log('hosts => ', hosts);
const queueUrl = 'Please hard code the queueUrl for now';
const sqsParams = {
MessageBody: hosts,
QueueUrl: queueUrl
};
addToQueueResults = await sqs.sendMessage(sqsParams).promise();
console.log('addToQueueResults => ', addToQueueResults);
return {
status: 200,
body: JSON.stringify(addToQueueResults)
};
} catch (error) {
console.log('error => ', error);
throw error;
}
};
IF.. that doesn't work.. then Step 3.. move the require of marklogic to below the require of AWS and setting the region in this last example.. (so any deeply nested marklogic AWS logic we're unaware of now overwrites your AWS require..) re-run it.. fingers crossed :-)

How to get the value of a variable out of the scope of the arrow function which created it

So, i have this code:
client.on("messageCreate", async (msg) => {
if (msg.content.startsWith("-p")) {
if (!msg.member.voice?.channel) return msg.channel.send("Por favor...");
const connection = joinVoiceChannel({
channelId: msg.member.voice.channel.id,
guildId: msg.guild.id,
adapterCreator: msg.guild.voiceAdapterCreator,
});
google
.youtube("v3")
.search.list({
key: process.env.YOUTUBE_TOKEN,
part: "id",
q: msg.content.slice(3),
maxResults: 1,
})
.then((response) => {
const { data } = response;
let link = "https://www.youtube.com/watch?v=" + data.items.id.videoId;
});
let args = msg.content.slice(3);
let stream = await play.stream(args);
let resource = createAudioResource(stream.stream, {
inputType: stream.type,
});
let player = createAudioPlayer({
behaviors: {
noSubscriber: NoSubscriberBehavior.Play,
},
});
player.play(resource);
connection.subscribe(player);
}
});
And I would like to use the value of the variable "link", which is a string, as a parameter for the method play.stre(args)
I'm don't fully understand how to get that done
Also, if possible I would like to find a way to store that value too inside an array outside the whole:
*client.on('messageCreate' , async msg => {... ...})*

You most likely want to call play.stream from within the then callback of list:
google.youtube("v3")
.search.list({
key: process.env.YOUTUBE_TOKEN,
part: "id",
q: msg.content.slice(3),
maxResults: 1,
})
.then(async (response) => {
const { data } = response;
let link = "https://www.youtube.com/watch?v=" + data.items.id.videoId;
let stream = await play.stream(link);
});
Like this, you'll make sure that you actually have a value set for link when using it to call stream.

Node callback style with promisify? "The 'original' argument must be of type function"

I'm using util.promisify in a Google Cloud Function to call IBM Watson Text-to-Speech, which returns a callback. My code works but I get an error message:
TypeError [ERR_INVALID_ARG_TYPE]: The "original" argument must be of type function
The documentation says
Takes a function following the common error-first callback style, i.e.
taking a (err, value) => ... callback as the last argument, and
returns a version that returns promises.
The IBM Watson callback is complicated and I can't figure out how to refactor it into the Node.js callback style. It's working, should I just ignore the error message? Here's my Google Cloud Function:
exports.IBM_T2S = functions.firestore.document('Users/{userID}/Spanish/IBM_T2S_Request').onUpdate((change) => {
let word = change.after.data().word;
let wordFileType = word + '.mp3';
function getIBMT2S(word, wordFileType) {
const {Storage} = require('#google-cloud/storage');
const storage = new Storage();
const bucket = storage.bucket('myProject.appspot.com');
const file = bucket.file('Audio/Spanish/Latin_America/' + wordFileType);
var util = require('util');
var TextToSpeechV1 = require('watson-developer-cloud/text-to-speech/v1');
var textToSpeech = new TextToSpeechV1({
username: 'groucho',
password: 'swordfish',
url: 'https://stream.watsonplatform.net/text-to-speech/api'
});
var synthesizeParams = {
text: word,
accept: 'audio/mpeg',
voice: 'es-LA_SofiaVoice',
};
const options = { // construct the file to write
metadata: {
contentType: 'audio/mpeg',
metadata: {
source: 'IBM Watson Text-to-Speech',
languageCode: 'es-LA',
gender: 'Female'
}
}
};
textToSpeech.synthesize(synthesizeParams).on('error', function(error) {
console.log(error);
}).pipe(file.createWriteStream(options))
.on('error', function(error) {
console.error(error);
})
.on('finish', function() {
console.log("Audio file written to Storage.");
});
};
var passGetIBMT2S = util.promisify(getIBMT2S(word, wordFileType))
passGetIBMT2S(word, wordFileType)
});

It's working because you are invoking getIBMT2S and passing the return value to util.promisfy and not the function itself.
There are a couple of issues here, firstly your getIBMT2S function doesn't look like it would be compatible with util.promisfy, as you've highlighted from the documents, a compatible function should follow the typical callback-style signature (getIBMT2S does not take a callback parameter).
Secondly, util.promisify expects a function - in your case you are passing the return value of the function instead. If getIBMT2S was updated to support a callback parameter then the correct usage would be
function getIBMT2S(word, wordFileType, cb) {
...
// be sure to invoke cb in here
}
var passGetIBMT2S = util.promisify(getIBMT2S); // <-- don't call getIBMT2S, pass it in directly
passGetIBMT2S(word, wordFileType) // <-- now invoke the wrapped function
.then(result => console.log('DONE'));
.catch(e => console.error(e));

Here's my finished code. There are two functions. getT2S calls IBM Watson Text-to-Speech, then writes the audiofile to Storage, then gets the download URL. writeDownloadURL checks if a Firestore document exists, then either sets or updates the download URL to Firestore.
exports.IBM_T2S = functions.firestore.document('Users/{userID}/Spanish/IBM_T2S_Request').onUpdate((change) => {
if (change.after.data().word != undefined) {
// get requested word object
let accent = change.after.data().accent;
let audioType = change.after.data().audioType;
let gender = change.after.data().gender;
let longLanguage = change.after.data().longLanguage;
let shortLanguage = change.after.data().shortLanguage;
let shortSource = change.after.data().shortSource;
let source = change.after.data().source;
let voice = change.after.data().voice;
let word = change.after.data().word;
console.log(word);
let wordFileType = word + '.' + audioType;
let pronunciation = `${accent}-${gender}-${shortSource}`;
const {Storage} = require('#google-cloud/storage');
const storage = new Storage();
const bucket = storage.bucket('myProject.appspot.com');
const file = bucket.file('Audio/' + longLanguage + '/' + pronunciation + '/' + wordFileType);
var TextToSpeechV1 = require('watson-developer-cloud/text-to-speech/v1');
var textToSpeech = new TextToSpeechV1({
username: 'groucho',
password: 'swordfish',
url: 'https://stream.watsonplatform.net/text-to-speech/api'
});
var synthesizeParams = {
text: word,
accept: 'audio/' + audioType,
voice: voice
};
const options = { // construct the file to write
metadata: {
contentType: 'audio/' + audioType,
metadata: {
accent: accent,
audioType: audioType,
gender: gender,
longLanguage: longLanguage,
shortLanguage: shortLanguage,
source: source,
voice: voice,
word: word
}
}
};
// check if Pronunciations collection exists, set or update to not destroy existing data
function writeDownloadURL(downloadURL) {
admin.firestore().collection('Dictionaries').doc(longLanguage).collection('Words').doc(word).collection('Pronunciations').doc(pronunciation).get()
.then(function(doc) {
if (doc.exists) {
return admin.firestore().collection('Dictionaries').doc(longLanguage).collection('Words').doc(word).collection('Pronunciations').doc(pronunciation).update({ audioFile: downloadURL })
.then(result => console.log('DONE'))
.catch(error => console.error(error));
} else {
return admin.firestore().collection('Dictionaries').doc(longLanguage).collection('Words').doc(word).collection('Pronunciations').doc(pronunciation).set({ audioFile: downloadURL })
.then(result => console.log('DONE'))
.catch(error => console.error(error));
} // close else
})
.catch(error => console.error(error));
} // close writeDownloadURL
// documentation at https://stackoverflow.com/questions/22519784/how-do-i-convert-an-existing-callback-api-to-promises
function getT2S(synthesizeParams) {
return new Promise(function(resolve, reject) {
// documentation at https://cloud.ibm.com/apidocs/text-to-speech?code=node#synthesize-audio-get
textToSpeech.synthesize(synthesizeParams).on('error', function(error) {
console.error(error);
reject(error);
}).pipe(file.createWriteStream(options))
.on('error', function(error) {
console.error(error);
reject(error);
})
.on('finish', function() {
resolve(file.getSignedUrl({
action: 'read',
expires: '03-17-2025'
}));
}); // close on finish
}); // close Promise
} // close getT2SAsync
async function getT2SAsync(synthesizeParams) {
var signedUrls = await getT2S(synthesizeParams);
var downloadURL = signedUrls[0];
await writeDownloadURL(downloadURL);
console.log("All done.");
}
return getT2SAsync(synthesizeParams);
} else { // if no word passed to function
console.error("Error.");
}
}); // close IBM_T2S
I mistakenly wrote
return file.getSignedUrl({
instead of
resolve(file.getSignedUrl({
The result was that no data came back from the promise, and the cloud function timed out after six seconds, without finishing execution. You have to do something with resolve. I used reject twice to be sure. :-)