I am trying out a demo with AWS lambda function using Axios and Cheerio, I am getting back response after calling the endpoint as {message: Internal Server Error}
exports.lambdaHandler = async (event, context) => {
try {
const axios = require('axios');
const cheerio = require('cheerio');
axios.get('https://www.kitco.com').then((response) => {
const html = response.data;
const $ = cheerio.load(html);
const ask = $('#AU-ask').text();
const bid = $('#AU-bid').text();
const resbid = bid.slice(0,7);
const resask = ask.slice(0,7);
const result = {
"ask": resask,
"bid": resbid
}
return result;
});
response = {
'statusCode': 200,
'body': result
}
} catch (err) {
console.log(err);
return err;
}
return response
};
result is clearly not in response scope, therefore this will result in a typical undefined error.
The solution would be to handle the logic inside axios.get callback, try this:
const axios = require('axios');
const cheerio = require('cheerio');
exports.lambdaHandler = (event, context) => {
axios.get('https://www.kitco.com')
.then((response) => {
const html = response.data;
const $ = cheerio.load(html);
const ask = $('#AU-ask').text();
const bid = $('#AU-bid').text();
const resbid = bid.slice(0, 7);
const resask = ask.slice(0, 7);
const result = {
statusCode: 200,
body: {
ask: resask,
bid: resbid
}
};
console.log(result);
})
.catch(err => {
console.log(err);
});
};
You can get error detail in monitor tab of Lambda console web. I guest you get back an error like response is undefined in return response line.
With your code, return response line will be execute immediately when you call the function, but response did not defined in the lambdaHandler scope.
I recommended that, don’t mix async/await syntax with Promise syntax (.then .catch), just use one of them, I suggest use async/await syntax.
The function will like:
exports.lambdaHandler = async (event, context) => {
try {
const axios = require('axios');
const cheerio = require('cheerio');
const response = await axios.get('https://www.kitco.com'); // wait until we get the response
const html = response.data;
const $ = cheerio.load(html);
const ask = $('#AU-ask').text();
const bid = $('#AU-bid').text();
const resbid = bid.slice(0, 7);
const resask = ask.slice(0, 7);
const result = {
"ask": resask,
"bid": resbid
}
return {
statusCode: 200,
body: JSON.stringify(result), // If you working with lambda-proxy-integrations, the `body` must be a string
}; // return to response the request
} catch (err) {
console.log(err);
return {
statusCode: 500, // Example, http status will be 500 when you got an exception
body: JSON.stringify({error: err}),
}
}
};
Related
I am seeing the following error when trying to extract an image's metadata information with the Sharp module: "Input file contains unsupported image format".
This is only happening for certain signed image urls, particularly ones that contain xmp information in the metadata.
I am hoping someone can help me spot where the issue might be occurring in this code snippet.
Here is the exact code snippet I am using (insert the signed image URL where specified in the doStuff function to test):
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
This code snippet above fails with the "Input file contains unsupported image format" on the line that extracts metadata (imageMetadata = await manipulator.metadata();)
So the strange thing is, I am able to properly extract the metadata (with no errors) with this same code if I add a short Sleep after this line: const fileStream = fs.createWriteStream(localPath);
So this code snippet (all I'm doing here is adding a short sleep after fs.createWriteSteam) allows the image metadata to be extracted without issue:
const sharp = require("sharp");
const fs = require('fs');
const fetch = require('node-fetch');
async function storeUrlToLocal(sourceUrl) {
const destPath = './';
const request = {
method: 'GET',
encoding: null,
};
response = await fetch(sourceUrl, request);
if (response.status >= 400)
throw new Error(`Failed to fetch data from ${sourceUrl}, status returned = ${response.status}`);
const localPath = `${destPath}test.png`;
const fileStream = fs.createWriteStream(localPath);
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
await sleep(1000);
return new Promise((resolve, reject) => {
response.body.pipe(fileStream);
response.body.on("error", reject);
response.body.on("end", async () => {
const fileExists = fs.existsSync(localPath);
console.log(`All the data in the file has been read ${localPath} = ${fileExists}`);
resolve(localPath);
});
response.body.on("finish",() => {
console.log('All writes are now complete.');
});
}).catch(error => {
console.log(error);
});
}
async function doStuff() {
const localFilePath = await storeUrlToLocal('<INSERT_IMAGE_URL_HERE>');
// Read image file and extract metadata
let manipulator;
let imageMetadata;
try {
manipulator = sharp(localFilePath, { limitInputPixels: 5000000000 });
console.log('Manipulator = ', manipulator);
imageMetadata = await manipulator.metadata();
console.log("ImageMetadata = ", imageMetadata);
} catch (error) {
console.log(`Image Metadata Extraction Error: ${error.message}`);
throw error;
}
}
doStuff();
Why would this Sleep resolve my issues? I don't see any asynchronous calls being run that I would need to be waiting for to complete. Perhaps fs.createWriteStream didn't have enough time to complete its operation? But I do not have the option to await the call to fs.createWriteStream, as it is not async.
I can't seem to figure out how to save the results of SomeQuery promise. Essentially I would like to take the value in res and pipe it into parseQuery function and return the final results. How do I make the parsed result accessible to an APIs response.
const neo4j = require('neo4j-driver')
var parser = require('parse-neo4j')
const astria_queries = require('./astriaQueries')
const uri = 'bolt://astria_graph:7687'
const user = 'xxx'
const password = 'xxx'
const someQuery = (query) => {
// run statement in a transaction
const driver = neo4j.driver(uri, neo4j.auth.basic(user, password))
const session = driver.session({ defaultAccessMode: neo4j.session.READ })
const tx = session.beginTransaction()
tx.run(query)
.then((res) => {
// Everything is OK, the transaction will be committed
parseQuery(res)
})
.then(() => {
// Everything is OK, the transaction will be committed
})
.catch((e) => {
// The transaction will be rolled back, now handle the error.
console.log(e)
})
.finally(() => {
session.close()
driver.close()
})
}
const parseQuery = (result) => {
try {
const test = parser.parse(result)
console.log(test)
} catch (err) {
console.log(err)
}
}
module.exports = {
someQuery,
}
It finally clicked with me. Here is the solution I came up with. Hopefully it will help others. If there is a better way please let me know. Thank you #fbiville for you help.
async actions
const neo4j = require('neo4j-driver')
var parser = require('parse-neo4j')
const astria_queries = require('./astriaQueries')
const uri = 'bolt://astria_graph:7687'
const user = 'neo4j'
const password = 'neo'
async function getRecords(query) {
// run statement in a transaction
const driver = neo4j.driver(uri, neo4j.auth.basic(user, password))
const session = driver.session({ defaultAccessMode: neo4j.session.READ })
const tx = session.beginTransaction()
try {
const records = await tx.run(query)
const parseRecords = await parseQuery(records)
return parseRecords
} catch (error) {
console.log(error)
} finally {
session.close()
driver.close()
}
}
async function parseQuery(result) {
try {
const parsedRes = await parser.parse(result)
// console.log(parsedRes)
return parsedRes
} catch (err) {
console.log(err)
}
}
// getRecords(astria_queries.get_data_sources)
module.exports = {
getRecords,
}
api send()
exports.get_data_sources = async (req, res) => {
try {
queryFuns.getRecords(astria_queries.get_data_sources).then((response) => {
res.send(response)
})
} catch (error) {
res.status(500).send(error)
console.log(error)
}
}
I'm trying to unit test this function.... following the documentation.
Any ideas why i'm getting this error? According to the documentation we can see it being called this way... What am I missing. https://jestjs.io/docs/en/bypassing-module-mocks
Thankx
async function isValid(pair) {
const path = process.env.PATH;
const query = '/check';
const requestOptions = ccyServiceRequestOptions(pair);
try {
const response = await fetch(path + query, requestOptions);
const json = await response.json();
return json.result.pair === pair;
} catch (err) {
if (err instanceof TypeError) {
return false;
}
else {
log.error(err);
throw err;
}
}
}
------------------------------------------------------------------------------------------
jest.mock('node-fetch');
const fetch = require('node-fetch');
const {Response} = jest.requireActual('node-fetch');
beforeEach(() => {
jest.clearAllMocks()
})
it('should call with valid url', async () => {
const pair = 'EURUSD';
fetch.mockReturnValue(Promise.resolve(new Response({})));
await ccyPairIsValid(pair);
expect(fetch).toHaveBeenCalled();
expect(fetch.mock.calls[0][0]).toHaveBeenCalledWith(process.env.PATH + '/check');
})
Error: expect(received).toHaveBeenCalledWith(...expected)
Matcher error: received value must be a mock or spy function
Received has type: string
Received has value: "http://localhost:8000/check"
You are calling fetch with two arguments, if you don’t care about your requestOptions argument then you can use expect.anything()
I am trying to execute this code:
UserSchema.methods.generateAuthToken = function () {
const user = this;
const access = 'auth';
const token = jwt.sign({_id: user._id.toHexString(), access}, 'abc123');
user.tokens.push({access, token});
user.save().then(() => {
return token;
});
};
In my express file, I have following code:
app.post('/users', (req, res) => {
const body = _.pick(req.body, ['email', 'password']);
const user = new User(body);
user.save().then(() => {
return user.generateAuthToken();
}).then((token) => {
res.header('x-auth', token).send(user);
}).catch((err) => {
console.log(err);
res.status(400).send(err);
});
});
The problem is that the token returned from the user.save().then() call in first file never reaches the express code user.generateAuthToken(). What is the reason and how should I solve it?
generateAuthToken has no return statement.
If you want to return the promise that then returns, then you have to do so explicitly.
you need to update your schema to the following:
UserSchema.methods.generateAuthToken = function () {
const user = this;
const access = 'auth';
const token = jwt.sign({_id: user._id.toHexString(), access}, 'abc123');
user.tokens.push({access, token});
user.save().then(() => {
return token;
});
};
The generateAuthToken does not return a Promise in your code
I just started working with Express and am currently lost on how to make an Axios request using route parameters and change some locals based on what the request returns. This is what I have so far:
helpers.js
const axios = require('axios');
const {
titleSuffix,
organizationPath,
varietyPath
} = require('./constants');
let organizationData = {};
let varietyData = {};
const Helpers = {
fetchOrganization: (organizationID) => {
axios.get(organizationPath + organizationID)
.then( (response) => {
//console.log(response);
organizationData = response.data.data;
})
.catch( (error) => {
//console.log(error);
});
return organizationData;
},
fetchVariety: (varietyID) => {
axios.get(varietyPath + varietyID)
.then( (response) => {
//console.log(response);
varietyData = response.data.data;
})
.catch( (error) => {
//console.log(error);
});
return varietyData;
},
setOrgOpenGraphTags: (growerHash, res) => {
Helpers.fetchOrganization(growerHash);
res.locals.meta.og.title = organizationData.name + titleSuffix;
console.log('Org = ' + organizationData.name);
},
setVarOpenGraphTags: (contextualId, res) => {
Helpers.fetchVariety(contextualId);
res.locals.meta.og.title = varietyData.name + titleSuffix;
console.log('Var = ' + varietyData.name);
}
};
module.exports = Helpers;
server.js
// Express
const express = require('express');
const app = express();
// Helpers
const {
setOrgOpenGraphTags,
setVarOpenGraphTags
} = require('./helpers');
// Organization
app.get(['/org/:growerHash/*', '/_org/:growerHash/*'], (req, res) => {
setOrgOpenGraphTags(req.params.growerHash, res);
res.render('org');
});
I'm fairly certain I'm missing something small but can't seem to get the following local changed based on the response from Axios:
res.locals.meta.og.title
Based on what I have so far how do I properly access the response from Axios in Express and change the locals? I really need an answer based around the code I've provided. Currently in my dev environment the request works but in production it returns "undefined". Thanks so much in advance.
The duplicate that I linked, Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference, discusses why and how writing asynchronous code means that you have to propagate being asynchronous.
Your code, as it is written right now, does not propagate asynchronicity. axios.get() returns a Promise. Unless everything that depends on the value that that Promise resolves to actually waits for the Promise chain to resolve, you aren't going to get what you are expecting.
Consider your code which I have commented below:
const axios = require('axios');
const Helpers = {
fetchOrganization: (organizationID) => {
// axios.get() will return a Promise
// You have to wait for the Promise to finish before
// you can use any data that it produces
// You must propogate the Proise of data up
// You should return axios.get(...)
axios.get(organizationPath + organizationID)
.then((response) => {
//console.log(response);
organizationData = response.data.data;
})
.catch((error) => {
//console.log(error);
});
// This won't be populated by the time you try to use it
return organizationData;
// Instead do
return axios
.get(organizationPath + organizationID)
.then(response => {
const organizationData = response.data.data;
return organizationData
})
.catch(err => console.error(err));
// Better yet, do
/*
return axios.get(organizationPath + organizationID)
.then(res => response.data.data) // Return is implied
.catch(err => console.error(err));
*/
},
setOrgOpenGraphTags: (growerHash, res) => {
// Nothing is coming out of this function and you aren't waiting on it
Helpers.fetchOrganization(growerHash);
// Instead do
return Helpers.fetchOrganization(growerHash)
.then(org => {
return org.name + titleSuffix;
});
//res.locals.meta.og.title = organizationData.name + titleSuffix;
//console.log('Org = ' + organizationData.name);
}
}
// Organization
app.get(['/org/:growerHash/*', '/_org/:growerHash/*'], (req, res) => {
// Below, you are starting the async process
// but you don't wait for the async to finish
// you just immediately res.render()
setOrgOpenGraphTags(req.params.growerHash, res);
res.render('org');
// Instead
setOrgOpenGraphTags(req.params.growerHash, res)
.then(orgTitle => {
res.locals.meta.og.title = orgTitle;
res.render('org');
});
});
After considering that, let's consider a distilled version of your code that will wait for the Promise chain to resolve:
// Let's boil your app down to it's core
const SOME_SUFFIX = "foobar";
// fetchOrganization
function getSomeData(id) {
return axios
.get(`http://www.example.com/things/${id}`)
.then(thatThing => thatThing.nested.property.i.want)
.catch(err => console.error(err));
}
// setOrgOpenGraphTags
function calculateDerivedData(id) {
return getSomeData(id)
.then(thatThingsProperty => `${thatThingsProperty}-${SOME_SUFFIX}`)
}
// Route
app.get("/some/endpoint/:id", (req, res) => {
calculateDerivedData(req.params.id)
.then(thatDerivedDataWeNeed => {
res.locals.whatever = thatDerivedDataWeNeed;
res.render("someTemplate");
})
});
If you want to write something that looks arguably cleaner, you can also consider async/await:
// Let's boil your app down to it's core
const SOME_SUFFIX = "foobar";
// fetchOrganization
async function getSomeData(id) {
try {
const thatThing = await axios.get(`http://www.example.com/things/${id}`);
return thatThing.nested.property.i.want;
} catch(err){
console.error(err);
}
}
// setOrgOpenGraphTags
async function calculateDerivedData(id) {
const thatThingsProperty = await getSomeData(id);
return `${thatThingsProperty}-${SOME_SUFFIX}`;
}
// Route
app.get("/some/endpoint/:id", async function(req, res) => {
res.locals.whatever = await calculateDerivedData(req.params.id);
res.render("someTemplate");
});