Develop Reference

Unit Test for AWS RedshiftData on node.js using jest

I am new to AWS Redshift Data. I was trying to write a jest unit test cases for AWS Redshift Data. I used aws-sdk for AWS Redshift Data and aws-sdk-mock to mock it and have a static response (success and error), But aws-sdk-mock seems not working to return mock response.
Here's my redshift-data.service.js:
const AWS = require('aws-sdk');
const rsd = new AWS.RedshiftData();
const logger = require('../config/logger');
const getDescribeStatement = async (queryId) => {
try {
return rsd
.describeStatement({ Id: queryId })
.promise()
.then((response) => {
return response;
})
.catch((error) => {
logger.info('DescribeStatement has failed.');
throw new Error(error);
});
} catch (error) {
logger.error(`Redshift error: ${error}`);
return { status: 'fail', error: error.message };
}
};
module.exports = {
getDescribeStatement,
};
and Here's my redshift-data.service.test.js:
const awsMock = require('aws-sdk-mock');
const redshiftDataService = require('../../../src/services/redshift-data.service');
describe('Redshift Data Service', () => {
it('should be able to describe statement/query', async () => {
awsMock.mock('RedshiftData', 'describeStatement.promise()', jest.fn().mockReturnValue(Promise.resolve(true)));
awsMock.describeStatement = jest.fn().mockReturnValue(Promise.resolve(true));
const queryId = 'd7a21d3b-5833-4793-a896-e430fb729f7b';
const service = await redshiftDataService.getDescribeStatement(queryId);
expect(service).toBeTruthy();
});
it('should not be able to describe statement/query', async () => {
awsMock.mock('RedshiftData', 'describeStatement.promise()', jest.fn().mockReturnValue(Promise.resolve(false)));
awsMock.describeStatement = jest.fn().mockReturnValue(Promise.resolve(false));
const queryId = 'd7a21d3b-5833-4793-a896-e430fb729f7b';
const service = await redshiftDataService.getDescribeStatement(queryId);
expect(service).toBeFalsy();
});
});
Thanks in advance for helping.

parsing data from an api call into a text file using axios

I am parsing data from an API call into a text file. However, I wanted to use async-await and break the call below call into 3 separate functions.
#!/usr/bin/env node
const yargs = require("yargs");
const axios = require("axios");
const fs = require("fs");
const options = yargs
.usage("Usage: -n <name>")
.option("n", {
alias: "name",
describe: "Your name",
type: "string",
demandOption: true,
})
.option("s", { alias: "search", describe: "Search Term", type: "string" })
.argv;
const greetings = `Hello ${options.name}!`;
console.log(greetings);
console.log("Here's a random joke for you: ");
const url = options.search
? `https://icanhazdadjoke.com/search?term${escape(options.search)}`
: " https://icanhazdadjoke.com/";
axios.get(url, { headers: { Accept: "application/json" } }).then((res) => {
if (options.search) {
res.data.results.forEach((j) => {
fs.appendFile("jokes.txt", "\n" + j.jokes, (err) => {});
});
if (res.data.results.length === 0) {
console.log("no joke found 😭");
}
} else {
fs.appendFile("jokes.txt", res.data.joke, (err) => {
if (err) throw err;
console.log("File Updated");
});
}
});
So the above code works absolutely fine and generates the file perfectly, however when I tried to break it into the following below functions, I just get undefined in the text file, I am not sure why this is happening.
const getJoke = async (url) => {
try {
const joke = await axios.get(url, {
headers: { Accept: "application/json" },
});
return joke;
} catch (error) {
console.error(error);
}
};
const parseJokes = (res) => {
if (options.search) {
res.data.results.forEach((j) => {
return `\n ${j.joke}`;
});
if (res.data.results.length === 0) {
console.log("no joke found 😭");
}
} else {
return res.data.joke;
}
};
const addJokeToFile = async () => {
const result = await getJoke(url)
.then((res) => {
parseJokes(res);
})
.catch((err) => {
console.error(`ERROR: ${err}`);
});
fs.appendFile("jokes.txt", result, (err) => {
console.error(err);
});
};

In the second (functional approach) addJokeToFile method, you are waiting for the promise to be resolved using both ways, await and .then, following modification to the code, might help you get through:
const addJokeToFile = async () => {
getJoke(url)
.then((res) => {
// Aside, we should also return some value from parseJokes function for "no joke found 😭" case, or return null and put a check here and only append to file when jokeString is not null.
const jokeString = parseJokes(res);
fs.appendFile("jokes.txt", jokeString, (err) => {
console.error(err);
});
})
.catch((err) => {
console.error(`ERROR: ${err}`);
});
};

Try using appendFile from 'fs/promises' so that you can stick with the async/await style. Since getJoke returns a promise I would expect result to be a Promise<string | undefined> depending on if any errors show up earlier in the chain.
const { appendFile } = require('fs/promises');
const addJokeToFile = async () => {
try {
const result = await getJoke(url);
const parsed = parseJokes(result);
await appendFile('jokes.txt', parsed);
} catch (err) {
console.error(err);
}
};

how to return value from a promise function

I have a function which checks whether a device is online or not. Below is the code.
const ping = require('ping');
export function findDevices(device) {
try {
const hosts = [device];
let result = null;
hosts.forEach((host) => {
ping.promise.probe(host)
.then((res) => {
console.log(res.alive)
result = res.alive;
return {
Status: result
}
});
});
} catch (err) {
logger.error(err, '[ config - findDevices() ]');
console.error(err);
return {
Status: "Failed"
}
}
}
I am calling this function in a redux action like this:
export function start(device) {
return dispatch => {
const status = Connectionstatus.findDevices(device);
return dispatch({
type: actionTypes.CONNECTIONSTATUS,
payload: {
ConnectionStatus: status
}
})
};
}
I am expective the status variable to be either true or false. But i am getting as undefined even though i am returning the value inside then of the promise function. i have tried awaiting this call and still its not working. Any help would be much appreciated. Thanks.

If that's the case you can do like this
const getStatus = async () => {
try {
const hosts = [device];
const promises = [];
hosts.forEach((host) => {
promises.push(ping.promise.probe(host));
});
const result = await Promise.all(promises);
const status = result.map((r) => { return r.alive; });
return status;
} catch (err) {
logger.error(err, '[ config - findDevices() ]');
return { status: 'Failed' };
}
};

Not 100% sure what all the vars are, but have you considered using async/await to simplify things a bit like this?
const getStatus122 = async device => {
return await Promise.all([device].map(ping.promise.probe))
.then(({ alive }) => alive)
.then(Status => ({ Status }))
.catch(error => {
logger.error(error, '[ config - findDevices() ]');
return { Status: 'Failed' };
})
}
More on that here: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function

With Promises you should inspect the promised result either in when or catch callback functions. With async/await the code may look a bit simpler. Here is the version with explicit promises.
const ping = require('ping');
const Connectionstatus = {
findDevices: (device) => {
return ping.promise.probe(device).then((res) => {
const result = res.alive;
console.log(result);
return {
Status: result,
};
}).catch((err) => {
logger.error(err, '[ config - findDevices() ]');
console.error(err);
return {
Status: "failed"
}
});
}
}
export function start(device) {
return dispatch => {
Connectionstatus.
findDevices(device).
then((status) => {
dispatch({
type: actionTypes.CONNECTIONSTATUS,
payload: {
ConnectionStatus: status
}
})
});
};
}
You may see that error handling moved to the catch callback function while the dispatch is done in the then callback function. And this is the answer to your question.

How do you access Axios response with Express?

I just started working with Express and am currently lost on how to make an Axios request using route parameters and change some locals based on what the request returns. This is what I have so far:
helpers.js
const axios = require('axios');
const {
titleSuffix,
organizationPath,
varietyPath
} = require('./constants');
let organizationData = {};
let varietyData = {};
const Helpers = {
fetchOrganization: (organizationID) => {
axios.get(organizationPath + organizationID)
.then( (response) => {
//console.log(response);
organizationData = response.data.data;
})
.catch( (error) => {
//console.log(error);
});
return organizationData;
},
fetchVariety: (varietyID) => {
axios.get(varietyPath + varietyID)
.then( (response) => {
//console.log(response);
varietyData = response.data.data;
})
.catch( (error) => {
//console.log(error);
});
return varietyData;
},
setOrgOpenGraphTags: (growerHash, res) => {
Helpers.fetchOrganization(growerHash);
res.locals.meta.og.title = organizationData.name + titleSuffix;
console.log('Org = ' + organizationData.name);
},
setVarOpenGraphTags: (contextualId, res) => {
Helpers.fetchVariety(contextualId);
res.locals.meta.og.title = varietyData.name + titleSuffix;
console.log('Var = ' + varietyData.name);
}
};
module.exports = Helpers;
server.js
// Express
const express = require('express');
const app = express();
// Helpers
const {
setOrgOpenGraphTags,
setVarOpenGraphTags
} = require('./helpers');
// Organization
app.get(['/org/:growerHash/*', '/_org/:growerHash/*'], (req, res) => {
setOrgOpenGraphTags(req.params.growerHash, res);
res.render('org');
});
I'm fairly certain I'm missing something small but can't seem to get the following local changed based on the response from Axios:
res.locals.meta.og.title
Based on what I have so far how do I properly access the response from Axios in Express and change the locals? I really need an answer based around the code I've provided. Currently in my dev environment the request works but in production it returns "undefined". Thanks so much in advance.

The duplicate that I linked, Why is my variable unaltered after I modify it inside of a function? - Asynchronous code reference, discusses why and how writing asynchronous code means that you have to propagate being asynchronous.
Your code, as it is written right now, does not propagate asynchronicity. axios.get() returns a Promise. Unless everything that depends on the value that that Promise resolves to actually waits for the Promise chain to resolve, you aren't going to get what you are expecting.
Consider your code which I have commented below:
const axios = require('axios');
const Helpers = {
fetchOrganization: (organizationID) => {
// axios.get() will return a Promise
// You have to wait for the Promise to finish before
// you can use any data that it produces
// You must propogate the Proise of data up
// You should return axios.get(...)
axios.get(organizationPath + organizationID)
.then((response) => {
//console.log(response);
organizationData = response.data.data;
})
.catch((error) => {
//console.log(error);
});
// This won't be populated by the time you try to use it
return organizationData;
// Instead do
return axios
.get(organizationPath + organizationID)
.then(response => {
const organizationData = response.data.data;
return organizationData
})
.catch(err => console.error(err));
// Better yet, do
/*
return axios.get(organizationPath + organizationID)
.then(res => response.data.data) // Return is implied
.catch(err => console.error(err));
*/
},
setOrgOpenGraphTags: (growerHash, res) => {
// Nothing is coming out of this function and you aren't waiting on it
Helpers.fetchOrganization(growerHash);
// Instead do
return Helpers.fetchOrganization(growerHash)
.then(org => {
return org.name + titleSuffix;
});
//res.locals.meta.og.title = organizationData.name + titleSuffix;
//console.log('Org = ' + organizationData.name);
}
}
// Organization
app.get(['/org/:growerHash/*', '/_org/:growerHash/*'], (req, res) => {
// Below, you are starting the async process
// but you don't wait for the async to finish
// you just immediately res.render()
setOrgOpenGraphTags(req.params.growerHash, res);
res.render('org');
// Instead
setOrgOpenGraphTags(req.params.growerHash, res)
.then(orgTitle => {
res.locals.meta.og.title = orgTitle;
res.render('org');
});
});
After considering that, let's consider a distilled version of your code that will wait for the Promise chain to resolve:
// Let's boil your app down to it's core
const SOME_SUFFIX = "foobar";
// fetchOrganization
function getSomeData(id) {
return axios
.get(`http://www.example.com/things/${id}`)
.then(thatThing => thatThing.nested.property.i.want)
.catch(err => console.error(err));
}
// setOrgOpenGraphTags
function calculateDerivedData(id) {
return getSomeData(id)
.then(thatThingsProperty => `${thatThingsProperty}-${SOME_SUFFIX}`)
}
// Route
app.get("/some/endpoint/:id", (req, res) => {
calculateDerivedData(req.params.id)
.then(thatDerivedDataWeNeed => {
res.locals.whatever = thatDerivedDataWeNeed;
res.render("someTemplate");
})
});
If you want to write something that looks arguably cleaner, you can also consider async/await:
// Let's boil your app down to it's core
const SOME_SUFFIX = "foobar";
// fetchOrganization
async function getSomeData(id) {
try {
const thatThing = await axios.get(`http://www.example.com/things/${id}`);
return thatThing.nested.property.i.want;
} catch(err){
console.error(err);
}
}
// setOrgOpenGraphTags
async function calculateDerivedData(id) {
const thatThingsProperty = await getSomeData(id);
return `${thatThingsProperty}-${SOME_SUFFIX}`;
}
// Route
app.get("/some/endpoint/:id", async function(req, res) => {
res.locals.whatever = await calculateDerivedData(req.params.id);
res.render("someTemplate");
});

s3.getObject().createReadStream() : How to catch the error?

I am trying to write a program to get a zip file from s3, unzip it, then upload it to S3.
But I found two exceptions that I can not catch.
1. StreamContentLengthMismatch: Stream content length mismatch. Received 980323883 of 5770104761 bytes. This occurs irregularly.
2. NoSuchKey: The specified key does not exist. This happens when I input the wrong key.
When these two exceptions occur, this program crashes.
I'd like to catch and handle these two exceptions correctly.
I want to prevent a crash.
const unzipUpload = () => {
return new Promise((resolve, reject) => {
let rStream = s3.getObject({Bucket: 'bucket', Key: 'hoge/hoge.zip'})
.createReadStream()
.pipe(unzip.Parse())
.on('entry', function (entry) {
if(entry.path.match(/__MACOSX/) == null){
// pause
if(currentFileCount - uploadedFileCount > 10) rStream.pause()
currentFileCount += 1
var fileName = entry.path;
let up = entry.pipe(uploadFromStream(s3,fileName))
up.on('uploaded', e => {
uploadedFileCount += 1
console.log(currentFileCount, uploadedFileCount)
//resume
if(currentFileCount - uploadedFileCount <= 10) rStream.resume()
if(uploadedFileCount === allFileCount) resolve()
entry.autodrain()
}).on('error', e => {
reject()
})
}
}).on('error', e => {
console.log("unzip error")
reject()
}).on('finish', e => {
allFileCount = currentFileCount
})
rStream.on('error', e=> {
console.log(e)
reject(e)
})
})
}
function uploadFromStream(s3,fileName) {
var pass = new stream.PassThrough();
var params = {Bucket: "bucket", Key: "hoge/unzip/" + fileName, Body: pass};
let request = s3.upload(params, function(err, data) {
if(err) pass.emit('error')
if(!err) pass.emit('uploaded')
})
request.on('httpUploadProgress', progress => {
console.log(progress)
})
return pass
}
This is the library I use when unzipping.
https://github.com/mhr3/unzip-stream
Help me!!

If you'd like to catch the NoSuchKey error thrown by createReadStream you have 2 options:
Check if key exists before reading it.
Catch error from stream
First:
s3.getObjectMetadata(key)
.promise()
.then(() => {
// This will not throw error anymore
s3.getObject().createReadStream();
})
.catch(error => {
if (error.statusCode === 404) {
// Catching NoSuchKey
}
});
The only case when you won't catch error if file was deleted in a split second, between parsing response from getObjectMetadata and running createReadStream
Second:
s3.getObject().createReadStream().on('error', error => {
// Catching NoSuchKey & StreamContentLengthMismatch
});
This is a more generic approach and will catch all other errors, like network problems.

You need to listen for the emitted error earlier. Your error handler is only looking for errors during the unzip part.
A simplified version of your script.
s3.getObject(params)
.createReadStream()
.on('error', (e) => {
// handle aws s3 error from createReadStream
})
.pipe(unzip)
.on('data', (data) => {
// retrieve data
})
.on('end', () => {
// stream has ended
})
.on('error', (e) => {
// handle error from unzip
});
This way, you do not need to make an additional call to AWS to find out if out if it exists.

You can listen to events (like error, data, finish) in the stream you are receiving back. Read more on events
function getObjectStream (filePath) {
return s3.getObject({
Bucket: bucket,
Key: filePath
}).createReadStream()
}
let readStream = getObjectStream('/path/to/file.zip')
readStream.on('error', function (error) {
// Handle your error here.
})
Tested for "No Key" error.
it('should not be able to get stream of unavailable object', function (done) {
let filePath = 'file_not_available.zip'
let readStream = s3.getObjectStream(filePath)
readStream.on('error', function (error) {
expect(error instanceof Error).to.equal(true)
expect(error.message).to.equal('The specified key does not exist.')
done()
})
})
Tested for success.
it('should be able to get stream of available object', function (done) {
let filePath = 'test.zip'
let receivedBytes = 0
let readStream = s3.getObjectStream(filePath)
readStream.on('error', function (error) {
expect(error).to.equal(undefined)
})
readStream.on('data', function (data) {
receivedBytes += data.length
})
readStream.on('finish', function () {
expect(receivedBytes).to.equal(3774)
done()
})
})

To prevent a crash, you need to asynchronously listen to the object's head metadata, where it does not return the whole object, which will take less time. Try this one!
isObjectErrorExists = async functions () => {
try {
const s3bucket = {
secret key: '',
client id: ''
}
const params = {
Bucket: 'your bucket name',
Key: 'path to object'
};
await s3bucket.headObject(params).promise(); // adding promise will let you add await to listen to process untill it completes.
return true;
} catch (err) {
return false; // headObject threw error.
}
throw new Error(err.message);
}
}
public yourFunction = async() => {
if (await this.isObjectErrorExists()) {
s3Bucket.getObject().createReadStream(); // works smoothly
}
}