Develop Reference

Decode a Uint8Array into a JSON

I am fetching data from an API in order to show sales and finance reports, but I receive a type gzip file which I managed to convert into a Uint8Array. I'd like to somehow parse-decode this into a JSON file that I can use to access data and create charts in my frontend with.
I was trying with different libraries (pako and cborg seemed to be the ones with the closest use cases), but I ultimately get an error Error: CBOR decode error: unexpected character at position 0
This is the code as I have it so far:
let req = https.request(options, function (res) {
console.log("Header: " + JSON.stringify(res.headers));
res.setEncoding("utf8");
res.on("data", function (body) {
const deflatedBody = pako.deflate(body);
console.log("DEFLATED DATA -----> ", typeof deflatedBody, deflatedBody);
console.log(decode(deflatedBody));
});
res.on("error", function (error) {
console.log("connection could not be made " + error.message);
});
});
req.end();
};
I hope anyone has stumbled upon this already and has some idea.
Thanks a lot!

Please visit this answer https://stackoverflow.com/a/12776856/16315663 to retrieve GZIP data from the response.
Assuming, You have already retrieved full data as UInt8Array.
You just need the UInt8Array as String
const jsonString = Buffer.from(dataAsU8Array).toString('utf8')
const parsedData = JSON.parse(jsonString)
console.log(parsedData)
Edit
Here is what worked for me
const {request} = require("https")
const zlib = require("zlib")
const parseGzip = (gzipBuffer) => new Promise((resolve, reject) =>{
zlib.gunzip(gzipBuffer, (err, buffer) => {
if (err) {
reject(err)
return
}
resolve(buffer)
})
})
const fetchJson = (url) => new Promise((resolve, reject) => {
const r = request(url)
r.on("response", (response) => {
if (response.statusCode !== 200) {
reject(new Error(`${response.statusCode} ${response.statusMessage}`))
return
}
const responseBufferChunks = []
response.on("data", (data) => {
console.log(data.length);
responseBufferChunks.push(data)
})
response.on("end", async () => {
const responseBuffer = Buffer.concat(responseBufferChunks)
const unzippedBuffer = await parseGzip(responseBuffer)
resolve(JSON.parse(unzippedBuffer.toString()))
})
})
r.end()
})
fetchJson("https://wiki.mozilla.org/images/f/ff/Example.json.gz")
.then((result) => {
console.log(result)
})
.catch((e) => {
console.log(e)
})

Thank you, I actually just tried this approach and I get the following error:
SyntaxError: JSON Parse error: Unexpected identifier "x"
But I managed to print the data in text format using the below function:
getFinancialReports = (options, callback) => {
// buffer to store the streamed decompression
var buffer = [];
https
.get(options, function (res) {
// pipe the response into the gunzip to decompress
var gunzip = zlib.createGunzip();
res.pipe(gunzip);
gunzip
.on("data", function (data) {
// decompression chunk ready, add it to the buffer
buffer.push(data.toString());
})
.on("end", function () {
// response and decompression complete, join the buffer and return
callback(null, buffer.join(""));
})
.on("error", function (e) {
callback(e);
});
})
.on("error", function (e) {
callback(e);
});
};
Now I would need to pass this into a JSON object.

simple GET request in mocha timeout

I have the following code
const https = require("https");
it("wait for some result", function (done) {
this.timeout(15000);
const options = {
hostname: "httpbin.org",
path: "/get",
headers: {
Authorization: "bearer ",
},
};
https.get(options, (resp) => {
let data = "";
// A chunk of data has been recieved.
resp.on("data", (chunk) => {
data += chunk;
console.log(data);
});
// The whole response has been received. Print out the result.
resp.on("end", () => {
console.log(JSON.parse(data).explanation);
});
resp.on("error", (err) => {
console.log("Error: " + err.message);
});
done();
});
});
this returns:
should respond with redirect on post:
Error: timeout of 15000ms exceeded. Ensure the done() callback is being called in this test.
I am trying to make this asynchronous because of mocha, which is why I am using callback. I am even waiting 15000 instead of the default 2000ms.
this code normally works, only fails with mocha.
Not sure how to fix this issue. Any help is appreciated.
Thanks in advance.

try to add .end() on your request object
const req = https.get(options, (resp) => {
let data = "";
// A chunk of data has been recieved.
resp.on("data", (chunk) => {
data += chunk;
console.log(data);
});
// The whole response has been received. Print out the result.
resp.on("end", () => {
console.log(JSON.parse(data).explanation);
done();
});
resp.on("error", (err) => {
console.log("Error: " + err.message);
});
});
req.end()

DynamoDB updateItem in Lambda fails silently

I'm attempting to implement a simple counter with a Lambda function, but whenever I test it, the updateItem below simply does not work: none of the log statements in the callback are run at all, and of course the relevant counter in the table is never updated. Here's my lambda function:
'use strict';
const AWS = require('aws-sdk');
const dynamodb = new AWS.DynamoDB({ apiVersion: '2012-08-10' });
let params = {
TableName: 'Counters',
Key: {
'name': { S: 'global' }
},
UpdateExpression: 'SET val = val + :inc',
ExpressionAttributeValues: {
':inc': { N: '1' }
},
ReturnValues: 'ALL_NEW'
};
exports.handler = async(event) => {
console.log("Invoked counter-test");
dynamodb.updateItem(params, function(err, data) {
console.log("In updateItem callback");
if (err)
console.log(err, err.stack);
else
console.log(data);
});
console.log("Updated counter");
const response = {
statusCode: 200,
body: JSON.stringify('Counter updated'),
};
return response;
};
And here's the output of the test:
Response:
{
"statusCode": 200,
"body": "\"Counter updated\""
}
Request ID:
"80e92299-2eea-45e4-9c68-54ccf87199c5"
Function Logs:
START RequestId: 80e92299-2eea-45e4-9c68-54ccf87199c5 Version: $LATEST
2019-05-07T11:34:21.931Z 80e92299-2eea-45e4-9c68-54ccf87199c5 Invoked counter-test
2019-05-07T11:34:21.934Z 80e92299-2eea-45e4-9c68-54ccf87199c5 Updated counter
END RequestId: 80e92299-2eea-45e4-9c68-54ccf87199c5
REPORT RequestId: 80e92299-2eea-45e4-9c68-54ccf87199c5 Duration: 275.91 ms Billed Duration: 300 ms Memory Size: 128 MB Max Memory Used: 67 MB
As you can see, no log statements from the updateItems callback ran.
If I try to update the counter from the command line using aws dynamodb it does work, however:
$ aws dynamodb update-item \
--table-name Counters \
--key '{"name": { "S": "global" }}' \
--update-expression 'SET val = val + :inc' \
--expression-attribute-values '{":inc": {"N": "1"}}' \
--return-values ALL_NEW \
--output json
{
"Attributes": {
"name": {
"S": "global"
},
"val": {
"N": "129"
}
}
}

This is due to the asynchronous nature of Javascript.
The method updateItem is asynchronous and you don't wait for the callback to be fired before returning (you start the updateItem operation and then immediately return a response).
If you want to maintain the callback pattern, you should do:
exports.handler = (event, context, callback) => {
console.log("Invoked counter-test");
dynamodb.updateItem(params, function(err, data) {
console.log("In updateItem callback");
if (err) {
console.log(err, err.stack);
callback(err);
} else {
console.log(data);
console.log("Updated counter");
const response = {
statusCode: 200,
body: JSON.stringify('Counter updated'),
};
callback(null, response);
}
});
};
Using promises:
exports.handler = (event, context, callback) => {
console.log("Invoked counter-test");
dynamodb.updateItem(params).promise()
.then((data) => {
console.log(data);
console.log("Updated counter");
const response = {
statusCode: 200,
body: JSON.stringify('Counter updated'),
};
callback(null, response);
});
.catch((err) => {
console.log(err, err.stack);
callback(err);
})
};
Using await (recommended):
exports.handler = async (event) => {
try {
console.log("Invoked counter-test");
const data = await dynamodb.updateItem(params).promise();
console.log(data);
console.log("Updated counter");
const response = {
statusCode: 200,
body: JSON.stringify('Counter updated'),
};
return response;
} catch (err) {
console.log(err, err.stack);
throw err;
}
};
See also Understanding Asynchronous JavaScript, Deeply Understanding JavaScript Async and Await with Examples and AWS SDK for Javascript - Using JavaScript Promises.

Using the results of a GET request in Express router

First Node/Express app.
I'm having a hard time wrapping my head around on how to retrieve data from an endpoint and rendering it in the browser.
I have a dataservice.js that gets a JSON object from an endpoint like this:
const http = require('http');
getFinhockeyData = function() {
http.get('http://tilastopalvelu.fi/ih/modules/mod_standings/helper/standings.php?statgroupid=3545', (res) => {
console.log(`Got response: ${res.statusCode}`);
var body = "";
res.on('data', function (chunk) {
body += chunk;
})
res.on('end', function () {
var data = JSON.parse(body);
console.log('data parsed.');
console.log('first team name: ' + data.teams[0].TeamName);
console.log(typeof data);
return data;
})
}).on('error', (e) => {
console.log(`Got error from Finhockey: ${e.message}`);
});
}
module.exports.getFinhockeyData = getFinhockeyData;
Up until now things work and the data object can be console.logged and its content is usable.
The router.js looks currently like this:
'use strict';
const express = require('express');
const async = require('async');
const router = express.Router();
const dataservice = require('./dataservice.js')
router.get('/', function(req, res) {
async.series([
function(callback) {
getFinhockeyData(callback)
}
],
function(err, results) {
console.log('start rendering');
res.render('index', { data: data });
})
});
module.exports = router;
When I run the app and refresh the / route, I can see from the console that the getFinhockeyData is called and the data object's content is available in dataservice.js's console.logs, but the browser window hangs and the res.render part is never reached.
I understand that the rendering should be done only after the endpoint data request has finished (async.series usage), but it seems that I lack a fundamental understanding on how to actually use the result data from the getFinhockeyData function in the main route.
Any advice on this? I'll be happy to provide more info if necessary.

Firstly, doing the request is asynchronous, so you'll have to use either a callback or a promise.
Even the async middleware won't let you just return data from an asynchronous call, it requires a callback, but using native promises seems easier here
const http = require('http');
getFinhockeyData = function() {
return new Promise( (resolve, reject) => {
http.get('http://tilastopalvelu.fi/ih/modules/mod_standings/helper/standings.php?statgroupid=3545', (res) => {
var body = "";
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
resolve( JSON.parse(body) );
});
}).on('error', reject);
});
}
module.exports.getFinhockeyData = getFinhockeyData;
Also note that you're exporting as a module with a property
module.exports.getFinhockeyData = getFinhockeyData;
when you're going to use that in the routes, you have to use the property
const dataservice = require('./dataservice.js');
router.get('/', function(req, res) {
dataservice.getFinhockeyData().then(function(data) {
res.render('index', { data: JSON.stringify(data) });
}).catch(function(err) {
// epic fail, handle error here
});
});

You are responding to your route call with
res.render('index', { data: data });
But there is no data variable. It should be
res.render('index', { data: results });
Which is the variable where you are storing your data when it comes from the callback

The reason for res.render() not being called is, http requests are async. To get the response a callback must be passed, which you did but forgot to call it in the dataservice.js
This should help...
Change your dataservice.js like the following...
const http = require('http');
getFinhockeyData = function(callback) {
http.get('http://tilastopalvelu.fi/ih/modules/mod_standings/helper/standings.php?statgroupid=3545', (res) => {
console.log(`Got response: ${res.statusCode}`);
var body = "";
res.on('data', function (chunk) {
body += chunk;
})
res.on('end', function () {
var data = JSON.parse(body);
console.log('data parsed.');
console.log('first team name: ' + data.teams[0].TeamName);
console.log(typeof data);
callback(null, data); //returning the data to the callback
})
}).on('error', (e) => {
console.log(`Got error from Finhockey: ${e.message}`);
callback(e, null);
});
}
module.exports.getFinhockeyData = getFinhockeyData;
Change your router.js like the following...
router.get('/', function(req, res) {
async.series([
function(callback) {
getFinhockeyData(callback)
}
],
function(err, results) {
if(err === null){
console.log('start rendering');
res.render('index', { data: results[0] });
}
})
});

Using Node.js to connect to a REST API

Is it sensible to use Node.js to write a stand alone app that will connect two REST API's?
One end will be a POS - Point of sale - system
The other will be a hosted eCommerce platform
There will be a minimal interface for configuration of the service. nothing more.

Yes, Node.js is perfectly suited to making calls to external APIs. Just like everything in Node, however, the functions for making these calls are based around events, which means doing things like buffering response data as opposed to receiving a single completed response.
For example:
// get walking directions from central park to the empire state building
var http = require("http");
url = "http://maps.googleapis.com/maps/api/directions/json?origin=Central Park&destination=Empire State Building&sensor=false&mode=walking";
// get is a simple wrapper for request()
// which sets the http method to GET
var request = http.get(url, function (response) {
// data is streamed in chunks from the server
// so we have to handle the "data" event
var buffer = "",
data,
route;
response.on("data", function (chunk) {
buffer += chunk;
});
response.on("end", function (err) {
// finished transferring data
// dump the raw data
console.log(buffer);
console.log("\n");
data = JSON.parse(buffer);
route = data.routes[0];
// extract the distance and time
console.log("Walking Distance: " + route.legs[0].distance.text);
console.log("Time: " + route.legs[0].duration.text);
});
});
It may make sense to find a simple wrapper library (or write your own) if you are going to be making a lot of these calls.

Sure. The node.js API contains methods to make HTTP requests:
http.request
http.get
I assume the app you're writing is a web app. You might want to use a framework like Express to remove some of the grunt work (see also this question on node.js web frameworks).

/*Below logics covered in below sample GET API
-DB connection created in class
-common function to execute the query
-logging through bunyan library*/
const { APIResponse} = require('./../commonFun/utils');
const createlog = require('./../lib/createlog');
var obj = new DB();
//Test API
routes.get('/testapi', (req, res) => {
res.status(201).json({ message: 'API microservices test' });
});
dbObj = new DB();
routes.get('/getStore', (req, res) => {
try {
//create DB instance
const store_id = req.body.storeID;
const promiseReturnwithResult = selectQueryData('tablename', whereField, dbObj.conn);
(promiseReturnwithResult).then((result) => {
APIResponse(200, 'Data fetched successfully', result).then((result) => {
res.send(result);
});
}).catch((err) => { console.log(err); throw err; })
} catch (err) {
console.log('Exception caught in getuser API', err);
const e = new Error();
if (err.errors && err.errors.length > 0) {
e.Error = 'Exception caught in getuser API';
e.message = err.errors[0].message;
e.code = 500;
res.status(404).send(APIResponse(e.code, e.message, e.Error));
createlog.writeErrorInLog(err);
}
}
});
//create connection
"use strict"
const mysql = require("mysql");
class DB {
constructor() {
this.conn = mysql.createConnection({
host: 'localhost',
user: 'root',
password: 'pass',
database: 'db_name'
});
}
connect() {
this.conn.connect(function (err) {
if (err) {
console.error("error connecting: " + err.stack);
return;
}
console.log("connected to DBB");
});
}
//End class
}
module.exports = DB
//queryTransaction.js File
selectQueryData= (table,where,db_conn)=>{
return new Promise(function(resolve,reject){
try{
db_conn.query(`SELECT * FROM ${table} WHERE id = ${where}`,function(err,result){
if(err){
reject(err);
}else{
resolve(result);
}
});
}catch(err){
console.log(err);
}
});
}
module.exports= {selectQueryData};
//utils.js file
APIResponse = async (status, msg, data = '',error=null) => {
try {
if (status) {
return { statusCode: status, message: msg, PayLoad: data,error:error }
}
} catch (err) {
console.log('Exception caught in getuser API', err);
}
}
module.exports={
logsSetting: {
name: "USER-API",
streams: [
{
level: 'error',
path: '' // log ERROR and above to a file
}
],
},APIResponse
}
//createlogs.js File
var bunyan = require('bunyan');
const dateFormat = require('dateformat');
const {logsSetting} = require('./../commonFun/utils');
module.exports.writeErrorInLog = (customError) => {
let logConfig = {...logsSetting};
console.log('reached in writeErrorInLog',customError)
const currentDate = dateFormat(new Date(), 'yyyy-mm-dd');
const path = logConfig.streams[0].path = `${__dirname}/../log/${currentDate}error.log`;
const log = bunyan.createLogger(logConfig);
log.error(customError);
}

A more easy and useful tool is just using an API like Unirest; URest is a package in NPM that is just too easy to use jus like
app.get('/any-route', function(req, res){
unirest.get("https://rest.url.to.consume/param1/paramN")
.header("Any-Key", "XXXXXXXXXXXXXXXXXX")
.header("Accept", "text/plain")
.end(function (result) {
res.render('name-of-the-page-according-to-your-engine', {
layout: 'some-layout-if-you-want',
markup: result.body.any-property,
});
});