Develop Reference

How to handle JSON data from XMLHttpRequest POST, using nodeJS

Overarching goal is to save some JSON data I create on a webpage to my files locally. I am definitely sending something to the server, but not in format I seem to able to access.
JsonData looks like:
{MetaData: {Stock: "UTX", Analysis: "LinearTrend2"}
Projections: [2018-10-12: 127.62, 2018-10-11: 126.36000000000001, 2018-10-10: 132.17, 2018-10-09: 140.12, 2018-10-08: 137.73000000000002, …]}
XMLHttpRequest on my webpage:
function UpdateBackTestJSON(JsonUpdate){ //JsonUpdate being the JSON object from above
var request = new XMLHttpRequest();
request.open('POST', 'UpdateBackTestJSON');
request.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
// request.setRequestHeader("Content-Type", "text/plain;charset=UTF-8");
request.onload = function() {
console.log("Updated JSON File");
};
console.log("about to send request");
console.log(JsonUpdate);
request.send(JSON.stringify(JsonUpdate));
}
and I handle posts on my server (rather carelessly I realize, just going for functionality as a start here)
var http = require('http')
, fs = require('fs')
, url = require('url')
, port = 8008;
var server = http.createServer (function (req, res) {
var uri = url.parse(req.url)
var qs = require('querystring');
if (req.method == 'POST'){
var body = '';
req.on('data', function (data){
body += data;
// 1e6 === 1 * Math.pow(10, 6) === 1 * 1000000 ~~~ 1MB
if (body.length > 1e6){
// FLOOD ATTACK OR FAULTY CLIENT, NUKE REQUEST
req.connection.destroy();
}
});
req.on('end', function () {
var POST = qs.parse(body);
console.log(POST); // PARSED POST IS NOT THE RIGHT FORMAT... or something, idk whats going on
UpdateBackTestData(POST);
});
}
function UpdateBackTestData(TheJsonData){
console.log("UpdateBackTestData");
console.log(TheJsonData);
JsonUpdate = JSON.parse(TheJsonData);
console.log(JsonUpdate["MetaData"]);
//var Stock = JsonUpdate["MetaData"]["Stock"];
// var Analysis = JsonUpdate["MetaData"]["Analysis"];
fs.writeFile("/public/BackTestData/"+Analysis+"/"+Stock+".json", TheJsonData, function(err){
if(err){
console.log(err);
}
console.log("updated BackTest JSON!!!");
});
}
Most confusing to me is that when I run this, the Json object Im am trying to pass, does go through to the server, but the entirety of the data is a string used as a key for a blank value in an object. when I parse the body of the POST, I get: {'{MetaData:{'Stock':'UTX','Analysis:'LinearTrend2'},'Projections':[...]}': ''}. So my data is there... but not in a practical format.
I would prefer not to use express or other server tools, as I have a fair amount of other services set up in my server that I don't want to go back and change if I can avoid it.
Thanks for any help

What is the output of a piped file stream?

Perhaps the question is not worded in the greatest way but here's some more context. Using GridFSBucket, I'm able to store a file in mongo and obtain a download stream for that file. Here's my question. Let's say I wanted to send that file back as a response to my http request.
I do:
downloadStream.pipe(res);
On the client side now when I print the responseText, I get some long string with some funky characters that look to be encrypted. What is the format/type of this string/stream? How do I setup my response so that I can get the streamed data as an ArrayBuffer on my client side?
Thanks
UPDATE:
I haven't solved the problem yet, however the suggestion by #Jekrb, gives exactly the same output as doing console.log(this.responseText). It looks like the string is not a buffer. Here is the output from these 2 lines:
console.log(this.responseText.toString('utf8'))
var byteArray = new Uint8Array(arrayBuffer);
UPDATE 2 - THE CODE SNIPPETS
Frontend:
var savePDF = function(blob){
//fs.writeFile("test.pdf",blob);
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200){
//TO DO: Handle the file in the response which we will be displayed.
console.log(this.responseText.toString('utf8'));
var arrayBuffer = this.responseText;
if (arrayBuffer) {
var byteArray = new Uint8Array(arrayBuffer);
}
console.log(arrayBuffer);
}
};
xhr.open("POST","/pdf",true);
xhr.responseType = 'arrayBuffer';
xhr.send(blob);
};
BACKEND:
app.post('/pdf',function(req,res){
MongoClient.connect("mongodb://localhost:27017/test", function(err, db) {
if(err) return console.dir(err);
console.log("Connected to Database");
var bucket = new GridFSBucket(db, { bucketName: 'pdfs' });
var CHUNKS_COLL = 'pdfs.chunks';
var FILES_COLL = 'pdfs.files';
// insert file
var uploadStream = bucket.openUploadStream('test.pdf');
var id = uploadStream.id;
uploadStream.once('finish', function() {
console.log("upload finished!")
var downloadStream = bucket.openDownloadStream(id);
downloadStream.pipe(res);
});
// This pipes the POST data to the file
req.pipe(uploadStream);
});
});

My guess is that either the response is being outputted as plain binary which is not base64 encoded (still a buffer) or it is a compressed (gzip) response that needs to be uncompressed first.
Hard to pinpoint the issue without seeing the code though.
UPDATE:
Looks like you're missing the proper response headers.
Try setting these headers before the downloadStream.pipe(res):
res.setHeader('Content-disposition', 'attachment; filename=test.pdf');
res.set('Content-Type', 'application/pdf');

Your stream is likely already a buffer. You might be able to call responseText.toString('utf8') to convert the streamed data into readable string.

I solved it!!!
Basically preset the response type to "arraybuffer" before you make the request using
req.responseType = "arraybuffer"
Now, once you receive the response, don't use responseText, instead use response. response contains the arraybuffer with the data for the file.

Sending binary data over websocket with cowboy and MessagePack

I'm trying to send a MessagePack-encoded message from Cowboy to a browser over WebSocket, and received data is always empty or invalid. I'm able to send binary data from JS to my cowboy handler, but not vice versa.
I'm using Cowboy 1.0.4 with official msgpack-erlang application. I also use msgpack-lite for my in-browser javascript.
Examples:
websocket_handler:
websocket_handle({text, <<"return encoded">>}, Req, State) ->
%% sends encoded message to client. Client is unable to decode and fails
{reply, {binary, msgpack:pack(<<"message">>)}, Req, State};
websocket_handle({binary, Encoded}, Req, State) ->
%% Works as expected
lager:info("Received encoded message: ~p", [msgpack:unpack(Encoded)]),
{ok, Req, State};
JS:
var host = "ws://" + window.location.host + "/websocket";
window.socket = new WebSocket(host);
socket.binaryType = 'arraybuffer';
socket.onmessage = function(event) {
var message = msgpack.decode(event.data);
console.log(message);
};
Browser returns an error inside msgpack.min.js:
Error: Invalid type: undefined
...ion n(t){var r=i(t),e=f[r];if(!e)throw new Error("Invalid type: "+(r?"0x"+r.toSt...
If I try to output raw event.data to console, here's what I'm getting:
ArrayBuffer {}
It seems to be empty for some reason. I'm new both to erlang and msgpack, and don't know what is going wrong. Thanks for your help!

Found the reason of my problem.
The way how I tried to decode message on the client was wrong:
socket.onmessage = function(event) {
var message = msgpack.decode(event.data);
console.log(message);
};
The right way:
socket.onmessage = function(event) {
var raw_binary_data = new Uint8Array(event.data);
var message = msgpack.decode(raw_binary_data);
console.log(message);
};

It seems like msgpack-lite doesn't support binary type. Try pack your data as a string.
{binary, msgpack:pack("message", [{enable_str, true}])}

Using Uint8Array is a valid solution for client side issue. On server to pack strings use:
msgpack:pack(<<"message">>,[{pack_str,from_binary}])
Source : Article

Handling Node.js socket data

I have server receiving data from a client [GPS Device]. I have problem presenting the data (i.e the results obtained from the client) in a readable format. Below are the things I have tried.
Doing:
console.log(data)
I get
<Buffer d0 d7 3d 00 c4 56 7e 81>
Also tried
console.log(data.toString())
But I get unwanted results:See below:
��A�V~�
Here is my full code:
var net = require('net');
var fs = require('fs');
var server = net.createServer(function (socket) {
console.log('Server started: Waiting for client connection ...');
console.log('Client connected:port,address: '+socket.remotePort, socket.remoteAddress);
socket.on('data', function (data) {
var date = new Date();
var today = date.getDate()+'_'+date.getMonth();
fs.appendFile(today+'_log.txt', data, function (err) {
if (err) throw err;
console.log(data.toString())
});
});
});
server.listen(my_port, my_ip);
Thanks for your input.

According to the documentation, you must specify an encoding to get a String instead of a Buffer:
Event: 'data'#
Buffer object
Emitted when data is received. The argument data will be a Buffer or String. Encoding of data is set by socket.setEncoding().
You could configure the socket to get the data in UTF-8, for example, with:
socket.setEncoding('utf8');

Assuming the data in buffer is 7 bit ASCII,
console.log(data.toString('ascii'))
would resolve the problem.

Do it like this
socket.on('data', function (data) {
var buff = Buffer.from(data);
But remember, lots of GPS devices use little-endian, so later you're going to have to decode the data as well.

NodeJS HTTP request POST ERROR socket hang up

Hi I'm having problems to perform HTTP request on NodeJS given a larger number array of json object. The request works fine given small array of json object. However, if I try to increase the size array of json, I received Error: socket hang up {"error":{"code":"ECONNRESET"}}. Is it required to perform multiple write? Or is it something wrong going on at the other end?
Thanks in advance for taking your time here!
// data is a json object
var post_data = JSON.stringify(data);
var buf = new Buffer(post_data);
var len = buf.length;
var options = {
hostname: address,
port: port,
path: pathName,
method: 'PUT',
headers: {
'Content-Type':'application/json',
'Content-Length': len,
'Transfer-Encoding':'chunked'
}
};
// http call to REST API server
var req = restHttp.request(options, function(res) {
console.log('server PUT response received.');
var resData = '';
res.on('data', function(replyData) {
// Check reply data for error.
console.log(replyData.toString('utf8'));
if(replyData !== 'undefined')
resData += replyData;
});
res.on('end', function() {
callback(JSON.parse(resData));
});
});
req.write(buf);
req.end();

You can stream the request body.
If the data in buf was in a readable stream then you can just do buf.pipe(req).
For example, if the current directory contains a file data.json with the JSON you can do
var buf = fs.createReadStream(__dirname + '/data.json');
to create a ReadStream object. Then you can pipe this to you req
buf.pipe(req);
The pipe command will call req.end once its done streaming.