Develop Reference

How to correctly send and receive deflated data

I'm using protobufs for serializing my data. So far I serialized my data on the server (node restify) send it, receive it (Request is made by XMLHttpRequest) and serialize it on the Client.
Now I want to employ zipping to reduce the transfered file size. I tried using the library pako, which uses zlib.
In a basic script that I used to compare the protobuf and zipping performance to json I used it this way, and there were no problems
var buffer = proto.encode(data,'MyType'); // Own library on top of protobufs
var output = pako.deflate(buffer);
var unpacked = pako.inflate(output);
var decoded = proto.decode(buffer,'MyType');
However if I try to do this in a client-server model I can't get it working.
Server:
server.get('/data', function (req, res) {
const data = getMyData();
const buffer = proto.encode(data, 'MyType');
res.setHeader('content-type', 'application/octet-stream;');
res.setHeader('Content-Encoding', 'gzip;');
return res.send(200,buffer);
});
My own protolibrary serializes the data in protobuf and then deflates it:
...
let buffer = type.encode(message).finish();
buffer = pako.deflate(buffer);
return buffer;
The request looks like this:
public getData(){
return new Promise((resolve,reject) => {
const request = new XMLHttpRequest();
request.open("GET", this.url, true);
request.responseType = "arraybuffer";
request.onload = function(evt) {
const arr = new Uint8Array(request.response);
const payload = proto.decode(request.response ,'MyType')
resolve(payload);
};
request.send();
});
}
The proto.decode method first inflates the buffer buffer = pako.inflate(buffer); and then deserializes it from Protobuf.
If the request is made i get following error: "Uncaught incorrect header check" returned by the inflate method of pako:
function inflate(input, options) {
var inflator = new Inflate(options);
inflator.push(input, true);
// That will never happens, if you don't cheat with options :)
if (inflator.err) { throw inflator.msg || msg[inflator.err]; }
return inflator.result;
}
Also I looked at the request in Postman and found following:
The deflated response looks like this: 120,156,60,221,119,64,21,237,119,39,240,247,246,242,10,49,191,244,178,73,54,157 and has a length of 378564
The same request without deflating (the protobuf) looks like this
�:�:
 (� 0�8#H
 (� 0�8#H
� (�0�8#�Hand has a length of 272613.
I'm assuming, that I'm doing something incorrectly on the server side, since the inflated request is larger than the one not using compression.
Is it the content-type Header? I'm out of ideas.

What is the output of a piped file stream?

Perhaps the question is not worded in the greatest way but here's some more context. Using GridFSBucket, I'm able to store a file in mongo and obtain a download stream for that file. Here's my question. Let's say I wanted to send that file back as a response to my http request.
I do:
downloadStream.pipe(res);
On the client side now when I print the responseText, I get some long string with some funky characters that look to be encrypted. What is the format/type of this string/stream? How do I setup my response so that I can get the streamed data as an ArrayBuffer on my client side?
Thanks
UPDATE:
I haven't solved the problem yet, however the suggestion by #Jekrb, gives exactly the same output as doing console.log(this.responseText). It looks like the string is not a buffer. Here is the output from these 2 lines:
console.log(this.responseText.toString('utf8'))
var byteArray = new Uint8Array(arrayBuffer);
UPDATE 2 - THE CODE SNIPPETS
Frontend:
var savePDF = function(blob){
//fs.writeFile("test.pdf",blob);
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200){
//TO DO: Handle the file in the response which we will be displayed.
console.log(this.responseText.toString('utf8'));
var arrayBuffer = this.responseText;
if (arrayBuffer) {
var byteArray = new Uint8Array(arrayBuffer);
}
console.log(arrayBuffer);
}
};
xhr.open("POST","/pdf",true);
xhr.responseType = 'arrayBuffer';
xhr.send(blob);
};
BACKEND:
app.post('/pdf',function(req,res){
MongoClient.connect("mongodb://localhost:27017/test", function(err, db) {
if(err) return console.dir(err);
console.log("Connected to Database");
var bucket = new GridFSBucket(db, { bucketName: 'pdfs' });
var CHUNKS_COLL = 'pdfs.chunks';
var FILES_COLL = 'pdfs.files';
// insert file
var uploadStream = bucket.openUploadStream('test.pdf');
var id = uploadStream.id;
uploadStream.once('finish', function() {
console.log("upload finished!")
var downloadStream = bucket.openDownloadStream(id);
downloadStream.pipe(res);
});
// This pipes the POST data to the file
req.pipe(uploadStream);
});
});

My guess is that either the response is being outputted as plain binary which is not base64 encoded (still a buffer) or it is a compressed (gzip) response that needs to be uncompressed first.
Hard to pinpoint the issue without seeing the code though.
UPDATE:
Looks like you're missing the proper response headers.
Try setting these headers before the downloadStream.pipe(res):
res.setHeader('Content-disposition', 'attachment; filename=test.pdf');
res.set('Content-Type', 'application/pdf');

Your stream is likely already a buffer. You might be able to call responseText.toString('utf8') to convert the streamed data into readable string.

I solved it!!!
Basically preset the response type to "arraybuffer" before you make the request using
req.responseType = "arraybuffer"
Now, once you receive the response, don't use responseText, instead use response. response contains the arraybuffer with the data for the file.

POST data from a client javascript page to a node server

Is it possible to POST data from a client javascript page to a node server (server.js) using an AJAX XMLHttpRequest()? What I am looking for is javascript code that will receive the data on the node server, specifically the values for member_nm ("newName") and member_email ("mail#google.com"). I control the server. I also understand that I can also use GET to send the text values by means of a querystring. Below is the request that is sent from the client javascript page by means of an event handler:
document.getElementById("btnAddMember").addEventListener("click", function()
{
var request = new XMLHttpRequest();
var path = "/Users/Admin/WebstormProjects/projectName/server.js";
request.onreadystatechange = function()
{
if ( request.readyState === 4 && request.status == 200 )
{
request.setRequestHeader("Content-Type", "text/plain; charset=UTF-8");
request.open("POST", path, true);
request.send("member_nm=newName&member_email=mail#google.com");
}
};
});

You need to setup your server to accept this post request, the easiest will be to use Express with bodyParser middleware, like this :
var express = require('express');
var server=express();
var bodyParser= require('body-parser');
server.use(bodyParser.json());
server.post('/', function(req, res){
if(req.body){
// get the params from req.body.paramName
}
});
server.listen(8222, function(){
console.log('listening for requests ..')
});
In your client code change the 'path' to point to the server url:port, and I will put these outside of the onReadyStateChange:
request.setRequestHeader("Content-Type", "text/plain; charset=UTF-8");
request.open("POST", path, true);
request.send("member_nm=newName&member_email=mail#google.com");

This is a working solution on how to POST variables from a client javascript file to a Node server using Express. The POST is initiated on the client by means of an event handler, btnAddMember. txtName.value and txtMembershipType.value contain the values to be posted. Note the syntax that is necessary to parse the values correctly. member_nm and member_type will be used to reference the properties on the Node server. First the client javascript:
document.getElementById("btnAddMember").addEventListener("click", function()
{
var request = new XMLHttpRequest();
var path = "http://0.0.0.0:0000"; // enter your server ip and port number
request.open("POST", path, true); // true = asynchronous
request.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
var text= '{"member_nm":"' + txtName.value + '","member_type":"' + txtMembershipType.value + '"}';
request.send ( text );
});
Next is the server side code. Note that bodyParser must now be added to your project as a node_module. This can be done through the node program manager (npm). The POST statement basically parses the req.body from a JSON format to a javascript object format using a variable called 'member'. The code then logs the posted values for the two variables, member_nm and member_type. Finally a response status is sent to the client if the POST is successful.
var bodyParser = require("body-parser");
...
app.use(bodyParser.text({ type: "application/json" }));
...
// receive the POST from the client javascript file
app.post("/", function (req, res)
{
if (req.body)
{
var member = JSON.parse( req.body ); // parse req.body as an object
console.log("member_nm: " + member.member_nm );
console.log("member_type: " + member.member_type );
res.sendStatus(200); // success status
}
else
{
res.sendStatus(400); // error status
}
});

NodeJS HTTP request POST ERROR socket hang up

Hi I'm having problems to perform HTTP request on NodeJS given a larger number array of json object. The request works fine given small array of json object. However, if I try to increase the size array of json, I received Error: socket hang up {"error":{"code":"ECONNRESET"}}. Is it required to perform multiple write? Or is it something wrong going on at the other end?
Thanks in advance for taking your time here!
// data is a json object
var post_data = JSON.stringify(data);
var buf = new Buffer(post_data);
var len = buf.length;
var options = {
hostname: address,
port: port,
path: pathName,
method: 'PUT',
headers: {
'Content-Type':'application/json',
'Content-Length': len,
'Transfer-Encoding':'chunked'
}
};
// http call to REST API server
var req = restHttp.request(options, function(res) {
console.log('server PUT response received.');
var resData = '';
res.on('data', function(replyData) {
// Check reply data for error.
console.log(replyData.toString('utf8'));
if(replyData !== 'undefined')
resData += replyData;
});
res.on('end', function() {
callback(JSON.parse(resData));
});
});
req.write(buf);
req.end();

You can stream the request body.
If the data in buf was in a readable stream then you can just do buf.pipe(req).
For example, if the current directory contains a file data.json with the JSON you can do
var buf = fs.createReadStream(__dirname + '/data.json');
to create a ReadStream object. Then you can pipe this to you req
buf.pipe(req);
The pipe command will call req.end once its done streaming.

Nodejs output -Domain name not found

Technically this is my first try in nodejs and frankly I am not sure if I am doing it right. I am creating a local server that will stream the output from a distant server. However, when I run my code and I enter a URL in the browser, the program fails with the following message:
events.js:45
throw arguments[1]; // Unhandled 'error' event
^
Error: ENOTFOUND, Domain name not found
at IOWatcher.callback (dns.js:74:15)
The URL I used was: 127.0.0.1:9000/http://www.yahoo.fr. And in the browser I had the following message:
No data received
Unable to load the webpage because the server sent no data.
Here are some suggestions:
Reload this web page later.
Error 324 (net::ERR_EMPTY_RESPONSE): The server closed the connection without sending any data.
Any help is greatly appreciated.
Here is the code:
var base, dest, node_client,
count = 0,
url = require('url'),
util = require('util'),
http = require('http'),
http_client = require('http'),
request = require('request'),
events = require('events'),
httpProxy = require('./lib/node-http-proxy'),
data_emitter = new events.EventEmitter();
httpProxy.createServer(9000, 'localhost').listen(8000);
http.createServer(function (req, res) {
if(!count)
{
base = url.parse(req.url).pathname;
node_client = http_client.createClient(80, base);
count++;
} else {
dest = req.url.substr(1, req.url.length -1);
}
request = node_client.request("GET", dest, {"host": base});
request.addListener("response", function (response) {
var body = "";
response.addListener("data", function (data) {
body +=data;
});
response.addListener("end", function () {
var out = JSON.parse(body);
if(out.length > 0) {
data_emitter.emit("out", out);
}
});
});
// request.close();
var listener = data_emitter.addListener("data", function(out) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write(JSON.stringify(out));
res.close();
});
}).listen(9000);

Wild guess : your browser automatically requests 127.0.0.1:9000/favicon.ico and your program then tries to resolve favicon.ico which obviously fails and makes your program crash before it can send any data for the real request.

Why such tangled code?
This is a scenario where it makes sense to avoid nested callbacks, and use named functions. If you refactor the code, then people are more likely to be help you.

Can you do console.log(out) in your listener callback? Let us know if Node.js has any response data to return.

Well, for any newbie like me in this area, here is how I solved it. It's not clean and can be implemented in better way. Feel free to change, give suggestions.
Code:
var url = require('url'),
http = require('http'),
request = require('request'),
httpProxy = require('./lib/node-http-proxy'),
des = '',
util = require('util'),
colors = require('colors'),
is_host = true;
httpProxy.createServer(9000, 'localhost').listen(8000);
var server = http.createServer(function (req, res) {
var pathname = '';
if(is_host) {
dest = req.url.substr(0, req.url.length -1);
pathname = dest;
is_host = false;
} else {
pathname = req.url.substr(0, req.url.length);
if(pathname.charAt(0) == "/") {
console.log('new request');
console.log(pathname);
pathname = dest + pathname;
}
}
console.log(pathname);
request.get({uri: pathname}, function (err, response, html) {
res.end(html);
});
console.log('fetched from ' + pathname);
});
server.listen(9000);