I'm new to Node.js and am working on a project and I'd like to use Request to stream from one endpoint to another. My goal is to use Request to get and post an edited JSON body using a pipe stream. I know that when doing so, content-type and content-length will be preserved in the POST headers. However, I would like to apply .forEach to all JSON objects in the body from the first url, and post them to the second url.
I'm not sure about the correct format, and I'd appreciate some clarification.
I know the basic syntax is this:
request.get('URL').pipe(request.post('URL'));
And so far my best guess is something like this:
request('FIRST_URL', function (error, response, body) {
body = JSON.parse(body);
body.forEach( function(arg) {
//return edited body
});
}).pipe(request.post('SECOND_URL'));
Am I missing something? Is there a better way to do this?
You could write your own transform stream. For example:
var Transform = require('stream').Transform;
var inherits = require('util').inherits;
function JSONTransform() {
Transform.call(this);
this._bufffer = '';
}
inherits(JSONTransform, Transform);
JSONTransform.prototype._transform = function(chunk, enc, cb) {
this._buffer += chunk;
cb();
});
JSONTransform.prototype._flush = function(cb) {
try {
var result = JSON.parse(this._buffer);
this._buffer = null;
// Do whatever transformations
// ...
this.push(JSON.stringify(result));
cb();
} catch (ex) {
cb(ex);
}
});
// Then just pipe
request.get('FIRST_URL')
.pipe(new JSONTransform())
.pipe(request.post('SECOND_URL'));
One other slightly different solution that may be worth considering would be to use a third-party streaming JSON parser module, which may or may not work for your use case.
Related
I have this code that I put in my console:
XMLHttpRequest.prototype.send = function(body) {
// modifies inputted request
newBody = JSON.parse(body);
newBody.points = 417;
// sends modified request
this.realSend(JSON.stringify(newBody));
}
It is supposed to make the points 417 every time it sends a request, but when I look at the request body, it still says the original amount of points. Any help?
Try to add an alert() or console.log() into your modified XMLHttpRequest.prototype.send to check if it actually works. There is a way to prevent this kind of modifications silently.
As others have noted, the error you are experiencing is hard to diagnose exactly without seeing how you created this.realSend.
However, this code will work:
const send = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.send = function (body) {
const newBody = JSON.parse(body);
newBody.points = 417;
send.call(this, JSON.stringify(newBody));
};
Note that instead of storing the original send method on XMLHttpRequest.prototype, I've kept in a separate variable and simply invoked it with the correct this value through send.call(). This seems like a cleaner implementation with less chance for conflicts with other code.
See this codesandbox for a working example.
If your function is not being called, possible fetch is used to make ajax requests.
So you can wrap both functions, like this
const send = XMLHttpRequest.prototype.send;
const _fetch = window.fetch;
XMLHttpRequest.prototype.send = function (body) {
const newBody = JSON.parse(body);
newBody.points = 417;
send.call(this, JSON.stringify(newBody));
};
window.fetch = function(url, options){
let newBody;
if(options.body) {
newBody = JSON.parse(options.body);
newBody.points = 417;
options.body = JSON.stringify(newBody);
}
_fetch.call(this, url, options);
}
I'm consuming a JSON stream and am trying to use fetch to consume it. The stream emits some data every few seconds. Using fetch to consume the stream gives me access to the data only when the stream closes server side. For example:
var target; // the url.
var options = {
method: "POST",
body: bodyString,
}
var drain = function(response) {
// hit only when the stream is killed server side.
// response.body is always undefined. Can't use the reader it provides.
return response.text(); // or response.json();
};
var listenStream = fetch(target, options).then(drain).then(console.log).catch(console.log);
/*
returns a data to the console log with a 200 code only when the server stream has been killed.
*/
However, there have been several chunks of data already sent to the client.
Using a node inspired method in the browser like this works every single time an event is sent:
var request = require('request');
var JSONStream = require('JSONStream');
var es = require('event-stream');
request(options)
.pipe(JSONStream.parse('*'))
.pipe(es.map(function(message) { // Pipe catches each fully formed message.
console.log(message)
}));
What am I missing? My instinct tells me that fetch should be able to mimic the pipe or stream functionality.
response.body gives you access to the response as a stream. To read a stream:
fetch(url).then(response => {
const reader = response.body.getReader();
reader.read().then(function process(result) {
if (result.done) return;
console.log(`Received a ${result.value.length} byte chunk of data`);
return reader.read().then(process);
}).then(() => {
console.log('All done!');
});
});
Here's a working example of the above.
Fetch streams are more memory-efficient than XHR, as the full response doesn't buffer in memory, and result.value is a Uint8Array making it way more useful for binary data. If you want text, you can use TextDecoder:
fetch(url).then(response => {
const reader = response.body.getReader();
const decoder = new TextDecoder();
reader.read().then(function process(result) {
if (result.done) return;
const text = decoder.decode(result.value, {stream: true});
console.log(text);
return reader.read().then(process);
}).then(() => {
console.log('All done!');
});
});
Here's a working example of the above.
Soon TextDecoder will become a transform stream, allowing you to do response.body.pipeThrough(new TextDecoder()), which is much simpler and allows the browser to optimise.
As for your JSON case, streaming JSON parsers can be a little big and complicated. If you're in control of the data source, consider a format that's chunks of JSON separated by newlines. This is really easy to parse, and leans on the browser's JSON parser for most of the work. Here's a working demo, the benefits can be seen at slower connection speeds.
I've also written an intro to web streams, which includes their use from within a service worker. You may also be interested in a fun hack that uses JavaScript template literals to create streaming templates.
Turns out I could get XHR to work - which doesn't really answer the request vs. fetch question. It took a few tries and the right ordering of operations to get it right. Here's the abstracted code. #jaromanda was right.
var _tryXhr = function(target, data) {
console.log(target, data);
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function () {
console.log("state change.. state: "+ this.readyState);
console.log(this.responseText);
if (this.readyState === 4) {
// gets hit on completion.
}
if (this.readyState === 3) {
// gets hit on new event
}
};
xhr.open("POST", target);
xhr.setRequestHeader("cache-control", "no-cache");
xhr.setRequestHeader("Content-Type", "application/json");
xhr.send(data);
};
I would like to know the good practice if I'm streaming data and want to have access to whole data after streaming;
I'm streaming like this:
res._oldWrite = res.write;
res.write = function (chunk, encoding, cb) {
var decoded = chunk.toString(encoding);
write.write(new Buffer(decoded, encoding), encoding, cb);
return res._oldWrite.call(res, new Buffer(decoded, encoding), encoding, cb);
}
Now that I want to access to my data I did something like:
res._oldWrite = res.write;
var jsonData = '';
res.write = function (chunk, encoding, cb) {
var decoded = chunk.toString(encoding);
jsonData += decoded;
write.write(new Buffer(decoded, encoding), encoding, cb);
return res._oldWrite.call(res, new Buffer(decoded, encoding), encoding, cb);
}
res.on('finish', function(){
// Now I can have access to jsopnData but it is gross ; what is the right way?
})
But isn't there any better way to do it?
So I'm not 100% sure I understand your question #Web Developer, but since you asked for code, below is all that I meant.
Note that there are probably other shorter ways of doing the same thing (but I'm not sure what you mean by "access whole data after streaming" --- store in memory? all at once? etc):
var dataStream = require('stream').Writable();
//I'm assuming the "real processing" is saving to a file
var fileStream = fs.createWriteStream('data.txt');
var masterStream = require('stream').Writeable();
masterStream._write = function (chunk, enc, next) {
dataStream.write(chunk);
fileStream.write(chunk);
next();
};
//if you now write to master stream, you get values in both dataStream and fileStream
//you can now listen to dataStream and "have access to the data"
I apologize if this is a stupid question, but I am new to Javascript and Node.js really hurts my head because it is asynchronous.
My goal is to query for a JSON object from an API and be able to work with it. I have tried to look for questions and answers on what I should be doing but none of it really makes sense to me, so I am hoping to learn by just seeing proper code.
var request = require('request');
var url = 'url here';
var temp;
var done = false;
request(url, function (error, response, body) {
if (!error) {
temp = body;
done = true;
console.log(temp);
} else {
console.log(error);
}
});
if (done){
console.log(temp);
}
Can someone please walk me through the proper way to restructure my code?
The function you are creating with the line
request(url, function (error, response, body) {
is not executed until the response is received. The rest of your code continues to run. Think of the flow something like this:
var request = require('request');
var url = 'url here';
var temp;
var done = false;
request(url, XXX);
if (done){
console.log(temp);
then when the response is received (perhaps much later on) the function XXX is executed.
As you can see, done will always be false when the line
if (done){
is executed.
While building a NNTP client in NodeJS, I have encountered the following problem. When calling the XZVER command, the first data I receive from the socket connection contains both a string and an inflated string;
224 compressed data follows (zlib version 1.2.3.3)
^*�u�#����`*�Ti���d���x�;i�R��ɵC���eT�����U'�|/S�0���� rd�
z�t]2���t�bb�3ѥ��>�͊0�ܵ��b&b����<1/ �C�<[<��d���:��VW̡��gBBim�$p#I>5�cZ�*ψ%��u}i�k�j
�u�t���8�K��`>��im
When I split this string and try to inflate it like this;
lines = chunk.toString().split('\r\n');
response = lines.shift();
zlib.inflate(new Buffer(lines.shift()), function (error, data) {
console.log(arguments);
callback();
});
I receive the following error;
[Error: invalid code lengths set] errno: -3, code: 'Z_DATA_ERROR'
Any help is welcome, I am kinda stuck here :(
UPDATE
After implementing the answer of mscdex, the whole function looks like this;
var util = require('util'),
zlib = require('zlib'),
Transform = require('stream').Transform;
function CompressedStream () {
var self = this;
this._transform = function (chunk, encoding, callback) {
var response = chunk.toString(),
crlfidx = response.indexOf('\r\n');
response = response.substring(0, crlfidx);
console.log(response);
zlib.gunzip(chunk.slice(crlfidx + 2), function (error, data) {
console.log(arguments);
callback();
});
};
Transform.call(this/*, { objectMode: true } */);
};
util.inherits(CompressedStream, Transform);
module.exports = CompressedStream;
You should probably avoid using split() in case those two bytes are in the raw data. You might try something like this instead:
var response = chunk.toString(),
crlfidx = response.indexOf('\r\n');
// should probably check crlfidx > -1 here ...
response = response.substring(0, crlfidx);
zlib.inflate(chunk.slice(crlfidx + 2), function (error, data) {
console.log(arguments);
callback();
});
However if you're doing this inside a 'data' event handler, you should be aware that you may not get the data you expect in a single chunk. Specifically you could get a CRLF split between chunks or you could get multiple responses in a single chunk.
It seems that my chunks were incorrectly encoded. By removing socket.setEncoding('utf8');, the problem was solved.