I have the following code, which will retrieve a text file from an external server, and search the file for a specific string.
The function:
function checkStringExistsInFile(String, cb) {
var opt = {
host: 'site.com',
port: 80,
path: '/directory/data.txt',
method: 'GET',
};
var request = http.request(opt, function(response){
response
.on('data', function(data){
var string = data+"";
var result = ((string.indexOf(" "+ String +" ")!=-1)?true:false);
cb(null, result);
})
.on('error', function(e){
console.log("-ERR: Can't get file. "+JSON.stringify(e));
if(cb) cb(e);
})
.on('end', function(){
console.log("+INF: End of request");
});
});
request.end();
}
And this is where I call the function, and do something with the results.
checkStringExistsInFile(String, function(err, result){
if(!err) {
if(result) {
//there was a result
} else {
//string not present in file
}
} else {
// error occured
}
});
This worked great in the beginning (small text file), but my textfile is getting larger (4000 characters+) and this is not working anymore.
What can I do to solve this? Should I safe the temporary save the file on my server first, should I open the file as a stream?
It would be appreciated if you can support your answer with a relevant example. Thanks in advance!
Documentation :
If you attach a data event listener, then it will switch the stream into flowing mode, and data will be passed to your handler as soon as it is available.
If you just want to get all the data out of the stream as fast as possible, this is the best way to do so.
http://nodejs.org/api/stream.html#stream_event_data
Data event is emitted as soon as there are data, even if the stream is not completely loaded. So your code just look for your string in the first lines of your stream, then callbacks.
What to do ?
Your function should only call callback on the end() event, or as soon as it finds something.
function checkStringExistsInFile(String, cb) {
var opt = {
host: 'site.com',
port: 80,
path: '/directory/data.txt',
method: 'GET',
};
var request = http.request(opt, function(response){
var result = false;
response
.on('data', function(data){
if(!result) {
var string = data+"";
result = (string.indexOf(" "+ String +" ")!=-1);
if(result) cb(null, result);
}
})
.on('error', function(e){
console.log("-ERR: Can't get file. "+JSON.stringify(e));
if(cb) cb(e);
})
.on('end', function(){
console.log("+INF: End of request");
cb(null, result)
});
});
request.end();
}
Related
This question is a follow-up from this answer. I'm asking it as a question because I don't have enough reputation to make comments.
I am trying to write an Android app that takes an image and sends it to a Node.js server.
I found this code in the post linked above, but it leaves me with a raw buffer of data. How can I save the contents of this buffer as a JPEG image?
app.post('/upload-image', rawBody, function(req, res) {
if (req.rawBody && req.bodyLength > 0) {
// TODO save image (req.rawBody) somewhere
// send some content as JSON
res.send(200, {
status: 'OK'
});
} else {
res.send(500);
}
});
function rawBody(req, res, next) {
var chunks = [];
req.on('data', function(chunk) {
chunks.push(chunk);
});
req.on('end', function() {
var buffer = Buffer.concat(chunks);
req.bodyLength = buffer.length;
req.rawBody = buffer;
next();
});
req.on('error', function(err) {
console.log(err);
res.status(500);
});
}
You can simply use writeFile
Pass the file path as first argument, your buffer as second argument, and a callback to execute code when the file has been written.
Alternatively, use writeFileSync, a synchronous version of writeFile which doesn't take a callback
I'm trying to do a post request onto my api, the api works perfectly ( I am able to post files, but not through a url), but now I'm trying to post through an url.
this is the code I have now, I removed some lines that aren't relevant to the question or were for testing.
request({
url: url + "gettoken"
, json: true
}, function (error, response, body) {
user = body;
var rs = fs.createReadStream(up.url);
var ws = request.post(url + "upload?token=" + `${user.token}&key=${user.key}&filename=${filename}`);
ws.on('drain', function () {
rs.resume();
});
rs.on('end', function () {
console.log(filename);
});
ws.on('error', function (err) {
console.error('cannot send file ' + err);
});
rs.pipe(ws);
})
Can anyone please help me.
So the idea is to upload a file that's located at up.url to another server at url + "upload?...".
Since fs.createReadStream is meant to read local files, and not URL's, you need something that can create a stream from a URL (or rather, retrieve that URL and stream the response).
You can also use request for that:
request({
url: url + "gettoken",
json: true
}, function (error, response, body) {
const user = body;
const rs = request.get(up.url);
const ws = request.post(url + "upload?token=" + `${user.token}&key=${user.key}&filename=${filename}`);
rs.on('end', function () {
console.log(filename);
});
ws.on('error', function (err) {
console.error('cannot send file ' + err);
});
rs.pipe(ws);
});
Typically, file uploads work through multipart/form-data, but your code doesn't suggest that being used here. If it is, the code would become something like this:
const ws = request.post(url + "upload?token=" + `${user.token}&key=${user.key}&filename=${filename}`, {
formData : {
the_file : rs
}
});
// no `rs.pipe(ws)`
I'm fairly new to Node and Javascript and I'm struggling with my first Node module. What I'm trying to do is export functions for specific API calls and I'd like to reuse my https.request function rather than duplicating the code in each function. For some reason I'm failing to wrap my head around how to pass the data back to my original function. Here's an abbreviated version - the listStuff function would be one of many to handle various api request actions.
'use strict';
const https = require('https');
const _ = require('underscore');
const hostStr = 'api.server.net';
function listStuff(){
var pathStr = '/release/api/stuff';
_apiCall(pathStr);
//Would like to handle the https response data here
};
function _apiCall(pathStr){
var options = {
host: hostStr,
path: pathStr
};
var req = https.get(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
var responseString = '';
res.on('data', function(d){
responseString += d;
});
res.on('end', function(){
var responseObject = JSON.parse(responseString);
});
});
req.end();
req.on('error', function(e){
console.log(e);
});
};
module.exports = {
listStuff: listStuff
};
Hope this helps. Register a callback in the apiCall function, and then check the callback params for error handling. Then, just make sure you return the callback when you want to end the function call (either in the on end or on error processing).
function listStuff(){
var pathStr = '/release/api/stuff';
_apiCall(pathStr, function(err, data) {
if (err) // handle err
//handle data.
});
};
function _apiCall(pathStr, callback){
var options = {
host: hostStr,
path: pathStr
};
var req = https.get(options, function(res) {
console.log("statusCode: ", res.statusCode);
console.log("headers: ", res.headers);
var responseString = '';
res.on('data', function(d){
responseString += d;
});
res.on('end', function(){
var responseObject = JSON.parse(responseString);
return callback(null, responseObject);
});
});
req.end();
req.on('error', function(e){
console.log(e);
return callback(e);
});
};
A slightly different approach using Promise objects. Note I looked into this as a learning exercise and hope it helps. I have not written all the code for you and the debugging is all yours!
Firstly make _apiCall returns a promise object.
function listStuff()
{
var pathStr = '/release/api/stuff';
var promise = _apiCall(pathStr);
promise.then( function( responceObject){
// handle response object data
});
promise.catch( function( error){
console.log( error.message); // report error
});
}
Next step is to make _apiCall return a promise object for the HTTPS request it will initiate inside the executor of promise creation.
function _apiCall(pathStr)
{ var options = {
host: hostStr,
path: pathStr
};
function beginGet( worked, failed)
{
// see below
}
return new Promise( beginGet);
}
Lastly write beginGet to initiate and call back success or fail functions depending on the outcome of the get request.
function beginGet( worked, failed)
{ var req;
var responseObj;
function getCallBack( res)
{ // all your get request handling code
// on error call failed( error)
// on sucessful completion, call worked(responseObj)
}
req = https.get(options, getCallBack);
}
Also please check with https.get documentation - I think it calls req.end() for you. All the other errors are mine :-)
In Node.js, say that I want to read a file from somewhere and stream the response (e.g., from the filesystem using fs.createReadStream()).
application.get('/files/:id', function (request, response) {
var readStream = fs.createReadStream('/saved-files/' + request.params.id);
var mimeType = getMimeTypeSomehow(request.params.id);
if (mimeType === 'application/pdf') {
response.set('Content-Range', ...);
response.status(206);
} else {
response.status(200);
}
readStream.pipe(response);
});
However, I want to detect if there is an error with the stream before sending my response headers. How do I do that?
Pseudocode:
application.get('/files/:id', function (request, response) {
var readStream = fs.createReadStream('/saved-files/' + request.params.id);
readStream.on('ready', function () {
var mimeType = getMimeTypeSomehow(request.params.id);
if (mimeType === 'application/pdf') {
response.set('Content-Range', ...);
response.status(206);
} else {
response.status(200);
}
readStream.pipe(response);
});
readStream.on('error', function () {
response.status(404).end();
});
});
Write stream is ended when readStream ends or has an error. You can prevent this default behaviour by passing end:false during pipe and end the write stream manually.
So even if the error occurs, your write stream is still open and you can do other stuff(e.g. sending 404 status) with writestream in the error callback.
var readStream = fs.createReadStream('/saved-files/' + request.params.id);
readStream.on('error', function () {
res.status(404).end();
});
readStream.on('end', function(){
res.end(); //end write stream manually when readstream ends
})
readStream.pipe(res,{end:false}); // prevent default behaviour
Update 1: For file streams, you can listen for open event to check if the file is ready to read:
readStream.on('open', function () {
// set response headers and status
});
Update 2: As OP mentioned there may be no open event for other streams, we may use the following if the stream is inherited from node's stream module. The trick is we write the data manually instead of pipe() method. That way we can do some 'initialization' on writable before starting to write first byte.
So we bind once('data') first and then bind on('data'). First one will be called before actual writing is happened.
readStream
.on('error',function(err) {
res.status(404).end();
})
.once('data',function(){
//will be called once and before the on('data') callback
//so it's safe to set headers here
res.set('Content-Type', 'text/html');
})
.on('data', function(chunk){
//now start writing data
res.write(chunk);
})
.on('end',res.end.bind(res)); //ending writable when readable ends
I am trying to make a function that returns the content of the webpage and this is what I have so far
var get_contents = function() {
var httpRequestParams =
{
host: "google.com",
port: 80,
path: "/?"
};
var req = http.get(httpRequestParams, function(res)
{
var data = '';
res.on('data', function(chunk) {
data += chunk.toString();
});
//console.log(data);
}).end();
return req;
}
This when I run this code, I see the html contents when the console logging is turned on but when I try to return the output, it just never works.
I can't figure out a way to return get_contents() anywhere. On the console, it just doesnt respond.
Thanks
Something like that: (dont forget to handle error and timeout)
var on_contents = function(cb) {
var httpRequestParams =
{
host: "google.com",
port: 80,
path: "/?"
};
var req = http.get(httpRequestParams, function(res)
{
var data = '';
res.on('data', function(chunk) {
data += chunk.toString();
});
res.on('end', function(){
cb(data);
});
//console.log(data);
}).end();
}
function onFinish(data) {
console.log(data);
}
on_contents(onFinish)
The short answer is: You can't return the data from that function. http.get is asynchronous, so it doesn't actually start running the callback until after your function ends. You'll need to have your get_contents function take a callback itself, check in the http.get handler whether you're done loading and, if you are, call the get_contents callback.
There is an awesome module [request][1] available in node.js.
var request = require('request'),
url = require('url');
var server = http.createServer(function (request, response) {
getPage("http://isohunt.com/torrents/?iht=-1&ihq=life+is+beautiful", function (body) {
console.log(body);
})
});
server.listen(3000);
More information can be found on http://www.catonmat.net/blog/nodejs-modules-request/