I am trying to read a csv file and write the contents to a new csv file but am having a few issues in doing so.
My code seems to be reading the file ok and if i log the results of body in the console it shows the complete file. However when I try and write to the file, it does not seem to write all the contents.
The code I am using is as follows...
var http = require('http');
var fs = require('fs');
var readfile = "http://datafeed.api.productserve.com/datafeed.csv";
var writefile = "app/data/file.csv";
http.get(readfile, function(response) {
var body = '';
response.on("data", function(chunk) {
body += chunk;
});
response.on('end', function(){
fs.writeFile(writefile, body, function(error) {
if(error) {
console.error("write error: " + error.message);
} else {
console.log('Successful');
}
});
});
});
Am I doing something incorrect? Or is there some sort of limit on how much you can write?
Any help is much appreciated :)
Related
I try to get text content from the webpage. For example Google.com
I write at console:
$ ('#SIvCob').innerText
and get:
"Google offered in: русский"
This is the text, what I find out. Now I want to save it to file (.txt).
Two moments: there is no only one item, that I search, actually 7-10. And, there is a refresh every second! I go to write a cycle.
I know about copy() function and about right click on the console and "Save As," but I need a CODE, which will do it automatically.
Thanks in advance.
The browser has no API to write to the file system since that would be a security risk. But you can use Nodejs and their File System API to write you text file.
You will also need to use the HTTP API to get the web content. And you will also need to parse your HTML, you can do it with fast-html-parser or any other module of your choice. (high5, htmlparser, htmlparser2, htmlparser2-dom, hubbub, libxmljs, ms/file, parse5, ...)
var http = require('http');
var fs = require('fs');
var parser = require('node-html-parser');
var options = {
host: 'www.google.com',
port: 80,
path: '/index.html'
};
var file = '/path/to/myFile.txt';
http.get(options, function(res) {
res.setEncoding('utf8');
var body = '';
res.on('data', function (chunk) {body += chunk});
res.on('end', function () {
var dom = parser.parse(body);
var text = dom.querySelector('#SIvCob').text;
fs.writeFile(file, text, function (err) {
if (err) throw err;
console.log('The file has been saved!');
});
});
});
How can I JSON stringify the response of a website and use FS to save the JSON stringified response to a file?
This is my code but it didn't convert to JSON:
var request = require("request")
var fs = require('fs');
var url = "http://rubycp.me/"
var http = require('http');
var file = fs.createWriteStream("file.json");
var url = http.get("http://rubycp.me/", function(response) {
JSON.stringify(response.pipe(file));
});
I simply want to convert the html of the page into JSON and store that
into a json file.
response is a stream object that does not hold any data. You first need to collect all the data of the stream using the data event. If all data is collect the end event is triggered, at that even you can stringify your collected data and write it to the file.
const fs = require('fs')
const http = require('http')
var url = 'http://rubycp.me/'
var file = fs.createWriteStream('file.json')
http.get(url, response => {
// list that will hold all received chunks
var result = []
response
.on('data', chunk => result.push(chunk)) // add received chunk to list
.on('end', () => {
file.write(JSON.stringify(Buffer.concat(result).toString())) // when all chunks are received concat, stringify and write it to the file
})
})
This is happening because you are trying to write the response object to the file, not the response.body, which is where the JSON actually resides. Try this:
var request = require("request")
var fs = require('fs');
var url = "http://rubycp.me/"
var http = require('http');
var url = http.get("http://rubycp.me/", function(res) {
res.on("data", function(chunk) {
console.log("BODY: " + chunk);
fs.writeFile("file.json", JSON.stringify(chunk), function(error) {
if (error) { return console.err(error) }
console.log('File was saved successfully');
});
});
});
Change it to JSON.stringify(response.body).pipe(file).
I recommend this:
var fs = require('fs');
var http = require('http');
var url = http.get("http://rubycp.me/", function(response) {
var dat="";
response.on("data", function(chunk) {
console.log("BODY: " + chunk);
dat+=chunk;});
var x = JSON.stringify(dat);
fs.writeFile("file.json", x, function(err) {
if(err) {
return console.log(err);
}
console.log("The file was saved!");
});
});
It looks like you are piping to the file before you have 'stringified' the response.
I'm certain I'm missing something obvious, but the gist of the problem is I'm receiving a PNG from a Mapbox call with the intent of writing it to the file system and serving it to the client. I've successfully relayed the call, received a response of raw data and written a file. The problem is that my file ends up truncated no matter what path I take, and I've exhausted the answers I've found skirting the subject. I've dumped the raw response to the log, and it's robust, but any file I make tends to be about a chunk's worth of unreadable data.
Here's the code I've got at present for the file making. I tried this buffer move as a last ditch after several failed and comparably fruitless iterations. Any help would be greatly appreciated.
module.exports = function(req, res, cb) {
var cartography = function() {
return https.get({
hostname: 'api.mapbox.com',
path: '/v4/mapbox.wheatpaste/' + req.body[0] + ',' + req.body[1] + ',6/750x350.png?access_token=' + process.env.MAPBOX_API
}, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var mapPath = 'map' + req.body[0] + req.body[1] + '.png';
var map = new Buffer(body, 'base64');
fs.writeFile(__dirname + '/client/images/maps/' + mapPath, map, 'base64', function(err) {
if (err) throw err;
cb(mapPath);
})
})
});
};
cartography();
};
It is possible to rewrite your code in more compact subroutine:
const fs = require('fs');
const https = require('https');
https.get(url, (response)=> { //request itself
if(response) {
let imageName = 'image.png'; // for this purpose I usually use crypto
response.pipe( //pipe response to a write stream (file)
fs.createWriteStream( //create write stream
'./public/' + imageName //create a file with name image.png
)
);
return imageName; //if public folder is set as default in app.js
} else {
return false;
}
})
You could get original name and extension from url, but it safer to generate a new name with crypto and get file extension like i said from url or with read-chunk and file-type modules.
Any help is welcome, I have been struggling for many hours now...
I have a somewhat straight forward code in which I ping the GitHub API to retrieve a JSON. If I execute this code synchronously it works very well. However, when I receiving multiple call at the same time (or if I run it in async.parallel), sometimes the result of aa is an invalid JSON (I can see it in my logs) and JSON.parse crash. It seems like there's a missing chunk in the final version of aa
app.get('/update-tests', function(req, res) {
const getOptionsGitHub = {
...
};
var req = https.get(getOptionsGitHub, function(res) {
aa = "";
res.on('data', function(chunk) { aa += chunk; });
res.on('end', function() {
try {
console.dir(aa);
callback(JSON.parse(aa));
} catch (e) {
console.dir(e);
}
});
});
res.send(200);
});
Any idea why I would have some missing chunk sometimes?
Right now I'm using this script in PHP. I pass it the image and size (large/medium/small) and if it's on my server it returns the link, otherwise it copies it from a remote server then returns the local link.
function getImage ($img, $size) {
if (#filesize("./images/".$size."/".$img.".jpg")) {
return './images/'.$size.'/'.$img.'.jpg';
} else {
copy('http://www.othersite.com/images/'.$size.'/'.$img.'.jpg', './images/'.$size.'/'.$img.'.jpg');
return './images/'.$size.'/'.$img.'.jpg';
}
}
It works fine, but I'm trying to do the same thing in Node.js and I can't seem to figure it out. The filesystem seems to be unable to interact with any remote servers so I'm wondering if I'm just messing something up, or if it can't be done natively and a module will be required.
Anyone know of a way in Node.js?
You should check out http.Client and http.ClientResponse. Using those you can make a request to the remote server and write out the response to a local file using fs.WriteStream.
Something like this:
var http = require('http');
var fs = require('fs');
var google = http.createClient(80, 'www.google.com');
var request = google.request('GET', '/',
{'host': 'www.google.com'});
request.end();
out = fs.createWriteStream('out');
request.on('response', function (response) {
response.setEncoding('utf8');
response.on('data', function (chunk) {
out.write(chunk);
});
});
I haven't tested that, and I'm not sure it'll work out of the box. But I hope it'll guide you to what you need.
To give a more updated version (as the most recent answer is 4 years old, and http.createClient is now deprecated), here is a solution using the request method:
var fs = require('fs');
var request = require('request');
function getImage (img, size, filesize) {
var imgPath = size + '/' + img + '.jpg';
if (filesize) {
return './images/' + imgPath;
} else {
request('http://www.othersite.com/images/' + imgPath).pipe(fs.createWriteStream('./images/' + imgPath))
return './images/' + imgPath;
}
}
If you can't use remote user's password for some reasons and need to use the identity key (RSA) for authentication, then programmatically executing the scp with child_process is good to go
const { exec } = require('child_process');
exec(`scp -i /path/to/key username#example.com:/remote/path/to/file /local/path`,
(error, stdout, stderr) => {
if (error) {
console.log(`There was an error ${error}`);
}
console.log(`The stdout is ${stdout}`);
console.log(`The stderr is ${stderr}`);
});