I'm fetching an website's content via phantomjs by including jquery with the page. Now i have to write them to a file via program. For that i used the following code
page.onLoadFinished = (function(status) {
if (status === 'success') {
page.includeJs('http://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js', function() {
page.evaluate(function() {
var mkdirp = require('mkdirp');
mkdirp(counter+'_folder', function(err) {
var html = $('pre[data-language="html"]').html();
var js = $('pre[data-language="js"]').html();
var css = $('pre[data-language="css"]').html();
var fs = require('fs');
fs.writeFile(counter+"_folder/"+"fiddle.html", html, function(err) {});
fs.writeFile(counter+"_folder/"+"fiddle.css", css, function(err) {});
fs.writeFile(counter+"_folder/"+"fiddle.js", js, function(err) {});
console.log("******* "+counter+" *************");
});
});
});
}
});
page.open(url[counter]);
Now what happening is inside evalute method when I'm using require the program is getting stopped there showing error cannot find variable require. Any idea why this is appering?
page.evaluate() is the sandboxed page context. It has no access to require, page, phantom ...
Furthermore, mkdirp is a node module which will not work with PhantomJS. If you want to use PhantomJS from node, you will have to use a bridge like phantom. See also: Use a node module from casperjs
Using that bridge, you have to pass the variables to the outside and save it from there:
page.open(url, function(){
var mkdirp = require('mkdirp');
mkdirp(counter+'_folder', function(err) {
page.evaluate(function() {
var html = $('pre[data-language="html"]').html();
var js = $('pre[data-language="js"]').html();
var css = $('pre[data-language="css"]').html();
return [html, js, css];
}, function(result){
var fs = require('fs');
fs.writeFile(counter+"_folder/"+"fiddle.html", stuff[0], function(err) {});
fs.writeFile(counter+"_folder/"+"fiddle.css", stuff[1], function(err) {});
fs.writeFile(counter+"_folder/"+"fiddle.js", stuff[2], function(err) {});
console.log("******* "+counter+" *************");
});
});
});
Note: PhantomJS' fs module doesn't have a writeFile function. Node and PhantomJS have different execution environments.
Related
I have come across plenty of resources online for this but haven't been able to find one that is straight forward enough for me to understand.
At the moment, I have multiple massive <script> tags in an HTML document that has handlebars content. The server sends this HTML document to the client where the client then renders the page with data from an AJAX call. I'd like to move this entire process server-side so that all the server has to do is send a static file and re-render the page when data is updated. Data changes a few times per day - which is why it isn't hard coded in and I would like to run the handlebars compiler on the HTML document when data is updated.
Is it possible to simply put the HTML document with handlebars templating in <script> tags through a function to generate a new HTML file with data filled in?
Here is the code I have within my app.js file that is runned the Node server that does not do what I want it to:
function registerHelpers(callback){
Handlebars.registerHelper('equal', function(lvalue, rvalue, options) {
if (arguments.length < 3)
throw new Error("Handlebars Helper equal needs 2 parameters");
if( lvalue!=rvalue ) {
return options.inverse(this);
} else {
return options.fn(this);
}
});
Handlebars.registerHelper('trim', function(text) {
text = text.replace(/ /g, '');
return new Handlebars.SafeString(text);
});
callback();
}
function buildHomePage() {
var source = require(__dirname + '/public/home.handlebars');
var template = Handlebars.precompile(source);
var collection = db.get('datalist'); //Monk call to MongoDB
collection.find({}, function (err, docs){
var result = template(docs);
console.log(result)
var fs = require('fs');
fs.writeFile("test.html", result, function(err) {
if(err) {
console.log(err);
}
});
});
};
registerHelpers(buildHomePage);
The following can render handlebars to static html. Run node example.js. You may need to run npm install --save handlebars prior.
var fs = require('fs');
var Handlebars = require('handlebars');
function render(filename, data)
{
var source = fs.readFileSync(filename,'utf8').toString();
var template = Handlebars.compile(source);
var output = template(data);
return output;
}
var data = JSON.parse(fs.readFileSync("./data/strings.json", 'utf8'));
var result = render('./templates/somefile.html', data);
console.log(result);
If your handlebars templates are simple, with only string replacement, you can do this with underscore.js. Assume this example is named 'generate.js'
var fs = require('fs');
var _ = require('underscore');
_.templateSettings.interpolate = /\{\{(.+?)\}\}/g;
function render(filename, data)
{
var source = fs.readFileSync(filename,'utf8').toString();
var compiled = _.template(source);
return compiled(data);
}
var data = JSON.parse(fs.readFileSync("./data/strings.json", 'utf8'));
var result = render('./templates/somefile.html', data);
console.log(result);
Then run node generate.js to output the rendered template to the console. You may need to do npm install --save underscore prior.
I have a PhantomJS script that works when I run it locally (Mac), but when I run it on my Linux server, it returns the following error:
ReferenceError: Can't find variable: $
https://fantasy.premierleague.com/a/statistics/value_form:5712 in global code
The code is:
var page = require('webpage').create();
var fs = require('fs');
var args = require('system').args;
page.settings.userAgent = 'SpecialAgent';
page.open('https://fantasy.premierleague.com/a/statistics/value_form', function (status) {
if (status !== 'success') {
console.log('Unable to access network');
} else {
var ua = page.evaluate(function () {
var result ="";
// ...
return result;
});
}
phantom.exit();
});
There may be a race condition between your code and jQuery being loaded on the page. Wrap the statements in your page.evaluate callback with a $(document).ready(function() { /* your statements here */ }); to ensure scripts on the page have loaded fully.
For anyone who is still using PhantomJS and encounters this problem, I solved it with
phantomjs --ignore-ssl-errors=yes
I don't intend for upvote.
I'm providing solution to solve some situations without simulating browser behavior with phantomjs just to retrieve data that can be handled directly by requesting to url.
You need the data from the page, so why not just do request to this url: https://fantasy.premierleague.com/drf/bootstrap-static
var request = require('request'); // install: npm i request
var fs = require('fs');
var args = require('system').args;
request.get({url: 'https://fantasy.premierleague.com/drf/bootstrap-static'}, function(err, response, body) {
console.log(body);
});
How I found this url?
Simple:
I am trying to write HTML file in nodejs. I need to create elements using all the DOM methods. I tried following, but throwing error document is undefined. Please suggest me what wrong I am doing and what is the right way to do this. Thanks
var fs = require('fs');
var myDiv= document.createElement("div");
myDiv.setAttribute("id","myDiv");'
var data = myDiv.outerHTML;
var writerStream = fs.createWriteStream('abcd.html');
writerStream.write(data,'UTF8');
writerStream.end();
writerStream.on('finish', function() { console.log("Write completed."); });
writerStream.on('error', function(err){ console.log(err.stack); });
console.log("Program Ended");
This issue is due to there is no window object available. Remember document object comes from window.
When we write document, it is actually window.document.
But nodejs does not provide the window object, as it runs on serverside.
So to do this you can use a module called jsdom. Go through jsdom here: https://www.npmjs.com/package/node-jsdom
Install jsdom and try something like:
var jsdom = require('jsdom');
var fs = require('fs');
var http = require("http");
http.createServer(function (request, response) {
response.writeHead(200, {'Content-Type': 'text/plain'});
response.end('Hello World\n');
}).listen(8081);
console.log('Server running at http://127.0.0.1:8081/');
jsdom.env({
html: 'http://127.0.0.1:8081/',
src: [],
done: function(errors, window) {
var document = window.document;
var myDiv= document.createElement("div");
myDiv.setAttribute("id","myDiv");'
var data = myDiv.outerHTML;
var writerStream = fs.createWriteStream('abcd.html');
writerStream.write(data,'UTF8');
writerStream.end();
writerStream.on('finish', function() { console.log("Write completed."); });
writerStream.on('error', function(err){ console.log(err.stack); });
console.log("Program Ended");
}
});
I know this question has been asked but my mind has been blown by my inability to get this working. I am trying to upload multiple images to my server with the following code:
var formidable = require('formidable');
var fs = require('fs');
...
router.post('/add_images/:showcase_id', function(req, res){
if(!admin(req, res)) return;
var form = new formidable.IncomingForm(),
files = [];
form.uploadDir = global.__project_dirname+"/tmp";
form.on('file', function(field, file) {
console.log(file);
file.image_id = global.s4()+global.s4();
file.endPath = "/img/"+file.image_id+"."+file.type.replace("image/","");
files.push({field:field, file:file});
});
form.on('end', function() {
console.log('done');
console.log(files);
db.get("SOME SQL", function(err, image_number){
if(err){
console.log(err);
}
var db_index = 0;
if(image_number) db_index = image_number.image_order;
files.forEach(function(file, index){
try{
//this line opens the image in my computer (testing)
require("sys").exec("display " + file.file.path);
console.log(file.file.path);
fs.renameSync(file.file.path, file.file.endPath);
}catch (e){
console.log(e);
}
db.run( "SOME MORE SQL"')", function(err){
if(index == files.length)
res.redirect("/admin/gallery"+req.params.showcase_id);
});
});
});
});
form.parse(req);
});
The line that opens the image via system calls works just fine, however I continue to get:
Error: ENOENT, no such file or directory '/home/[username]/[project name]/tmp/285ef5276581cb3b8ea950a043c6ed51'
by the rename statement.
the value of file.file.path is:
/home/[username]/[project name]/tmp/285ef5276581cb3b8ea950a043c6ed51
I am so confused and have tried everything. What am I doing wrong?
Probably you get this error because the target path does not exist or you don't have write permissions.
The error you get is misleading due to a bug in nodejs, see:
https://github.com/joyent/node/issues/5287
https://github.com/joyent/node/issues/685
Consider adding:
console.log(file.file.endPath);
before the fs.renameSync call and check if the target path exist and is writable by your application
You stated form. Therefore note that Formidable doesn't work out of the box with just NodeJS. Unless you were to use something like the prompt module for input. If you are using HTML, you'll need something like Angular, React or Browserify to be able to give it access to your interface.
I'm trying to use the new version of the node-csv node module to do some CSV manipulation.
node-csv
I've used perl in the past, but would like to try JavaScript this time. I'm having trouble figuring out how to import a local CSV file instead of using the built in generator. The documentation for node-csv doesn't show how to do this as far as I can tell(although it does provide an example for the previous version).
Here is the example code, which works as expected.
var csv = require('csv');
var generator = csv.generate({seed: 1, columns: 2, length: 20});
var parser = csv.parse();
var transformer = csv.transform(function(data){
return data.map(function(value){return value.toUpperCase()});
});
var stringifier = csv.stringify();
generator.on('readable', function(){
while(data = generator.read()){
parser.write(data);
}
});
parser.on('readable', function(){
while(data = parser.read()){
transformer.write(data);
}
});
transformer.on('readable', function(){
while(data = transformer.read()){
stringifier.write(data);
}
});
stringifier.on('readable', function(){
while(data = stringifier.read()){
process.stdout.write(data);
}
});
I plan on using the FS module, but am not sure how to pass the local file into the node-csv functions.
var fs = require('fs');
Here is an example for the PREVIOUS version, which uses completely different syntax:
// node samples/sample.js
var csv = require('csv');
var fs = require('fs');
csv()
.from.stream(fs.createReadStream(__dirname+'/sample.in'))
.to.path(__dirname+'/sample.out')
.transform( function(row){
row.unshift(row.pop());
return row;
})
.on('record', function(row,index){
console.log('#'+index+' '+JSON.stringify(row));
})
.on('end', function(count){
console.log('Number of lines: '+count);
})
.on('error', function(error){
console.log(error.message);
});
Any suggestions?