It would be wonderful if you could help me with this!
I'm using node.js and heroku to deply an app and I'm changing my web.js file everything I add a new post. This was fine earlier, but with more pages, it's starting to get out of hand. I'm wondering if I could automate this?
Eg, this is what my web.js file looks like now:
var express = require('express');
var fs = require('fs');
var htmlfile = "index.html";
var app = express(express.logger());
//day 7 and so on.. till day 15.
var day6 = "./100/day6/day6.html";
app.get('/day6', function(request, response) {
var html = fs.readFileSync(day6).toString();
response.send(html);
});
var day5 = "./100/day5/day5.html";
app.get('/day5', function(request, response) {
var html = fs.readFileSync(day5).toString();
response.send(html);
});
var day4 = "./100/day4/day4.html";
app.get('/day4', function(request, response) {
var html = fs.readFileSync(day4).toString();
response.send(html);
});
var day3 = "./100/day3/day3.html";
app.get('/day3', function(request, response) {
var html = fs.readFileSync(day3).toString();
response.send(html);
});
var day2 = "./100/day2/day2.html";
app.get('/day2', function(request, response) {
var html = fs.readFileSync(day2).toString();
response.send(html);
});
var challengehtmlfile = "100.html";
app.get('/100-day-challenge', function(request, response) {
var html = fs.readFileSync(challengehtmlfile).toString();
response.send(html);
});
var bloghtmlfile = "blog.html";
app.get('/blog', function(request, response) {
var html = fs.readFileSync(bloghtmlfile).toString();
response.send(html);
});
app.get('/', function(request, response) {
var html = fs.readFileSync(htmlfile).toString();
response.send(html);
});
var port = process.env.PORT || 8080;
app.listen(port, function() {
console.log("Listening on " + port);
});
If you are planning to add much more pages, it is not a good idea to add one route per page. Better way is to use pattern matching in your routing. For example,
app.get('/day/:num', function(request, response) {
var day = "./100/day" + request.params.num + "/day" + request.params.num + ".html";
var html = fs.readFileSync(day).toString();
response.send(html);
});
Now this route would match any number of days. Only thing you would have to change is the URL. Instead of '/day3', you should access '/day/3'.
In /day/:num, :num is just a variable name in the url. You access that variable through request.params.num. Then you construct the filename with that information. Hope this would clean up your script. :)
For more info, check express api here (http://expressjs.com/api.html#req.params).
Copy this code and save it as "insert.sh".
#!/bin/bash
count=$1
day=day$1
sed -i "/express(express.logger());/avar ${day} = \"./100/${day}/${day}.html\";\napp.get('/${day}', function(request, response) {\nvar html = fs.readFileSync(${day}).toString();\nresponse.send(html);\n});\n" $2
This script takes two parameters: # of days and file name. For example,
bash insert.sh 3 yournode.js
This command will insert the necessary code for day3 in your node.js file. I assumed you want to add those lines after express.logger() line. If you want to put the code after a different line, simply replace express(express.logger()); in the script to anything you want. Please let me know.
Related
I have this in Node.JS file.
var express = require('express');
var app = express();
var http = require('http').Server(app);
var cfenv = require("cfenv");
var appEnv = cfenv.getAppEnv();
http.listen(appEnv.port, appEnv.bind);
var PersonalityInsightsV2 = require('watson-developer-cloud/personality-insights/v2');
var personality_insights = new PersonalityInsightsV2({
username: '<YOUR-USERNAME>',
password: '<YOUR-PASSWORD>'
});
personality_insights.profile({
text: "<YOUR-100-UNIQUE-WORDS>",
language: 'en' },
function (err, response) {
if (err)
console.log('error:', err);
else
console.log(JSON.stringify(response, null, 2));
});
I am sending an API call but as you can see, it shows me the result in JSON in the console.
How can I make this result in JSON that shows me in the console, show it to me in an HTML?
Thank you very much!
I supose that the problem is in console.log(JSON.stringify(res,null, 2));, but, I don't know how put this in HTML.
You can't just turn JSON into HTML. JSON is a data format. HTML is a markup language. You'll manually have to create some HTML with the way you want it, and then drop in values from the JSON.
For example, you could do something like this:
else {
const html =
`<!DOCTYPE html>
<body>
<p>${response.name}</p>
`;
console.log(html);
}
That would give you some HTML like:
<!DOCTYPE html>
<body>
<p>Bob</p>
assuming response has a value of name.
It sounds like you're wanting to view the JSON on an HTML page in a browser. Something like this should help. It will start your Express server listening on whatever port you specified using appEnv.port, and will serve up myJson (which will then be assigned in your code)
var express = require('express');
var app = express();
var http = require('http').Server(app);
var cfenv = require("cfenv");
var appEnv = cfenv.getAppEnv();
var myJson;
// respond with JSON when a GET request is made to the index
app.get('/', function (req, res) {
res.send(myJson)
})
app.listen(appEnv.port);
var PersonalityInsightsV2 = require('watson-developer-cloud/personality-insights/v2');
var personality_insights = new PersonalityInsightsV2({
username: '<YOUR-USERNAME>',
password: '<YOUR-PASSWORD>'
});
personality_insights.profile({
text: "<YOUR-100-UNIQUE-WORDS>",
language: 'en' },
function (err, response) {
if (err)
console.log('error:', err);
else
myJson = JSON.stringify(response, null, 2);
});
To try this, you would open your browser to "http://localhost:appEnv.port/" (where appEnv.port is the port you chose). You should see your JSON output
I am scraping the webpage https://www.g2a.com/rising-storm-2-vietnam-steam-cd-key-global.html
I need to get the title from the table data.
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/scrape', function(req, res) {
url = 'https://www.g2a.com/rising-storm-2-vietnam-steam-cd-key-global.html';
request(url, function(error, response, body) {
if (!error) {
var $ = cheerio.load(body);
var arr = [];
var title = $('.mp-user-rating tr').each(function() {
var tableData = $('.marketplace-name > .mp-rating-popup');
arr.push({ 'title': tableData.text() });
});
}
res.send('Check your console!')
});
})
app.listen('8081');
console.log('Magic happens on port 8081');
exports = module.exports = app;
Here the data is in third column and cannot able to get .mp-user-rating tr data what is expected.
The image shows the structure of the table
Any help would be appreciated.
So, I went to the page and ran this in the console.
var arr = [];
var title = jQuery('.mp-user-rating tr').each(function(i, element) {
var tableData = jQuery(element).find('.mp-rating-popup');
arr.push({ 'title': tableData.text() });
});
console.log(arr);
The array consists of 8 objects that each have the titles within them.
UPDATE:
I pulled in the html information using your code. I think the issue is, the html is loaded asynchronously by the website, as a result, pulling the html will only retrieve the static markup. You will need to use PhantomJS or chrome's headless browser in order to load the website and allow the asyncronous information to load, then you can grab the html.
See here for some good docs on PhantomJS: https://github.com/Medium/phantomjs
https://socketstream.github.io/socketstream/docs/#/tutorials/authentication
when i put the codes in this section to the demo project(real time chat), the chat function doesn't work. can any one help me with this?
here's my code:
// ===================== SEPARATOR ============
var ss = require('socketstream');
var redirect = require('connect-redirection');
var everyauth = require('everyauth');
var http = require('http');
var conf = require('./conf/conf');
var auth_twitter = conf.oauth.twitter;
// Define a single-page client called 'main'
ss.client.define('main', {
view: 'app.html',
css: ['../node_modules/normalize.css/normalize.css', 'app.css'],
code: ['../node_modules/es6-shim/es6-shim.js', 'libs/jquery.min.js', 'app'],
tmpl: 'chat'
});
ss.http.middleware.prepend(redirect());
// Serve this client on the root URL
ss.http.route('/', function(req, res){
// if(!req.session.userId){ return res.redirect('/login'); }
res.serveClient('main');
});
// Use server-side compiled Hogan (Mustache) templates. Others engines available
ss.client.templateEngine.use(require('ss-hogan'));
// Minimize and pack assets if you type: SS_ENV=production node app.js
if (ss.env === 'production') ss.client.packAssets();
everyauth.twitter
.consumerKey(auth_twitter.KEY)
.consumerSecret(auth_twitter.SEC)
.findOrCreateUser( function (session, accessToken, accessTokenSecret, twitterUserMetadata) {
var userName = twitterUserMetadata.screen_name;
console.log('Twitter Username is', userName);
session.userId = userName;
session.save();
return true;
})
.redirectPath('/');
var bodyParser = require('body-parser');
//ss.http.middleware.prepend(bodyParser.urlencoded());
//ss.http.middleware.append(everyauth.middleware());
var server = http.Server(ss.http.middleware);
server.listen(3000);
ss.start(server);
After reading some doc, i tried this and it worked:
replace the last three lines codes:
var server = http.Server(ss.http.middleware);
server.listen(3000);
ss.start(server);
with this:
ss.start();
I am a beginner at node js and I'm trying to write a web scraping script. I got permission from the site admin to scrape their products if I make less then 15 requests a minute. When I started out it used to request all the URLs at once but after some tooling around, I was able to go through each item in the array, but the script doesn't stop when there is no more items in the array? I'm not really happy with my result and feel like there is a better way to do this.
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
var async = require('async');
app.get('/scrape', function(req, res){
productListing = ['ohio-precious-metals-1-ounce-silver-bar','morgan-1-ounce-silver-bar']
var i = 0;
async.eachLimit(productListing, 1, function (product, callback) {
var getProducts = function () {
var url = 'http://cbmint.com/' + productListing[i];
request(url, function(error, response, html) {
if(!error){
var $ = cheerio.load(html);
var title;
var json = { title : ""};
$('.product-name').filter(function(){
var data = $(this);
title = data.children().children().first().text();
json.title = title;
})
}
var theTime = new Date().getTime();
console.log(i);
console.log(json.title);
console.log(theTime);
i++;
});
}
setInterval(getProducts,10000);
})
res.send('Check your console!')
})
app.listen('8081')
console.log('Magic happens on port 8081');
exports = module.exports = app;
You aren't calling callback inside the iterator function. Take a look at the docs for eachLimit.
Technically this is my first try in nodejs and frankly I am not sure if I am doing it right. I am creating a local server that will stream the output from a distant server. However, when I run my code and I enter a URL in the browser, the program fails with the following message:
events.js:45
throw arguments[1]; // Unhandled 'error' event
^
Error: ENOTFOUND, Domain name not found
at IOWatcher.callback (dns.js:74:15)
The URL I used was: 127.0.0.1:9000/http://www.yahoo.fr. And in the browser I had the following message:
No data received
Unable to load the webpage because the server sent no data.
Here are some suggestions:
Reload this web page later.
Error 324 (net::ERR_EMPTY_RESPONSE): The server closed the connection without sending any data.
Any help is greatly appreciated.
Here is the code:
var base, dest, node_client,
count = 0,
url = require('url'),
util = require('util'),
http = require('http'),
http_client = require('http'),
request = require('request'),
events = require('events'),
httpProxy = require('./lib/node-http-proxy'),
data_emitter = new events.EventEmitter();
httpProxy.createServer(9000, 'localhost').listen(8000);
http.createServer(function (req, res) {
if(!count)
{
base = url.parse(req.url).pathname;
node_client = http_client.createClient(80, base);
count++;
} else {
dest = req.url.substr(1, req.url.length -1);
}
request = node_client.request("GET", dest, {"host": base});
request.addListener("response", function (response) {
var body = "";
response.addListener("data", function (data) {
body +=data;
});
response.addListener("end", function () {
var out = JSON.parse(body);
if(out.length > 0) {
data_emitter.emit("out", out);
}
});
});
// request.close();
var listener = data_emitter.addListener("data", function(out) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write(JSON.stringify(out));
res.close();
});
}).listen(9000);
Wild guess : your browser automatically requests 127.0.0.1:9000/favicon.ico and your program then tries to resolve favicon.ico which obviously fails and makes your program crash before it can send any data for the real request.
Why such tangled code?
This is a scenario where it makes sense to avoid nested callbacks, and use named functions. If you refactor the code, then people are more likely to be help you.
Can you do console.log(out) in your listener callback? Let us know if Node.js has any response data to return.
Well, for any newbie like me in this area, here is how I solved it. It's not clean and can be implemented in better way. Feel free to change, give suggestions.
Code:
var url = require('url'),
http = require('http'),
request = require('request'),
httpProxy = require('./lib/node-http-proxy'),
des = '',
util = require('util'),
colors = require('colors'),
is_host = true;
httpProxy.createServer(9000, 'localhost').listen(8000);
var server = http.createServer(function (req, res) {
var pathname = '';
if(is_host) {
dest = req.url.substr(0, req.url.length -1);
pathname = dest;
is_host = false;
} else {
pathname = req.url.substr(0, req.url.length);
if(pathname.charAt(0) == "/") {
console.log('new request');
console.log(pathname);
pathname = dest + pathname;
}
}
console.log(pathname);
request.get({uri: pathname}, function (err, response, html) {
res.end(html);
});
console.log('fetched from ' + pathname);
});
server.listen(9000);