Develop Reference

Connect node app and server + post image to server

I have a very basic question about a node application, and a question about HTTP requests. It's the first time I create a node app with server, and I just can't seem to get the different components to work together.
This is my server.js
var express = require('express');
var multer = require('multer');
const request = require('request');
const upload = multer({dest: __dirname + '/uploads/images'});
const app = express();
const PORT = 3000;
app.use(express.static('public'));
app.post('/upload', upload.single('photo'), (req, res) => {
if(req.file) {
res.json(req.file);
}
else throw 'error';
});
app.listen(PORT, () => {
console.log('Listening at ' + PORT );
});
Then I have a file app.js with a motion-detection system. Every time motion is detected, a picture is taken. This all works fine.
Then the picture should be sent to the server. This is what I can't figure out.
I created a function toServer() that should post the detected data to the server
const request = require('request');
function toServer(data) {
const formData = {
// Pass data via Buffers
my_buffer: data,
// Pass optional meta-data with an 'options' object with style: {value: DATA, options: OPTIONS}
// Use case: for some types of streams, you'll need to provide "file"-related information manually.
// See the `form-data` README for more information about options: https://github.com/form-data/form-data
};
request.post({url:'http://localhost:3000/upload', formData: formData}, function optionalCallback(err, httpResponse, body) {
if (err) {
return console.error('Upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
});
};
Problem 1: when running the server.js on localhost:3000, it doesn't find any of the scripts loaded in index.html nor my app.js.
Problem 2: when running the index.html on live-server, all scripts are found, but i get the error "request is not defined".
I am pretty sure there is some basic node setup thing I'm missing.
The solution for toServer() might be more complicated.
Thanks for your time,
Mustard Shaper

Problem 1:
this could happen because you have not specified to render your index.html.
for example:
res.render('index')
if it's not because of the single quotes in upload.single('photo') try double quotes.
Another possible error could be that you are missing a default display engine setting.
an example: https://www.npmjs.com/package/hbs
Problem 2:
it may be because you are missing the header
var request = require('request');
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: 'http://localhost',
body: "example"
}, function(error, response, body){
console.log(body);
});
See more at https://expressjs.com/

Why two times the callback invoked in CreateServer of http in nodejs

i am trying with following code:
const http = require('http');
const fs = require('fs');
var hServer = http.createServer( (req, res) => {
console.log ("Received Connection..");
fs.readFile('./index.html', function(err, page) {
res.writeHeader(200, {"Content-Type": "text/html"});
res.write(page);
res.end();
});
});
hServer.listen(8989);
When i connect from browser http://localhost:8989,
I received two times the console print "Received Connection." Why?

const http = require('http');
const fs = require('fs');
var hServer = http.createServer( (req, res) => {
console.log ("Received Connection...");
console.log('URL: ' + req.url);
fs.readFile('./index.html', function(err, page) {
res.writeHeader(200, {"Content-Type": "text/html"});
res.write(page);
res.end();
});
});
hServer.listen(8989);
Will print:
Received Connection...
URL: /
Received Connection...
URL: /favicon
It is because the browser automatically ask for the favicon, the little icon you see in your tabs.
If you fire your request from POSTMan, wget, curl, or other http tools, you'll only see one request.

This can be traced down by logging out req using console.log(req).
Looking at the raw request we see that the browser additionally requests /favicon for each request.
url: '/',
url: '/favicon.ico',

Express session not working just with IE

Running my third party js app, it has seen so weird when play with session on the IE .
From here no problem using Chrome, FF and Safari.
My apps has the following design:
Any client can use my third party js code.
So, this widget.js just does a call for my app node using express.
This widget.js is a IIFE where on my app get the request and create the session using the code:
app.js
var emptygif = require('emptygif');
var express = require('express');
var uuid = require('node-uuid');
var expressSession = require('express-session');
var app = express();
app.set('trust proxy', 1)
app.use(expressSession({
genid: function(req) {
return uuid.v4(); // use UUIDs for session IDs
},
secret: '1234567890QWERTY',
cookie: { expires: new Date(Date.now() + 900000) },
secure: false,
httpOnly: true
}));
app.use(function(req, res, next) {
res.header('Access-Control-Allow-Credentials', true);
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE');
res.header('Access-Control-Allow-Headers', 'X-Requested-With, X-HTTP-Method-Override, Content-Type, Accept');
next();
});
app.get('/widget.js', function(request, response){
response.sendfile('widget.js');
});
app.get('/ping', function(req, res) {
var msg_out = load(req);
console.log( msg_out );
emptygif.sendEmptyGif(req, res, {
'Content-Type': 'image/gif',
'Content-Length': emptygif.emptyGifBufferLength,
'Cache-Control': 'public, max-age=0' // or specify expiry to make sure it will call everytime
});
});
function load(req){
var msg = {
user_token: req.session.token_bility,
time_stamp: new Date(),
user_agent: req.headers['user-agent'],
language: req.headers["accept-language"],
referrer: req.headers['referer']
}
return msg;
}
app.listen();
widget.js
(function(window, undefined) {
var _jq;
function drawWidget() {
console.log('drawing');
}
function loadSupportingFiles(_url,callback) {
var path = 'http://myserver/' + _url;
var oImg=document.createElement("img");
oImg.setAttribute('src', path);
var body = document.getElementsByTagName('body')[0];
body.appendChild(oImg);
callback();
};
function scriptLoadHandler() {
console.log('loading pixel');
}
loadSupportingFiles('ping', function() {
scriptLoadHandler();
});
})(window);
I already have done some searching by google for this questions but still without solution.
I will share with pleajure about any hint and knowledge from how to fix this.
Pay attention:
I forgotte to say some keys informations.
The token has been generated well for all browsers (IE, Chrome, FF and Safari).
But each hint for the server generate different tokens only for IE.
The behavior shows that IE expires every request.
IIFE ref.
https://en.wikipedia.org/wiki/Immediately-invoked_function_expression

Solution
After trying some parameters options this resolved.
cookie: { maxAge: 60*10000 },
secure: false,
httpOnly: true
I changed to maxAge instead expires.
These links helped me.
How to keep session in expressjs to not expire
http://mrcoles.com/blog/cookies-max-age-vs-expires/

Loopback IO OAuth not working

I am trying to get a https loopback server up and running protected by OAuth. I am using the loopback gateway sample project as a reference. But for some reason I can't get the OAuth piece to work. What I mean is, even after adding in the OAuth bits and pieces, the APIs don't seem to be protected. I get a response back even if there is no token in my request. This is what my server.js looks like
var loopback = require('loopback');
var boot = require('loopback-boot');
var https = require('https');
var path = require('path');
var httpsRedirect = require('./middleware/https-redirect');
var site = require('./site');
var sslConfig = require('./ssl-config');
var options = {
key: sslConfig.privateKey,
cert: sslConfig.certificate
};
var app = module.exports = loopback();
// Set up the /favicon.ico
app.middleware('initial', loopback.favicon());
// request pre-processing middleware
app.middleware('initial', loopback.compress());
app.middleware('session', loopback.session({ saveUninitialized: true,
resave: true, secret: 'keyboard cat' }));
// -- Add your pre-processing middleware here --
// boot scripts mount components like REST API
boot(app, __dirname);
// Redirect http requests to https
var httpsPort = app.get('https-port');
app.middleware('routes', httpsRedirect({httpsPort: httpsPort}));
var oauth2 = require('loopback-component-oauth2')(
app, {
// Data source for oAuth2 metadata persistence
dataSource: app.dataSources.pg,
loginPage: '/login', // The login page url
loginPath: '/login' // The login processing url
});
app.set('view engine', 'ejs');
app.set('views', path.join(__dirname, 'views'));
// Set up login/logout forms
app.get('/login', site.loginForm);
app.get('/logout', site.logout);
app.get('/account', site.account);
app.get('/callback', site.callbackPage);
var auth = oauth2.authenticate({session: false, scope: 'demo'});
app.use(['/protected', '/api', '/me', '/_internal'], auth);
app.get('/me', function(req, res) {
// req.authInfo is set using the `info` argument supplied by
// `BearerStrategy`. It is typically used to indicate scope of the token,
// and used in access control checks. For illustrative purposes, this
// example simply returns the scope in the response.
res.json({ 'user_id': req.user.id, name: req.user.username,
accessToken: req.authInfo.accessToken });
});
signupTestUserAndApp();
//var rateLimiting = require('./middleware/rate-limiting');
//app.middleware('routes:after', rateLimiting({limit: 100, interval: 60000}));
//var proxy = require('./middleware/proxy');
//var proxyOptions = require('./middleware/proxy/config.json');
//app.middleware('routes:after', proxy(proxyOptions));
app.middleware('files',
loopback.static(path.join(__dirname, '../client/public')));
app.middleware('files', '/admin',
loopback.static(path.join(__dirname, '../client/admin')));
// Requests that get this far won't be handled
// by any middleware. Convert them into a 404 error
// that will be handled later down the chain.
app.middleware('final', loopback.urlNotFound());
// The ultimate error handler.
app.middleware('final', loopback.errorHandler());
app.start = function(httpOnly) {
if(httpOnly === undefined) {
httpOnly = process.env.HTTP;
}
server = https.createServer(options, app);
server.listen(app.get('port'), function() {
var baseUrl = (httpOnly? 'http://' : 'https://') + app.get('host') + ':' + app.get('port');
app.emit('started', baseUrl);
console.log('LoopBack server listening # %s%s', baseUrl, '/');
});
return server;};
// start the server if `$ node server.js`
if (require.main === module) {
app.start();
}
function signupTestUserAndApp() {
// Create a dummy user and client app
app.models.User.create({username: 'bob',
password: 'secret',
email: 'foo#bar.com'}, function(err, user) {
if (!err) {
console.log('User registered: username=%s password=%s',
user.username, 'secret');
}
// Hack to set the app id to a fixed value so that we don't have to change
// the client settings
app.models.Application.beforeSave = function(next) {
this.id = 123;
this.restApiKey = 'secret';
next();
};
app.models.Application.register(
user.username,
'demo-app',
{
publicKey: sslConfig.certificate
},
function(err, demo) {
if (err) {
console.error(err);
} else {
console.log('Client application registered: id=%s key=%s',
demo.id, demo.restApiKey);
}
}
);
});
}
I don't get any errors when the server starts up. Thoughts?

Got it figured. More information here https://github.com/strongloop/loopback-gateway/issues/17, but basically I had my rest-api middleware not configured right.

How can I scrape sites that require authentication using node.js?

I've come across many tutorials explaining how to scrape public websites that don't require authentication/login, using node.js.
Can somebody explain how to scrape sites that require login using node.js?

Use Mikeal's Request library, you need to enable cookies support like this:
var request = request.defaults({jar: true})
So you first should create a username on that site (manually) and pass the username and the password as params when making the POST request to that site. After that the server will respond with a cookie which Request will remember, so you will be able to access the pages that require you to be logged into that site.
Note: this approach doesn't work if something like reCaptcha is used on the login page.

I've been working with NodeJs Scrapers for more than 2 years now
I can tell you that the best choice when dealing with logins and authentication is to NOT use direct request
That is because you just waste time on building manual requests and it is way slower,
Instead, use a high lever browser that you control via an API like Puppeteer or NightmareJs
I have a good starter and in-depth guide on How to start scraping with Puppeteer, I'm sure it will help!

Or using superagent:
var superagent = require('superagent')
var agent = superagent.agent();
agent is then a persistent browser, which will handle getting and setting cookies, referers, etc. Just agent.get, agent.post() as normal.

You can scrape the data from sites that require authentication like csrf token.
Using the cookies for each request like this:
var j = request.jar(); // this is to set the jar of request for session and cookie persistence
request = request.defaults({ jar: j }); //here we are setting the default cookies of request
Here is small code to elaborate it further:
var express = require('express');
var bodyParser = require('body-parser');
var querystring = require('querystring');
var request = require('request'); //npm request package to send a get and post request to a url
const cheerio = require('cheerio'); //npm package used for scraping content from third party sites
var cookieParser = require('cookie-parser')
var http = require('http');
var app = express();
app.use(cookieParser());
var _csrf; //variable to store the _csrf value to be used later
app.use(bodyParser.json());
var html = '';
var j = request.jar(); // this is to set the jar of request for session and cookie persistence
request = request.defaults({ jar: j }); //here we are setting the default cookies of request
//___________________API CALL TO VERIFY THE GMS NUMBER_______________________
app.get('/check', function(req, response) {
var schemeId = null;
if (req.query.schemeId) {
schemeId = req.query.schemeId;
console.log(schemeId);
} else {
response.send('false');
response.end();
}
getCsrfValue(function(err, res) {
if (!err) {
_csrf = res;
console.log(_csrf);
request.post({
headers: {
'Authorization': '',
'Content-Type': 'application/x-www-form-urlencoded',
},
uri: 'https://www.xyz.site',
body: "schemeId=" + schemeId + "&_csrf=" + _csrf
}, function(err, res, body) {
if (err) {
console.log(err);
} else {
console.log("body of post: " + res.body);
const $ = cheerio.load(body.toString());
var txt = $('.schemeCheckResult').text();
console.log(txt);
if (txt) {
response.send('true');
} else {
response.send('false');
}
html += body;
}
});
} else {
response.send(err);
}
})
});
//______________FUNCTION TO SCRAPE THE CSRF TOKEN FROM THE SITE____________
function getCsrfValue(callback) {
request.get({
headers: {
'Authorization': '',
'Content-Type': 'application/x-www-form-urlencoded',
},
uri: 'https://www.xyz.site'
}, function(err, res, body) {
if (err) {
return callback(err);
} else {
const $ = cheerio.load(body.toString());
var txt = $('input[name=_csrf]').val();
_csrf = txt;
return callback(null, _csrf);
}
});
}
module.exports = app;