I'm trying to set up an express service for a program that I'm writing that contacts an external API and then returns the results so it can be stored in a Mongo I have set up.
This seems like it should be fairly straightforward, but I'm new to Node.js/Express and I'm getting a "Can't set headers after they are sent" error.
I'm getting the data that I want from the external API, but how to send that data properly back to my Angular app.js so it can update in my table?
"addSelected()" is the function I'm calling in my app.js to kick off the process. The "data" prints part of the way through the full response but then cuts off and gives me the "Can't set Headers after they are sent" error. From what I understand this is from sending the response and then trying to modify the response header after the fact.. but I'm unsure of a workaround or if I'm just formatting everything wrong as this is my first swing at MEAN stack in general.
I know the problem is on the line "res.send(data)" in server.js but I don't know how to correctly format the response.
My code:
server.js
//server.js
//setup ==============================
var express = require ('express');
var request = require('request');
var app = express();
var mongoose = require('mongoose');
var https = require('https');
//config ============================
app.use(express.static(__dirname + '/public/'));
console.log("running PipeHelper");
mongoose.connect('mongoedit');
var Schema = mongoose.Schema;
var opSchema = new Schema({
title: String,
description: String,
company: String,
post_date: String,
close_date: String,
contact: String,
location: String,
url: String,
notice_type: String
});
var item = mongoose.model('item', opSchema);
//routes===========================
//returns full database
app.get('/api/db', function(req, res){
item.find({},function(err, items){
if (err) res.send(err);
res.json(items);
});
});
//searches FBO for opportunities to add to database
app.get('/api/search:FBO_key', function(req, res){
var data;
console.log("2");
var baseURL = "api.data.gov"
var params = "/gsa/fbopen/v0/opps?q=" + req.params.FBO_key;
params += "&api_key="+"keyyyy";
params += "&all";
params += "&start=0";
params += "&p=1";
params += "&limit=10";
url = baseURL+params;
var options = {
port: 443,
host: 'api.data.gov',
path: params,
method: 'GET'
};
//get FBO data
var request = https.request(options, function(response){
console.log("4");
response.on('data', function (chunk){
//response data to send back to app.js
data += chunk.toString();
res.send(data);
});
});
console.log("3");
request.end();
request.on('error', function(e){
console.error(e);
});
});
app.get('/', function(req,res){
res.sendfile('./public/index.html');
});
app.listen(8000);
app.js
var app = angular.module("pipeHelper", ['smart-table']);
app.controller('mainCtrl', [
'$scope', '$http', function($scope, $http){
$scope.selected = [];
$scope.displayData= [];
$scope.items=[];
$scope.FBOsearch;
//populates table on startup with whole DB
$http.get('./api/db')
.success(function(data){
$scope.items=data;
$scope.displayData = [].concat($scope.items);
})
.error(function(data){
console.log('Error: '+data);
});
$scope.addSelected = function(){
//search FBO, add opportunities, update table
console.log("1");
$http.get('./api/search'+'NGA')
.success(function(data){
console.log("5");
console.log(data);
$scope.items=data;
$scope.displayData= [].concat($scope.items);
})
.error(function(data){
console.log('Error: ' +data);
});
};
$scope.isSelected = function(item){
//if its selected, remove it
// if its unselected, add it
if ($scope.selected.indexOf(item)==-1){
$scope.selected.push(item);
}
else{
$scope.selected.splice($scope.selected.indexOf(item), 1);
}
console.log($scope.selected);
//temp placeholder function. Eventually add to array of selected objects for placement in Pipeliner/deletion
};
}]);
solved the issue. I was unaware that response.on('data') gets called multiple times, thus calling my res.send(data) multiple times and incompletely causing it to crash with the error. I added the following to the request function:
response.on('end'function(){
res.send(data);
};
basically when the external API data is finished coming in, send it with express. Learn by doing I suppose. Hope this helps someone eventually.
I can't leave a comment, so I will just make it an answer.
I would recommend installing node-inspector, npm install -g node-debug. Then run your app with node-debug server.js. This will spawn a new instance of Firefox or Chrome dev tools and allows you to debug your nodeJS code. Very useful.
The error you are seeing is most likely related to request.end(), if I were to guess. After .end() is called, you can no longer modify the header content. I doubt it would make a difference, but try putting the request.end() after you have the request.on('error') call.
EDIT: 10/15/15
I would highly recommend installing VS Code. It has a built-in debugger for node apps.
Related
I have a very basic question about a node application, and a question about HTTP requests. It's the first time I create a node app with server, and I just can't seem to get the different components to work together.
This is my server.js
var express = require('express');
var multer = require('multer');
const request = require('request');
const upload = multer({dest: __dirname + '/uploads/images'});
const app = express();
const PORT = 3000;
app.use(express.static('public'));
app.post('/upload', upload.single('photo'), (req, res) => {
if(req.file) {
res.json(req.file);
}
else throw 'error';
});
app.listen(PORT, () => {
console.log('Listening at ' + PORT );
});
Then I have a file app.js with a motion-detection system. Every time motion is detected, a picture is taken. This all works fine.
Then the picture should be sent to the server. This is what I can't figure out.
I created a function toServer() that should post the detected data to the server
const request = require('request');
function toServer(data) {
const formData = {
// Pass data via Buffers
my_buffer: data,
// Pass optional meta-data with an 'options' object with style: {value: DATA, options: OPTIONS}
// Use case: for some types of streams, you'll need to provide "file"-related information manually.
// See the `form-data` README for more information about options: https://github.com/form-data/form-data
};
request.post({url:'http://localhost:3000/upload', formData: formData}, function optionalCallback(err, httpResponse, body) {
if (err) {
return console.error('Upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
});
};
Problem 1: when running the server.js on localhost:3000, it doesn't find any of the scripts loaded in index.html nor my app.js.
Problem 2: when running the index.html on live-server, all scripts are found, but i get the error "request is not defined".
I am pretty sure there is some basic node setup thing I'm missing.
The solution for toServer() might be more complicated.
Thanks for your time,
Mustard Shaper
Problem 1:
this could happen because you have not specified to render your index.html.
for example:
res.render('index')
if it's not because of the single quotes in upload.single('photo') try double quotes.
Another possible error could be that you are missing a default display engine setting.
an example: https://www.npmjs.com/package/hbs
Problem 2:
it may be because you are missing the header
var request = require('request');
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: 'http://localhost',
body: "example"
}, function(error, response, body){
console.log(body);
});
See more at https://expressjs.com/
I'm building my first node/express app and am following this tut.
I am at a point where I am trying to get all JSON data and put it in an array to be sent to the template and rendered. When I try to run the app via CLI, I get the following error:
Directory Structure
The data output at the var blogsurlall location
hellotest.js
var routes = require('./routes/index');
var express = require('express');
var app = express();
var request = require("request");
var blogsurlall = "https://[JSON export URL location configured in a Drupal 8 view]";
app.set('view engine','ejs');
var server = app.listen (2000, function(){ console.log('Waiting for you on port 2000'); });
/* Get all global blogs data */
request({
url: blogsurlall,
json: true
}, function (error, response, body) {
if (!error && response.statusCode === 200) {
blogsdata_all = body;
}
// Create blogs array for footer.
var blogs = [];
// Fill up the array with blogs.
blogsdata_all.blogs.forEach(function(item){
blogs = blogs.concat(item);
});
app.locals.blogsdata = blogs;
});
app.use('/', routes);
index.js
var express = require('express');
var routes = express.Router();
routes.get('/', function(req, res){ res.render('default',{title: 'Home', body: 'blogsdata'}); });
routes.get('/about-us', function(req, res){ res.send('<h1>Lucius Websystems</h1>Amsterdam, The Netherlands'); });
routes.get('/about/:name?', function(req, res){ var name = req.params.name; res.send('<h1>' +name +'</h1>About text'); });
/* GET Blog detail page. */
routes.get('/blog/:blogid', function(req, res, next) {
// Place json data in a var.
var blogsdata = req.app.locals.blogsdata;
// Create array.
var blogItem = [];
// Check and build current URL
var currentURL = '/blog/' + req.params.blogid;
// Lop through json data and pick correct blog-item based on current URL.
blogsdata.forEach(function (item) {
if (item.title == currentURL) {
blogItem = item;
}
});
if (blogItem.length == 0) {
// Render the 404 page.
res.render('404', {
title: '404',
body: '404'
});
} else {
// Render the blog page.
res.render('blog-detail', {
blog: blogItem
});
}
});
module.exports = routes;
From the CLI error, it appears no blog data is even returned to be read into the array.
I have carefully gone through the tutorial several times and I think there are steps that may be implied that I am missing.
Can someone please help me understand how to get the blog data so that it can be read into the array and output to my template?
Also open to troubleshooting suggestions in comments.
Thanks for reading!
The error is raising in this line:
blogsdata_all.blogs.forEach(function(item){
As the error says, blogs is undefined.
If there is an error in the request or status code isn't 200, the body is not assigned to the variable, but you are not finishing the execution, so the variable in that case would be undefined.
Other possible problem is the json received doesn't have blogs as key of the body.
Check this both things and let us know if you found the problem
I am building rest api app using express.js and mssql module and I'm stacked with problem:
I need flexible architecture for my app. So I have this structure:
controller
--products.js
router
--api
---products.js
-settings.js
-app.js
In app.js I start express server, in router/api/products.js I set some api endpoints. But in controller I need to
1) start connection
2) start request
3) add data to request from router
And the problem is that in each file in controller I need to copy-paste same code as:
var sqlDb = require('mssql');
var settings = require('./settings');
exports.executeQuery = function (bdRole,sql,callback) {
var config = [];
switch(bdRole) {
case 'Main' : config = settings.sqlDbConfigMain; break
case 'Child' : config = settings.sqlDbConfigChild; break
default : config = null;
}
var conn = new sqlDb.Connection(config);
conn.connect()
.then(function(){
var req = new sqlDb.Request(conn);
req.query(sql)
.then(function (recordset) {
callback(recordset);
})
.catch(function (err) {
console.log(err);
callback(null,err);
});
})
.catch(function (err) {
console.log(err);
callback(null,err);
});
};
And I think that it is vary bad to copy-past code. But all my attempts to split this code to make its more flexible failed because I'm newbie.
I not found any examples which can help me(
I need not only execute query, but I need to execute procedure with table variable inside etc.
So the question is: Do I need to put the
var sqlDb = require('mssql');
var settings = require('./settings');
in each controller and continue copy-past or a more correct approach(way) exist?
I fairly new to MEAN, so sorry if this question is so obvious. I want to send an email to a contact when they click a send button. My code for handling a send email is using a post I am currently using a SendGrid Nodejs API to send the email. The problem is I keep running into a 400 Post Error.
This is the error I get in my Google Chrome Console
This is the error I get in my server terminal
This is in my controller.js:
$scope.send = function(contact) {
console.log("Controller: Sending message to:"+ contact.email);
$http.post('/email', contact.email).then(function (response) {
// return response;
refresh();
});
};
this code is in my server.js:
var express = require("express");
var app = express();
//require the mongojs mondule
var mongojs = require('mongojs');
//which db and collection we will be using
var db = mongojs('contactlist', ['contactlist']);
//sendgrid with my API Key
var sendgrid = require("sendgrid")("APIKEY");
var email = new sendgrid.Email();
var bodyParser = require('body-parser');
//location of your styles, html, etc
app.use(express.static(__dirname + "/public"));
app.use(bodyParser.json());
app.post('/email', function (req, res) {
var curEmail = req.body;
console.log("Hey I am going to send this person a message:" + curEmail);
var payload = {
to : 'test#gmail.com',
from : 'test1#gmail.com',
subject : 'Test Email',
text : 'This is my first email through SendGrid'
}
sendgrid.send(payload, function(err, json) {
if (err) {
console.error(err);
}
console.log(json);
});
});
Currently the email is hard coded but I will make the change after I fix the post issue. If you could point me in the right direction that would be very helpful. Thank you.
Looks like you're expecting the request body to contain JSON, with this line:
app.use(bodyParser.json());
Your error in your console says Unexpected token, which leads me to believe that body-parser encountered something it couldn't parse as JSON... probably a string. Which means that you sent your email as a string in the request body.
The easy fix would be to change how you're sending the request client-side:
var data = { email: 'some#email.com' }; // as opposed to just 'some#email.com'
$http.post('/email', data).then(refresh);
use this code
$scope.send = function(contact) {
console.log("Controller: Sending message to:"+ contact.email);
$http.post('/email', contact).then(function (response) {
// return response;
refresh();
});
};
and at server side
app.use(bodyParser.urlencoded({ extended: false }))
app.use(bodyParser());
I am trying to scrape a website using nodejs and it works perfectly on sites that do not require any authentication. But whenever I try to scrape a site with a form that requires username and password I only get the HTML from the authentication page (that is, if you would click 'view page source' on the authentication page it self, that is the HTML I get). I am able to get the desired HTML using curl
curl -d "username=myuser&password=mypw&submit=Login" URL
Here is my code...
var express = require('express');
var fs = require('fs'); //access to file system
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/scrape', function(req, res){
url = 'myURL'
request(url, function(error, response, html){
// check errors
if(!error){
// Next, we'll utilize the cheerio library on the returned html which will essentially give us jQuery functionality
var $ = cheerio.load(html);
var title, release, rating;
var json = { title : "", release : "", rating : ""};
$('.span8 b').filter(function(){
// Let's store the data we filter into a variable so we can easily see what's going on.
var data = $(this);
title = data.first().text();
release = data.text();
json.title = title;
json.release = release;
})
}
else{
console.log("Error occurred: " + error);
}
fs.writeFile('output.json', JSON.stringify(json, null, 4), function(err){
console.log('File successfully written! - Check your project directory for the output.json file');
})
res.send('Check your console!')
})
})
app.listen('8081')
console.log('Magic happens on port 8081');
exports = module.exports = app;
I have tried the following...
var request = require('request',
username:'myuser',
password:'mypw');
This just returns the authentication page's HTML
request({form: {username:myuser, password:mypw, submit:Login}, url: myURL}, function(error, response, html){
...
...
...
}
This also just returns the authentication page's HTML
So my question is how do I achieve this using nodejs?
you shouldn't use .get but .post and put the post param (username and password) in your call
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: url,
body: "username=myuser&password=mypw&submit=Login"
}, function(error, response, html){
//do your parsing...
var $ = cheerio.load(html)
});