How to load JSON data on Node.js - javascript

I am working to load JSON data using Node/express.js and plot it on map. As first start, I am inspired by the example presented in the repp leaflet-geojson-stream. https://github.com/tmcw/leaflet-geojson-stream/tree/master/example
Client:
https://github.com/tmcw/leaflet-geojson-stream/blob/master/example/client.js
var L = require('leaflet'),
leafletStream = require('../');
L.Icon.Default.imagePath = 'http://leafletjs.com/dist/images';
window.onload = function() {
var div = document.body.appendChild(document.createElement('div'));
div.style.cssText = 'height:500px;';
var map = L.map(div).setView([0, 0], 2);
L.tileLayer('http://a.tile.openstreetmap.org/{z}/{x}/{y}.png').addTo(map);
var gj = L.geoJson().addTo(map);
leafletStream.ajax('/points.geojson', gj)
.on('end', function() {
});
};
Server :
https://github.com/tmcw/leaflet-geojson-stream/blob/master/example/server.js
var express = require('express'),
browserify = require('browserify'),
app = express();
app.get('/', function(req, res) {
res.send('<html><head><link rel="stylesheet" href="http://cdn.leafletjs.com/leaflet-0.6.4/leaflet.css" /></head><body><script src="bundle.js"></script></html>');
});
app.get('/bundle.js', function(req, res) {
var b = browserify();
b.add('./client.js');
b.bundle().pipe(res);
});
app.get('/points.geojson', function(req, res) {
res.write('{"type":"FeatureCollection","features":[');
var i = 0, die = 0;
function send() {
if (++i > 20) {
res.write(JSON.stringify(randomFeature()) + '\n,\n');
i = 0;
} else {
// it seems like writing newlines here causes the buffer to
// flush
res.write('\n');
}
if (die++ > 1000) {
res.write(JSON.stringify(randomFeature()));
res.write(']');
res.end();
return;
}
setTimeout(send, 10);
}
send();
});
app.listen(3000);
function randomFeature() {
return {
type: 'Feature',
geometry: {
type: 'Point',
coordinates: [
(Math.random() - 0.5) * 360,
(Math.random() - 0.5) * 180
]
},
properties: {}
};
}
In the project, they create random json file. I wanted to read json file, then plot it. The reason I want to "Stream data" is to deal with the size of file (I know that there is better and easier ways to load json data), But I wanted to use this module.
I modified the server script :
var express = require('express'),
browserify = require('browserify'),
app = express();
app.get('/', function(req, res) {
res.send('<html><head><link rel="stylesheet" href="http://cdn.leafletjs.com/leaflet-0.6.4/leaflet.css" /></head><body><script src="bundle.js"></script></html>');
});
app.get('/bundle.js', function(req, res) {
var b = browserify();
b.add('./client.js');
b.bundle().pipe(res);
});
var o = require('../output_.geojson');
app.get('/points.geojson', function(req, res) {
res.json(o);
});
app.listen(3000);
res.write('');
But I am getting error :
/Users/macbook/leaflet-geojson-stream/output_.geojson:1
(function (exports, require, module, __filename, __dirname) { "type":"FeatureC
^
SyntaxError: Unexpected token :
at Module._compile (module.js:439:25)
at Object.Module._extensions..js (module.js:474:10)
at Module.load (module.js:356:32)
at Function.Module._load (module.js:312:12)
at Module.require (module.js:364:17)
at require (module.js:380:17)
at Object.<anonymous> (/Users/macbook/leaflet-geojson-stream/example/server.js:15:9)
at Module._compile (module.js:456:26)
at Object.Module._extensions..js (module.js:474:10)
at Module.load (module.js:356:32)
Can anyone give a hint on what should I do in order to read and load json external file to plot the data.

The : isn't expected, because you're in a function at that point.
To parse JSON, you simply have to call JSON.parse(), as stated in How to parse JSON using Node.js?. So you read the file, get the JSON String that's in there, and put it through JSON.parse()

You are currently loading the whole JSON file into memory by 'requiring' it.
Instead you want to stream the file because it is big and so use the fs.createReadStream function:
var fs = require('fs');
app.get('/points.geojson', function(req, res) {
res.setHeader('Content-Type', 'application/json');
fs.createReadStream(__dirname + '../output_.geojson').pipe(res);
});
Also make sure that the contents of ../output_.geojson is actually valid JSON. You can use JSONLint to check - the file should with '{' or '[' and NOT have Javascript functions inside.

Related

Object has no method 'createApp'

I'm new to Javascript. I'm trying to load the following script using Node.js (ver. 0.8.8, it has to be done in this specific version), OS - Windows 10:
//require('look').start();
var main = require('skilap-core');
if (process.argv[2]=="automated") {
process.on("message",function (msg) {
if (msg.c == "startapp") {
var app = main.createApp(msg.data);
app.startApp(__dirname+"/data",function (err) {
process.send({c:"startapp_repl",data:err})
});
}
})
} else {
var app = main.createApp();
app.startApp(__dirname+"/data",function (err) {
if (err) console.log(err);
});
module.exports = app;
}
But I get the following error:
> TypeError: Object #<Object> has no method 'createApp'
at Object.<anonymous> D:\Downloads\skilap\skilap\app.js:13:17)
at Module._compile (module.js:449:26)
at Object.Module._extensions..js (module.js:467:10)
at Module.load (module.js:356:32)
at Function.Module._load (module.js:312:12)
at Module.runMain (module.js:492:10)
at process.startup.processNextTick.process._tickCallback(node.js:244:9)
As I understand, the module loads correctly, but doesn't compile. I've done research in the web and also saw other related questions on this site, but I cannot see what the problem actually is. What does the error mean and what needs to be changed to launch the app? Any help would be appreciated.
You require a module I've never heard of:
var main = require('skilap-core');
This is a javascript object. In your code you are trying to call a createApp() method on that object:
var app = main.createApp();
The main object does not have the createApp() function.
I would use a debugger or insert a console.log(main) after the require to see what the object actually looks like.
I used npm pack skilap-core to download it as a tgz file and examine it. There doesn't appear to be a createApp() function at all. It appears to want to take some kind of custome webapp as a parameter:
module.exports = function (webapp) {
var app = webapp.web;
var ctx = webapp._ctx;
var prefix = webapp.prefix;
var api = webapp;
app.get(prefix, function (req, res, next) {
res.redirect(prefix+"/user");
})
app.get("/", webapp.layout(), function(req, res, next) {
...
I changed the line var main = require('skilap-core') to var main = require('./modules/core/lib/core.js') (the path to the module file) and the module started working.

throw new TypeError('Router.use() requires middleware function but got a ' + gettype(fn));

I have created an express application with MongoJS. I am returning a function, still it's showing an object has been returned. I have even added module.exports=router in my JavaScript page. Can someone please point out the missing export here?
app.js:
var express = require('express');
var define = require('./user/userSlots');
var app = express();
app.set("views", "./view");
app.use('/', define);
app.listen(3111, function () {
console.log('Example app listening on port 3111!');
});
userSlots.js:
var express = require('express');
var module = require('../admin/dbModule');
var router = express.Router();
router.get('/login', function(request, response) {
var username = request.params.username;
var email = request.params.email;
module.authenticateUser(username, email, response);
});
module.exports = router;
dbModule.js:
var mongojs = require("../../node_modules/mongojs");
var databaseUrl = "localhost/user";
var db = mongojs(databaseUrl);
var test = db.collection('user_information');
var message = null;
db.user_information.ensureIndex({ email: 1 }, { unique: true });
exports.authenticateUser = function(username, email, response) {
db.user_information.find({ username: username, email: email }, function(err, users) {
if (err || !users) {
console.log("Failure");
message = "Failure";
reponse.render('index', { title: 'Hello', message: message });
} else if (users.length == 0) {
console.log("Not Valid user");
message = "Failure";
reponse.render('index', { title: 'Hello', message: message });
} else {
console.log("valid");
message = "Success";
reponse.render('index', { title: 'Hello', message: message });
}
});
}
Error:
E:\Vehicles\vehicles.com\node_modules\express\lib\router\index.js:458
throw new TypeError('Router.use() requires middleware function but got a '
+ gettype(fn));
^
TypeError: Router.use() requires middleware function but got a Object
at Function.use (E:\Vehicles\vehicles.com\node_modules\express\li
b\router\index.js:458:13)
at EventEmitter.<anonymous> (E:\Vehicles\vehicles.com\node_module
s\express\lib\application.js:220:21)
at Array.forEach (native)
at EventEmitter.use (E:\Vehicles\vehicles.com\node_modules\expres
s\lib\application.js:217:7)
at Object.<anonymous> (E:\Vehicles\vehicles.com\application\app.j
s:12:5)
at Module._compile (module.js:570:32)
at Object.Module._extensions..js (module.js:579:10)
at Module.load (module.js:487:32)
at tryModuleLoad (module.js:446:12)
at Function.Module._load (module.js:438:3)
You are causing a problem by redefining module in this line:
var module = require('../admin/dbModule');
And, then expecting it to be the original, built-in module later with this:
module.exports = router;
Change this:
var module = require('../admin/dbModule');
to a different name:
var dbModule = require('../admin/dbModule');
And, then use dbModule to refer to your database module and then the built-in module will still be available for normal use in module.exports.

Express middleware error

I'm making an API that accepts a video and uploads it to cloudinary. Meta data about the upload is then returned to the user. The problem that I am having occurs when I try to handle the video in middleware. I'm not sure why I keep getting this error.
The error:
/Users/name/test/cloudinary_upload/node_modules/express/lib/router/route.js:196
throw new Error(msg);
^
Error: Route.post() requires callback functions but got a [object Object]
at Route.(anonymous function) [as post] (/Users/name/test/cloudinary_upload/node_modules/express/lib/router/route.js:196:15)
at Function.proto.(anonymous function) [as post] (/Users/name/test/cloudinary_upload/node_modules/express/lib/router/index.js:510:19)
at Object.<anonymous> (/Users/name/test/cloudinary_upload/routes.js:8:8)
at Module._compile (module.js:541:32)
at Object.Module._extensions..js (module.js:550:10)
at Module.load (module.js:458:32)
at tryModuleLoad (module.js:417:12)
at Function.Module._load (module.js:409:3)
at Module.require (module.js:468:17)
at require (internal/module.js:20:19)
PROJECT CODE BELOW:
app.js
'use strict';
var express = require('express');
var formidable = require('express-formidable');
var port = process.env.PORT || 3000;
var routes = require('./routes');
var app = express();
// Parse incoming request
app.use(formidable());
app.use('/routes', routes);
// Catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// Error handler
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.json({
error: {
message: err.message
}
});
});
app.listen(port, function() {
console.log('Express server is listening on port ' + port);
});
routes.js
'use strict';
var express = require('express');
var createThumbnail = require('./middleware/generatevideothumbnail');
var uploadToCloudinary = require('./middleware/uploadtocloudinary');
var router = express.Router();
router.post('/upload', uploadToCloudinary, createThumbnail, function(req, res, next) {
console.log(req.cloudinaryObject);
console.log(req.videoThumbnail);
res.json(req.cloudinaryObject);
});
module.exports = router;
uploadtocloudinary.js
'use strict';
var cloudinary = require('../cloudinary_config.js').cloudinary;
/************************************************************
MIDDLEWARE: Accepts the request object. Uploads the video
file to cloudinary. Passes the video's public id, version, file
type, time of upload, file size and url to the next middleware
on the request object.
************************************************************/
function uploadToCloudinary(req, res, next) {
cloudinary.uploader.upload(req.files.fileToUpload.path, function(resultOfUpload) {
var cloudinaryObject = {
public_id: resultOfUpload.public_id,
version: resultOfUpload.version,
typeOfFile: resultOfUpload.format, // Type of file
uploadTime: resultOfUpload.created_at, // Upload time
fileSize: resultOfUpload.bytes, // File size
videoUrl: resultOfUpload.secure_url // Video URL
}
req.cloudinaryObject = cloudinaryObject;
next();
}, {
resource_type: "video"
});
}
module.exports.uploadToCloudinary = uploadToCloudinary;
generatevideothumbnail.js
'use strict';
/************************************************************
MIDDLEWARE: Accepts the request object. Use the version and
public id of video to create thumbnail url string. Pass
thumbnail url to next middleware on the request object.
************************************************************/
function generateVideoThumbnail(req, res, next) {
req.videoThumbnail = "https://res.cloudinary.com/xxxxxxxxx/video/upload/v"
+ req.cloudinaryObject.version
+ "/"
+ req.cloudinaryObject.public_id
+ ".jpg";
next()
}
module.exports.generateThumbnail = generateVideoThumbnail;
There are a few different choices of what to send with your required file when you use module.exports. You can send an entire object, or you can send specific functions.
With module.exports.uploadToCloudinary= and module.exports.generateThumbnail= you're sending an object that includes the functions uploadToCloudinary and generateThumbnail. When you require it with, for example,
var createThumbnail = require('./middleware/generatevideothumbnail');
you are able to use the function generateThumbnail on the object you're exporting. In other words, to access the function you want, you'd have to write:
createThumbnail.generateThumbnail
If you just want to be able to use only one specific function when you bring in the required file, just give the name of the function to module.exports, like this:
module.exports = generateVideoThumbnail;
Reference: https://www.sitepoint.com/understanding-module-exports-exports-node-js/ and https://nodejs.org/api/modules.html
Let me know if this makes sense.

SyntaxError: Unexpected token >

I just installed node and tried to write and run some programs.
When I tried this example progra, I get an error.
Maybe node and npm were installed incorrectly?
Maybe some necessary packages should be install?
const http = require('http');
const net = require('net');
const url = require('url');
// Create an HTTP tunneling proxy
var proxy = http.createServer( (req, res) => {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('okay');
});
proxy.on('connect', (req, cltSocket, head) => {
// connect to an origin server
var srvUrl = url.parse(`http://${req.url}`);
var srvSocket = net.connect(srvUrl.port, srvUrl.hostname, () => {
cltSocket.write('HTTP/1.1 200 Connection Established\r\n' + 'Proxy-agent: Node.js-Proxy\r\n' + '\r\n');
srvSocket.write(head);
srvSocket.pipe(cltSocket);
cltSocket.pipe(srvSocket);
});
});
Why does the below error appear?
var proxy = http.createServer( (req, res) => {
^
SyntaxError: Unexpected token >
at Module._compile (module.js:439:25)
at Object.Module._extensions..js (module.js:474:10)
at Module.load (module.js:356:32)
at Function.Module._load (module.js:312:12)
at Function.Module.runMain (module.js:497:10)
at startup (node.js:119:16)
at node.js:902:3 ##
Try it like this;
var requestListener = function (req, res) {
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('okay');
}
var proxy = http.createServer(requestListener);

Can't get plotly + node.js to stream data coming through POST requests

I'm trying to get plotly to stream data received by my server through a POST request to http://localhost:3000/step.
Building on the rest-example.js in plotly-nodejs/examples, here's my server code (I've blurred out my username, apikey, and token):
'use strict';
var express = require('express');
var logger = require('morgan');
var bodyParser = require('body-parser');
var events = require('events');
var eventEmitter = new events.EventEmitter();
var app = express();
var server = require('http').Server(app);
var port = process.env.PORT || 3000;
server.listen(port);
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.post('/step', function(req, res) {
var data = req.body.data;
eventEmitter.emit('step', data);
res.end('ok');
});
var plotly = require('plotly')('username', 'apikey');
var token = 'token';
var dataInit = [{x:[], y:[], stream: { token: token, maxpoints: 10 } }];
var layout = {fileopt : "extend", filename : "REST-test"};
plotly.plot(dataInit, layout, function (err, msg) {
if(err) return console.error('step data error', err.stack);
var stream = plotly.stream(token, function() {});
eventEmitter.on('step', function(data) {
console.log('sending to plotly: ' + data + ' steps');
var streamObject = JSON.stringify({ x: getDateString(), y: data });
stream.write(streamObject+'\n');
});
});
function getDateString() {
var d = new Date();
return d.toLocaleString();
};
When I POST data using cURL, for example curl http://localhost:3000/step --data "data=5", I can see that the data reaches the callback inside the plotly.plot block, but plotly never starts up and streams the data.
In some slightly more complex server code I was working on earlier, I also get the error which may or may not be related and which always points to the beginning of the plotly.plot block.
cb(null, body);
^
SyntaxError: Unexpected end of input
This is the full error stack:
/home/plotly-testing/node_modules/plotly/index.js:305
cb(null, body);
^
SyntaxError: Unexpected end of input
at Object.parse (native)
at /home/plotly-testing/node_modules/plotly/index.js:72:25
at IncomingMessage.<anonymous> (/home/plotly-testing/node_modules/plotly/index.js:305:9)
at IncomingMessage.emit (events.js:129:20)
at _stream_readable.js:908:16
at process._tickCallback (node.js:355:11)
---------------------------------------------
at IncomingMessage.Readable.on (_stream_readable.js:671:33)
at parseRes (/home/plotly-testing/node_modules/plotly/index.js:304:9)
at ClientRequest.<anonymous> (/home/plotly-testing/node_modules/plotly/index.js:71:9)
at ClientRequest.emit (events.js:107:17)
at HTTPParser.parserOnIncomingClient (_http_client.js:426:21)
at HTTPParser.parserOnHeadersComplete (_http_common.js:111:23)
at TLSSocket.socketOnData (_http_client.js:317:20)
---------------------------------------------
at new ClientRequest (_http_client.js:93:10)
at Object.exports.request (http.js:49:10)
at Object.exports.request (https.js:136:15)
at Plotly.plot (/home/plotly-testing/node_modules/plotly/index.js:70:21)
at Object.<anonymous> (/home/plotly-testing/index.js:175:8)
at Module._compile (module.js:460:26)
at Object.Module._extensions..js (module.js:478:10)
at Module.load (module.js:355:32)
at Function.Module._load (module.js:310:12)
Line 305 of plotly/index.js points to the following method, which seems to indicate something was wrong in one of my callbacks, but I'm not sure.
// response parse helper fn
function parseRes (res, cb) {
var body = '';
if ('setEncoding' in res) res.setEncoding('utf-8');
res.on('data', function (data) {
body += data;
if (body.length > 1e10) {
// FLOOD ATTACK OR FAULTY CLIENT, NUKE REQ
res.connection.destroy();
res.writeHead(413, {'Content-Type': 'text/plain'});
res.end('req body too large');
return cb(new Error('body overflow'));
}
});
res.on('end', function () {
cb(null, body);
});
}
So I've modified the code to include a console.log inside the Plotly.plot callback.
See gist here:
https://gist.github.com/alexander-daniel/b36f9be78abbbaa4847e#file-index-js-L33
And that way we can see that Plotly returned a graph URL that we can look at.
https://gist.github.com/alexander-daniel/b36f9be78abbbaa4847e#file-console_output-L5
That should resolve the first issue.
As far as the second issue goes, it seems the problem is two fold:
- JSON.parse calls inside the library are not wrapped in try/catch, so it looks like if the stream-server returns anything that is not JSON, this will break.
We're looking into the streaming-server error returns, but I have opened this issue here re: the try/catch blocks in the API library.
github.com/plotly/plotly-nodejs/issues/37

Categories

Resources