I'm working on a small web framework to run a HCI study in and came across the following problem:
I have a Node server running with express to serve my local host data from JSON files. Not the best db but since it's a single user system (only one participant will ever be using the system at any time) it really didn't make sense to add any other technology. The following Get request code works just fine:
function getUser(id,last) {
return $.ajax({
type: "GET",
url: "/data/user/"+id,
async: false
}).responseJSON;
}
Which is handled by the following node code:
app.get('/data/:asset/:id', function (req, res) {
var accJSN
res.setHeader('Content-Type', 'application/json');
if(req.params.asset === "user")
{
accJSN = JSON.parse(fs.readFileSync(path.join(__dirname,'/public/data/users.json')));
res.send(JSON.stringify(accJSN.users[req.params.id]));
}
The above code produces a response which I can use/print and contains the responseJSON attribute. The following code does not, I'll note the node code is in the same server.js file and the functions with jquery/ajax calls are in my client page:
Client side code:
function getUserset(ids,last) {
userQuery = "";
for(i=0;i<ids.length;i++)
{
userQuery += ids[i] + ",";
}
userQuery = userQuery.slice(0,-1);
return $.ajax({
type: "GET",
url: "/userset?users=["+userQuery+"]",
async: false
}).responseJSON;
}
Server code:
app.get('/userset', function (req, res) {
var accJSN = JSON.parse(fs.readFileSync(path.join(__dirname,'/public/data/users.json')));
accJSN = accJSN.users;
var users = JSON.parse(req.query.users);
var resJSN = new Array;
for(var i=0;i<users.length;i++)
{
var temp = {};
temp["id"] = users[i];
temp["fName"] = accJSN[users[i]].fName;
temp["lName"] = accJSN[users[i]].lName;
temp["profilePic"] = accJSN[users[i]].profilePic;
resJSN.push(temp);
}
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(resJSN));
})
The data I need is actually available in the response text but I cannot for the life of me figure out why the second example doesn't also include the responseJSON attribute, or, if I'm totally wrong, why the first one does. Any thoughts or solutions appreciated, thanks!
Instead of manually setting the application/json header and sending a response just use
res.json(resJSN);
instead of
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify(resJSN));
Related
Overarching goal is to save some JSON data I create on a webpage to my files locally. I am definitely sending something to the server, but not in format I seem to able to access.
JsonData looks like:
{MetaData: {Stock: "UTX", Analysis: "LinearTrend2"}
Projections: [2018-10-12: 127.62, 2018-10-11: 126.36000000000001, 2018-10-10: 132.17, 2018-10-09: 140.12, 2018-10-08: 137.73000000000002, …]}
XMLHttpRequest on my webpage:
function UpdateBackTestJSON(JsonUpdate){ //JsonUpdate being the JSON object from above
var request = new XMLHttpRequest();
request.open('POST', 'UpdateBackTestJSON');
request.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
// request.setRequestHeader("Content-Type", "text/plain;charset=UTF-8");
request.onload = function() {
console.log("Updated JSON File");
};
console.log("about to send request");
console.log(JsonUpdate);
request.send(JSON.stringify(JsonUpdate));
}
and I handle posts on my server (rather carelessly I realize, just going for functionality as a start here)
var http = require('http')
, fs = require('fs')
, url = require('url')
, port = 8008;
var server = http.createServer (function (req, res) {
var uri = url.parse(req.url)
var qs = require('querystring');
if (req.method == 'POST'){
var body = '';
req.on('data', function (data){
body += data;
// 1e6 === 1 * Math.pow(10, 6) === 1 * 1000000 ~~~ 1MB
if (body.length > 1e6){
// FLOOD ATTACK OR FAULTY CLIENT, NUKE REQUEST
req.connection.destroy();
}
});
req.on('end', function () {
var POST = qs.parse(body);
console.log(POST); // PARSED POST IS NOT THE RIGHT FORMAT... or something, idk whats going on
UpdateBackTestData(POST);
});
}
function UpdateBackTestData(TheJsonData){
console.log("UpdateBackTestData");
console.log(TheJsonData);
JsonUpdate = JSON.parse(TheJsonData);
console.log(JsonUpdate["MetaData"]);
//var Stock = JsonUpdate["MetaData"]["Stock"];
// var Analysis = JsonUpdate["MetaData"]["Analysis"];
fs.writeFile("/public/BackTestData/"+Analysis+"/"+Stock+".json", TheJsonData, function(err){
if(err){
console.log(err);
}
console.log("updated BackTest JSON!!!");
});
}
Most confusing to me is that when I run this, the Json object Im am trying to pass, does go through to the server, but the entirety of the data is a string used as a key for a blank value in an object. when I parse the body of the POST, I get: {'{MetaData:{'Stock':'UTX','Analysis:'LinearTrend2'},'Projections':[...]}': ''}. So my data is there... but not in a practical format.
I would prefer not to use express or other server tools, as I have a fair amount of other services set up in my server that I don't want to go back and change if I can avoid it.
Thanks for any help
var http = require('http');
var qhttp = require('q-io/http');
var formidable = require('formidable');
var categories;
qhttp.read("https://api.myjson.com/bins/509wa")
.then(function (json) {
categories = JSON.parse(json).categories;
})
.then(null, console.error);
module.exports.putCat = function(req, res){
var form = new formidable.IncomingForm();
form.parse(req, function(error, fields, files){
if(error){console.log(error)}
fields["catProd"] = [];
categories.push(fields);
var dataString = JSON.stringify({categories: categories});
console.log(dataString);
var options = {
host : "api.myjson.com",
path : "/bins/509wa.json",
method: "PUT",
headers: {"Content-Type": "application/json", "Content-Length": dataString.length}
};
function callback(response){
var body = "";
response.on('data', function(chunk){
body+=chunk;
});
response.on('end', function(){
console.log('Received data: '+body);
});
}
http.request(options, callback).write(dataString);
res.end();
});
};
screenshot
It works perfectly with something like JSON.stringify("hello":"world");. However, when I tried with my data that needs to be stored (which is much longer), it doesn't send anything to the API. Thanks in advance!
You have a race condition with the categories variable. If some external code calls putCat() quickly after loading the module, then the categories data may not be available yet.
If you have async module loading things to do, then you should probably expose a module constructor which returns a promise and you can then do the putCat() after that promise resolves.
For my front-end (angular) app, I need to connect to an external API, which does not support CORS.
So my way around this is to have a simple proxy in Node.JS / Express.JS to pass the requests. The additional benefit is that I can set my api-credentials at proxy level, and don't have to pass them to the front-end where the user might steal/abuse them.
This is all working perfectly.
Here's the code, for the record:
var request = require('request');
var config = require('./config');
var url = config.api.endpoint;
var uname = config.api.uname;
var pword = config.api.pword;
var headers = {
"Authorization" : 'Basic ' + new Buffer(uname + ':' + pword).toString('base64'),
"Accept" : "application/json"
};
exports.get = function(req, res) {
var api_url = url+req.url;
var r = request({url: api_url, headers: headers});
req.pipe(r).pipe(res);
};
The API-endpoint I have to use has XML as only output format. So I use xml2js on the front-end to convert the XML reponse to JSON.
This is also working great, but I would like to lighten the load for the client, and do the XML -> JSON parsing step on the server.
I assume I will have to create something like:
req.pipe(r).pipe(<convert_xml_to_json>).pipe(res);
But I don't have any idea how do create something like that.
So basically I'm looking to create an XML to JSON proxy as a layer on top of an already existing API.
There are a lot of questions on SO regarding "how do I make a proxy" and "how do I convert XML to JSON" but I couldn't find any that combine the two.
you need to use transform stream and for xml to json conversion you need some library i use this xml2json
..then u use it like this (simplified but it should work with request too)
var http = require('http');
var fs = require('fs');
var parser = require('xml2json');
var Transform = require('stream').Transform;
function xmlParser () {
var transform = new Transform();
transform._transform = function(chunk, encoding, done) {
chunk = parser.toJson(chunk.toString())
console.log(chunk);
this.push(chunk);
done();
};
transform.on('error', function (err) {
console.log(err);
});
return transform;
}
var server = http.createServer(function (req, res) {
var stream = fs.createReadStream(__dirname + '/data.xml');
stream.pipe(xmlParser()).pipe(res);
});
server.listen(8000);
I have the following code, and node.js can't resolve the url:
const request = require('request')
const teamURL = `/users/${user._id}/teams`;
const req = request({
url: teamURL,
json: true
},
function(error, response, body) {
if (!error && response.statusCode == '200') {
res.render('userHome.html', {
user: user,
teams: body
});
}
else {
console.error(error);
next(error);
}
});
is there a good way to use relative paths/urls with the request library on a server-side node.js Express app?
Giving just a relative url only works if it is clear from context what the root part of the url should be. For instance, if you are on stackoverflow.com and find a link /questions, it's clear from context the full url should be stackoverflow.com/questions.
The request library doesn't have this kind of context information available, so it needs the full url from you to do be able to make the request. You can build the full url yourself of course, for instance by using url.resolve():
var url = require('url');
var fullUrl = url.resolve('http://somesite.com', '/users/15/teams');
console.log(fullUrl); //=> 'http://somesite.com/users/15/teams');
But of course this will still require you to know the root part of the url.
Jasper 's answer is correct -- the request module needs full URL. if you are in a situation where you have a single page application, with lots of requests to an API with the same base URL, you can save a lot of typing by creating a module like this:
var url = require('url');
var requestParser = (function() {
var href = document.location.href;
var urlObj = url.parse(href, true);
return {
href,
urlObj,
getQueryStringValue: (key) => {
let value = ((urlObj && urlObj.query) && urlObj.query[key]) || null;
return value;
},
uriMinusPath: urlObj.protocol + '//' + urlObj.hostname
};
})();
then, to grab the base URL anytime you need it: requestParser.uriMinusPath
and grab the value of an arbitrary query parameter: RequestParser.getQueryStringValue('partner_key');
I have a Node.js / Express app working, that receives routes like so:
app.get('/resource/:res', someFunction);
app.get('/foo/bar/:id', someOtherFunction);
This is great and works fine.
I am also using Socket.IO, and want to have some server calls use websockets instead of traditional RESTful calls. However, I want to make it very clean and almost use the same syntax, preferrably:
app.sio.get('/resource/:res', someFunction);
This will give a synthetic 'REST' interface to Socket.IO, where, from the programmer's perspective, he isn't doing anything different. Just flagging websockets: true from the client.
I can deal with all the details, such as a custom way to pass in the request verbs and parse them and so and so, I don't have a problem with this. The only thing I am looking for is some function that can parse routes like express does, and route them properly. For example,
// I don't know how to read the ':bar',
'foo/:bar'
// Or handle all complex routings, such as
'foo/:bar/and/:so/on'
I could dig in real deep and try to code this myself, or try to read through all of express' source code and find where they do it, but I am sure it exists by itself. Just don't know where to find it.
UPDATE
robertklep provided a great answer which totally solved this for me. I adapted it into a full solution, which I posted in an answer below.
You can use the Express router class to do the heavy lifting:
var io = require('socket.io').listen(...);
var express = require('express');
var sioRouter = new express.Router();
sioRouter.get('/foo/:bar', function(socket, params) {
socket.emit('response', 'hello from /foo/' + params.bar);
});
io.sockets.on('connection', function(socket) {
socket.on('GET', function(url) {
// see if sioRouter has a route for this url:
var route = sioRouter.match('GET', url);
// if so, call its (first) callback (the route handler):
if (route && route.callbacks.length) {
route.callbacks[0](socket, route.params);
}
});
});
// client-side
var socket = io.connect();
socket.emit('GET', '/foo/helloworld');
You can obviously pass in extra data with the request and pass that to your route handlers as well (as an extra parameter for example).
robertklep provided a great answer which totally solved this for me. I adapted it into a full solution, which is below in case others want to do something similar:
Node (server side):
// Extend Express' Router to a simple name
app.sio = new express.Router();
app.sio.socketio = require('socket.io').listen(server, { log: false });
// Map all sockets requests to HTTP verbs, which parse
// the request and pass it into a simple callback.
app.sio.socketio.sockets.on('connection', function (socket) {
var verbs = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'];
for (var i = 0; i < verbs.length; ++i) {
var go = function(verb) {
socket.on(verb, function (url, data) {
var route = app.sio.match(verb, url);
if (route && route.callbacks.length) {
var req = {url: url, params: route.params, data: data, socket:socket}
route.callbacks[0](req);
}
});
}(verbs[i]);
}
});
// Simplify Socket.IO's 'emit' function and liken
// it traditional Express routing.
app.sio.end = function(req, res) {
req.socket.emit('response', req.url, res);
}
// Here's an example of a simplified request now, which
// looks nearly just like a traditional Express request.
app.sio.get('/foo/:bar', function(req) {
app.sio.end(req, 'You said schnazzy was ' + req.data.schnazzy);
});
Client side:
// Instantiate Socket.IO
var socket = io.connect('http://xxxxxx');
socket.callbacks = {};
// Similar to the server side, map functions
// for each 'HTTP verb' request and handle the details.
var verbs = ['get', 'post', 'put', 'path', 'delete'];
for (var i = 0; i < verbs.length; ++i) {
var go = function(verb) {
socket[verb] = function(url, data, cb) {
socket.emit(String(verb).toUpperCase(), url, data);
if (cb !== undefined) {
socket.callbacks[url] = cb;
}
}
}(verbs[i]);
}
// All server responses funnel to this function,
// which properly routes the data to the correct
// callback function declared in the original request.
socket.on('response', function (url, data) {
if (socket.callbacks[url] != undefined) {
socket.callbacks[url](data);
}
});
// Implementation example, params are:
// 1. 'REST' URL,
// 2. Data passed along,
// 3. Callback function that will trigger
// every time this particular request URL
// gets a response.
socket.get('/foo/bar', { schnazzy: true }, function(data){
console.log(data); // -> 'You said schnazzy was true'
});
Thanks for your help, robertklep!