How do I load my variable "vSave" with data. This is where Javascript confuses me. It seems the way I wrote this vSave is undefined when it is returned but I know the response.on('end') runs and has the data I am looking for. I just don't know how to get it returned to my router so I can use it on my client side.
var express = require('express');
var router = express.Router();
var parseString = require('xml2js').parseString;
var config = require('../config_bartapi');
var http = require('http');
var vTemp; // [DEBUG]
var vSave; // [DEBUG]
// Real Time Departure from a given station
router.route('/departTimeStation')
.get(function(req, res) {
var vParsed = '';
vCmd = 'etd';
vOrig = req.query.vOriginStation;
vDir = 'n'; // [NOTE] - 'n' or 's', north or south, OPTIONAL
vPlat = 1; // [NOTE] - 1 to 4, number of platform, OPTIONAL
var xoptions = {
host: 'api.bart.gov',
path: '/api/etd.aspx?cmd=' + vCmd + '&orig=' + vOrig + '&key=' + config.bart.client
};
var xcallback = function(response) {
response.on('data', function(chunk) {
vParsed += chunk;
});
response.on('end', function() {
parseString(vParsed, function(err, result) {
vSave = JSON.stringify(result.root.station);
});
});
};
var vTestHttp = http.request(xoptions, xcallback).end();
return res.send (vSave);
});
// list all BART stations
router.route('/listAllStations')
.get(function(req, res) {
var vParsed = '';
vCmd = 'stns';
var options = {
host: 'api.bart.gov',
path: '/api/stn.aspx?cmd=' + vCmd + '&key=' + config.bart.client
};
var callback = function(response) {
response.on('data', function(chunk) {
vParsed += chunk;
});
response.on('end', function() {
parseString(vParsed, function(err, result) {
vTemp = result.root.stations[0].station;
});
});
};
var vTestHttp2 = http.request(options, callback).end();
return res.send (vTemp)
});
module.exports = router;
Thanks for your help. It seems like an easy concept but I just can't seem to get it.
I had to edit. This is the full bit of code. I originally posted only the one module. Sorry for the extreme edit.
Try to send it in the callback you pass to parseString() function.
Related
I am trying to build a file of json data from repeated calls to a restAPI. The final file to be written is the sum of the data received from all the calls. At present the file is being written with contents of the first call then overwritten by the contents of the first + second call (see console output below code).
As I have to make many calls, once the code is working, I would like to only write the file once the request has finished and the json string has been built. Does anyone now how I would go about doing this? Maybe with a callback(?), which I still don't have the hang of, once the requests have finished or the json string has finished being built.
"use strict";
const fs = require('fs');
const request = require('request');
var parse = require('csv-parse');
const path = "../path tocsv.csv";
const pathJSON = "../pathtoJSON.json";
var shapes = "https://url";
var options = {
url: '',
method: 'GET',
accept: "application/json",
json: true,
};
var csvData = [];
var jsonData = "[";
fs.createReadStream(path)
.pipe(parse({delimiter: ','}))
.on('data', function(data) {
csvData.push(data[1]);
})
.on('end',function() {
var start = Date.now();
var records = csvData.length //2212 objects
console.log(records);
var dataLength = 2 //set low at moment
for (var i = 0; i < dataLength; i += 1) {
var url = shapes + csvData[i];
options.url = url; //set url query
request(options, function(error, response, body) {
var time = Date.now() - start;
var s = JSON.stringify(body.response);
console.log( '\n' + (Buffer.byteLength(s)/1000).toFixed(2)+
" kilobytes downloaded in: " + (time/1000) + " sec");
console.log(i)
buildJSON(s);
});
}
function buildJSON(s) {
var newStr = s.substring(1, s .length-1);
jsonData += newStr + ',';
writeFile(jsonData);
}
function writeFile(jsonData) {
fs.writeFile(pathJSON, jsonData, function(err) {
if (err) {
return console.log(err);
} else {
console.log("file complete")
}
});
}
});
128.13 kilobytes downloaded in: 2.796 sec
2
file complete
256.21 kilobytes downloaded in: 3.167 sec
2
file complete
Perhaps writing to the file after all requests are complete will help. In the current code, the writeFile function is called each time a request is completed (which overwrites the file each time)
A quick way to fix this is to count requests (and failures) and write to file only after all the requests are complete.
"use strict";
const fs = require('fs');
const request = require('request');
var parse = require('csv-parse');
const path = "../path tocsv.csv";
const pathJSON = "../pathtoJSON.json";
var shapes = "https://url";
var options = {
url: '',
method: 'GET',
accept: "application/json",
json: true,
};
var csvData = [];
var jsonData = "[";
fs.createReadStream(path)
.pipe(parse({
delimiter: ','
}))
.on('data', function (data) {
csvData.push(data[1]);
})
.on('end', function () {
var start = Date.now();
var records = csvData.length //2212 objects
console.log(records);
var dataLength = 2 //set low at moment
var jsonsDownloaded = 0; // Counter to track complete JSON requests
var jsonsFailed = 0; // Counter to handle failed JSON requests
for (var i = 0; i < dataLength; i += 1) {
var url = shapes + csvData[i];
options.url = url; //set url query
request(options, function (error, response, body) {
if(error){
jsonsFailed++;
writeFile(jsonData);
return;
}
jsonsDownloaded++;
var time = Date.now() - start;
var s = JSON.stringify(body.response);
console.log('\n' + (Buffer.byteLength(s) / 1000).toFixed(2) +
" kilobytes downloaded in: " + (time / 1000) + " sec");
console.log(i)
buildJSON(s);
});
}
function buildJSON(s) {
var newStr = s.substring(1, s.length - 1);
jsonData += newStr + ',';
writeFile(jsonData);
}
function writeFile(jsonData) {
if(dataLength - (jsonsDownloaded + jsonsFailed) > 0){
return;
}
fs.writeFile(pathJSON, jsonData, function (err) {
if (err) {
return console.log(err);
} else {
console.log("file complete")
}
});
}
});
Note:
Requests being fired in quick succession like (2000 requests in a for loop) in my experience does not work well.. Try batching them. Also, doing it this way does not guarantee order (if that is important in your usecase)
An alternative would be to open your file in append mode. You can do this by passing an extra options object with flag set to your fs.writeFile call.
fs.writeFile(pathJSON, jsonData, {
flag: 'a'
}, function (err) {
if (err) {
return console.log(err);
}
});
References:
fs.writeFile Docs
File system flags
Ok, I do not understand what is going here, works locally but not on my server.
I have a angular controller that post to my node server.
each time I try and run the function that triggers the post I get
POST http://www.mysite.co.uk/mm3/back-end/savephotos 404 (Not Found)
Im honestly lost, ive rewritten the post 5 times I cant find the problem.
If anyone can see where ive gone wrong please help.
angular controller
mm3\js\controller.js
//all photos've been pushed now sending it to back end
$timeout(function () {
$http.post('back-end/savephoto', $scope.photosToPhp).then(function (success) {
$scope.generating = false;
$scope.generateBtn = 'Generate';
//creating mock up gallery
for (var x = 0; x < success.data.photos; x++) {
var file = '/mm3/tmp/' + success.data.folder + "/out" + x + ".png";
$scope.gallery.push(file);
}
$scope.photosToPhp = [];
}, function (error) {
});
}, 800);
then my node back-end
UPDATED:
So I have added a few console logs in my function to see where its going wrong and where it is getting to.
I keep getting:
test 1 function started error saving photo
mm3\back-end\controller.js
app.post('/mm3/back-end/savePhoto', function (req, res) {
console.log('test 1 function started');
var folder = Math.random().toString(36).substr(2, 20);
var photos = req.body;
var counts = 0;
var callback = function(counts){
if(counts < photos.length){
saveBase64(photos[counts],folder,counts,callback);
console.log('test 2 save photo');
}else{
var counts = 0;
var response = {"folder":folder, "photos": photos.length};
console.log('test 3 save photo else');
res.send(response)
}
};
saveBase64(photos[counts],folder,counts,callback);
});
app.post('/mm3/downloadZip', function(req, res){
var photos = req.body;
var out = photos[0];
var test = out.split('/');
var loc = test.pop();
var end = test.join('/');
console.log('test 3 function Generate zip file');
console.log(end);
var outName = '/' + end +'/mm3/MockUp.zip';
var output = fs.createWriteStream(outName);
var archive = archiver('zip', {store: true });
var zip = function(photos, f){
for(var t = 0; t < photos.length; t++){
var file = 'mockUp'+ t +'.jpg';
var from = '/var/www/html' + photos[t];
archive.file( from, { name: file });
}
f();
};
output.on('close', function() {
var photos = req.body;
var out = photos[0];
var test = out.split('/');
var loc = test.pop();
var end = test.join('/');
res.send(end + '/MockUp.zip');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
archive.on('error', function(err) {
throw err;
});
archive.pipe(output);
zip(photos, f);
function f(){
archive.finalize();
}
});
function saveBase64(photo,folder, counts, callback){
var result = photo.split(',')[1];
var path = '/mm3/tmp/' + folder;
var filename = path + "/out"+ counts + ".png";
mkdirp( path, function() {
fs.writeFile(filename, result, 'base64', function(error){
if (error) {
console.log('error saving photo');
}else{
console.log('photo saved');
counts ++;
callback(counts);
}
});
});
}
I think this is the problem:
app.post('back-end/savephoto', function (req, res) {
// skipped some lines
});
change it to
app.post('/back-end/savephoto', function (req, res) {
// skipped some lines
});
In Angular, the below:
$http.post('back-end/savephoto......
Becomes:
$http.post('/back-end/savephoto.....
In Node, the below:
app.post('back-end/savephoto.....
Becomes:
app.post('back-end/savephoto....
Then, you need to add a console.log under the Node route to see if it even is executed. This will narrow it down. Also, you can remove the $http.post call outside of the timeout to eliminate the obvious.
Let me know how you get on.
Shayan
Querying a database for a list of servers to perform a command on. The array is populated properly and echos out as planned, but none of the connections occur. I tried both passing the array directly into rexec and looping through a forEachAsync. Neither process the server list properly. Am I referencing the array elements improperly?
Mind the syntax errors at the end, I was just trying to include both methods I tried.
#!
var mysql = require('mysql');
var resultset = require('node-array');
var rexec = require('remote-exec');
var fs = require('fs');
var _ = require('lodash');
//var streamBuffers = require('stream-buffers');
var moment = require('moment');
var util = require('util');
var now = moment().format('YYYYMMDD_HHmmss');
var logStdout = process.stdout;
var errStderr = process.stderr;
console.log = function () {
logStdout.write(util.format.apply(null, arguments) + '\n');
}
console.error = function () {
errStderr.write(util.format.apply(null, arguments) + '\n');
}
var connection = mysql.createConnection({
host : 'abc',
user : 'user',
password : '******',
database : 'db'
});
var ssh_options = {
port: 22,
username: 'e109gh',
privateKey: fs.readFileSync('R:/nodeJS/sshkey.priv'),
stdout: fs.createWriteStream('./out.txt'),
stderr: fs.createWriteStream('./err.txt')
}
var my_conn_options = _.clone(ssh_options);
var cmds = ['hostname -i'];
connection.query('SELECT name FROM server', function(err, rows) {
rows.forEachAsync(function(element, index, array) {
console.log(element.name);
rexec(element.name,cmds,my_conn_options,function(err){
if (err) {
now = moment().format('YYYY-MM-DD HH:mm:ss');
console.error(err);
} else {
console.log("it worked for "+element.name);
}
});
});
});
// var buffer = new streamBuffers.WritableStreamBuffer();
connection.end(function(err) {});
// my_conn_options.stdout = buffer;
//
// rexec(rows,cmds,my_conn_options,function(err){
// if (err) {
// now = moment().format('YYYY-MM-DD HH:mm:ss');
// console.error(err);
// } else {
// console.log()
// }
// });
//
//});
I am a newbie and trying to create an application based on poloniex.js API getting error-TypeError: curl.setopt is not a function] set node-curl(not working) and node-libcurl (partially works,but the function seems incorrectly expressed) slightly confused between the two curl) node-curl is outdated and maybe that's the problem-can you tell what is wrong?
'use strict';
var autobahn = require('autobahn'),
crypto = require('crypto'),
async = require('async'),
https = require('https'),
nonce = require('nonce')(),
querystring = require('querystring'),
Curl = require('node-libcurl').Curl,
microtime = require('microtime'),
events = require('events'),
util = require('util');
var Poloniex = function Poloniex() {};
Poloniex._query_tradeApi = function (req, callback) {
var post_data,
hash = crypto.createHmac('sha512', "key-key-key"),
sign,
received,
headers;
nonce = (new Date()).getTime() * 1000;
post_data = querystring.stringify(req);
hash.update(post_data);
sign = hash.digest("hex");
try {
headers = [ 'Key: ' + "SECRET-SECRET-SECRET", 'Sign: ' + sign ];
var curl = new Curl(),
close = curl.close.bind( curl );
curl.setopt('URL', 'https://poloniex.com/tradingApi/');
curl.setopt('POST', 1);
curl.setopt('POSTFIELDS', post_data);
curl.setopt('HTTPHEADER', headers);
received = '';
curl.on('data', function (chunk) {
received += chunk;
return chunk.length;
});
curl.on('header', function (chunk) {
return chunk.length;
});
curl.on('error', curl.close.bind( curl ),function (e) {
console.error('exchanges/poloniex', '_query_tradeApi', e,
req, e.stack);
callback(e, undefined);
curl.perform();
curl.close();
});
curl.on('end', function () {
try {
var data = JSON.parse(received);
callback(undefined, data);
} catch (ex) {
console.error('exchanges/poloniex', '_query_tradeApi',
ex, req, ex.stack);
callback(ex, received);
}
curl.close();
});
curl.perform();
} catch (ee) {
console.error('exchanges/poloniex', '_query_tradeApi', ee,
req, ee.stack);
callback(ee, received);
}
};
The syntax required is curl.setOpt, not curl.setopt.
For my project I have a server.js that calls a helper function place-search.js as shown below.
var express = require('express');
var server = express.Router();
var placeSearch = require("./helpers/place-search");
var obj = "hello";
server.use(function(req, res, next) {
console.log(req.method, req.url);
next();
});
server.post('/', function(req, res) {
/* get the object passed by the client's post request */
obj = req.body;
//console.log("Obj: " + obj);
/* send the confirmation back to the client */
res.status(200).send("body");
placeSearch.placeSearch(obj);
});
module.exports.server = server;
Here is my place-search.js :
var config = require("./config.js");
var Promise = require('bluebird');
var DistanceMatrix = require("./distance-matrix.js");
var GooglePlaces = Promise.promisifyAll(require("googleplaces"));
var googlePlaces = new GooglePlaces(config.apiKey, config.outputFormat);
var extract = require('./extract.js');
var combination = require('./combination_ver2.js');
var permutation = require('./permutation.js');
function placeSearch(obj) {
console.log("Inside place search!");
/**
* Place search - https://developers.google.com/places/documentation/#PlaceSearchRequests
*/
var arr = [];
var count = 0;
var rad = obj["radius"];
console.log("radius: " + rad);
var loc = obj["location"];
console.log("Location: " + loc);
var mode = obj["mode"];
var params = obj["params"];
/* client's keywords */
var arr;
var ar = [];
for (var i = 0; i < params; i++) {
arr[i] = obj[i];
console.log(arr[i]);
var param = {
location: loc,
radius: rad,
mode: mode,
keyword: arr[i]
};
ar.push(param);
}
console.log("before promises");
var promises = ar.map(function(name) {
return googlePlaces.placeSearch(name, function(response) {
arr.push(response);
console.log(response);
console.log(count++);
//If all responses have been returned
//Find combos and pass to distance-matrix
if (count == ar.length) {
var Matrix = new Array();
var result = new Array();
//to extract only lat and lng from arr.results
//Matrix = extract.extract(arr);
result = combination.combination(arr);
// NOW RESULT IS THE ARRAY OF ALL COMBINATION
// NOW RESULT IS THE ARRAY OF COMBINATIONS OF latlng pairs AND PASS IT TO FRONTEND
/*result.forEach(function(combo, index) {
console.log("combo" + combo)
DistanceMatrix.distanceMatrix(mode, combo, result.length);
});*/
// IF YOU WANT TO SEE PERMUTATION
//permutation.permutation(result);
console.log("combination results: " + result);
}
})
});
}
module.exports.placeSearch = placeSearch;
My problem is I do not know how to pass the result variable back to the server.js so that I can use that result as an input for another helper function. I can not for the life of me figure out how to do this. Any help at all would be greatly appreciated.
Well, I don't see your placeSearch function returning anything at all right now, nor doing any kind of callback. Your placeSearch function should expose a callback parameter, which then gets called once you have the answer you want to send back.
Your server file will then take action on that callback. Abbreviating your code, it'd look something like this:
server.post('/', function(req, res) {
/* get the object passed by the client's post request */
obj = req.body;
//console.log("Obj: " + obj);
placeSearch.placeSearch(obj, function(error, data){
/* send the data back to the client */
res.status(200).send(data);
});
});
To support that, your placeSearch function will have to call its callback when appropriate:
function placeSearch(obj, callback){
/* all the stuff you do to assemble your data */
// if (there_is_some_error):
if(err) return cb(err);
// when you have all your data available, no error has occurred
return cb(null, data);
}
Something else you might notice is that your ar.map won't work as you seem to expect. ar.map is a synchronous function, you're calling async code inside... not gonna work the way you think. It's a bit long for this post, but you should look at the async library from npm to manage an array of asynchronous requests to collect one combined result.
use callback your code looks like this:
function placeSearch(obj,callback) {
console.log("Inside place search!");
/**
* Place search - https://developers.google.com/places/documentation/#PlaceSearchRequests
*/
var arr = [];
var count = 0;
var rad = obj["radius"];
console.log("radius: " + rad);
var loc = obj["location"];
console.log("Location: " + loc);
var mode = obj["mode"];
var params = obj["params"];
/* client's keywords */
var arr;
var ar = [];
for (var i = 0; i < params; i++) {
arr[i] = obj[i];
console.log(arr[i]);
var param = {
location: loc,
radius: rad,
mode: mode,
keyword: arr[i]
};
ar.push(param);
}
console.log("before promises");
var promises = ar.map(function(name) {
return googlePlaces.placeSearch(name, function(response) {
arr.push(response);
console.log(response);
console.log(count++);
//If all responses have been returned
//Find combos and pass to distance-matrix
if (count == ar.length) {
var Matrix = new Array();
var result = new Array();
//to extract only lat and lng from arr.results
//Matrix = extract.extract(arr);
result = combination.combination(arr);
// NOW RESULT IS THE ARRAY OF ALL COMBINATION
// NOW RESULT IS THE ARRAY OF COMBINATIONS OF latlng pairs AND PASS IT TO FRONTEND
/*result.forEach(function(combo, index) {
console.log("combo" + combo)
DistanceMatrix.distanceMatrix(mode, combo, result.length);
});*/
// IF YOU WANT TO SEE PERMUTATION
//permutation.permutation(result);
console.log("combination results: " + result);
callback(null,result);
}
})
});
}
in server.js:
server.post('/', function(req, res) {
/* get the object passed by the client's post request */
obj = req.body;
//console.log("Obj: " + obj);
/* send the confirmation back to the client */
res.status(200).send("body");
placeSearch.placeSearch(obj,function(err,result){
if(!err){
console.log(result);
}
})
});
It seems like you're having trouble with the async operation. You'll want to return the promise from your place-search module. You'll also need to convert the callbacks from placeSearch into a promise.
EDIT: updated since googlePlaces.placeSearch doesn't return a promise
inside placeSearch
function placeSearch(obj) {
//...
var promises = ar.map(function(name) {
var placeDefer = Q.defer();
return googlePlaces.placeSearch(name, function(response) {
placeDefer.resolve(response); // or placeDefer.reject if a failure occurs
});
return placeDefer.promise;
});
return promises;
}
and in your route:
// I'm going to just assume Q promise library here
var Q = require("q");
server.post('/', function(req, res) {
/* get the object passed by the client's post request */
obj = req.body;
//console.log("Obj: " + obj);
/* send the confirmation back to the client */
res.status(200).send("body");
Q.all(placeSearch.placeSearch(obj))
.spread(function() {
var places = Array.prototype.slice.call(arguments);
// possibly call res.status(200).send("body"); here?
// only if you're trying to use the places in your response
});
});