Develop Reference

How to SEND the content when making a GET in Javascript?

I need to GET all the data of a workbook, and I tried one method, but it does not work properly.. The problem is that the Promise is returning the first line of the first worksheet, but it does not continue the process for the rest of the rows of the worksheet, respectively, all the worksheets and I don't know what to do. Maybe you know..
So I wrote this code:
const SheetGet = {
getSheet: (req, res) => {
return new Promise ((resolve, reject) => {
XlsxPopulate.fromFileAsync(filePath)
.then(workbook => {
const wb = xlsx.readFile(filePath, {cellDates: true});
const sheetNames = wb.SheetNames;
sheetNames.forEach(function (element){
let endOfData = false;
let i = 2;
let dataArr = [];
while (endOfData === false){
let taskId = workbook.sheet(element).cell("A" + i.toString()).value();
if (taskId !== undefined){
res.send({
type: 'GET',
list_name: element,
task_id: taskId,
task_name: workbook.sheet(element).cell("B" + i.toString()).value(),
task_description: workbook.sheet(element).cell("C" + i.toString()).value(),
task_due_date: workbook.sheet(element).cell("D" + i.toString()).value(),
task_priority: workbook.sheet(element).cell("E" + i.toString()).value(),
task_status: workbook.sheet(element).cell("F" + i.toString()).value(),
task_notes: workbook.sheet(element).cell("G" + i.toString()).value()
});
i++;
}
else {
endOfData = true;
}
}
})
});
})
}
}
It only gets this an then it stops, and I need to get all of the data from the worksheet.
Do you have any idea on how to resolve this issue? Or the proper way to make it work? Thank you very very much for your time and help!!! Much appreciated every help!!
P.S. I tried this code with "console.log", and it works very well, but the problem is when I changed to res.send, in order to get the info to Postman.

I assume you are using express as the framework, the problem is when you use res.send method, the server already send the data to client, while the rest of the code still running in the background. What I'm gonna do with this case are like this.
const SheetGet = {
getSheet: (req, res) => {
return new Promise ((resolve, reject) => {
XlsxPopulate.fromFileAsync(filePath)
.then(workbook => {
const wb = xlsx.readFile(filePath, {cellDates: true});
const sheetNames = wb.SheetNames;
sheetNames.forEach(function (element){
let endOfData = false;
let i = 2;
let dataArr = [];
while (endOfData === false){
let taskId = workbook.sheet(element).cell("A" + i.toString()).value();
if (taskId !== undefined){
dataArr.push({ // this one
type: 'GET',
list_name: element,
task_id: taskId,
task_name: workbook.sheet(element).cell("B" + i.toString()).value(),
task_description: workbook.sheet(element).cell("C" + i.toString()).value(),
task_due_date: workbook.sheet(element).cell("D" + i.toString()).value(),
task_priority: workbook.sheet(element).cell("E" + i.toString()).value(),
task_status: workbook.sheet(element).cell("F" + i.toString()).value(),
task_notes: workbook.sheet(element).cell("G" + i.toString()).value()
});
i++;
}
else {
endOfData = true;
}
}
})
return res.json({ data: dataArr }); // this one
});
})
}
}
While there is some unnecessary code, the least change you can do to make it works are the code above.
Cheers.

Write final json to a file from repeated requests to rest API

I am trying to build a file of json data from repeated calls to a restAPI. The final file to be written is the sum of the data received from all the calls. At present the file is being written with contents of the first call then overwritten by the contents of the first + second call (see console output below code).
As I have to make many calls, once the code is working, I would like to only write the file once the request has finished and the json string has been built. Does anyone now how I would go about doing this? Maybe with a callback(?), which I still don't have the hang of, once the requests have finished or the json string has finished being built.
"use strict";
const fs = require('fs');
const request = require('request');
var parse = require('csv-parse');
const path = "../path tocsv.csv";
const pathJSON = "../pathtoJSON.json";
var shapes = "https://url";
var options = {
url: '',
method: 'GET',
accept: "application/json",
json: true,
};
var csvData = [];
var jsonData = "[";
fs.createReadStream(path)
.pipe(parse({delimiter: ','}))
.on('data', function(data) {
csvData.push(data[1]);
})
.on('end',function() {
var start = Date.now();
var records = csvData.length //2212 objects
console.log(records);
var dataLength = 2 //set low at moment
for (var i = 0; i < dataLength; i += 1) {
var url = shapes + csvData[i];
options.url = url; //set url query
request(options, function(error, response, body) {
var time = Date.now() - start;
var s = JSON.stringify(body.response);
console.log( '\n' + (Buffer.byteLength(s)/1000).toFixed(2)+
" kilobytes downloaded in: " + (time/1000) + " sec");
console.log(i)
buildJSON(s);
});
}
function buildJSON(s) {
var newStr = s.substring(1, s .length-1);
jsonData += newStr + ',';
writeFile(jsonData);
}
function writeFile(jsonData) {
fs.writeFile(pathJSON, jsonData, function(err) {
if (err) {
return console.log(err);
} else {
console.log("file complete")
}
});
}
});
128.13 kilobytes downloaded in: 2.796 sec
2
file complete
256.21 kilobytes downloaded in: 3.167 sec
2
file complete

Perhaps writing to the file after all requests are complete will help. In the current code, the writeFile function is called each time a request is completed (which overwrites the file each time)
A quick way to fix this is to count requests (and failures) and write to file only after all the requests are complete.
"use strict";
const fs = require('fs');
const request = require('request');
var parse = require('csv-parse');
const path = "../path tocsv.csv";
const pathJSON = "../pathtoJSON.json";
var shapes = "https://url";
var options = {
url: '',
method: 'GET',
accept: "application/json",
json: true,
};
var csvData = [];
var jsonData = "[";
fs.createReadStream(path)
.pipe(parse({
delimiter: ','
}))
.on('data', function (data) {
csvData.push(data[1]);
})
.on('end', function () {
var start = Date.now();
var records = csvData.length //2212 objects
console.log(records);
var dataLength = 2 //set low at moment
var jsonsDownloaded = 0; // Counter to track complete JSON requests
var jsonsFailed = 0; // Counter to handle failed JSON requests
for (var i = 0; i < dataLength; i += 1) {
var url = shapes + csvData[i];
options.url = url; //set url query
request(options, function (error, response, body) {
if(error){
jsonsFailed++;
writeFile(jsonData);
return;
}
jsonsDownloaded++;
var time = Date.now() - start;
var s = JSON.stringify(body.response);
console.log('\n' + (Buffer.byteLength(s) / 1000).toFixed(2) +
" kilobytes downloaded in: " + (time / 1000) + " sec");
console.log(i)
buildJSON(s);
});
}
function buildJSON(s) {
var newStr = s.substring(1, s.length - 1);
jsonData += newStr + ',';
writeFile(jsonData);
}
function writeFile(jsonData) {
if(dataLength - (jsonsDownloaded + jsonsFailed) > 0){
return;
}
fs.writeFile(pathJSON, jsonData, function (err) {
if (err) {
return console.log(err);
} else {
console.log("file complete")
}
});
}
});
Note:
Requests being fired in quick succession like (2000 requests in a for loop) in my experience does not work well.. Try batching them. Also, doing it this way does not guarantee order (if that is important in your usecase)
An alternative would be to open your file in append mode. You can do this by passing an extra options object with flag set to your fs.writeFile call.
fs.writeFile(pathJSON, jsonData, {
flag: 'a'
}, function (err) {
if (err) {
return console.log(err);
}
});
References:
fs.writeFile Docs
File system flags

Iterating with callback/anonymous functions

I'm new to Node.JS and advanced Javascript in general, but I'm trying to build a schedule manager application on my own and I faced a problem (I will detail it later) when trying to execute the following code:
router.get('/', function (req, res) {
var day = new Date(req.query.day);
Location.getLocations(function (err, locations) {
if (locations.length > 0) {
var i;
for (i = 0; i < locations.length; i++) {
var location = locations[i];
Appointment.getAppointments(day, location, function (err, appointments) {
if (err) throw err;
if (appointments.length == 0) {
// CREATE APPOINTMENTS
for (var j = location.available_time_start; j <= location.available_time_end; j += location.appointment_duration) {
var newAppointment = new Appointment();
newAppointment.start_date = new Date(day.getFullYear(), day.getMonth() + 1, day.getDate(), j);
newAppointment.appointment_duration = location.appointment_duration;
newAppointment.location = location.id;
newAppointment.booked = false;
newAppointment.locked = false;
Appointment.createAppointment(newAppointment, function (err, appointment) {
if (err) throw err;
console.log(appointment.location + ' - ' + appointment.start_date);
});
}
}
});
}
} else {
// THERE ARE NO LOCATIONS
}
res.render('appointments', { locations: locations });
});
The problem is:
When I try to iterate the locations object and then execute the getAppointments function the code isn't executed at this exact moment. Later, when it's executed, location object is always the same (the iteration doesn't work), resulting on a unexpected result (all appointments with the same/last location).
I tried using IIFE (Immediately-invoked function expression) to execute the code instantly, but when I did this I couldn't get the appointments callback object and my logic is broken too.
Thanks in advance!

The problem was solved by using let instead of var as suggested by #JaromandaX.

Your code seems to be saving appointments but doesn't do anything with the saved appointments (are you mutating locations?).
When saving an appointment goes wrong the requestor doesn't know about it because createAppointment is asynchronous and by the time the callback is called back res.render('appointments', { locations: locations }); is already executed.
You could try converting your callback based functions to promises:
const asPromise = (fn,...args) =>
new Promise(
(resolve,reject)=>
fn.apply(undefined,
args.concat(//assuming one value to resole
(err,result)=>(err)?reject(err):resolve(result)
)
)
);
const savedAppointmentsForLocation = (day,location,appointments) => {
const savedAppointments = [];
if (appointments.length == 0) {
// CREATE APPOINTMENTS
for (var j = location.available_time_start; j <= location.available_time_end; j += location.appointment_duration) {
var newAppointment = new Appointment();
newAppointment.start_date = new Date(day.getFullYear(), day.getMonth() + 1, day.getDate(), j);
newAppointment.appointment_duration = location.appointment_duration;
newAppointment.location = location.id;
newAppointment.booked = false;
newAppointment.locked = false;
savedAppointments.push(
asPromise(
Appointment.createAppointment.bind(Appointment),
newAppointment
)
);
}
}
//you are not doing anything with the result of the saved appointment
// I'll save it as promise to see if something went wrong to indicate
// to the requestor of the api that something went wrong
return Promise.all(savedAppointments);
}
router.get('/', function (req, res) {
var day = new Date(req.query.day);
asPromise(Location.getLocations.bind(Location))
.then(
locations=>
promise.all(
locations.map(
location=>
asPromise(Appointment.getAppointments.bind(Appointment),[day,location])
.then(appointments=>[location,appointments])
)
)
)
.then(
results=>//results should be [ [location,[appointment,appointment]],...]
Promise.all(
results.map(
([location,appointments])=>
savedAppointmentsForLocation(day,location,appointments)
.then(ignoredSavedAppointment=>location)
)
)
)
.then(locations=>res.render('appointments', { locations: locations }))
.catch(
error=>{
console.log("something went wrong:",error);
res.status(500).send("Error in code");
}
)
});

Apigee Usergrid: Mass delete option missing

I am using usergrid to store data for a customer project. It's got two collections carShowrooms and cars. So far I am good. But I have a scenario where I have refresh the masterdata of the collection cars. Everytime I do this, I have to delete all the existing data in cars and replace it with incoming cars data from the master inventory system.
Now, with the docu in https://www.npmjs.org/package/usergrid, I see that I can only destroy one car at a time.
car.destroy(function(err){
if (err){
//error - car not deleted
//winston log - tbd
} else {
//success - car deleted
}
});
This is ok for smaller showrooms, but bigger multibrand showrooms have variety of cars - sometimes even upto 50 different varieties (8 car brands * approx. 8 different options).
Is there a mass delete option? can someone please point me to a docu if I am missing something here.
P.S. I am new to usergrid, if this is a repeated question, please mark so and point me to the right url

If you're so inclined, I've written a Node.js bulk deleter that runs delete requests in parallel. It takes approximately 3 minutes to delete 1000 entities.
Here's an always up-to-date gist, and a copy for SO:
// Installation
// 1. Install Node.js http://nodejs.org/download/
// 2. In Terminal, cd (navigate) to the directory where you saved this file
// 3. Run 'npm install request async'
// 4. Edit the script config below with your token, org, app, and collection name.
// 5. To run the script, at the Terminal prompt, run 'node api_baas_deleter.js'
// Config
var access_token = "{token}";
var as_basepath = "http://api.usergrid.com/{org}/{app}/"; // You need the trailing slash!
var collection = "{collection_name}";
// End Config
var request = require('request');
var async = require('async');
var authstring = "access_token=" + access_token;
var total = 0;
var startTime = Date.now();
function deleteRecords(callback) {
request.get({
url: as_basepath + collection + "?" + authstring,
json: true
}, function(e, r, body) {
if (body.count === undefined) {
var err = "Error: invalid endpoint. Check your basepath and collection name.";
console.log(err);
if (typeof(callback) === 'function') {
callback(err)
}
} else {
// console.log("Found " + body.count + " entities");
if (body.count > 0) {
var deletes = [];
for (var i = 0; i < body.count; i++) {
deletes.push({
url: as_basepath + collection + "/" + body.entities[i].uuid + "?" + authstring,
json: true
});
console.log("Deleting " + body.entities[i].uuid)
}
async.each(deletes, function(options, callback) {
request.del(options, function(e, r, body) {
if (r.statusCode === 200) {
total++;
}
callback(e);
});
}, function(err) {
setTimeout(function() {
deleteRecords(collection, function(e) {
callback(e);
});
}, 600); // Mandatory, since it seems to not retrieve entities if you make a request in < 600ms
});
} else {
var timeInMinutes = minutesFromMs(Date.now() - startTime);
console.log("Deleted " + total + " entities in " + timeInMinutes + " minute" + ((timeInMinutes > 1 || timeInMinutes < 1) ? "s" : ""));
if (typeof(callback) === 'function') {
callback()
}
}
}
});
}
function minutesFromMs(time) {
return Math.round(((time % 86400000) % 3600000) / 60000).toString();
}
deleteRecords();

There currently isn't a mass delete function in the Usergrid Node SDK, but you can create one. This is how I added a monkey-patched delete-by-query function into the Node SDK:
Usergrid.client.prototype.delete = function(opts, callback) {
if (_.isFunction(opts)) { callback = opts; opts = undefined; }
if (!opts.qs.q) { opts.qs.q = '*'; }
var options = {
method: 'DELETE',
endpoint: opts.type,
qs: opts.qs
};
var self = this;
this.request(options, function (err, data) {
if (err && self.logging) {
console.log('entities could not be deleted');
}
if (typeof(callback) === 'function') {
callback(err, data);
}
});
};
Hope that helps!
Scott

How do run multiple asynchronous functions in this node code?

I'm pulling text from N urls. First I get the N urls in linksOnPage and then i run a doOnPage function to get the text from each url. When i run code only 1 of the N urls gets processed through the function. I assume it's because the processing function is running asynchronously. How do I stack these up in a queue and run them all/ whats a better way to do this?
Here's the main JS code:
var nodeio, linksOnPage, lyricsFromLink, db;
nodeio = require('node.io');
db = require('./db');
db.loadDB();
var loadSong = function(artist, title, lyrics){
console.log("loadSong being called");
var newSongObj = {};
newSongObj['artist'] = artist;
newSongObj['title'] = title;
newSongObj['lyrics'] = lyrics;
//store the lyrics in a mongo table
var newSong = new db.Song(newSongObj);
newSong.save(function(err) {
if(err){
throw err;
} else{
console.log("saved with no errors!");
}
});
};
// generic utility for getting links on a page and running a function on each one
exports.linksOnPage = function(pageObj, linkSelector, doOnPage, contentSelector) {
nodeio.scrape(function(){
this.getHtml(pageObj.pageUrl, function(err, $) {
var links = [];
var i = 0;
$(linkSelector).each(function(link) {
var fullLink = pageObj.rootUrl + link.attribs.href
links.push(fullLink);
//run a function on each link
console.log('getting lyrics for song: ', i);
doOnPage(pageObj.artist, fullLink, contentSelector);
i = i+1;
});
//this.emit(links);
});
});
}
// get the lyrics for a specific song
exports.lyricsFromLink = function(artist, pageUrl, lyricsSelector) {
nodeio.scrape(function(){
this.getHtml(pageUrl, function(err, $) {
var lyrics = "";
console.log('before each statement');
$(lyricsSelector).each(function(lyricParagraph) {
lyrics = lyrics + " " + lyricParagraph.text;
});
console.log('after each statement');
loadSong(artist, pageUrl, lyrics);
this.emit(lyrics)
});
});
}