Develop Reference

How to pass Google/sheetsAPI data to website?

Sooooo,
I want to get data from a google sheet to a website which appends the passed data dynamically to the website so the amount of itme's on the website is equal to the amount of rows. I pretty much got everything done separately but I just cant seem to figure out how to combine them.
I set up a node.js project which uses my Api key to retrieve the data, manipulate and push it into an array.
also maybe there's a more efficient way to push the data but i couldn't figure anything else out but saving it to a file and then reading it back. I then pumped that into a AWS api gateway+Lambda funtion which didn't really work because I couldn't get Lambda to require google/api but i think i could sort that out.
I also already did all the HTML / JS on the website to push the data to the page I just cant seem to figure out how to get the data from my AWS-http-link into my website. No matter if I use async or promise or none of both It just can't get the data.
'ISSUES: using google/api inside Lambda, getting data from Api to array, maybe diffrent way to get the data from google api'
Thank you in advance!
AWS-link: https://po7bu16g2i.execute-api.eu-central-1.amazonaws.com/live/celldata
//JS code on website
var receivearray =[];
function createEntry(data){
for(i=0;i<data.length;i++){
var outdt = document.createElement('div');
outdt.className='outwrap';
var pers_name = document.createElement('div');
pers_name.className='name';
var pers_name_txt= document.createElement('h3');
pers_name_txt.className='name_head';
pers_name_txt.innerHTML=data[i][0];
outdt.append(pers_name);
pers_name.append(pers_name_txt);
for(k=1;k<data[i].length;k++){
var top =document.createElement('div');
top.className='text_wrap';
var q = document.createElement('p');
q.className='frage';
q.innerHTML=data[i][k][0];
var a= document.createElement('p');
a.innerHTML=data[i][k][1];
outdt.append(top);
top.append(q,a);
}
document.body.appendChild(outdt);
}
};
createEntry(receivearray);
//Node.js Code
//requirements
var GoogleSpreadsheet = require('google-spreadsheet');
var crds = require('./credt.json');
const fs = require('fs');
//updates the data-tfile
function update(sheetid){
//google api shapens
var doc = new GoogleSpreadsheet(sheetid);
doc.useServiceAccountAuth(crds,
function (err) {
doc.getRows(1,
function (err, rows) {
//saves all retrieved data into textfile
fs.writeFile('ting.txt', JSON.stringify(rows),
function (err) {
//checks if file was saved succesfully
if (err) throw err;
console.log('updated')
}
)
}
)
}
)
}
function prepdata(){
//define variables
var text = fs.readFileSync("ting.txt", "utf8");
var findata =[];
var hold=[];
//singleout all row-elements
for(x=0;x<text.length;x++){
if(text.slice(x,x+7)=='"name":'){
hold.push(text.slice(x+7,x+120));
}
}
//split data by , into sets
for(x=0;x<hold.length;x++){
findata.push(hold[x].split(','));
}
//push sets into arrays
for (var k = 0; k < findata.length; k++){
for (var i = 0; i < findata[k].length; i++){
findata[k][i] = findata[k][i].replace(/"/g, " ");
findata[k][i] = findata[k][i].split(':');
}
}
return findata;
}
update("1W6tVuj0krrwI7PyTRJha2ZOX72kGGfFAI8eqXOirWHo");
console.log(prepdata());

How to use multiple callbacks when looping data

I'm trying to get HTML form data, loop it through, change it a bit and insert it to database. I have tried like below app.js.
How can I make callbacks so that formdata what I have modified is available for .create function?
I have searched from everywhere and I always end up in dead end and undefined variable somehow.
app.js:
//Find the day where to save
Day.findById(req.params.id, function(err, day) {
if (err) {
console.log(err);
res.redirect("/diary");
} else {
// Search function to find data with _id
function ingredientIdQuery(reqBodyId) {
var ingQuery = Ingredient.find({_id:reqBodyId});
return dbQuery;
}
// This loops through HTML formdata and formats it for mongoose model
for (var i = 0; i < req.body.amount.length; i++) {
if (req.body.amount[i] !== "") {
var amount = Number(req.body.amount[i]);
var singleMealTempObj = {};
singleMealTempObj.amount = amount;
var _id = req.body.id[i];
var query = ingredientIdQuery(_id);
// Executing the query for the data I need with id
query.exec(function(err, ingr){
if(err) {
return console.log(err);
} else {
singleMealTempObj.ingredient = ingr[0];
singleMealTempArr.push(singleMealTempObj);
}
});
}
}
}
// This inserts data into day
Meal.create(singleMealTempArr, function(err, singleMealObject) {
if (err) {
console.log(err);
} else {
day.meals.push(singleMealObject);
day.save();
res.redirect("/day/" + day._id + "/dayshow");
}
});
});
});
Edit:
Thanks for reply and notices! While I was trying to do everything to get this work I missed those few things like declaring variables. Sorry for that. I threw the towel in to the cage at this point.
flow goes like this:
User sends HTML form data to app.js which is inside object of two arrays (id[] and amount[]). Amount array needs to be looped through if it has value other than 0. Same index id array value is used to fetch data from database. This data what is found from database with id from id[] is used with same index amount[] and it should be saved to mongo.
I can get the values from HTML form ok. but I have tried to make a search in Mongo in a for loop (query.exec in the code) I get the data ok. When I log the data outside the database query, variable is undefined.
I hope this clarifys a bit what I'm trying to achieve.
I'll continue this later... :)

I guess issue originates because of this function.
function ingredientIdQuery(reqBodyId) {
var ingQuery = Ingredient.find({_id:reqBodyId});
return dbQuery;
}
Is find function asynchronous or synchronous?
Also you are returning dbQuery but dbQuery does not seem to be changed inside the function.

Couple I noticed that may fix this:
You never define singleMealTempArr, so when you try to push data to it, you are gonna run into problems.
Your ingredientIdQuery function returns dbquery - which also isn't defined. You actually call it ingQuery. Even so...are you positive that this will return the data that you want?
// lets loop through all the form fields in req.body.amount
for (var i = 0; i < req.body.amount.length; i++) {
// keep going unless the form field is empty
if (req.body.amount[i] !== "") {
// assign all the form fields to the following vars
var amount = Number(req.body.amount[i]);
var singleMealTempObj = {};
singleMealTempObj.amount = amount;
var _id = req.body.id[i];
var query = ingredientIdQuery(_id);
// we are executing the ingredientIdQuery(_id), better
// double-check that this query returns the result we are
// looking for!
query.exec(function(err, ingr){
if(err) {
return console.log(err);
} else {
singleMealTempObj.ingredient = ingr[0];
// now that we've gone through and mapped all the form
// data we can assign it to the singleMealTempArr
// WOOPS! Looks like we forgot to assign it!
singleMealTempArr.push(singleMealTempObj);
}
});
}
}
}

Passing data from mongo to ejs

(This is my first question here, I'm excited :))
2 files: index.js (what I'm using to connect to the mongo, find and toArray the data), and admin.ejs where I want to display the data (user records: username, first name, last name...).
index.js:
var db = MongoClient.connect('mongodb://127.0.0.1:27017/test', function(err, db) {
if(err)
throw err;
db.collection("Users").find().toArray(function (err, result) {
var i, count;
for (i = 0, count = result.length; i < count; i++) {
myArr.push(result[i]);
}
myArr = JSON.stringify(myArr);
});
console.log(myArr); // just for testing
console.log("connected to the mongoDB !");
});
app.get('/', function (req, res) {
res.render('Admin', {
myVar: myArr
});
});
admin.ejs:
var myOtherVar = JSON.parse('<%-myVar%>');
In the broswer "view source code" I can see
var myOtherVar = JSON.parse('[{"_id":"567a6fd307200cb90f7af961","Username":"Yogev" ...
so I know that the data passes correctly, but any atempt to use it fails - the JS "see" it as object Object and not as an array...
What should I do?

You'll just need to use object notation to access the data.
console.log(myOtherVar[0].Username)
// Returns 'Yogev'

This way you send a object to the view
res.view('Admin',{ myVar : myArr});
This way you use it in the view
<h1> <%= myVar[0].property %></h1>
In this case you use <%= because <% does not print the returned value by the expression just after it.

Multiple MongoDB queries without using a loop?

I'm trying to figure out if I can make my MongoDB queries (fron Node.js) more quicker and more efficient.
Basically I have an array of "Players", each Player has a "player_id" property. I want to iterate over each Player and use the player_id to find data about the Player in a MongoDB database.
Because of the Asynchronous nature of Node.js I have to guarantee that when I send in a query to the database, the data that I get back corresponds to the player_id that was used to query.
So far I have the following code (that I simplified). What I'm most interested in is how I can achieve this without using a for loop. Thanks.
var playersArray = new Array({"player_id":1234567}, {"player_id":9847621}, {"player_id":0946783}, {"player_id":8712890});
queryDataForPlayers(playersArray, function(data){
//Done - Each Player now has a "new_data_property" property
})
function queryDataForPlayers(playersArray){
var newPlayersArray = new Array();
var counter = 0;
for(var i=0; i<playersArray.length; i++)
{
retrievePlayerData(playersArray[i].player_id, playersArray[i], function(err,data)
{
newPlayersArray.push(data);
if(++counter == playersArray.length)
{
callback(newPlayersArray);
}//end if
});
}
}
var Schema = mongoose.model('Schema');
var ObjectID = require('mongodb').ObjectID;
function retrievePlayerData(playerID, obj, callback){
Schema.find({_id:ObjectID(String(playerID))}, function(err,data){
obj["new_data_property"] = data;
callback(err,obj);
});
}

I can't really test this, but you can pass in an array of player ID's directly to mongo, and get a document set with the related data back in just one query, something like
var playersArray = new Array({"player_id":1234567}, {"player_id":9847621}, {"player_id":0946783}, {"player_id":8712890});
var Schema = mongoose.model('Schema');
var ObjectID = require('mongodb').ObjectID;
function queryDataForPlayers(playersArray, callback){
var player_ids = playersArray.map(function(player) {
return ObjectID(String(player.player_id));
});
Schema.find({
'_id': { $in: player_ids}
}, function(err, docs){
callback(err, docs);
});
}

Use $in operator... You can use it like
Schema.find({_id:{$in:[array_of_playerid]} }).exec(function(error,results)){
}

Self referencing Parse.com class

I've imported a table containing a self referencing parent/child relationship into a class in Parse.com. Parse creates a new ID which I would now like to replace the UID with. I cannot seem to update all of the corresponding parent ID fields.
I've almost managed to get it going with some cloud code:
Parse.Cloud.define("updatePID", function(request, response) {
var _results={};
var query = new Parse.Query("ww_notes");
query
.limit(1000)
.find({
success: function(results){
for (var i=0; i<results.length; i++){
_results[results[i].get("uid")] = results[i].id;
};
for (var i=0; i<results.length; i++){
if (_results[results[i].get("pid")]){
results[i].set("test", _results[results[i].get("pid")]);
results[i].save();
};
};
response.success(results);
},
error: function(){
response.error("failed PID update");
}
});
})
This returns the corrected recordset but doesn't save it in the database. Note: test is just a test field to get this function working before switching to updating the PID. I've tried to simplify the problem by moving a static write into the first loop:
for (var i=0; i<results.length; i++){
_results[results[i].get("uid")] = results[i].id;
results[i].set("test", "things");
results[i].save();
};
This only updated 4 of 245 records. The recordset sent back was correctly updated but the Parse data store was not. I've created an equivalent job which resulted in 27 of the 245 records being updated. The job returned successfully.
Stephen

The problem you're having is almost certainly related to how you handle the asynch saves.
The reason you're getting only a few saves does is that the code launches them off asynchronously in a tight loop, then returns. You're getting a few saves done while the loop is finishing, then no more once you run response.whatever.
The fix is to batch the saves and use promises like this...
// ...
query.limit(1000);
query.find().then(function(results) {
for (var i=0; i<results.length; i++){
_results[results[i].get("uid")] = results[i].id;
};
// we'll save everything in this array
var toSave = [];
for (var i=0; i<results.length; i++){
if (_results[results[i].get("pid")]){
results[i].set("test", _results[results[i].get("pid")]);
toSave.push(results[i]);
};
};
// return a promise that's complete when everything is saved
return Parse.Object.saveAll(toSave);
}).then(function(results) {
response.success(results);
}, function(error) {
response.error(error);
});