I have a list of 600 url's that i want to loop over requests to.
Would it be possible to console.log the content of each one as i receive it as opposed to waiting for the 600 to finish then return it all as 1?
Sorry if this seems a bit vague, not sure on the correct terms to use to describe this.
Meteor.methods({
getNations: function () {
this.unblock();
var result = Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:1%7D');
var totalPages = JSON.parse(result.content).totalPages;
for (var i = 1; i < totalPages; i++) {
try {
var page = Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:' + i + '%7D');
console.log(JSON.parse(page.content));
} catch(e) {
console.log('Cannot get page', e);
}
}
return result;
}
})
This will get all the pages, insert them into a collection and make them available on the client. There are a few caveats though. If you call the method multiple times you will get duplicates in the database, not sure if that's what you intended. Also, the error logging only happens on the server and is never displayed to the client, dont know if that's what you want either. Note that the second Meteor.http.get is passed a callback which makes it run async.
# Shared code
Nations = new Mongo.Collection('nations')
# Server
Meteor.methods({
getNations: function () {
this.unblock();
var result = Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:1%7D');
var totalPages = JSON.parse(result.content).totalPages;
var callback = function(err, page) {
if (err)
console.log('Cannot get page', e);
else
Nations.insert(page);
}
for (var i = 1; i < totalPages; i++) {
Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:' + i + '%7D', callback);
}
return result;
}
});
Meteor.publish('nations', function() {
return Nations.find();
});
# Client
Meteor.subscribe('nations');
Related
A few days ago I did a project and I had some problems, which where solved in this question, let me try to resume it.
I need to insert multiple objects into a DB in SQLServer, for that, I did a function that loops another function, which opens a connection, inserts and closes the connection, then, repeats it over and over again.
It worked fine, till today that was tested in a collegue PC, in the server of the job, I get this error:
Error: Requests can only be made in the LoggedIn state, not the LoggedInSendingInitialSql state
Error: Requests can only be made in the LoggedIn state, not the SentLogin7WithStandardLogin state
Here's the code we tested (the same in my last question), it works in my PC, but not in the other:
var config = {
...
};
function insertOffice(index) {
var connection = new Connection(config);
connection.on("connect", function (err) {
console.log("Successful connection");
});
connection.connect();
let url = `https://api.openweathermap.org/data/2.5/weather?lat=${offices[index].latjson}&lon=${offices[index].lonjson}&appid=${api_key}&units=metric&lang=sp`;
fetch(url)
.then((response) => { return response.json(); })
.then(function (data) {
var myObject = {
Id_Oficina: offices[index].IdOficina,
...
};
const request = new Request(
"EXEC USP_BI_CSL_insert_reg_RegistroTemperaturaXidOdicina #IdOficina, ...",
function (err) {
if (err) {
console.log("Couldnt insert data (" + index + "), " + err);
} else {
console.log("Data with ID: " + myObject.Id_Oficina +" inserted succesfully(" + index + ").")
}
}
);
request.addParameter("IdOficina", TYPES.SmallInt, myObject.Id_Oficina);
...
request.on("row", function (columns) {
columns.forEach(function (column) {
if (column.value === null) {
console.log("NULL");
} else {
console.log("Product id of inserted item is " + column.value);
}
});
});
request.on("requestCompleted", function () {
connection.close();
});
connection.execSql(request);
});
}
function functionLooper() {
for (let i = 0; i < offices.length; i++) {
let response = insertOffice(i);
}
}
functionLooper();
So, I thought it would be a good idea to use a setTimeOut, to:
Run functionLooper().
Open connection, insert and close.
Wait a few seconds.
Repeat.
So, I changed to this:
setTimeout(functionLooper, 2000);
function functionLooper() {
for (let i = 0; i < offices.length; i++) {
let response = insertOffice(i);
}
}
It works, but, as you can see, only waits when I first run it, so tried to make a function that runs setTimeout(functionLooper, 2000); like functionLooper() does, but it didn't work either.
function TimerLooper() {
for (let i = 0; i < offices.length; i++) {
setTimeout(functionLooper, 500);
}
}
function functionLooper() {
for (let i = 0; i < offices.length; i++) {
let response = insertOffice(i);
}
}
TimerLooper();
This shows me this error:
Error: Validation failed for parameter 'Descripcion'. No collation was set by the server for the current connection.
file:///...:/.../.../node_modules/node-fetch/src/index.js:95
reject(new FetchError(request to ${request.url} failed, reason: ${error.message}, 'system', error));
^ FetchError: request to https://api.openweathermap.org/data/2.5/weather?lat=XX&lon=XX&appid=XX&units=metric&lang=sp failed, reason: connect ETIMEDOUT X.X.X.X:X
So, I have some questions
How can I use properly setTimeOut? I did this function based on what I watch here in SO, but I just can't get it and I don't know what I'm doing wrong.
Why it works in my PC and the other don't? Do we have to change some kind of config or something?
Using setTimeOut, is the correct way to solve this problem? if not, what would you suggest me?
Could you do something like:
//edit: not disconnect but end
connection.on("end", function(){
functionLopper(index++)
})
function functionLooper(i) {
if(i<offices.length) insertOffice(i)
}
Edit: according to tidious doc
There is an end event emitted on connection.close()
Event: 'end'
function () { }
The connection has ended. This may be as a result of the client calling close(), the server closing the connection, or a network error.
My suggestion from above
var config = {
...
};
function insertOffice(index) {
var connection = new Connection(config);
connection.on("connect", function (err) {
console.log("Successful connection");
});
connection.connect();
let url = `...`;
fetch(url)
.then((response) => { return response.json(); })
.then(function (data) {
...
});
connection.on("end", function(){
functionLopper(index++)
})
}
function functionLooper(i) {
if(i<offices.length) insertOffice(i)
}
``
i dont know how i can pass some information into a GET in node. I have a .txt file and sometimes have a change in this file, so, i want to show this change when i enter in my webpage, but the change only occurs when the server refresh.
var itens = func.readfile();
app.get("/inventario", function(req, res){
res.format({
html: function(){
res.render('inventario', {itenss: func.readfile()});
}
});
i have a variable that receive a data from a function in other file.
exports.readfile = function() {
var texto = [];
fs.readFile('./static/itens.txt', function(err, data) {
if (err) {
return console.error(err);
}
var palavra = '';
var linha = [];
for (var i = 0; i < data.length; i++) {
if (data[i] != 10) {
palavra = palavra + (String.fromCharCode(data[i]));
} else if (data[i] == 10) {
texto.push(palavra);
palavra = [];
}
}
console.log(texto);
});
return texto;
}
so if the variable is out of scope, the page can receive the data and show in html file, but, i want to refresh the data in array case i refresh the page and the new information in .txt file is shown too
fs.readFile() is asynchronous. That means that when you call the function, it starts the operation and then immediately returns control back to the function and Javascript busily executes more Javascript in your function. In fact, your readFile() function will return BEFORE fs.readFile() has called its callback. So, you cannot return the data directly from that function. Instead, you will need to either return a promise (that resolves when the data is eventually available) or you will need to communicate the result back to the caller using a callback function.
See this answer: How do I return the response from an asynchronous call? for a lot more info on the topic.
You can modify your readFile() function to return a promise (all async functions return a promise) like this:
const fsp = require('fs').promises;
exports.readfile = async function() {
let texto = [];
let data = await fsp.readFile('./static/itens.txt');
let palavra = '';
let linha = [];
for (var i = 0; i < data.length; i++) {
if (data[i] != 10) {
palavra = palavra + (String.fromCharCode(data[i]));
} else if (data[i] == 10) {
texto.push(palavra);
palavra = [];
}
}
console.log(texto);
return texto;
}
And, then you can use that function like this:
app.get("/inventario", function(req, res){
res.format({
html: function(){
func.readFile().then(data => {
res.render('inventario', {itenss: data});
}).catch(err => {
console.log(err);
res.sendStatus(500);
});
}
});
For performance reasons, you could also cache this result and cache the time stamp of the itens.txt file and only reread and reprocess the file when it has a newer timestamp. I'll leave that as an exercise for you to implement if it is necessary (may not be needed).
I'm new to promises and I'm sure there's an answer/pattern out there but I just couldn't find one that was obvious enough to me to be the right one. I'm using node.js v4.2.4 and https://www.promisejs.org/
This should be pretty easy I think...I need to do multiple blocks of async in a specific order, and one of the middle blocks will be looping through an array of HTTP GETs.
//New Promise = asyncblock1 - FTP List, resolve the returned list array
//.then(asynchblock2(list)) - loop through list array and HTTP GET needed files
//.then(asynchblock3(list)) - update local log
I tried creating a new Promise, resolving it, passing the list to the .then, doing the GET loop, then the file update. I tried using a nested promise.all inside asynchblock2, but it's actually going in reverse order, 3, 2, and 1 due to the timing of those events. Thanks for any help.
EDIT: Ok, this is the pattern that I'm using which works, I just need a GET loop in the middle one now.
var p = new Promise((resolve, reject) => {
setTimeout(() => {
console.log('2 sec');
resolve(1);
},
2000);
}).then(() => {
return new Promise((resolve) => {
setTimeout(() => {
console.log('1.5 sec');
// instead of this section, here I'd like to do something like:
// for(var i = 0; i < dynamicarray.length; i++){
// globalvar[i] = ftpclient.getfile(dynamicarray[i])
// }
// after this loop is done, resolve
resolve(1);
},
1500);
});
}).then(() => {
return new Promise((resolve) => {
setTimeout(() => {
console.log('1 sec');
resolve(1);
},
1000);
});
});
EDIT Here is the almost working code!
var pORecAlert = (function(){
var pa;
var newans = [];
var anstodownload = [];
var anfound = false;//anfound in log file
var nexttab;
var lastchar;
var po;
var fnar = [];
var antext = '';
//-->> This section works fine; it's just creating a JSON object from a local file
try{
console.log('trying');
porfile = fs.readFileSync('an_record_files.json', 'utf8');
if(porfile == null || porfile == ''){
console.log('No data in log file - uploaded_files_data.json being initialized!');
plogObj = [];
}
else{
plogObj = JSON.parse(porfile);
}
}
catch(jpfp){
console.log('Error parsing log file for PO Receiving Alert: ' + jpfp);
return endPORecAlertProgram();
};
if((typeof plogObj) === 'object'){
console.log('an_record_files.json log file found and parsed for PO Receiving Alert!');
}
else{
return mkError(ferror, 'pORecAlert');
};
//finish creating JSON Object
pa = new Client();
pa.connect(ftpoptions);
console.log('FTP Connection for FTP Check Acknowledgement begun...');
pa.on('greeting', function(msg){
console.log('FTP Received Greeting from Server for ftpCheckAcknowledgement: ' + msg);
});
pa.on('ready', function(){
console.log('on ready');
//START PROMISE LIST
var listpromise = new Promise((reslp, rejlp) => {
pa.list('/public_html/test/out', false, (cerr, clist) => {
if(cerr){
return mkError(ferror, 'pORecAlert');
}
else{
console.log('Resolving clist');
reslp(clist);
}
});
});
listpromise.then((reclist) => {
ftpplist:
for(var pcl = 0; pcl < reclist.length; pcl++){
console.log('reclist iteration: ' + pcl);
console.log('checking name: ', reclist[pcl].name);
if(reclist[pcl].name.substring(0, 2) !== 'AN'){
console.log('Not AN - skipping');
continue ftpplist;
}
else{//found an AN
for(var plc = 0; plc < plogObj.length; plc++){
if(reclist[pcl].name === plogObj[plc].anname){
//console.log('Found reclist[pcl].name in local log');
anfound = true;
};
};
if(anfound === false){
console.log('Found AN file to download: ', reclist[pcl].name);
anstodownload.push(reclist[pcl].name);
};
};
};
console.log('anstodownload array:');
console.dir(anstodownload);
return anstodownload;
}).then((fnar) => {
//for simplicity/transparency, here is the array being overwritten
fnar = new Array('AN_17650_37411.699.txt', 'AN_17650_37411.700', 'AN_17650_37411.701', 'AN_17650_37411.702.txt', 'AN_17650_37411.801', 'AN_17650_37411.802.txt');
return Promise.all(fnar.map((gfname) => {
var nsalertnames = [];
console.log('Getting: ', gfname);
debugger;
pa.get(('/public_html/test/out/' + gfname), function(err, anstream){//THE PROBLEM IS THAT THIS GET GETS TRIGGERED AN EXTRA TIME FOR EVERY OTHER FILE!!!
antext = '';
console.log('Get begun for: ', gfname);
debugger;
if(err){
ferror.nsrest_trace = 'Error - could not download new AN file!';
ferror.details = err;
console.log('Error - could not download new AN file!');
console.log('************************* Exiting *************************')
logError(ferror, gfname);
}
else{
// anstream.on('data', (anchunk) => {
// console.log('Receiving data for: ', gfname);
// antext += anchunk;
// });
// anstream.on('end', () => {
// console.log('GET end for: ', gfname);
// //console.log('path to update - gfname ', gfname, '|| end text.');
// fs.appendFileSync(path.resolve('test/from', gfname), antext);
// console.log('Appended file');
// return antext;
// });//end end
};
});//get end
}));//end Promise.all and map
}).then((res99) => {
// pa.end();
// return Promise(() => {
console.log('end all. res99: ', res99);
// //res4(1);
// return 1;
// });
});
});
})();
-->> What happens here:
So I added the almost working code. What is happening is that for every other file, an additional Get request gets made (I don't know how it's being triggered), which fails with an "Unable to make data connection".
So for my iteration over this array of 6, there ends up being 9 Get requests. Element 1 gets requested (works and expected), then 2 (works and expected), then 2 again (fails and unexpected/don't know why it was triggered). Then 3 (works and expected), then 4 (works and expected), then 4 again (fails and unexpected) etc
what you need is Promise.all(), sample code for your app:
...
}).then(() => {
return Promise.all(arry.map(item => ftpclient.getFile(item)))
}).then((resultArray) => {
...
So thanks for the help (and the negative votes with no useful direction!)
I actually reached out to a good nodejs programmer and he said that there seemed to be a bug in the ftp module I was using, and even when trying to use a blackbird .map, the quick succession of requests somehow kicked off an error. I ended up using promise-ftp, blackbird, and promiseTaksQueue - the kicker was that I needed interval. Without it the ftp would end up causing a strange illogical error in the ftp module.
You need the async library. Use the async.eachSeries in situations where you need to use asynchronous operations within a loop, then execute a function when all of those are complete. There are many variations depending on the flow you want but this library does it all.
https://github.com/caolan/async
async.each(theArrayToLoop, function(item, callback) {
// Perform async operation on item here.
doSomethingAsync(item).then(function(){
callback();
})
}, function(err){
//All your async calls are finished continue along here
});
Okay, so my code is pulling data from a yelp business using their official API. My problem is that I can't seem to get the data to return out of the function. The problem isn't in ejs, it's that the data doesn't return when I tell it to! I just get undefined with some attempts, and with others (including the one I'm going to show here), I get an empty array. I'm pasting only the code that's important, let me know if you need more!
function yelp(){
var b = [];
var i = 0;
(struck the initialization of client)
client.business("(struck)", function(error, data) {
if (error != undefined){
res.send("an error occured. exiting");
process.process.reallyExit();
}
b[i++] = data.name;
b[i++] = data.display_phone;
b[i++] = data.rating;
console.log(b); //NEW!!
});
console.log(b);
return b;
}
app.get('/yelp', function(req,res){
var arr = yelp();
console.log(arr);
res.render('yelp.ejs', {title: 'Yelp!', arr: arr});
});
}
I added one more line of code, that I THINK may have narrowed down the problem to being related to my poor internet connection. I added ANOTHER console.log(b), this time inside of the business API call. the console.log(arr) is shows second, the console.log(b); just before the reutrn shows first, and LAST is the console.log(b) INSIDE the API call. It also took a good 30 seconds for that log to appear, and it appeared AFTER the page loaded. So, how do I go about making the page wait for the data? Or is this unrelated to my problem?
Without knowing their API, I do recognize the callback style. The result of the call to your client would then be in the data parameter in the callback, and thats where you want to render your view.
The following is not tested.
function yelp(cb) {
var b = [];
var i = 0;
// (struck the initialization of client)
client.business("(struck)", function(error, data) {
if (error) {
return cb(error);
}
b[i++] = data.name;
b[i++] = data.display_phone;
b[i++] = data.rating;
});
console.log(b);
cb(null, b)
}
app.get('/yelp', function(req, res) {
yelp(function(err, arr) {
if (err) {
res.send("an error occured. exiting");
process.process.reallyExit();
return;
}
console.log(arr);
res.render('yelp.ejs', {
title: 'Yelp!',
arr: arr
});
});
});
The thing to notice here, is the callback passing. This is normally how you do
async work in Node.js. It can be made a bit prettier using promises.
nodejs is async, meaning the app won't wait for the yelp() function to return. you can pass the yelp function a callback like so:
function yelp(callback){
var b = [];
var i = 0;
(struck the initialization of client)
client.business("(struck)", function(error, data) {
if (error != undefined){
res.send("an error occured. exiting");
process.process.reallyExit();
}
b[i++] = data.name;
b[i++] = data.display_phone;
b[i++] = data.rating;
});
console.log(b);
callback(b);
}
yelp(funtion(arr) {
res.render('yelp.ejs', {title: 'Yelp!', arr: arr});
})
You are expecting sync things to happen, but it's async. Your client.business method takes in a callback as it's second argument which isn't returning by the time res.render gets called.
Try this:
function yelp(callback) {
var b = [];
var i = 0;
client.business("(struck)", function(error, data) {
if (error != undefined){
res.send("an error occured. exiting");
process.process.reallyExit();
}
b[i++] = data.name;
b[i++] = data.display_phone;
b[i++] = data.rating;
// No returns in async. Just call the callback.
callback('yelp.ejs', { {title: 'Yelp!', arr: b}})
});
}
app.get('/yelp', function(req,res){
yelp(res.render);
});
I have a function that GETs a JSON object from a remote server, or from a local cache on-disk.
In a use-case, i have to call this function several thousand times with varying arguments, but when i do so, i get max stack overflow errors. I must be making a recursive call somewhere, but i can't see where it could be as my process.nextTick function calls seem to be in the right place.
I get none of my log.error readouts in the console, which would be evident if any of the recursive calls to retry the request were made.
The console output shows a repeated occurrence of
(node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral.
then...
RangeError: Maximum call stack size exceeded
Then the program exits.
Can anyone offer any help regarding what i may be doing wrong? I'm completely stumped.
Below is the function that invokes the problematic function "tf2inv.loadInventory()"
function refreshInventories(accounts, force, callback) {
//job executes download function, then pushes to inventories object
var inventories = {};
var Qinv = async.queue(function (task, invCallback) {
tf2inv.loadInventory(
task.force,
task.steamid,
function(inv, alias) {
inventories[alias] = inv;
process.nextTick(invCallback);
}
);
}, 100)
//when all queue jobs have finished, callback with populated inventories object
Qinv.drain = function (err) {
log.info('All inventories downloaded');
callback(inventories);
}
//adding jobs to the queue
for (var i = accounts.length - 1; i >= 0; i--) {
Qinv.push({
force: force,
steamid: accounts[i]
});
};
}
Shown here is the function that either parses from the cache, or requests from the remote server.
//tf2inv
var loadInventory = function(force, sid, callback) {
var invLoc = invFolder+sid
if(force) {
if(fs.existsSync(invLoc)) {
fs.unlinkSync(invLoc);
}
}
if(fs.existsSync(invLoc)) {
var body = fs.readFileSync(invLoc);
try {
var inventory = JSON.parse(body);
} catch (e) {
fs.unlinkSync(invLoc);
log.error("parsing " + sid+"'s inventory");
loadInventory(true, sid, invFolder, callback);
return;
}
process.nextTick(function() { callback(inventory, sid) })
return;
} else {
var urlPre = "http://api.steampowered.com/IEconItems_440/GetPlayerItems/v0001/?key=";
var urlSidPre = "&steamid=";
var urlInvSuf = "&inventory=yes";
var URL = urlPre+steam_API+urlSidPre+sid+urlInvSuf;
http.get(URL, function (res) {
var body = '';
res.on('data', function (data) {
body+=data;
fs.appendFile(invLoc, data);
});
res.on('end', function() {
try {
inventory = JSON.parse(body);
} catch (e) {
if(fs.existsSync(invLoc)) {
fs.unlinkSync(invLoc);
}
log.error("parsing " + sid+"'s downloaded inventory");
loadInventory(force, sid, invFolder, callback)
return;
}
process.nextTick(function() { callback(inventory, sid) })
return;
});
res.on('error', function (e, socket) {
log.error(sid + " inventory error")
if(fs.existsSync(invLoc)) {
fs.unlinkSync(invLoc);
}
log.debug('Retrying inventory')
loadInventory(force, sid, invFolder, callback);
return;
})
res.on('close', function () {res.emit('end'); log.error('connection closed')})
})
.on('error', function(e) {
log.error(JSON.stringify(e));
if(fs.existsSync(invLoc)) {
fs.unlinkSync(invLoc);
}
log.debug('Retrying inventory')
loadInventory(force, sid, invFolder, callback)
return;
})
}
};
It is likely to be failing to parse the body coming back from the server. It then immediately calls itself again, failing again, infinitely looping and causing a stack overflow.
I suggest you do not retry automatically on a failed parse - if it fails once, it is likely to fail again. It would be best to call back with the error, and let the part of your programming calling this handle the error, or passing it back to the point where it can let the user know that something is wrong.