i dont know how i can pass some information into a GET in node. I have a .txt file and sometimes have a change in this file, so, i want to show this change when i enter in my webpage, but the change only occurs when the server refresh.
var itens = func.readfile();
app.get("/inventario", function(req, res){
res.format({
html: function(){
res.render('inventario', {itenss: func.readfile()});
}
});
i have a variable that receive a data from a function in other file.
exports.readfile = function() {
var texto = [];
fs.readFile('./static/itens.txt', function(err, data) {
if (err) {
return console.error(err);
}
var palavra = '';
var linha = [];
for (var i = 0; i < data.length; i++) {
if (data[i] != 10) {
palavra = palavra + (String.fromCharCode(data[i]));
} else if (data[i] == 10) {
texto.push(palavra);
palavra = [];
}
}
console.log(texto);
});
return texto;
}
so if the variable is out of scope, the page can receive the data and show in html file, but, i want to refresh the data in array case i refresh the page and the new information in .txt file is shown too
fs.readFile() is asynchronous. That means that when you call the function, it starts the operation and then immediately returns control back to the function and Javascript busily executes more Javascript in your function. In fact, your readFile() function will return BEFORE fs.readFile() has called its callback. So, you cannot return the data directly from that function. Instead, you will need to either return a promise (that resolves when the data is eventually available) or you will need to communicate the result back to the caller using a callback function.
See this answer: How do I return the response from an asynchronous call? for a lot more info on the topic.
You can modify your readFile() function to return a promise (all async functions return a promise) like this:
const fsp = require('fs').promises;
exports.readfile = async function() {
let texto = [];
let data = await fsp.readFile('./static/itens.txt');
let palavra = '';
let linha = [];
for (var i = 0; i < data.length; i++) {
if (data[i] != 10) {
palavra = palavra + (String.fromCharCode(data[i]));
} else if (data[i] == 10) {
texto.push(palavra);
palavra = [];
}
}
console.log(texto);
return texto;
}
And, then you can use that function like this:
app.get("/inventario", function(req, res){
res.format({
html: function(){
func.readFile().then(data => {
res.render('inventario', {itenss: data});
}).catch(err => {
console.log(err);
res.sendStatus(500);
});
}
});
For performance reasons, you could also cache this result and cache the time stamp of the itens.txt file and only reread and reprocess the file when it has a newer timestamp. I'll leave that as an exercise for you to implement if it is necessary (may not be needed).
Related
Update !!
I fixed my initial issue with the help of Dacre Denny answer below however when writing tests for my code it turned out that the changes were not being saved before the server responded therefor the company collection in my test database was empty, I fixed this issue with the following code
Companies.find({ company_name: company.company_name }).then(found => {
if (found.length !== 0) {
return res.status(400).json({ error: "Company already exists" });
}
var userForms = company.users;
company.users = [];
const finalCompany = new Companies(company);
console.log(finalCompany);
var userPromises = [];
for (var x = 0; x < userForms.length; x++) {
var user = userForms[x].user;
user.company = finalCompany._id;
userPromises.push(userCreation(user));
}
return Promise.all(userPromises).then(responses => {
for (var x in responses) {
if (!responses[x].errors) {
finalCompany.addUser(responses[x]._id);
} else {
res.status(400).json(responses[x]);
}
}
return finalCompany;
});
})
// I moved the save in here !!!
.then((finalCompany) => {
finalCompany.save().then(()=>{
res.status(200).json({signup:"Successful"});
})
},(err) => {
res.json({error: err});
});
});
Original Issue
I am trying to create a mongoose document to represent a company, this code saves the model in my db however it does not seem to be responding with a status code or reply to postman when I make a request
I've used a debugger to step through the code but I am very rusty on my JS and I am afraid I've gone into deep water with promises thats gone over my head.
router.post('/c_signup', auth.optional, (req, res, next) => {
const { body: { company } } = req;
var error_json = cbc(company);
if( error_json.errors.length > 0 ){
return res.status(422).json(error_json);
}
Companies.find({company_name: company.company_name})
.then((found) => {
if (found.length !== 0) {
return res.status(400).json({error: "Company already exists"});
}
var userForms = company.users;
company.users = [];
const finalCompany = new Companies(company);
var userPromises = [];
for (var x =0; x < userForms.length; x ++) {
var user = userForms[x].user;
user.company = finalCompany._id;
userPromises.push(userCreation(user));
}
Promise.all(userPromises).then((responses) => {
for (var x in responses){
if (!responses[x].errors){
finalCompany.addUser(responses[x]._id);
}
else {
res.status(400).json(responses[x]);
}
}
console.log("h2");
finalCompany.save(function () {
console.log("h3");
return res.status(200);
});
})
});
return res.status(404);
});
This is the output from the debug but the execution is hanging here
h2
h3
There are a few issues here:
First, the save() function is asynchronous. You'll need to account for that by ensuring the promise that save() returns, is returned to the handler that it's is called in.
The same is true with the call to Promise.all() - you'll need to add that promise to the respective promise chain by returning that promise to the enclosing handler (see notes below).
Also, make sure the request handler returns a response either via res.json(), res.send(), etc, or by simply calling res.end(). That signals that the request has completed and should address the "hanging behaviour".
Although your code includes res.json(), there are many cases where it's not guaranteed to be called. In such cases, the hanging behaviour would result. One way to address this would be to add res.end() to the end of your promise chain as shown below:
Companies.find({ company_name: company.company_name }).then(found => {
if (found.length !== 0) {
return res.status(400).json({ error: "Company already exists" });
}
var userForms = company.users;
company.users = [];
const finalCompany = new Companies(company);
var userPromises = [];
for (var x = 0; x < userForms.length; x++) {
var user = userForms[x].user;
user.company = finalCompany._id;
userPromises.push(userCreation(user));
}
/* Add return, ensure that the enclosing then() only resolves
after "all promises" here have completed */
return Promise.all(userPromises).then(responses => {
for (var x in responses) {
if (!responses[x].errors) {
finalCompany.addUser(responses[x]._id);
} else {
res.status(400).json(responses[x]);
}
}
console.log("h2");
/* Add return, ensure that the enclosing then() only resolves
after the asnyc "save" has completed */
return finalCompany.save(function() {
console.log("h3");
return res.status(200);
});
});
})
.then(() => {
res.end();
},(err) => {
console.error("Error:",err);
res.end();
});
I have a couple of APIs I need to call to collect and merge information.
I make the first API call and, based on the result, I make several calls to the second one (in a loop).
Since http requests are asynchronous I'm loosing the information. By the time the second step is finished the server (nodejs) already sent the response back to the client.
I've already tried to, somehow, use the callback functions. This managed to keep the response to the client waiting but the information of the second call was still lost. I guess somehow the variables are not being synchronized.
I also did a quick test with away/async but my Javascript mojo was not enough to make it run without errors.
/* pseudo code */
function getData(var1, callback){
url= "http://test.server/bla?param="+var1;
request.get(url, function (error, response, body){
var results = [];
for(var item of JSON.parse(body).entity.resultArray) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function(secondStepData){
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
});
results.push(o);
}
callback(results);
});
}
function getSecondStep(object, callback){
url = "http://othertest.server/foobar?param=" + object.data1;
request.get(url, function (error, response, body){
var results = [];
if(response.statusCode == 200){
for(var item of JSON.parse(body).object.array) {
var o = {}
o['data4'] = item.data4;
o['data5'] = item.data5;
results.push(o);
}
callback(results);
}
});
}
What I would like is to be able to collect all the information into one JSON object to return it back to the client.
The client will then be responsible for rendering it in a nice way.
I recommend using the async / await pattern with the request-promise-native library.
This makes API calls really easy to make and the code is cleaner when using this pattern.
In the example below I'm just calling a httpbin API to generate a UUID but the principle applies for any API.
const rp = require('request-promise-native');
async function callAPIs() {
let firstAPIResponse = await rp("https://httpbin.org/uuid", { json: true });
console.log("First API response: ", firstAPIResponse);
// Call several times, we can switch on the first API response if we like.
const callCount = 3;
let promiseList = [...Array(callCount).keys()].map(() => rp("https://httpbin.org/uuid", { json: true }));
let secondAPIResponses = await Promise.all(promiseList);
return { firstAPIResponse: firstAPIResponse, secondAPIResponses: secondAPIResponses };
}
async function testAPIs() {
let combinedResponse = await callAPIs();
console.log("Combined response: " , combinedResponse);
}
testAPIs();
In this simple example we get a combined response like so:
{
{
firstAPIResponse: { uuid: '640858f8-2e69-4c2b-8f2e-da8c68795f21' },
secondAPIResponses: [
{ uuid: '202f9618-f646-49a2-8d30-4fe153e3c78a' },
{ uuid: '381b57db-2b7f-424a-9899-7e2f543867a8' },
{ uuid: '50facc6e-1d7c-41c6-aa0e-095915ae3070' }
]
}
}
I suggest you go over to a library that supports promises (eg: https://github.com/request/request-promise) as the code becomes much easier to deal with than the callback method.
Your code would look something like:
function getData(var1){
var url = "http://test.server/bla?param="+var1;
return request.get(url).then(result1 => {
var arr = JSON.parse(body).entity.resultArray;
return Promise.all( arr.map(x => request.get("http://othertest.server/foobar?param=" + result1.data1)))
.then(result2 => {
return {
data1: result1.data1,
data2: result1.data2,
data3: result1.data3,
secondStepData: result2.map(x => ({data4:x.data4, data5:x.data5}))
}
})
});
}
And usage would be
getData("SomeVar1").then(result => ... );
The problem is that you are calling the callback while you still have async calls going on. Several approaches are possible, such us using async/await, or reverting to Promises (which I would probably do in your case).
Or you can, well, call the callback only when you have all the information available. Pseudo code follows:
function getData(var1, callback){
url= "http://test.server/bla?param="+var1;
request.get(url, function (error, response, body){
var results = [];
var items = JSON.parse(body).entity.resultArray;
var done = 0, max = items.length;
for(var item of items) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function(secondStepData){
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
results.push(o);
done += 1;
if(done === max) callback(results);
});
}
});
}
(note that since this is pseudo code, I am not checking for errors or handling a possible empty result from request.get(...))
You need to call the callback of first function only when all the second callback functions have been called. Try this changes:
function getData(var1, callback) {
url = "http://test.server/bla?param=" + var1;
request.get(url, function (error, response, body) {
var results = [],count=0;
var arr = JSON.parse(body).entity.resultArray;
for (let [index, value] of arr.entries()) {
var o = {};
o['data1'] = item.data1;
o['data2'] = item.data2;
o['data3'] = item.data3;
getSecondStep(o, function (secondStepData) {
//console.log("Callback object");
//console.log(o);
o['secondStepData'] = secondStepData;
results[index] = o;
count++;
if (count === arr.length) {
callback(results);
}
});
}
});
}
I'm trying to write the following code and make it work synchronously, but the only problem that it works correctly with console.log, which prints me every item in array with delay in 1 second, but don't work with the following structure:
for (let i = 0; i < array.length; i++) {
setTimeout(function () {
1.http request via rp or request.get (I receive a huge data array)
2. .map results
3.insert to Mongo via mongoose
}
}
as for now I have the following code inside:
request.get({url: array[i].url}), function (error, body) {
body.map(element => {
//do stuff, it works fine
});
collection.insertMany(body, function (err, docs) {
//#justloggerthings
}
Or I have almost the same version with rp instead of request.get
By default I have mongoose.Promise = global.Promise;
Why this cause a problem? Because body.length is very huge dataset which eat a lot of RAM. (Now imagine 20+ arrays with insertMany)
So Mongo trying to insertMany all responses from request at once (when they ready, w/o 1000s delay). Actually that's why I choose request instead of rp (request-promise) but it seems look async too. So should I choose another http get module from npm and switch to it. And not to worry about it?
Or should I wrap this operations to promise || made an async function and recall it inside loop every time (1000s for example) when I it's correctly finished. In this case, the only thing which I found actual on StackOverflow is:
How to insert data to mongo synchronously (Nodejs, Express)
Bit it's a bit outdated. So any ideas?
Well, i dont have your actual code so i will write in pseudo code what you can do.
const chunkArray = (array, chunkSize) => {
let i,j,chunks = [];
for (i = 0, j = array.length; i < j; i += chunkSize) {
chunks.push(array.slice(i, i + chunkSize));
}
return chunks;
}
for (let i = 0; i < array.length; i++) {
let bigArray = await request('your data url');
// do some mapping or whatever
// then break your large array into smaller chunks
let chunks = chunkArray(bigArray, 10);
let j;
for (j = 0; j < chunks.length; j++) {
await collection.insertMany(chunks[j]);
}
}
Actual code that solve my problem is:
async function test (name, url, lastModified) {
try {
const response = await rp({uri: url, json: true});
response.map(async (element) => {
if (element.buyout > 0) {
element.price = (element.buyout / element.quantity);
}
element.lastModified = lastModified
});
return collection.insertMany(response);
} catch (err) {
console.error(err)
}
}
async function addAsync() {
const z = await test();
console.log(z);
}
addAsync();
I have a list of 600 url's that i want to loop over requests to.
Would it be possible to console.log the content of each one as i receive it as opposed to waiting for the 600 to finish then return it all as 1?
Sorry if this seems a bit vague, not sure on the correct terms to use to describe this.
Meteor.methods({
getNations: function () {
this.unblock();
var result = Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:1%7D');
var totalPages = JSON.parse(result.content).totalPages;
for (var i = 1; i < totalPages; i++) {
try {
var page = Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:' + i + '%7D');
console.log(JSON.parse(page.content));
} catch(e) {
console.log('Cannot get page', e);
}
}
return result;
}
})
This will get all the pages, insert them into a collection and make them available on the client. There are a few caveats though. If you call the method multiple times you will get duplicates in the database, not sure if that's what you intended. Also, the error logging only happens on the server and is never displayed to the client, dont know if that's what you want either. Note that the second Meteor.http.get is passed a callback which makes it run async.
# Shared code
Nations = new Mongo.Collection('nations')
# Server
Meteor.methods({
getNations: function () {
this.unblock();
var result = Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:1%7D');
var totalPages = JSON.parse(result.content).totalPages;
var callback = function(err, page) {
if (err)
console.log('Cannot get page', e);
else
Nations.insert(page);
}
for (var i = 1; i < totalPages; i++) {
Meteor.http.get('https://www.easports.com/uk/fifa/ultimate-team/api/fut/item?jsonParamObject=%7B%22page%22:' + i + '%7D', callback);
}
return result;
}
});
Meteor.publish('nations', function() {
return Nations.find();
});
# Client
Meteor.subscribe('nations');
I am learning node.js with learnyounode.
I am having a problem with JUGGLING ASYNC.
The problem is described as follows:
You are given three urls as command line arguments. You are supposed to make http.get() calls to get data from these urls and then print them in the same order as their order in the list of arguments.
Here is my code:
var http = require('http')
var truecount = 0;
var printlist = []
for(var i = 2; i < process.argv.length; i++) {
http.get(process.argv[i], function(response) {
var printdata = "";
response.setEncoding('utf8');
response.on('data', function(data) {
printdata += data;
})
response.on('end', function() {
truecount += 1
printlist.push(printdata)
if(truecount == 3) {
printlist.forEach(function(item) {
console.log(item)
})
}
})
})
}
Here is the questions I do not understand:
I am trying to store the completed data in response.on('end', function(){})for each url using a dictionary. However, I do not know how to get the url for that http.get(). If I can do a local variable inside http.get(), that would be great but I think whenever I declare a variable as var url, it will always point to the last url. Since it is global and it keeps updating through the loop. What is the best way for me to store those completed data as the value with the key equal to the url?
This is how I would go about solving the problem.
#!/usr/bin/env node
var http = require('http');
var argv = process.argv.splice(2),
truecount = argv.length,
pages = [];
function printUrls() {
if (--truecount > 0)
return;
for (i = 0; i < pages.length; i++) {
console.log(pages[i].data + '\n\n');
}
}
function HTMLPage(url) {
var _page = this;
_page.data = '### [URL](' + url + ')\n';
http.get(url, function(res) {
res.setEncoding('utf8');
res.on('data', function(data) {
_page.data += data;
});
res.on('end', printUrls);
});
}
for (var i = 0; i < argv.length; i++)
pages.push(new HTMLPage(argv[i]));
It adds the requests to an array on the start of each request, that way once done I can iterate nicely through the responses knowing that they are in the correct order.
When dealing with asynchronous processing, I find it much easier to think about each process as something with a concrete beginning and end. If you require the order of the requests to be preserved then the entry must be made on creation of each process, and then you refer back to that record on completion. Only then can you guarantee that you have things in the right order.
If you were desperate to use your above method, then you could define a variable inside your get callback closure and use that to store the urls, that way you wouldn't end up with the last url overwriting your variables. If you do go this way though, you'll dramatically increase your overhead when you have to use your urls from process.argv to access each response in that order. I wouldn't advise it.
I went about this challenge a little differently. I'm creating an array of functions that call http.get, and immediately invoking them with their specifc context. The streams write to an object where the key is the port of the server which that stream is relevant to. When the end event is triggered, it adds to that server to the completed array - when that array is full it iterates through and echos in the original order the servers were given.
There's no right way but there are probably a dozen or more ways. Wanted to share mine.
var http = require('http'),
request = [],
dataStrings = {},
orderOfServerInputs = [];
var completeResponses = [];
for(server in process.argv){
if(server >= 2){
orderOfServerInputs[orderOfServerInputs.length] = process.argv[server].substr(-4);
request[request.length] = function(thisServer){
http.get(process.argv[server], function(response){
response.on("data", function(data){
dataStrings[thisServer.substr(-4)] = dataStrings[thisServer.substr(-4)] ? dataStrings[thisServer.substr(-4)] : ''; //if not set set to ''
dataStrings[thisServer.substr(-4)] += data.toString();
});
response.on("end", function(data){
completeResponses[completeResponses.length] = true;
if(completeResponses.length > 2){
for(item in orderOfServerInputs){
serverNo = orderOfServerInputs[item].substr(-4)
console.log(dataStrings[serverNo]);
}
}
});
});
}(process.argv[server]);
}
}
Immediately-Invoked Function Expression (IIFE) could be a solution to your problem. It allows us to bind to function a specific value, in your case, the url which gets the response. In the code below, I bind variable i to index and so, whichever url gets the response, that index of print list will be updated. For more information, refer to this website
var http = require('http')
var truecount = 0;
var printlist = [];
for(var i = 2; i < process.argv.length; i++) {
(function(index){
http.get(process.argv[index], function(response) {
response.setEncoding('utf8');
response.on('data', function(data) {
if (printlist[index] == undefined)
printlist[index] = data;
else
printlist[index]+= data;
})
response.on('end', function() {
truecount += 1
if(truecount == 3) {
printlist.forEach(function(item) {
console.log(item)
})
}
})
})
})(i)
}