I have problem when I try to use data from another file in my nodejs program
the example is like this.
const Search = require('./model/Search');
getData = () => {
let result = []
Search.find({}, (err, data) => {
if (err) {
console.log(err);
}
for (let i = 0; i < data.length; i++) {
result.push = data[i].title;
}
});
return result;
}
and I will get the result in another file
const coba = require('./data');
console.log(coba);
and I get undefined in my terminal. Can you help me solve this problem?
getData = () => {
let result = []
Search.find({}, (err, data) => {
if (err) {
console.log(err);
}
for (let i = 0; i < data.length; i++) {
result.push = data[i].title;
}
// return statement should be inside Search.find callback
return result;
});
}
Hi, your Search.find is asynchronous function as it is fetching results from DB. and you have your return statement outside that asynchronous function because of which you are getting undefined. Please find the correction in above snippet. By the way Sorry, I completely got it wrong in my first attempt to answer. Edited the same post. Cheers!
Related
Hope you're having a good day. I recently discovered that it is not as easy to handle callbacks inside a for loop. I have tried a few things but couldn't find a solution.
Here is the code:
var book = new Array;
var chapters = Object.keys(epub.spine.contents).length;
for (let i = 0; i < chapters; i++) {
let cacheArray = [];
epub.getChapter(epub.spine.contents[i].id, function (err, data) {
if (err) {
return console.log(err);
}
//remove html tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
})
}
console.log(book)//returns empty array ofc
After this code is executed, I need to loop over the array to search its contents. If that was not the case, my approach would be to just send it to a db.
My approach was the following:
var recursiveChapter = function (n) {
var book = new Array;
if (n < chapters) {
// chapter function
epub.getChapter(epub.spine.contents[n].id, function (err, data) {
if (err) {
throw err
}
//remove HTML tags
let str = data.replace(/<\/?[\w\s]*>|<.+[\W]>/g, '');
book.push(str)
recursiveChapter(n + 1)
});
}
}
//start the recursive function
recursiveChapter(0)
console.log(book)//returns an empty array
I am stuck and can't think of a way of using this data without storing it in a db.
Any help would be appreciated .
There are a few ways you can tackle this. One way is to use the async library, this allows to to run async. calls in parallel, or in series.
For example, we can use async.mapSeries() to get the result of a series of asynchronous calls as an array.
You input your array of ids, then the callback will return an array of the data returned, for example, I've mocked out the getChapter() function to give you an idea of how it would work:
// Mock out epub object
const epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
console.log("Calling async.mapSeries for ids:", ids);
async.mapSeries(ids, epub.getChapter, (err, result) => {
if (err) {
console.error(err);
} else {
console.log("Result:", result)
}
})
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/3.2.0/async.min.js" integrity="sha512-6K6+H87tLdCWvY5ml9ZQXLRlPlDEt8uXmtELhuJRgFyEDv6JvndWHg3jadJuBVGPEhhA2AAt+ROMC2V7EvTIWw==" crossorigin="anonymous"></script>
You could also promisify the epub call and use Promise.all to get the result, like so:
epub = {
getChapter(id, callback) {
setTimeout(() => callback(null, "Data for id " + id), 250);
}
}
let ids = [1,2,3,4];
function getChapterPromisified(id) {
return new Promise((resolve, reject) => {
epub.getChapter(id, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
})
})
}
// Create a list of promises, one for each call
const promises = ids.map(id => getChapterPromisified(id))
Promise.all(promises)
.then(result => console.log("Result:", result))
.catch(error => console.error("An error occurred:", err));
Here is my code:
app.post('/update', async (req, res, done) => {
let drive = req.app.get('drive');
let fileId = req.body.id;
let subject = req.body.subject;
let grade = req.body.grade;
let industry = req.body.industry;
await TagFile.findOne({id: fileId}, (err, res) => {
if (err) return console.log(err);
//console.log(res);
//console.log(subject);
takeRes(res);
})
function takeRes(res) {
for (let i = 0; i > subject.length; i++) {
if(!res[0].subject.includes(subject[i])) {
res[0].subject.push(subject[i]);
console.log(res[0].subject)
console.log("first was called")
}
else {
console.log(res[0].subject)
console.log("else was called.")
}
}
}
})
This is a post request that is made from my site. When it gets to the step where the for loop needs to be used, I don't get anything in the console from the if else statements. I know this probably has something to do with async, but I don't fully understand it, and would love a push in the right direction. Thank you!
When you are using await you don't need to use the callback function. Use following snippet to query with mongoose, rest of the parts depends on your business logic:
const res = await TagFile.findOne({id: fileId}).exec();
takeRes(res)
result of findOne() is object and result of find() is array and when you use findOne() the res[0] is incorrect and you can use lean()to convert model to object, so use find like this:
let res = await TagFile.find({ id: fileId }).lean();
for (let i = 0; i > subject.length; i++) {
if (!res[0].subject.includes(subject[i])) {
res[0].subject.push(subject[i]);
console.log(res[0].subject);
console.log("first was called");
} else {
console.log(res[0].subject);
console.log("else was called.");
}
}
when you use findOne try like this:
let res = await TagFile.findOne({ id: fileId });
for (let i = 0; i > subject.length; i++) {
if (!res.subject.includes(subject[i])) {
res.subject.push(subject[i]);
console.log(resizeTo.subject);
console.log("first was called");
} else {
console.log(res.subject);
console.log("else was called.");
}
}
Sorry for the very confusing question, I have this code that gets information from a website without any node modules or libraries. It is a list of users separated into different pages use ?page= at the end of the URL. I have managed to iterate through the pages and split up the raw HTML just right. However, my promise resolves before all the data is collected. How can I wait for everything to finish before I resolve the promise? I have tried countless solutions, but none seem to work. Please don't ask to use a node package, as my goal is to not use one :) A friend helped with the regex and splitting it up. Here is the code I am using:
function getData() {
return new Promise((resolve, reject) => {
let final = [] //the array of users returned in the end
const https = require("https"), url = "https://buildtheearth.net/buildteams/121/members";
https.get(url + "?page=1", request => { //initial request, gets the number of user pages.
let rawList = '';
request.setEncoding("utf8"),
request.on("data", data => {rawList += data}),
request.on("end", () => {
if(request = (request = (request = rawList.substring(rawList.indexOf('<div class="pagination">'))).substring(0, request.indexOf("</div>"))).match(/<a(.+)>(.+)<\/a>/g)) {
for(let t = parseInt(request[request.length - 1].match(/(\d+)(?!.*\d)/g)), a = 1; a < t + 1; a++) { //iterates through member pages
https.get(url + "?page=" + a, request2 => { //https request for each page of members
let rawList2 = '';
request2.setEncoding('utf8'),
request2.on("data", data => {rawList2 += data}),
request2.on("end", () => {
let i = rawList2.match(/<td>(.+)<\/td>/g); //finds table in HTML
if (i)
for (var t = 1; t < i.length; t += 3) //iterates through rows in table
console.log(i[t].replace(/<td>/g, "").replace(/<\/td>/g, "")), /* logs element to the console (for testing) */
final.push(i[t].replace(/<td>/g, "").replace(/<\/td>/g, "")); //pushes element to the array that is resolved in the end
})
})
}
}
resolve(final) //resolves promise returning final array, but resolves before elements are added with code above
})
})
})
}
If this helps, here is the website I am trying to get info from.
I am still a little new to JS so if you could help, I would really appreciate it :)
I ended up turning each action into an async function with a try and catch block and then chained the functions together with .then() For the base (getting data from a website) I took inspiration from an article on Medium. Here is the site I am pulling data from, and here is the function to get data from a website:
const getData = async (url) => {
const lib = url.startsWith('https://') ? https : http;
return new Promise((resolve, reject) => {
const req = lib.get(url, res => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error(`Status Code: ${res.statusCode}`));
}
const data = [];
res.on('data', chunk => data.push(chunk));
res.on('end', () => resolve(Buffer.concat(data).toString()));
});
req.on('error', reject);
req.end();
});
};
and then I got the number of pages (which can be accessed by appending ?page=<page number> to the end of the url) with this this function:
const pages = async () => {
try {
let html = await getData('https://buildtheearth.net/buildteams/121/members',);
let pages = await (html = (html = html.substring(html.indexOf('<div class="pagination">'))).substring(0, html.indexOf("</div>"))).match(/<a(.+)>(.+)<\/a>/g)
let pageCount = await parseInt(pages[pages.length - 1].match(/(\d+)(?!.*\d)/g))
return pageCount
} catch (error) {
console.error(error);
}
}
and then I used the page count to iterate through the pages and add the HTML of each to an array with this function:
const getPages = async pageCount => {
let returns = []
try {
for (page = 1; page <= pageCount; page++) {
try {
let pageData = await getData('https://buildtheearth.net/buildteams/121/members?page=' + page)
returns.push(pageData)
} catch (error) {
return error
}
}
} catch (error) {
return error
} finally {return returns}
}
and then I iterated through the array of strings of HTML of each page, and extracted the data I needed out of each with this function which would return the list of members I need:
const iteratePages = async pages => {
if (!Array.isArray(pages)) return
try {
let returns = []
await pages.forEach(page => {
let list = page.match(/<td>(.+)<\/td>/g);
if (list)
for (var element = 1; element < list.length; element += 3)
returns.push(list[element].replace(/<td>/g, "").replace(/<\/td>/g, ""));
})
return returns
} catch (error) {
return error
}
}
And then it was a matter of chaining each together to get the array I needed:
pages().then(pageCount => getPages(pageCount)).then(pages => iteratePages(pages)).then(finalList => {console.log(finalList); console.log(finalList.length)})
I'm trying to write the following code and make it work synchronously, but the only problem that it works correctly with console.log, which prints me every item in array with delay in 1 second, but don't work with the following structure:
for (let i = 0; i < array.length; i++) {
setTimeout(function () {
1.http request via rp or request.get (I receive a huge data array)
2. .map results
3.insert to Mongo via mongoose
}
}
as for now I have the following code inside:
request.get({url: array[i].url}), function (error, body) {
body.map(element => {
//do stuff, it works fine
});
collection.insertMany(body, function (err, docs) {
//#justloggerthings
}
Or I have almost the same version with rp instead of request.get
By default I have mongoose.Promise = global.Promise;
Why this cause a problem? Because body.length is very huge dataset which eat a lot of RAM. (Now imagine 20+ arrays with insertMany)
So Mongo trying to insertMany all responses from request at once (when they ready, w/o 1000s delay). Actually that's why I choose request instead of rp (request-promise) but it seems look async too. So should I choose another http get module from npm and switch to it. And not to worry about it?
Or should I wrap this operations to promise || made an async function and recall it inside loop every time (1000s for example) when I it's correctly finished. In this case, the only thing which I found actual on StackOverflow is:
How to insert data to mongo synchronously (Nodejs, Express)
Bit it's a bit outdated. So any ideas?
Well, i dont have your actual code so i will write in pseudo code what you can do.
const chunkArray = (array, chunkSize) => {
let i,j,chunks = [];
for (i = 0, j = array.length; i < j; i += chunkSize) {
chunks.push(array.slice(i, i + chunkSize));
}
return chunks;
}
for (let i = 0; i < array.length; i++) {
let bigArray = await request('your data url');
// do some mapping or whatever
// then break your large array into smaller chunks
let chunks = chunkArray(bigArray, 10);
let j;
for (j = 0; j < chunks.length; j++) {
await collection.insertMany(chunks[j]);
}
}
Actual code that solve my problem is:
async function test (name, url, lastModified) {
try {
const response = await rp({uri: url, json: true});
response.map(async (element) => {
if (element.buyout > 0) {
element.price = (element.buyout / element.quantity);
}
element.lastModified = lastModified
});
return collection.insertMany(response);
} catch (err) {
console.error(err)
}
}
async function addAsync() {
const z = await test();
console.log(z);
}
addAsync();
in the "LearnYouNode" excercise i have following question to the task 9. In the hints, there is mentioned, that this excercise could also be done with the packages "asycn" or "after". I tried it with both and both failed :|
(i have already done the recommended solution with no additional package)
Here are my miserable attempts:
Shared code of both variants:
var http = require("http");
var bl = require("bl");
var after = require("after");
var async = require("async");
var results = [];
//getting the 3 arguments
var urls = []
for(var i = 2; i < process.argv.length; i++){
urls.push(process.argv[i]);
}
function printResults(){
for(var i = 0; i < results.length; i++){
console.log(results[i]);
}
}
"after" attempt:
//Understood it that way printResults is called after the var next is 3 times generated, no result at all
var next = after(3, printResults)
for(i = 0; i<urls.length;i++){
next(i);
}
var next = function (i){
http.get(urls[i], response => {
response.setEncoding('utf8');
var singleString = "";
response.on("data", data =>{
singleString += data;
}).on("end",function() {
results.push(singleString);
});
response.on("error", err => {
return console.log(err);
});
});
}
"asycn" attempt:
// I get output but in the wrong order
async.map(urls,function(url, printResults) {
http.get(url, response => {
response.setEncoding('utf8');
var singleString = "";
response.on("data", data =>{
singleString += data;
}).on("end",function() {
console.log(singleString);
});
response.on("error", err => {
console.log(err);
});
});
}, function(err){
console.log(err);
});
I really don´t get what i am doing wrong. Thank you very much for your help. Kind regards,
SirSandmann
It's all about syntax, you should read docs more carefully.
For async:
var http = require('http');
var urls = process.argv.slice(2);
var async = require('async');
// the second parameter is a callback provided by async
// it allows async to know when your async task is done
async.map(urls, (url, next) => {
http.get(url, response => {
var str = '';
response
.on('data', data => str += data)
.on('end', () => next(null, str)); // we use the callback here to give the data back
});
}, (err, results) => {
// the second callback is not an error handler
// it's called when all async jobs are done
// so the logging of the results goes here
results.forEach(res => console.log(res));
});
For the after solution, you declared next() twice and you don't use the original next() in your http.get():
var http = require('http');
var urls = process.argv.slice(2);
var after = require('after');
var results = [];
function printResults() {
results.forEach(res => console.log(res));
}
// when you call next(), it tells after 'hey, my async job is done'
var next = after(3, printResults);
urls.forEach((url, i) => {
http.get(url, response => {
var str = '';
response
.on('data', data => str += data)
.on('end', () => {
// after is very "dumb", you have to store your result yourself at the right index
results[i] = str;
// this is very important, 'hey, my async job is done'
next();
});
});
});
In both solutions, you noticed there is a callback (I called it next in both), in asynchronous Javascript, a callback is the only way to notify an asynchronous job is done. That why it's important in both examples to call the provided callback.