Develop Reference

Best way to turn JSON change into event

I'm creating a YouTube upload notification bot for a Discord Server I am in using the YouTube RSS Feed and am having problems with it. I have issues with the bot sending the same video twice even though I've tried everything to fix it. The bot cycles through different users in a for loop and checks the user's latest video's ID with one stored in a JSON file. If they do not match, it sends a message and updates the JSON. Here is my current code:
function update(videoId, n) {
var u = JSON.parse(fs.readFileSync("./jsons/uploads.json"))
u[n].id = videoId
fs.writeFile("./jsons/uploads.json", JSON.stringify(u, null, 2), (err) => {
if (err) throw err;
// client.channels.cache.get("776895633033396284").send()
console.log('Hey, Listen! ' + n + ' just released a new video! Go watch it: https://youtu.be/' + videoId + "\n\n")
});
}
async function uploadHandler() {
try {
var u = require('./jsons/uploads.json');
var users = require('./jsons/users.json');
for (i = 0; i < Object.keys(users).length; i++) {
// sleep(1000)
setTimeout(function(i) {
var username = Object.keys(users)[i]
let xml = f("https://www.youtube.com/feeds/videos.xml?channel_id=" + users[username]).text()
parseString(xml, function(err, result) {
if (err) {} else {
let videoId = result.feed.entry[0]["yt:videoId"][0]
let isMatch = u[username].id == videoId ? true : false
if (isMatch) {} else {
if (!isMatch) {
u[username] = videoId
update(videoId, username)
}
}
}
});
}, i * 1000, i)
}
} catch (e) {
console.log(e)
}
}
My code is rather simple but I've had the same issue with other codes that use this method; therefore what would be the best way to accomplish this? Any advice is appreciated

There are a few issues with your code that I would call out right off the bat:
Empty blocks. You use this especially with your if statements, e.g. if (condition) {} else { // Do the thing }. Instead, you should negate the condition, e.g. if (!condition) { // Do the thing }.
You declare the function uploadHandler as async, but you never declare that you're doing anything asynchronously. I'm suspecting that f is your asynchronous Promise that you're trying to handle.
You've linked the duration of the timeout to your incrementing variable, so in the first run of your for block, the timeout will wait zero seconds (i is 0, times 1000), then one second, then two seconds, then three...
Here's a swag at a refactor with some notes that I hope are helpful in there:
// Only require these values once
const u = require('./jsons/uploads.json');
const users = require('./jsons/users.json');
// This just makes the code a little more readable, I think
const URL_BASE = 'https://www.youtube.com/feeds/videos.xml?channel_id=';
function uploadHandler() {
Object.keys(users).forEach(username => {
// We will run this code once for each username that we find in users
// I am assuming `f` is a Promise. When it resolves, we'll have xml available to us in the .then method
f(`${URL_BASE}${username}`).then(xml => {
parseString(xml, (err, result) => {
if (!err) {
const [videoId] = result.feed.entry[0]['yt:videoId']; // We can use destructuring to get element 0 from this nested value
if (videoId !== u[username].id) {
// Update the in-memory value for this user's most recent video
u[username].id = videoId;
// Console.log the update
console.log(`Hey listen! ${username} just released a new video! Go watch it: https://youtu.be/${videoId}\n\n`);
// Attempt to update the json file; this won't affect the u object in memory, but will keep your app up to date
// when you restart it in the future.
fs.writeFile('./jsons/uploads.json', JSON.stringify(u, null, 2), err => {
if (err) {
console.err(`There was a problem updating uploads.json with the new videoId ${videoId} for user ${username}`);
}
});
}
}
});
})
// This .catch method will run if the call made by `f` fails for any reason
.catch(err => console.error(err));
});
}
// I am assuming that what you want is to check for updates once every second.
setInterval(uploadHandler, 1000);

Extracting data from an object inside of another function without writing to JSON?

I'm working on writing a Discord bot with music functionality using discord.js and node, along with a handful of other packages like yt-search and ytdl-core.
The problem I'm trying to solve is related to the code below (newVar was just a placeholder while testing):
let regex = /^https/i;
let isUrl = regex.test(checkUrl);
let songInfo;
if (!isUrl) {
yts(suffix, function (err, r) {
if(err) console.error(err);
const videos = r.videos;
let data = JSON.stringify(videos[0])
fs.writeFileSync('youtube.json', data)
})
let newVar = require('../youtube.json');
let {url, title} = newVar;
songInfo = await ytdl.getInfo(newVar.url)
} else {
songInfo = await ytdl.getInfo(args[1]);
}
const song = {
title: songInfo.title,
url: songInfo.video_url,
};
What I'm trying to do,
Is to check whether or not the 'suffix' is a URL, and if not, run suffix through the yts() (yt-search) function, and get the URL from the returned object.
Then pass that url value through the ytdl.getInfo() function.
It works as intended to an extent, but writing to the JSON is causing a problem in that it is returning the same URL even when a new search is completed, until the program is restarted,
Then it will repeat the process with whatever value was stored in the JSON file when the program was executed. However, I get the results when I console.log(videos[0].url), and the value changes with each query, but I have no way to pass that data outside of the yts() function without writing to the JSON first.
Any ideas?
I'm sorry if I'm not specific enough, or confused in my understanding, this is one of my first "complex" projects. It could also be that the issue exists elsewhere in the module, but from what I've done so far I think it's somewhere in the code shown above. Thanks!

here is something you can do to get it right.
const getSongInfo = (url, suffix) => {
return new Promise(async (resolve, reject) => {
let regex = /^https/i;
let isUrl = regex.test(url);
if (!isUrl) {
// netween where is the suffix variable ?
yts(suffix, async (err, r) => {
if(err) reject(err);
const videos = r.videos;
let data = JSON.stringify(videos[0]);
// still don't know why bother save it and access it again.
fs.writeFileSync('youtube.json', data);
let newVar = require('../youtube.json');
resolve(await ytdl.getInfo(newVar.url));
});
} else {
resolve(await ytdl.getInfo(args[1]));
}
});
}
// hope the outer function is async
let songInfo = await getSongInfo(checkUrl, suffix);
const song = {
title: songInfo.title,
url: songInfo.video_url,
};
Between make sure to check that suffix variable which is not in scope.

Retry nodejs http.request (post,put,delete)

What is the correct way to implement a retry on error/condition without using any third party modules in nodejs, please?
I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
Do I need to destroy/end the sockets?
I've tried looking for examples but have only found reference to third party modules and http.get samples which don't seem to work. How does one test this?
I have attempted the below without success:
async pingApi(cb) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, /ect do I reference this path?
}
};
let request = await http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', function () {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.result != 'OK') {
setTimeout(pingApi, 1000); //cant pass callback
} else {
cb(null, json.result) //works
}
}
});
})
request.end(); //does the socket need to be closed if retry is this function?
}
Any help, pointing in the right direction or criticism will be greatly appreciated as I think this is a very important learning curve for me.
Thank you in advance,

I'm not sure how to call the same function on the error and how to then pass the original callback/data to the newly called function?
I don't know for sure that everything else in your function is correct, but you can fix the recursion that you're asking about by changing this:
setTimeout(pingApi, 1000); //cant pass callback
to this:
setTimeout(() => {
this.pingApi(cb);
}, 1000);
You aren't showing the whole context here, but if pingApi() is a method, then you also need to keep track of the this value to you can call this.pingApi(db). You can preserve the value of this by using arrow function callbacks like this:
response.on('end', () => { ... });
Other things I notice that look off here:
There's no reason to use await http.request(). http.request() does not return a promise so using await with it does not do anything useful.
Without the await, there's then no reason for your function to be declared async since nobody is using a returned promise from it.
It's not clear what if (this.complete) is meant to do. Since this is inside a regular function callback, the value of this won't be your pingApi object. You should either save this higher in the scope typically with const self = this or all callbacks internally need to be arrow functions so the value of this is preserved.
You should probably put try/catch around JSON.parse() because it can throw if the input is not perfect JSON.
You should probably not retry forever. Servers really hate clients that retry forever because if something goes wrong, the client may just be bashing the server every second indefinitely. I'd suggest a certain number of max retries and then give up with an error.
Do I need to destroy/end the sockets?
No, that will happen automatically after the request ends.
How does one test this?
You have to create a test route in your server that returns the error condition for the first few requests and then returns a successful response and see if your code works with that.
Here's an attempt at a code fixup (untested):
const maxRetries = 10;
pingApi(cb, cnt = 0) {
let options = {
"method":"post",
"path": `/API/pingAPI?${this.auth}`, // ect do I reference this path?
};
let request = http.request(options, (response) => {
let body = new Buffer(0);
response.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
response.on('end', () => {
if (this.complete) {
let decoded = new Buffer(body, 'base64').toString('utf8')
try {
let json = JSON.parse(decoded);
if (json.result != 'OK') {
if (cnt < maxRetries)
setTimeout(() => {
this.pingApi(cb, ++cnt);
}, 1000);
} else {
cb(new Error("Exceeded maxRetries with error on pingApi()"));
}
} else {
cb(null, json.result) //works
}
} catch(e) {
// illegal JSON encountered
cb(e);
}
}
});
})
request.end();
}
Remaining open questions about this code:
What is this.complete doing and what this should it be referencing?
Why is there no request.write() to send the body of the POST request?
I know you ask for no external modules, but my preferred way of doing this would be to use promises and to use the request-promise wrapper around http.request() because it handles a lot of this code for you (checks response.status for you, parses JSON for you, uses promise interface, etc...). You can see how much cleaner the code is:
const rp = require('request-promise');
const maxRetries = 5;
pingApi(cnt = 0) {
let options = {
method: "post",
url: `http://somedomain.com/API/pingAPI?${this.auth}`,
json: true
};
return rp(options).then(result => {
if (result.result === "OK") {
return result;
} else {
throw "try again"; // advance to .catch handler
}
}).catch(err => {
if (cnt < maxRetries) {
return pingApi(++cnt);
} else {
throw new Error("pingApi failed after maxRetries")
}
});
}
And, then sample usage:
pingApi().then(result => {
console.log(result);
}).catch(err => {
console.log(err);
})

your use of async/await with core node server intrigued me and I've tried to use much as possible of this new async features.
This is what I end up with: https://runkit.com/marzelin/pified-ping
const pify = require("util").promisify;
const http = require("http");
const hostname = "jsonplaceholder.typicode.com";
const failEndpoint = "/todos/2";
const goodEndpoint = "/todos/4";
let options = {
method: "get",
path: `${failEndpoint}`,
hostname
};
async function ping(tries = 0) {
return new Promise((res) => {
const req = http.request(options, async (response) => {
let body = new Buffer(0);
response.on("data", (chunk) => {
body = Buffer.concat([body, chunk]);
})
const on = pify(response.on.bind(response));
await on("end");
let decoded = new Buffer(body, 'base64').toString('utf8')
let json = JSON.parse(decoded);
if (json.completed) {
return res("all good");
}
if (tries < 3) {
console.log(`retrying ${tries + 1} time`);
return res(ping(tries + 1));
}
return res("failed");
})
req.on('error', (e) => {
console.error(`problem with request: ${e.message}`);
});
// write data to request body
req.end();
})
}
const status = await ping();
"status: " + status

Iterate save in Node.JS across an array

I couldn't use a simple for loop because request.save is a function. So I tried forEach. It works perfectly! Until I add in the request.save part and I get the following error message that breaks my app.
Error: Can't set headers after they are sent.
exports.submit = function (req, res) {
Person.find({
cellPhone: req.body.phone
}).exec(function (err, people) {
people.forEach(saveRequest);
}
function saveRequest(item, index) {
var request = new Requests();
request.start = req.body.start.value;
request.finish = req.body.finish.value;
request.phone = req.body.phone;
request.offDay = req.body.date;
request.user = people[index]._id;
request.name = people[index].name;
request.group = people[index].group;
request.save(function (err) {
if (err) {
console.log('request.save');
return res.status(400);
} else {
// Remove sensitive data before login
//user.password = undefined;
//user.salt = undefined;
console.log(request);
res.json(request);
}
});
}
});

The problem is when you perform the .save() you pass an anonymous function that complete the response in case of error.
So you finish on the first save event error.
You should complete the response outside the save callback.
Maybe use events to sync your code, or better the generators.
Before your forEach loop:
let savedResponses = [];
let savedErrors = [];
...
Then your savedRequest:
function saveRequest(item, index) {
var request = new Requests();
request.start = req.body.start.value;
request.finish = req.body.finish.value;
request.phone = req.body.phone;
request.offDay = req.body.date;
request.user = people[index]._id;
request.name = people[index].name;
request.group = people[index].group;
request.save(function (err) {
if (err) {
console.log('request.save error');
savedErrors.push(err);
// return res.status(400);
} else {
// Remove sensitive data before login
//user.password = undefined;
//user.salt = undefined;
console.log(request);
savedResponses.push(request);
}
});
}
Then after the forEach loop, you should wait the end of the asynchronous staff in the .save() callbacks.
You could use the event package or the generators or the promise pattern.
It depend on the version of your node.
When you have the code synched you could just complete your response checking for errors first:
if (savedErrors.length > 0) {
res.status = 400;
// ... report errors
}
Or just complete the response with the savedResponses.

Async / Callstack confusion

Alrighty, I'm pretty sure I know what the issue is, but I can't for the life of me figure out how to resolve it.
The way the below code works is the front-end sends two words back to the server, some sanitization happens and breaks the string into an array. That array is then iterated over, an async request is made for each word to the Wordnik API for synonyms. The resulting data structure sent back to the client is an object with {word1: [...synonyms], word2: [...synonyms]}.
With two words, this works exactly how I want 4 out of 5 times. That fifth time, the synonyms for the second word get applied to the first word and the second word has no data. Obviously, send it more words and the data confusion occurs more often.
So, I'm pretty sure this is a call stack issue, but I can't figure out how to resolve it. I keep thinking if I wrap the wordnikClient in setTimeout(..., 0); it's a step in the right direction, but feel like I'm misapplying the pattern. Any words of wisdom out there?
EDIT: https://github.com/ColinTheRobot/tweetsmithy-node/blob/master/server.js This was the prior version it has the same async issue. I had initially designed it with a Promise, but realized over the last couple days, that it wasn't really doing anything/I had also probably misapplied it so took it out for now.
app.get('/get-synonyms', (req, res) => {
var tweetWords = sanitizeTweet(req.query.data);
getDefs(tweetWords, res);
});
var getDefs = function(tweetWords, res) {
var i = 0;
var serialized = {};
tweetWords.forEach((word) => {
wordnikClient(word, (body) => {
var wordToFind = tweetWords[i];
var shortenedWords = [];
i++;
if (body[0]) {
shortenedWords = _.filter(body, (syn) => {
return syn.length < wordToFind.length;
});
serialized[wordToFind] = shortenedWords;
}
if (tweetWords.length == i) {
res.send(serialized);
}
});
});
}
var sanitizeTweet = function(tweet) {
var downcasedString = tweet.toLowerCase();
var punctuationless = downcasedString.replace(/[.,-\/#!$%\^&\*;:{}=\-_`~()]/g,"");
var finalString = punctuationless.replace(/\s{2,}/g," ");
return finalString.split(' ');
}
var wordnikClient = function(word, callback) {
var url = `http://api.wordnik.com:80/v4/word.json/${word}/relatedWords?useCanonical=false&relationshipTypes=synonym&limitPerRelationshipType=10&api_key=${process.env.WORDNIK_API_KEY}`
console.log('calling client');
request(url, (err, response, body) => {
if (!err && response.statusCode == 200 && response.body != '[]') {
callback(JSON.parse(body)[0].words);
} else if (!err && response.statusCode == 200 && response.body == '[]') {
callback([false]);
}
});
}

Yes, what is happening is that your second async call is completing first and because fo
if (tweetWords.length == i) {
res.send(serialized);
}
});
is returning to the client. One alternative is to use https://github.com/caolan/async to cooridnate your async calls, but I would suggest you convert wordnikClient to promises and then use Promise.all to control res.send
var wordnikClient = function(word) {
var url = `http://api.wordnik.com:80/v4/word.json/${word}/relatedWords?useCanonical=false&relationshipTypes=synonym&limitPerRelationshipType=10&api_key=${process.env.WORDNIK_API_KEY}`
console.log('calling client');
return new Promise( (resolve, reject) => {
request(url, (err, response, body) => {
if (!err && response.statusCode == 200 && response.body != '[]') {
resolve(JSON.parse(body)[0].words);
} else if (!err && response.statusCode == 200 && response.body == '[]') {
reject([false]);
}
});
});
and
Promise.all(tweetWords.map((word) => wordnikClient(word)))
.then(serialized => res.send(serialized))
.catch(err => res.status(500).send(err))
I've probably lost a little functionality along the way but you can re-add that

What the asynchronous callbacks do inside getDefs is not clear. The i variable counts the order of the replies, so I don't see why to use that to index tweetWords. I suggest you to use just word instead. A somewhat clearer solution could be made using Promises:
function getDefs(tweetWords, res) {
var serialized = {};
Promise.all(tweetWords.map(word => {
return wordnikClientAsync(word).then(body => {
if (body[0]) {
serialized[word] = _.filter(body, syn => syn.length < word.length);
}
});
})).then(() => {
res.send(serialized);
}, () => {
res.send("Error");
});
function wordnikClientAsync(word) {
return new Promise(resolve => wordnikClient(word, resolve));
}
}

Change tweetWords[i]; for word because the variable is outside the callback and iteration may don't run currently.