We use the following code to stream the results of a query back to the client:
app.get('/events', (req, res) => {
try {
const stream = db('events')
.select('*')
.where({ id_user: 'foo' })
.stream()
stream.pipe(JSONStream.stringify()).pipe(res)
} catch (err) {
next(err)
}
})
While the code seems to have an excellent memory usage profile (stable/low memory usage) it creates random DB connection acquisition timeouts:
Knex: Timeout acquiring a connection. The pool is probably full. Are
you missing a .transacting(trx) call?
This happens in production at seeming random intervals. Any idea why?
This happens because aborted requests (i.e client closes the browser mid-request) don't release the connection back to the pool.
First, ensure you're on the latest knex; or at least v0.21.3+ which has introduced fixes to stream/pool handling.
From the on you have a couple options:
Either use stream.pipeline instead of stream.pipe which handles aborted requests correctly like so:
const { pipeline } = require('stream')
app.get('/events', (req, res) => {
try {
const stream = db('events')
.select('*')
.where({ id_session: req.query.id_session })
.stream()
return pipeline(stream, JSONStream.stringify(), res, err => {
if (err) {
return console.log(`Pipeline failed with err:`, err)
}
console.log(`Pipeline ended succesfully`)
})
} catch (err) {
next(err)
}
})
or listen to the [close][close] event on req and destroy the DB stream yourself, like so:
app.get('/events', (req, res) => {
try {
const stream = db('events')
.select('*')
.where({ id_session: req.query.id_session })
.stream()
// Not listening to this event will crash the process if
// stream.destroy(err) is called.
stream.on('error', () => {
console.log('Stream was destroyed')
})
req.on('close', () => {
// stream.end() does not seem to work, only destroy()
stream.destroy('Aborted request')
})
stream.pipe(JSONStream.stringify()).pipe(res)
} catch (err) {
next(err)
}
})
Useful reading:
knex Wiki: Manually close streams. Careful, the stream.end mentioned here doesn't seem to work.
knex Issue: stream.end() does not return connection to pool
Related
I'm having a problem with socket.io at the moment I try to send a second time from the server to the client
here is the server code with express and socket.io
io.on('connection', async function (socket) {
let socketId = socket.id;
const mta = new Client("20.64.24.144", 22005, "*", "*");
mta.resources.evokestats.getPlayerCount()
.then((result) => {
console.log("result", result);
socket.emit("players-start", { players: result })
})
.catch((err) => {
console.error(`Ooops! Something went wrong ${err}`);
});
app.post('/player_connect', async function (req, res) {
let ip = req.body[0];
let player = await players.findOne({ ip: ip })
if (player) {
await socket.emit("players", { players: req.body[1] })
} else {
try {
player = await players.create({ ip: ip, name: req.body[2] })
await socket.emit("players", { players: req.body[1] })
await socket.emit("last_24_players", { players: 1 });
} catch (error) {
console.log("error", error)
}
}
res.send("connected")
});
});
and here is my client with reactjs and socket.io
useEffect(() => {
getStats();
}, [])
async function getStats(params) {
socket.on("players-start", function (data) {
setNowPlayers(data.players)
});
socket.on("players", function (data) {
console.log("players", data)
setNowPlayers(data.players)
});});
And in my client using react, in useEffect I listen to the "players-start" and the "players" that was emit.
players-start: It is for every first time that I enter my client he only calls once, to bring all players connected
players: Every time someone connects to the game server, a post call is made to my server where I use the express with socket, in the url '/player_connect' and then immediately emit
The problem: whenever I issue an issue on 'players-start' and then immediately enter the game server that calls the url '/player_connect' it is not triggering the issue of 'players' or at least the client is not receiving.
Test I've done:
My first attempt was to stick everything to the listener "players" but it still doesn’t work
I really appreciate everyone's help.
I am trying to make a discord bot from NodeJS that utilizes MongoDB for its database. When I try to delete or update a document, sometimes, it returns mongoError: Topology was destroyed. I have read up on this error before and it says that the connection was being interrupted.
Here is the code for my Database Handler:
class DatabaseHandler {
constructor(client) {
this.client = client;
}
async connect(callback) {
try {
await this.client.connect();
await callback(this.client);
} catch (err) {
console.error(err);
} finally {
this.client.close();
console.log("CLIENT CLOSED");
}
}
}
module.exports = DatabaseHandler;
Here is the place that the error occurs:
DB.connect(async (client) => {
console.log(ObjectId(this._id));
let DBList = await client.db("Giveaways").collection("giveawayData");
let delVal = {
_id: ObjectId(this._id)
};
await DBList.deleteOne(delVal); // error occurs here
})
I do not think it is because of the this.client.close() because it is executed after all of the operations are finished.
I want to kill the worker threads after the 'done' event is fired in Threads module of Nodejs. How do I achieve this?
const Threads = require('threads');
const Pool = Threads.Pool;
const workerPool = new Pool();
module.exports = class JobManager {
static bufferedJob(pathToScript, params, callback){
workerPool
.run(pathToScript)
.send(params)
.on('done', (result, input) => {
console.log(`Worker Job done: ${pathToScript} `);
callback(null, result);
})
.on('error', (job, error) => {
console.log(`Error in executing Worker Job: ${pathToScript}`);
callback(job || error);
})
}
}
port.close()
Added in: v10.5.0
Disables further sending of messages on either side of the connection. This method can be called when no further communication will happen over this MessagePort.
The 'close' event will be emitted on both MessagePort instances that are part of the channel.
I do not quite understand how to properly break the logic on the controllers and models in nodeJS when working with the backend application. Suppose I have an example
This code is in the model of my application, and logically I understand that the model is only responsible for choosing from the database, and the controller and everything else should be done by the controller, but I don’t quite understand how to do this and I tried to transfer part of the code to the controller and export it, but I did not succeed (Please, help, at least with this example! The main thing for me is to understand the principle of working with MVC in the node !!!
exports.currentPostPage = function(req, res){
db.query('SELECT * FROM `posts`', function (err, result) {
if (err){
console.log(err);
}
var post = result.filter(item => {return (item.id == req.params.id)? item: false})[0];
if (post === undefined){
res.render('pages/404');
} else {
res.render('pages/post-page', {postId: req.params.id, item: post});
}
});
};
So, you're on the right track. There's a lot of different ways to do it depending on preferences, but one pattern I've seen pretty commonly is to use the callback as a way to integrate. For example, let's say you have your model file:
exports.getPostById = (id, cb) => {
db.query('SELECT * FROM `posts` WHERE id=?', [id], function (err, result) {
if (err){
return cb(err); // or, alternatively, wrap this error in a custom error
}
// here, your logic is just returning whatever was returned
return cb(null, result);
});
};
Note I also am letting the DB handling the ID lookup, as it's probably more efficient at doing so for larger data sets. You didn't say what DB module you're using, but all the good ones have some way of doing parametrized queries, so use whatever works w/ your DB driver.
Anyway, the Model file therefore handles just the data interaction, the controller then handles the web interaction:
// postController.js
const model = require('../models/postModel.js'); // or whatever you named it
exports.populatePost = (req, res, next, id) => {
model.getPostById(id, (err, post) => {
if (err) return next(err); // centralized error handler
req.post = post;
next();
});
}
export.getOnePost = (req, res, next) => {
if (req.post) {
return res.render('pages/post-page', req.post);
}
// again, central error handling
return next({ status: 404, message: 'Post not found' });
}
I have mentioned central error handling; I vastly prefer it to scattering error handling logic all over the place. So I either make custom errors to represent stuff, or just do like above where I attach the status and message to an anonymous object. Either will work for our purposes. Then, in a middleware file you can have one or more handler, the simplest like this:
// middleware/errors.js
module.exports = (err, req, res, next) => {
console.error(err); // log it
if (err.status) {
return res.status(err.status).render(`errors/${err.status}`, err.message);
}
return res.status(500).render('errors/500', err.message);
}
Finally, in your routing setup you can do things like this:
const postController = require('../controllers/postController');
const errorHandler = require('../middleware/errors.js');
const postRouter = express.Router();
postRouter.param('postId', postController.populatePost);
postRouter.get('/:postId', postController.getOnePost);
// other methods and routes
app.use('/posts', postRouter)
// later
app.use(errorHandler);
As was pointed out in the comments, some folks prefer using the Promise syntax to callbacks. I don't personally find them that much cleaner, unless you also use the async/await syntax. As an example, if your db library supports promises, you can change the model code to look like so:
exports.getPostById = async (id, cb) => {
// again, this assumes db.query returns a Promise
return await db.query('SELECT * FROM `posts` WHERE id=?', [id]);
}
Then your controller code would likewise need to change to handle that as well:
// postController.js
const model = require('../models/postModel.js'); // or whatever you named it
exports.populatePost = async (req, res, next, id) => {
try {
const post = await model.getPostById(id)
req.post = post
return next()
} catch (err) {
return next(err)
}
}
When I make multiple post requests to my nodejs server and all of the parameters are correct, everything works fine (and doesnt freeze) but when I make multiple post requests with incorrect parameters that gives an error, my nodejs server just freezes/hangs for a few minutes. Why is this?
Here is my code btw
app.post('/pushtransaction', function(req, res) {
console.log(req.body);
console.log(5);
if (req.body.sigs) {
let sigver = xmf.modules.ecc.Signature.from(req.body.sigs).toString();
let lasig = [sigver];
console.log(req.body.packedTr);
let transi = JSON.parse(req.body.packedTr);
//let sigver = req.body.sigs;
let package = {
compression: 'none',
transaction: transi,
signatures: lasig
}
console.log(package);
//Pushes tx in correct format
xmf.pushTransaction(package).then(result=>{
res.send(result);
res.end();
console.log(result);
}).catch(err => {
console.log(err)
});
}
})
When your error is encountered, your Node server does not know what to do other than console.log() the error. It needs to end that request and send some response. You can res.status(400).send({ error: err }) when you're within the catch.
Make sure res.send() method gets called every time in your request.
Updated Javascript:
app.post('/pushtransaction', function(req, res) {
console.log(req.body);
console.log(5);
if (req.body.sigs) {
let sigver = xmf.modules.ecc.Signature.from(req.body.sigs).toString();
let lasig = [sigver];
console.log(req.body.packedTr);
let transi = JSON.parse(req.body.packedTr);
//let sigver = req.body.sigs;
let package = {
compression: 'none',
transaction: transi,
signatures: lasig
}
console.log(package);
//Pushes tx in correct format
xmf.pushTransaction(package).then(result=>{
res.send(result);
res.end();
console.log(result);
}).catch(err => {
console.log(err);
res.status(400).send();
});
}
res.status(400).send();
})
Additionally you don't have to call res.end() if you call res.send(). see Must res.end() be called in express with node.js?
Adding to other answers, you can add a middleware for timeouts, if any service fails to respond in some time, like
var timeout = require('connect-timeout');
app.use(timeout('5s'));