I was requested to create a simple rest api with node js and make a script that creates 10000 elements in the database through api calls. I created the server with the Hapi framework. If I send a single or 100 'PUT' requests to the API it creates a new element without problem but if I try to make 1000 requests or more it wont be able to create all of them or anything at all. I would like to know what may be the problem, if I'm not doing the script correctly or is the server the problem. So far I've received 2 errors:
{ [Error: connect ECONNRESET] code: 'ECONNRESET', errno: 'ECONNRESET', syscall: 'connect' }
and (libuv) kqueue(): Too many open files in system
The first one trying to call the api 1000 times and the second one trying with 10000 times
The code of the server is the following
var Hapi = require('hapi');
var server = new Hapi.Server();
var joi = require("joi");
var dbOpts = {
"url" : "mongodb://localhost:27017/songsDB",
"settings" : {
"db" : {
"native_parser" : false
}
}
};
server.register({
register: require('hapi-mongodb'),
options: dbOpts
},function (err) {
if (err) {
console.error(err);
throw err;
}
});
server.connection({
host: 'localhost',
port: 8080
});
server.route({
method: 'POST',
path: '/songs',
config: {
handler: function (request, reply) {
var newSong = {
name: request.payload.name,
album: request.payload.album,
artist: request.payload.artist
};
var db = request.server.plugins['hapi-mongodb'].db;
db.collection('songs').insert(newSong, {w:1}, function (err, doc){
if (err){
return reply(Hapi.error.internal('Internal MongoDB error', err));
}else{
reply(doc);
}
});
},
validate:{
payload: {
name: joi.string().required(),
album: joi.string().required(),
artist: joi.string().required()
}
}
}
});
server.start(function () {
console.log('Server running at:', server.info.uri);
});
The code for the request is the following
var unirest = require('unirest');
for(var i = 1; i<=10000; i++){
unirest.post('http://localhost:8080/songs')
.header('Accept', 'application/json')
.send({ "name": "song"+i, "artist": "artist"+i, "album":"album"+i})
.end(function (response) {
console.log(response.body);
});
}
If running under OSX, open terminal and then try using:
sudo launchctl limit maxfiles 1000000 1000000
and then try again.
For "too many open files in system", looks like you reach the limit of your system. If you are using Linux, you can do a ulimit -a to display all settings.
There is one which may limit your number of open files.
open files (-n) 1024
Assuming you are on a Mac or Linux, you need to increase the maximum number of open files allowed by the system.
If you insert this into the terminal it will show you what your settings are:
lsof | wc -l
You will see that your 'open files' setting is likely smaller than the number you are trying to work with.
To change this setting, use the following command:
ulimit -n #####
where ##### is some arbitrary number (but higher than what you have).
If you are on a Windows machine, the answer is slightly more complicated. It seems Windows has a per-process limit, which can be modified (though it doesn't sound easy). Look here for some more details:
Windows equivalent of ulimit -n
When I ran the code, the first 11 POSTs would throw errors. Apparently it is because the script begins sending them before the mongodb connection is active. All I did was added a brief timeout to the POSTing to give mongodb a chance to start breathing. When I did that it worked fine. 10000 records upon completion.
All I changed was this:
setTimeout(function () {
for(var i = 1; i<=10000; i++){
unirest.post('http://localhost:8080/songs')
.header('Accept', 'application/json')
.send({ "name": "song"+i, "artist": "artist"+i, "album":"album"+i})
.end(function (response) {
//console.log(response.body);
});
}
}, 5000);
Related
This is (hopefully) the end of a god-awful journey. I have a long history of communicating with serial ports via tty terminal programs and it usually takes 5 mins to set them all up.
What am I trying do?
1. Write commands to the TH-D72A via javascript
2. Read data from the TH-D72A via javascript
Successes:
A: Can read and write in python. However the TH-D72A get into a weird mode where it doesn't respond.
B: Contacted Kenwood Customer support and have a very lengthy discussion with them. Yet, I have put in 3 weeks of something that should take 5 mins. and they don't seem to understand what's wrong either.
C: This operational mode will be on Raspberry Pis but my dev OS is Mac OS. I'm able to do the python scripts in either platform.
Code
I'm not sure if I must use a parser or simply read from the port. port.read() does not display any data. This code compiles but there are no data sent to the func:
'use strict';
var express = require('express');
var router = express.Router();
var util = require("util");
const SerialPort = require('serialport');
const Readline = SerialPort.parsers.Readline;
var devicePath = '/dev/ttyUSB0';
var osvar = process.platform;
console.log(osvar);
if (osvar == 'darwin') {
console.log("Using Mac OS");
devicePath = '/dev/tty.SLAB_USBtoUART';
}else{
devicePath = '/dev/ttyUSB0';
}
const port = new SerialPort(devicePath, {
baudRate: 9600,
parity: 'none',
stopBits: 1,
dataBits: 8,
flowControl: false
}, function(err) {
if (err){
console.log('error: ', err.message);
port.close();
}});
const parser = new Readline({delimiter: '\n'});
port.pipe(parser);
console.log('parser setup');
parser.on('data', function(data) {
console.log('data received: ', data);
});
port.on('data', function(data) { console.log('Date:' +data);
});
port.on('open', function() { console.log('Port Opened');
});
port.on('close', function() { console.log('Port Closed');
});
port.write('DISP\r\n');
router.get('/', function (req,res) {
res.render('terminal', { title: 'Terminal Page' });
});
module.exports = router;
Other
The projects are open at github: https://github.com/kcw-grunt/nodetest
And, I forked a module (npm) to add more methods: https://www.npmjs.com/package/th-d72-ax25
I hope there is some javascript guru that can see the problem (which must be really simple)
Background
I have become a forensics specialist trying to figure why this simple task has taken literally 6 weeks...and I have found a solution. I consider it a workaround but I can move forward with it to continue code my bigger solution
Each of my supporters have suggested that javascript was sound and I tend to agree with them. But, the system was not responding at all. This includes the serialport 2.0.6, 2.1.2 and even v 6.2.0.
This code actually works (as well as #psiphi75 said it should):
'use strict';
var express = require('express');
var router = express.Router();
var serialport = require('serialport');
var SerialPort = serialport.SerialPort;
var devicePath = '/dev/ttyUSB2';
var port = new SerialPort( devicePath, { // change path
baudrate: 9600
, function ( err ) {
if ( err ) {
console.error('error opening serial port:' , err);
}
}
});
port.on('data', function(data) {
console.log('Data:'+data)
});
port.write('KISS ON\r\n', function(err) {
console.log('KISS ON Turned on');
});
port.write('RESTART\r\n', function(err) {
console.log('Restarted');
});
port.on('open', function() {
console.log('Port Opened');
});
port.on('closed', function() {
console.log('Port Closed');
});
var start = Date.now();
setInterval(function() {
var delta = Date.now() - start; // milliseconds elapsed since start
// alternatively just show wall clock time:
console.log(new Date().toUTCString());
port.write('I\r\n');
}, 5000); // update
router.get('/', function (req,res) {
res.render('terminal', { title: 'Terminal Page' });
});
module.exports = router;
The problem has actually been the Kenwood TH-D72A serial (USB) port. In fact when using it in a dev enviroment, it remains in a state where it does not emit data after the second serialport 'connection'. This is a serious bug Kenwood should fix especially that the HT is considered one of the most expensive on the market.
The workaround has been to simply power cycle the TH-D72A each time a build + run is started.
I may spend some time finding out if there is some persistent state causing this. But, the code restarts the HT in each build.. This will be a serious problem for a non-tech user in the field let alone someone without 2 Raspberry Pis, a Linux, windows and Mac
I've used SerialPort many times without issues (gprs-signal-strength, super-duper-serial-gps-system. Here is some example code.
Note: These use SerialPort v2.0.x. Which looks like it used the new SerialPort.SerialPort(...) syntax as opposed to the udpated new SerialPort(...) syntax.
const port = new SerialPort.SerialPort('/dev/ttyS2', {
baudrate: 115200,
parser: SerialPort.parsers.readline( '\r\n' ) // you may just want \r or \n
}, function ( err ) {
if ( err ) {
console.error( 'Error opening GPRS serial port: ', err );
}
} );
port.on('data', function(data) {
console.log(data)
});
If anyone is still trying to do this, I got it working with the following code and serialport#7.1.5:
const SerialPort = require("serialport");
const port = new SerialPort("/dev/tty.SLAB_USBtoUART", {
baudRate: 9600,
dataBits: 8,
parity: "none",
stopBits: 1,
rtscts: true
});
I think the trick is to turn on hardware flow control with rtscts: true.
I need to be able to run '/etc/init.d/mongod status' or 'service mongod status' from wihtin a node js file, in order to store the response in the database.
When I run the above commands in the command line, I get the following response:
● mongod.service - SYSV: Mongo is a scalable, document-oriented database.
Loaded: loaded (/etc/rc.d/init.d/mongod)
Active: active (running) since Thu 2017-02-02 08:07:42 UTC; 3h 27min ago
Docs: man:systemd-sysv-generator(8)
Process: 793 ExecStart=/etc/rc.d/init.d/mongod start (code=exited, status=0/SUCCESS)
Main PID: 1027 (mongod)
CGroup: /system.slice/mongod.service
└─1027 /usr/bin/mongod -f /etc/mongod.conf
However, I want to include this status in an API response that I write. Therefore, when a user request my API, I want it to return the mongoDB status check as seen above.
I have tried the following ways:
router.get('/status', function(req, res) {
var databaseCheck = service mongod status // not sure how to do this
res.json({
mongoResponse: '//to have the above status check response here'
});
});
I am new to all this, so any help would be appreciated. I may understand that my thinking is wrong - do let me know if there is a different way of doing this please
Connect a database and then check connection like db.serverConfig.isConnected(). The below code is a full example.
const app = express();
let dbClient;
let db;
let collection;
MongoClient.connect(configuration.mongoDbUri, { useNewUrlParser: true, poolSize: 30 }, (error, client) => {
if (error) {
console.log("Connection failed for some reason. Err: ", error);
return error;
}
db = client.db("myDB");
dbClient = client;
collection = db.collection('myCollection');
app.locals.mongoDb = db;
});
app.get("/status", (req, res) => {
if (db.serverConfig.isConnected()) {
console.log("db.serverConfig.isConnected :", db.serverConfig.isConnected());
return res.send({ result: true});
}
return res.send({ result: false});
});
app.listen(configuration.app.port, error => {});
You can use nodejs child-process module to run a shell command like you would from a terminal. In a terminal you would "service mongod status", in the nodejs child-process you would do the same by putting that command as an argument to the child-process execute function, like so:
const exec = require('child_process').exec;
exec('service mongod status', (error, stdout, stderr) => {
if (error) {
console.error(`exec error: ${error}`);
return;
}
console.log(`stdout: ${stdout}`);
console.log(`stderr: ${stderr}`);
});
Try code like this into you app:
db.serverConfig.isConnected();
I am using WebDriverIO for the UI Automation and I am trying to automate for a login page.
When I click the Login button, *.com/user/login request will get trigger in the background. Here, I want to catch that *.com/user/login call response, because I have to get a token from the response to proceed to next test case.
Can anyone please help me?
Thanks in advance!
Capture network traffic using BrowserMob, selenium server and below code. For more details , refer my this answer. Then parse stuff.har , to get desired result. Har file is in json format.
var Proxy = require('browsermob-proxy').Proxy
, webdriverio = require("./node_modules/webdriverio/")
, fs = require('fs')
, proxy = new Proxy()
;
proxy.cbHAR('webdriver.io', doSeleniumStuff, function(err, data) {
if (err) {
console.error('ERR: ' + err);
} else {
fs.writeFileSync('stuff.har', data, 'utf8');
}
});
function doSeleniumStuff(proxy, cb) {
var browser = webdriverio.remote({
host: 'localhost'
, port: 4444
, desiredCapabilities: { browserName: 'firefox', seleniumProtocol: 'WebDriver', proxy: { httpProxy: proxy } }
});
browser
.init()
.url("http://webdriver.io")
//.setValue("#yschsp", "javascript")
//.submitForm("#sf")
.end().then(cb);
}
I'm having a sporadic issue with my Node/loopback server. My setup is as follows:
EDIT: My node version is v4.2.4, I'm running Node on Windows 10 Professional.
Client side:
nw.js client that traverses the local file-system and computes the MD5 value of files.
Using a request.post, the client sends the hash of the file to the server in the format 'fileHash: d41d8cd98f00b204e9800998ecf8427e' (that's just an example hash, I know it's an empty file)
function checkHash (fileHash){
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: 'http://localhost:3000/api/checkBoths/hashcheck',
method: 'POST',
form: {
fileHash: fileHash
}
}, function(error, response, body){
if(error) {
console.log(error);
} else {
console.log(response.statusCode, body);
}
});
}
Server side:
Node/loopback server running at localhost:3000.
The hashCheck function is meant to read the data that's posted from the client and query a MySQL database which is for a match of the hash.
The response from the server will be in the format goodResult : true if the hash exists in the database or goodResult : false if not.
var request = require('request');
module.exports = function (CheckBoth) {
var goodResult;
CheckBoth.hashCheck = function (fileHash, cb) {
requestGood(fileHash);
function requestGood (fileHash) {
request.get('http://127.0.0.1:3000/api/Goodhashes/' + fileHash + '/exists', function (error, response, body) {
if (!error && response.statusCode == 200) {
goodResult = JSON.parse(body).exists;
}
if (error) {
console.error(error);
}
});
console.log(goodResult);
}
cb( goodResult);
};
CheckBoth.remoteMethod(
'hashCheck',
{
accepts: {arg: 'fileHash', type: 'string'},
returns: [{arg: 'goodResult', type: 'string'}]
}
);
};
Issue:
The server can respond with ~1000 queries before the following appears throughout the responses:
{ [Error: connect ECONNREFUSED 127.0.0.1:3000]
code: 'ECONNREFUSED',
errno: 'ECONNREFUSED',
syscall: 'connect',
address: '127.0.0.1',
port: 3000 }
I've tried putting in different callbacks within the server code but that has not made any difference. I think I should probably throttle the requests to the server but I don't know how to achieve this.
Any help would be greatly appreciated.
The ulimit default for most systems is 1024. See the manual for limits.conf (http://linux.die.net/man/5/limits.conf).
While #migg is correct about ensuring no memory leaks in your application and its process handling tuning the system for high loads is also normal procedure for quite a few applications.
Try this and see if it helps;
$ ulimit -n 65535
Edit: I have not tested this but here is a document from ibm regarding Windows; http://www-01.ibm.com/support/docview.wss?uid=swg21392080
I'm looking for a way to check which files are present in a remote directory i want to access via ssh or similar and write the filenames into an array.
So far I had no luck. unix rsync has an -n flag which can print every file which is present at the destinated location, but I don't get how to use the rsync-output in grunt.
Here's how you might do it via sftp with ssh2:
var SSH2 = require('ssh2');
var conn = new SSH2();
conn.on('ready', function() {
conn.sftp(function(err, sftp) {
if (err) throw err;
sftp.readdir('/tmp', function(err, list) {
if (err) throw err;
console.dir(list);
conn.end();
});
});
}).connect({
host: '192.168.100.100',
port: 22,
username: 'frylock',
// password: 'foobarbaz',
privateKey: require('fs').readFileSync('/here/is/my/key')
});