I desperately need some help.I am trying to upload large file(8 GB) to gridfs using mongoose and nodeJS. But as the file is very large it takes some time to upload. And after a while I get the following error:
home/user/FileUpload/node_modules/mongodb/lib/utils.js:98
process.nextTick(function() { throw err; });
^
MongoError: connection 0 to 127.0.0.1:27017 timed out
at Function.MongoError.create (/home/user/FileUpload/node_modules/mongodb-core/lib/error.js:29:11)
at Socket.<anonymous> (/home/user/FileUpload/node_modules/mongodb-core/lib/connection/connection.js:186:20)
at Object.onceWrapper (events.js:314:30)
at emitNone (events.js:105:13)
at Socket.emit (events.js:207:7)
at Socket._onTimeout (net.js:402:8)
at ontimeout (timers.js:488:11)
at tryOnTimeout (timers.js:323:5)
at Timer.listOnTimeout (timers.js:283:5)
I have tried to resolve this by increasing connectTimeoutMS but the error still persist. I am using MongoDB 3.4.5 mongoose 4.8.4 nodejs 8.1.4 and npm 5.0.3.
Following is app.js:
var mongoose = require('mongoose');
var schema = mongoose.schema;
mongoose.connect('mongodb://127.0.0.1/gridFS'),{
server: {
socketOptions: {
socketTimeoutMS: 3000000,
connectionTimeoutMS: 3000000,
keepAlive:3000000
}
},
replset: {
socketOptions: {
keepAlive: 3000000,
connectTimeoutMS: 3000000
}
}
};
var conn = mongoose.connection;
var path = require('path');
var Grid = require('gridfs-stream');
var fs = require('fs');
var videoPath = path.join(__dirname, 'readFrom/bio seq test1.txt');
Grid.mongo = mongoose.mongo;
conn.once('open', function(){
console.log('- connection open -');
var gfs = Grid(conn.db);
var writestream = gfs.createWriteStream({
filename: 'bio seq test 1'
});
fs.createReadStream(videoPath).pipe(writestream);
writestream.on('close', function(file){
console.log(file.filename + 'Written to DB');
});
});
Following is package.json file:
{
"name": "file-upload-gridfs",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo '' && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"body-parser": "^1.16.1",
"cookie-parser": "^1.4.3",
"express": "^4.14.1",
"gridfs-stream": "^1.1.1",
"mongoose": "^4.8.4",
"morgan": "^1.8.2",
"multer": "1.3.0",
"multer-gridfs-storage": "1.0.0",
"path.join": "^1.0.0",
"serve-favicon": "^2.4.3"
}
}
Ok. I figured out the problem using this really helpful discussion.The default socket connection time for MongoDB is 30 seconds. If any query/operation takes longer than this the connection is aborted and connection timeout error occurs.
With this change I was able to upload a 32GB file to GridFS without any interruption.
https://github.com/Automattic/mongoose/issues/4789
I was passing the timeout parameter in following way.
server: {
socketOptions: {
socketTimeoutMS: 3000000,
connectionTimeoutMS: 3000000,
keepAlive:3000000
}
},
replset: {
socketOptions: {
keepAlive: 3000000,
connectTimeoutMS: 3000000
}
}
};
But it needs to be set in the following way:
const serverOptions = {
poolSize: 100,
socketOptions: {
socketTimeoutMS: 6000000
}
};
mongoose.createConnection(dbpath, {
server: serverOptions,
replset: serverOptions //if you are using replication
});
In my case I have used localhost.
const serverOptions = {
poolsize:100 ,
socketOptions:{
socketTimeoutMS: 6000000
}
};
var mongodbUri = 'mongodb://localhost:27017/gridFS';
mongoose.connect(mongodbUri, {
server: serverOptions
});
Hope this will help anyone with similar issue.
Related
I'm using the mssql package for node in a MacOs Monterey:
import mssql from "mssql"
connect = async () => {
try {
var cfg = {
server: "10.0.0.180",
authentication: {
type: "default",
options: {
userName: "username",
password: "password",
},
},
options: {
encrypt: true,
enableArithAbort: true,
integratedSecurity: true,
trustServerCertificate: true,
rowCollectionOnDone: true,
database: "db",
cryptoCredentialsDetails: {
// Required for SQL Server 2012 connection,
// as node 18 uses a newer TLS protocol
minVersion: "TLSv1",
},
},
};
let conn = await mssql.connect(cfg);
return conn;
} catch (error) {
return null;
}
return false;
};
Working with node v18.13.0 I'm getting the following error:
ConnectionError: Failed to connect to 10.0.0.180:1433 - 0036761901000000:error:0A000102:SSL routines:ssl_choose_client_version:unsupported protocol:../deps/openssl/openssl/ssl/statem/statem_lib.c:1986:
My project config:
"dependencies": {
"mssql": "^9.0.1",
"nconf": "^0.12.0",
"qs": "^6.11.0"
},
"devDependencies": {
"#babel/core": "^7.20.2",
"#babel/node": "^7.20.2",
"#babel/preset-env": "^7.20.2",
"babel-loader": "^9.1.0",
"webpack": "^5.75.0",
"webpack-cli": "^4.10.0"
}
I'm stuck on that as I cannot connect to the database. What might be causing this?
I am trying to seed my db and this issue is preventing that. It keeps saying "Inventory.create is not a function
at seed". I am not sure if this is a syntactical issue, a logistical problem or even a dependency issue so please point me in the wrong direction.
The tech stack that I am using is Express, Node.js, Sequelize, PostgreSQL
My files are structured as such:
--script
-seed.js
--server
-api
-db
-Model.js
-index.js
-db.js
-app.js
-index.js
Model.js
const db = require('./db');
const Model = db.define('model', {
name: {
type: Sequelize.STRING,
allowNull: false,
},
total: {
type: Sequelize.FLOAT,
allowNull: false,
},
})
module.export = Model;
index.js
const db = require('./db')
const Inventory = require('./Inventory')
module.exports = {
db,
models: {
Inventory,
},
}
db.js
const Sequelize = require('sequelize')
const pkg = require('../../package.json')
const databaseName = pkg.name + (process.env.NODE_ENV === 'test' ? '-test' : '')
if(process.env.LOGGING === 'true'){
delete config.logging
}
if(process.env.DATABASE_URL){
config.dialectOptions = {
ssl: {
rejectUnauthorized: false
}
};
}
const db = new Sequelize(
process.env.DATABASE_URL || `postgres://localhost:5432/${databaseName}`, {
logging: false,
})
module.exports = db
seed.js
'use strict'
const {db, models: {Model}} = require('../server/db')
async function seed() {
await db.sync({ force: true })
console.log('db synced!')
// Creating Inventory
const stock = await Promise.all([
Model.create({ name: 'First', amount: 200 }),
Model.create({ name: 'Second', amount: 200 }),
])
console.log(`seeded successfully`)
}
async function runSeed() {
console.log('seeding...')
try {
await seed()
} catch (err) {
console.error(err)
process.exitCode = 1
} finally {
console.log('closing db connection')
await db.close()
console.log('db connection closed')
}
}
if (module === require.main) {
runSeed()
}
module.exports = seed
I included this just in case
package.json
{
"name": "ShopifyInventory",
"version": "1.0",
"description": "Simple backend heavy app for inventory management",
"main": "app.js",
"scripts": {
"build": "webpack",
"build:dev": "npm run build -- --watch --mode=development",
"seed": "node script/seed.js",
"start": "node server"
},
"dependencies": {
"axios": "^0.21.1",
"bcrypt": "^5.0.0",
"compression": "^1.7.3",
"express": "^4.18.1",
"history": "^4.9.0",
"jsonwebtoken": "^8.5.1",
"morgan": "^1.9.1",
"pg": "^8.7.3",
"pg-hstore": "^2.3.4",
"react": "^18.1.0",
"react-dom": "^18.1.0",
"react-router-dom": "^6.3.0",
"sequelize": "^6.19.2"
},
"author": "Galilior :)",
"devDependencies": {
"#babel/core": "^7.17.12",
"#babel/preset-react": "^7.17.12",
"babel-loader": "^8.2.5",
"webpack": "^5.72.1",
"webpack-cli": "^4.9.2"
}
}
I am seeing this error in my index.js file in VSC which looks to be caused by the app.post request.
I've attempted different parsing options in regards to eslint and emca but still aren't getting anywhere.
Appreciate any input.
Parsing error: Unexpected token => eslint
index.js
const functions = require("firebase-functions");
const express = require("express");
const cors = require("cors");
const stripe = require("stripe")("hidden");
// Setting Up the API
// - App Config
const app = express();
// - Middlewares
app.use(cors({ origin: true }));
app.use(express.json());
// - API routes
app.get("/", (request, response) => response.status(200).send("hello world"));
app.post("/payments/create", async (request, response) => {
const total = request.query.total;
console.log("Payment Request Recieved for this amount >>> ", total);
const paymentIntent = await stripe.paymentIntents.create({
amount: total, // subunits of the currency
currency: "usd",
});
// OK - Created
response.status(201).send({
clientSecret: paymentIntent.client_secret,
});
});
// - Listen command
exports.api = functions.https.onRequest(app);
This is my package.json
{
"name": "functions",
"description": "Cloud Functions for Firebase",
"scripts": {
"lint": "eslint .",
"serve": "firebase emulators:start --only functions",
"shell": "firebase functions:shell",
"start": "npm run shell",
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log"
},
"engines": {
"node": "14"
},
"main": "index.js",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.17.1",
"firebase-admin": "^9.8.0",
"firebase-functions": "^3.14.1",
"stripe": "^8.174.0"
},
"devDependencies": {
"#babel/core": "^7.15.5",
"#babel/eslint-parser": "7.15.4",
"eslint": "^7.6.0",
"eslint-config-google": "^0.14.0",
"eslint-plugin-react": "^7.25.1",
"firebase-functions-test": "^0.2.0"
},
"private": true
}
And then this is my .eslintrc.js file
module.exports = {
root: true,
env: {
es6: true,
node: true,
},
extends: [
"eslint: recommended",
"google",
"plugin: react/recommended",
],
rules: {
quotes: ["error", "double"],
},
};
Seems like you're missing parser: #babel/eslint-parser from your eslintrc
All right. So. I have a really basic API running on Node.js. I'm also using MongoDB as a database and I'm using Mongoose to build queries. My problem is that I'm trying to query for a sorted list of data using using the mongoose 'sort' method. For some reason the method isn't working and Node tries to use the Javascript sort method instead. This is my code:
Query handling:
const mongoose = require("mongoose");
const express = require("express");
const Tour = require("../models/tourModel.js");
exports.getAllTours = async (req, res) => {
try {
// eslint-disable-next-line node/no-unsupported-features/es-syntax
const queryObject = { ...req.query };
const excludedFields = ["page", "sort", "limit", "fields"];
excludedFields.forEach((el) => delete queryObject[el]);
let queryStr = JSON.stringify(queryObject);
queryStr = queryStr.replace(/\b(gte|gt|lte|lt)\b/g, (match) => `$${match}`);
let query = await Tour.find(JSON.parse(queryStr));
// query.sort((a, b) => (a[req.query.sort] > b[req.query.sort] ? 1 : -1));
if (req.query.sort) {
query = query.sort(req.query.sort);
}
const tours = await query;
res.status(200).json({
status: "success",
data: { tours },
});
} catch (error) {
res.status(400).json({ status: "failed", message: error.message });
}
};
Router
const tourController = require("../controllers/tourController");
const router = express.Router();
// router.param("id", tourController.checkID);
router
.route("/")
.get(tourController.getAllTours)
.post(tourController.createTour);
router
.route("/:id")
.get(tourController.getTourById)
.patch(tourController.updateTour)
.delete(tourController.deleteTour);
module.exports = router;
Package.json
"name": "natours",
"version": "1.0.0",
"description": "fdsfd",
"main": "index.js",
"scripts": {
"start:dev": "nodemon server.js",
"start:prod": "nodemon server.js NODE_ENV=production "
},
"author": "me",
"license": "ISC",
"dependencies": {
"dotenv": "^8.2.0",
"express": "^4.17.1",
"mongoose": "^5.12.7",
"morgan": "^1.10.0"
},
"devDependencies": {
"eslint": "^7.25.0",
"eslint-config-airbnb": "^18.2.1",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-jsx-a11y": "^6.4.1",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-prettier": "^3.4.0",
"eslint-plugin-react": "^7.23.2",
"prettier": "^2.2.1"
}
}
When I try to query for 127.0.0.1:3000/api/v1/tours?sort=price,
I get a response
{
"status": "failed",
"message": "The comparison function must be either a function or undefined"
}
I'm a real noob in this so does anyone have any ideas what is causing this or how to fix it?
let query = await Tour.find(JSON.parse(queryStr));
This is the result of your query since await executes the query. If you want to store the query, don't use await
let query = Tour.find(JSON.parse(queryStr));
// query.sort((a, b) => (a[req.query.sort] > b[req.query.sort] ? 1 : -1));
if (req.query.sort) {
query = query.sort(req.query.sort);
}
const tours = await query;
I'm over a month into development of a MEVN app with MySQL. I'm using a framework called MEVN-CLI but that shouldn't really matter.
I keep having to change my server port ever few days whenever I get this error:
[nodemon] 2.0.3
[nodemon] to restart at any time, enter `rs`
[nodemon] watching path(s): *.*
[nodemon] watching extensions: js,html,css,json,md
[nodemon] starting `node run server.js --port 9000/api --open`
events.js:288
throw er; // Unhandled 'error' event
^
Error: listen EADDRINUSE: address already in use :::3000
at Server.setupListenHandle [as _listen2] (net.js:1309:16)
at listenInCluster (net.js:1357:12)
at Server.listen (net.js:1445:7)
at Function.listen (C:\Dev\Projects\node-mysql\server\node_modules\express\lib\application.js:618:24)
at Object.<anonymous> (C:\Dev\Projects\node-mysql\server\server.js:258:5)
at Module._compile (internal/modules/cjs/loader.js:1158:30)
at Object.Module._extensions..js (internal/modules/cjs/loader.js:1178:10)
at Module.load (internal/modules/cjs/loader.js:1002:32)
at Function.Module._load (internal/modules/cjs/loader.js:901:14)
at Function.executeUserEntryPoint [as runMain] (internal/modules/run_main.js:74:12)
Emitted 'error' event on Server instance at:
at emitErrorNT (net.js:1336:8)
at processTicksAndRejections (internal/process/task_queues.js:84:21) {
code: 'EADDRINUSE',
errno: 'EADDRINUSE',
syscall: 'listen',
address: '::',
port: 3000
}
I change between 9000, 3000, 3002, 8000, etc. I want to keep it on a single port but this keeps happening and I can't find what's causing it.
I have a client and a server, they communicate with Axios, my Server communicates with a MySQL server I host elsewhere but I also think this is irrelevant but I figure it can't hurt to share that.
Here is my server:
const port = process.env.PORT || 4000;
const env = require("./env.json");
const express = require("express");
const mysql = require("mysql");
const path = require("path");
const bodyParser = require("body-parser");
const favicon = require("serve-favicon");
const app = express();
const cors = require("cors");
const {
testColumns,
showUsers,
searchResultColumns,
showAccountColumns
} = require("./mysql/columns");
app.use(bodyParser.json());
app.use(favicon(path.join(__dirname, "public", "favicon.ico")));
app.use(cors());
//* Build MySQL Connection
const db = mysql.createConnection({
host: env.server,
user: env.user,
password: env.pass,
database: env.schema
});
//* Connect to Database
db.connect((err) => {
if (err) {
console.log("Error Connecting:", err);
}
console.log("MySQL Connected...");
});
app.get(`/api/v1/:query/:x`, (req, res) => {
[Business Logic - Works fine]
if (err) {
// If error
console.log(err);
}
// If no results
else if (results == 0) {
res.send(`No Results\n ${results} \n - ${err}`);
return;
}
// If successful
res.send(results);
});
});
app.get(`/api/v1/:query/`, (req, res) => {
[Business Logic - Works fine]
if (err) {
// If error
console.log(err);
}
// If no results
else if (results == 0) {
res.send(`No Results\n ${results} \n - ${err}`);
return;
}
// If successful
res.send(results);
});
});
//* Root Handler
app.get("/", (req, res) =>
res.sendFile(path.join(__dirname + "/public/index.html"))
);
//? Error Handler
app.get("/404", (req, res) =>
res.sendFile(path.join(__dirname + "/public/404.html"))
);
//! Error Handler
app.get("/sample", (req, res) =>
res.sendFile(path.join(__dirname + "/public/sample.html"))
);
//? Listen for server port
app.listen(port, () => {
console.log(`Sample Dev Server at:${port}`);
});
I deleted some stuff that's not really relevant because it was hundreds of similar lines, I haven't abstracted my routing away yet.
This is my router:
import Vue from "vue";
import Router from "vue-router";
import Home from "./views/Home.vue";
import NotFound from "./views/NotFound.vue";
Vue.use(Router);
export default new Router({
mode: "history",
base: process.env.BASE_URL,
routes: [
{
path: "/",
name: "Home",
component: Home
},
{
path: "/NotFound",
name: NotFound,
component: () => import("./views/NotFound.vue")
},
{
path: "/search",
component: searchPage,
children: [
{ path: "/search/byName", component: searchByName },
{ path: "/search/china", component: searchByNumber }
]
}
]
});
And if it's relevant, this is my packages.json file:
{
"name": "vmax",
"version": "0.1.0",
"private": true,
"scripts": {
"serve": "vue-cli-service serve",
"build": "vue-cli-service build",
"lint": "vue-cli-service lint"
},
"dependencies": {
"axios": "^0.19.2",
"core-js": "^3.6.4",
"vue": "^2.6.11",
"vue-router": "^3.1.6",
"vuetify": "^2.2.11"
},
"devDependencies": {
"#vue/cli-plugin-babel": "~4.3.0",
"#vue/cli-plugin-eslint": "~4.3.0",
"#vue/cli-service": "~4.3.0",
"babel-eslint": "^10.1.0",
"eslint": "^6.7.2",
"eslint-plugin-vue": "^6.2.2",
"sass": "^1.19.0",
"sass-loader": "^8.0.0",
"vue-cli-plugin-vuetify": "~2.0.5",
"vue-template-compiler": "^2.6.11",
"vuetify-loader": "^1.3.0"
},
"eslintConfig": {
"root": true,
"env": {
"node": true
},
"extends": [
"plugin:vue/essential",
"eslint:recommended"
],
"parserOptions": {
"parser": "babel-eslint"
},
"rules": {}
},
"browserslist": [
"> 1%",
"last 2 versions",
"not dead"
]
}
I changed just now to port 4000 and it's working just find but something is wrong that's causing me to have to change server ports every few days.
Probably the PORT is in use.
To terminate program PORT,
$ killall -9 node
Node / Express: EADDRINUSE, Address already in use happens when closing VS Code, particularly on Windows when a server is running with WSL.
I'm currently using something called CurrPorts which is free to view which ports are in use, and force close them.
To be clear, it's not specific to Windows or WSL, but that is where it seems most prevalent. If you're in the habit of using port 3000, 5500, or 8080 for example - it's likely in use by Node.
Add this in your "package.json" file
"stop-win": "Taskkill /IM node.exe /F",
"stop-linux": "killall node"