Develop Reference

How to use express-fileuploader-s3 ? (For Object Storage in Linode)

I use framework Expressjs for upload file Object Storage using module express-fileupload-s3 . In here, I use code example :
var http = require('http');
var express = require('express');
var mutilpart = require('connect-multiparty');
var uploader = require('express-fileuploader');
var S3Strategy = require('express-fileuploader-s3');
var app = express();
app.use('/upload/image', mutilpart());
uploader.use(new S3Strategy({
uploadPath: '/',
headers: {
'x-amz-acl': 'public-read'
},
options: {
domain: 'myDomainBucketObjectStorage',
key: 'accessKey',
secret: 'secretKey',
bucket: 'bucketname'
}
}));
app.post('/upload/image', function (req, res, next) {
uploader.upload('s3', req.files['images'], function (err, files) {
if (err) {
return next(err);
}
res.send(JSON.stringify(files));
});
});
http.createServer(app).listen(8000);
Reference : https://www.npmjs.com/package/express-fileuploader-s3
Output :
[{"fieldName":"images","originalFilename":"example.png","path":"/tmp/example.png","headers":{"content-disposition":"form-data;
name=\"images\";
filename=\"example.png\"","content-type":"image/png"},"size":58261,"name":"example.png","type":"image/png","url":"https:[slash2]domain[dot]com//50cbadc0-151a-11ea-a82c-6fb771b09257.png"}]
Output I think is success, but after I check to Object Storage is empty file. Does anyone have a solution ?

How to render object from REST response

I want to render to the ui / print to console log some object value from GET response.
I'm using Node JS for my server side and HTML + JS for my client side.
Because my goal is to render data and the request type is cross domain I can't use "fetch" function.
My only alternative to execute it is to send it by "JSONP" dataType.
Actually, the request is sent and the response receives by callback as well, but my code is print "null" to the console and not the response data.
When I've tried to used JSON.parse() it received a "parseerror".
The expected result it's to get only the image tag value (2.0.90) and to print this inside the console log / render it to the UI.
async function uiChecking() {
let index;
const hostsDock = [qa + dockers];
let lengthVal = hostsDock.length;
for (let hostIndxD = 0; hostIndxD < lengthVal; hostIndxD++) {
index = hostIndxD;
let url = hostsDock[hostIndxD];
$.ajax({
url: url,
dataType: 'jsonp',
}).done( function(data) {
console.log("A " + data);
});
}
}
**Server.js **
var express = require('express');
var cors = require('cors');
var app = express();
var path = require("path");
var fetch = require('fetch-cookie')(require('node-fetch'));
var btoa = require('btoa');
var http = require('http');
var corsOptionsDelegate = function (req, callback) {
var corsOptions;
if (whitelist.indexOf(req.header('Origin')) !== -1) {
corsOptions = { origin: true } // reflect (enable) the requested origin in the CORS response
}else{
corsOptions = { origin: false } // disable CORS for this request
}
callback(null, data , corsOptions) // callback expects two parameters: error and options
};
app.engine('.html', require('ejs').__express);
app.set('views', __dirname + '/view');
app.set('view engine', 'html');
app.use(express.static(path.join(__dirname, 'public')));
app.get('/', function(req, res){
res.render('index');
res.render('logo');
res.writeHead(200, {'Content-Type': 'application/json'});
});
// app.get('/products/:id', cors(corsOptionsDelegate), function (req, res, next) {
// res.json({msg: 'This is CORS-enabled for a whitelisted domain.'})
// });
app.get('/data/:id', function (req, res, next) {
var opts = {
host: config.alertService.host,
port: config.alertService.port,
method: 'GET',
path: '/DataService/rest/receiveData/' + req.params.id
}
var reqGet = http.request(opts, function (dataResponse) {
var responseString = '';
dataResponse.on('data', function (data) {
responseString += data;
});
var response = {x:[],y:[],z:[],t:[]};
dataResponse.on('end', function () {
var responseObject = JSON.parse(responseString);
var accs = responseObject.data.listPCS;
for(var i in accs){
response.x.push(accs[i].accX);
response.z.push(accs[i].accY);
response.y.push(accs[i].accZ);
response.t.push(accs[i].timestamp);
}
res.jsonp(response);
});
});
reqGet.end();
reqGet.on('error', function (e) {
console.error(e);
});
});
if (app.settings.env === 'production') {
app.error(function(err, req, res) {
res.render('new404.html', {
status: 500,
locals: {
error: error
}
});
});
}
app.listen(8033, function () {
console.log('CORS-enabled web server listening on port 8033')
});

You need to iterate through the response to return the result e.g..
$.each(data, function(index) {
console.log(data[index].ui);
console.log(data[index].id); console.log(data[index].Name);
});

Node js url shortener is looping rather than redirecting

I am writing a url shortener service in Node JS using mongo to connect to mLab.
Right now the user can send a request to the service with a url to shorten, and it returns a shortened url. However, if the user then sends the shortened url as a request, the redirect does not happen. Rather, the service goes into a loop.
1) How do I see what exactly is getting grabbed from the db? (Knowing how to do this would help out in trouble-shooting)
2) And what may be the cause of the looping issue?
var express = require('express')
var app = express()
var path = require('path');
var port = process.env.PORT || 8080;
var crypto = require("crypto");
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var UserSchema = new Schema({ // this schema is used for writing to the db
url : String,
key : String
});
var urlcntrctr = new Schema( // this schema is used for reading from the db
{ key: String, url : String, _id: String },
{ collection: 'urlcntrctr'}
);
const SchemaName = mongoose.model('SchemaName', urlcntrctr); // for reading from the db
app.get('/', (req, res, next) => res.sendFile(path.join(__dirname, '/index.html')) ) ;
app.set('port', (process.env.PORT || 5000));
app.get('/new/:url(*)', function(req, res) {
var shortenme = req.params[0];
var showme = req.params[0];
console.log("User's request: " +shortenme);
var amItrue = validateURL(shortenme);
if (amItrue){
connectmongoviamongoose();
var shortenmeObj = yncryptyyn(shortenme);
shortenme = shortenmeObj.key;
writeToDb(shortenmeObj); b
closetheconnection();
var contractedurl = 'http://firstappever-olddognewtrix123.c9users.io/' + shortenme;
var responseObject = ({"Original url: ": showme, "Contracted url: ": shortenme });
res.send(responseObject);
}
else{console.log("You need to enter a url, beginning with 'http' or 'https' and ending in '.com' or '.org' or whatever!");};
})
app.get('/:tag(*)', function(req, res) {
var targetnumber = req.params.tag;
sendforRedirect(req, res);
sendforRedirect(req, res);
})
function sendforRedirect(req, res){
var target = req.params.tag;
console.log("The value of target is " + target)
; var options = { server: { socketOptions: { keepAlive: 1, connectTimeoutMS: 30000 } },
replset: { socketOptions: { keepAlive: 1, connectTimeoutMS : 30000 } } };
var mongodbUri = 'mongodb://<dbusername>:<dbuserpassword>#ds159988.mlab.com:59988/urlcntrctr';
mongoose.connect(mongodbUri, options);
mongoose.Promise = global.Promise;
var conn = mongoose.connection;
conn.on('error', console.error.bind(console, 'connection error:'));
conn.once('open', function() {
console.log("OK, you are connected for the redirect. ")
var query = {
key: {
$eq: target
}
}
SchemaName.find(query, function (err, doc) {
if(err){
console.log(err);
conn.close();
};
if(doc){
res.redirect(doc.url); // rather than redirecting, it is looping *****************
conn.close();
} else {
res.send("Sorry, we don't recognize that url");
conn.close();
}
});
});
}
function writeToDb(dataObject){
mongoose.model('Document', UserSchema);
var urlFromUser = mongoose.model('Document');
var urlfromuser = new urlFromUser();
urlfromuser.url = dataObject.url;
urlfromuser.key = dataObject.key;
urlfromuser.save();
};
function validateURL(textval) { //copied from http://stackoverflow.com/questions/1303872/trying-to-validate-url-using-javascript
var urlregex = /^(https?|ftp):\/\/([a-zA-Z0-9.-]+(:[a-zA-Z0-9.&%$-]+)*#)*((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|([a-zA-Z0-9-]+\.)*[a-zA-Z0-9-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(:[0-9]+)*(\/($|[a-zA-Z0-9.,?'\\+&%$#=~_-]+))*$/;
return urlregex.test(textval);
}
function connectmongoviamongoose(){
var mongoose = require('mongoose');
var options = { server: { socketOptions: { keepAlive: 300000, connectTimeoutMS: 30000 } },
replset: { socketOptions: { keepAlive: 300000, connectTimeoutMS : 30000 } } };
var mongodbUri = 'mongodb://<dbusername>:<dbuserpassword>#ds159988.mlab.com:59988/urlcntrctr';
mongoose.createConnection(mongodbUri, options);
var conn = mongoose.connection;
conn.on('error', console.error.bind(console, 'connection error:'));
conn.once('open', function() {
console.log("OK, you are connected. ")
});
}
function closetheconnection(){
var mongoose = require('mongoose');
mongoose.connection.close();
}
function yncryptyyn(incryptme){
var ulimit = 6;
var key = crypto.createHash('md5').update(incryptme).digest("base64");
key = key.slice(0,ulimit);
var obj = {
url: incryptme,
key: key
};
return obj;
}
app.listen(app.get('port'), function() {
console.log('Node app is running on port', app.get('port'));
});

Better than console.log statements, you can use the package node-inspector to actually set breakpointsnin your code via chrome devtools and step through the code Much more robust process.
I would note that it is not clear to me what kind of urls you are shortening (internal to your site or external), but at present it looks like you're calling the redirect function twice, which should cause an error unto itself, and second if you are redirecting to internal urls your routes are probably going to match a lot that you don't want them to.
Finally, your code is kind of a jumble right now, which will make debugging harder no matter what you do. Try and break it out into different files based on what they do and test bits independently as much as possible.

For question 1: just put in a console.log, for example like this:
if(doc){
console.log(doc);
res.redirect(doc.url);
....
Even better put the whole functionality of the look up of the url into an own function, so you can check the working of the lookup and the working of the redirect independently.

Data not correctly loaded with Node.js using express.js, fs and handlebars

I am kinda new to Node.js and experienced a strange behavior in my application.
At the beginning I am reading all uploaded files to my server using fs.readdir. Then the data is getting stored like this:
// Store all files and their data
var uploads = [];
var docData = [];
//read all files when server starts
fs.readdir(dir, function(err, items) {
if(err) {
console.log("Could not read files from directory " + dir);
}
else {
uploads = items;
uploads.forEach(function(item) {
var path = dir + item;
textract.fromFileWithPath(path, function( err, text ) {
if(err) {
console.log("Could not parse file " + path);
}
else {
var val = {
name : item,
data: text
};
docData.push(val);
}
});
});
}
});
So I store on the one hand the raw files and on the other hand the extracted text as json. But it does not load the files properly. It looks like the items are handed out to the frontend before these arrays are set, but I am not sure about this problem. The data is given to the frontend in this way:
app.get('/:search', function(req, res){
var result = [];
var invalidItems = 0;
for (var i = 0; i < docData.length; i++) {
if(!(stringStartsWith(docData[i].name,"."))) {
var index = i + 1 - invalidItems;
result.push({id: index, doc: docData[i].name, count: 3});
}
else{
invalidItems++;
}
}
res.send(result);
});
And the handlebars template using jade looks like this:
// Template for search results
script#result-template(type='text/x-handlebars-template')
table#data.table.table-striped.table-bordered
thead
tr
th ID
th Document
th Count
th Delete
tfoot
tr
th ID
th Document
th Count
th Delete
tbody
| {{#each this}}
tr
td {{id}}
td
a(href="/uploads/{{doc}}") {{doc}}
td {{count}}
td
span.glyphicon.glyphicon-trash
| {{/each}}
Is there any "clear" way to ensure that all the uploaded data has been parsed before sending the result to the frontend or do you think there is another stupid mistake?
Edit: The call in the js looks like that:
var needle = $('input[name=srch-term]').val();
$.ajax({
url: "http://localhost:1234/"+needle,
type: 'GET',
success: function (resp)
var source = $("#result-template").html();
var template = Handlebars.compile(source);
$("#result").html(template(resp));
},
error: function(e) {
alert('Error: '+e.text);
}
});
The complete app.js looks like this:
var express = require('express');
var path = require('path');
var http = require('http');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var elasticsearch = require('elasticsearch');
var multer = require("multer");
var fs = require('fs');
var textract = require('textract');
var app = express();
var upload = multer({ dest : './public/uploads'});
var dir = './public/uploads/';
// Store all files and their data
var uploads = [];
var docData = [];
//read all files when server starts
fs.readdir(dir, function(err, items) {
if(err) {
console.log("Could not read files from directory " + dir);
}
else {
uploads = items;
uploads.forEach(function(item) {
var path = dir + item;
textract.fromFileWithPath(path, function( err, text ) {
if(err) {
console.log("Could not parse file " + path);
}
else {
var val = {
name : item,
data: text
};
docData.push(val);
}
});
});
}
});
require('dotenv').load();
var client = new elasticsearch.Client({
host: process.env.ES_HOST,
log: 'trace'
});
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
// uncomment after placing your favicon in /public
//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', routes);
app.use(multer({ dest: './public/uploads/',
rename: function (fieldname, filename) {
//add the current date to the filename to allow multiple uploads
return filename + Date.now();
},
onFileUploadStart: function (file) {
console.log(file.originalname + ' is starting ...');
},
onFileUploadComplete: function (file) {
console.log(file.fieldname + ' uploaded to ' + file.path);
//add the uploaded file to the stored files
uploads.push(file);
textract.fromFileWithPath(file.path, function( error, text ) {
if(error) {
console.log("Could not parse file " + file.path);
}
else {
//add the uploaded file's text to the docData
var len = dir.length - 2;
var val = {
name : file.path.substring(len),
data: text
};
docData.push(val);
}
});
}
}));
app.post('/upload',function(req,res){
upload(req,res,function(err) {
if(err) {
return res.end("Error uploading file.");
}
res.redirect('/');
});
});
app.get('/:search', function(req, res){
var result = [];
var invalidItems = 0;
console.log("Data = " + docData.length);
for (var i = 0; i < docData.length; i++) {
if(!(stringStartsWith(docData[i].name,"."))) {
var index = i + 1 - invalidItems;
result.push({id: index, doc: docData[i].name, count: 3});
}
else{
invalidItems++;
}
}
console.log("data; " + docData.length);
res.send(result);
});
// catch 404 and forward to error handler
app.use(function(req, res, next) {
var err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handlers
// development error handler
// will print stacktrace
if (app.get('env') === 'development') {
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: err
});
});
}
// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
res.status(err.status || 500);
res.render('error', {
message: err.message,
error: {}
});
});
app.listen(process.env.SRV_PORT);
module.exports = app;
function stringStartsWith (string, prefix) {
var res;
try {
res = string.slice(0, prefix.length) == prefix;
}
catch(err) {
res = false;
}
return res;
}

child_process.fork - process.send does not return message

I am running this script with node child_process.fork api.
That is my express application script, from where I start my application:
/**
* Module dependencies.
*/
var express = require('express')
, routes = require('./routes')
, http = require('http')
, path = require('path');
var app = express();
//database connection
var connection = require('express-myconnection');
var mysql = require('mysql');
//all environments
app.set('port', process.env.PORT || 3000);
app.set('views', __dirname + '/views');
app.set('view engine', 'jade');
app.use(express.favicon());
app.use(express.logger('dev'));
app.use(express.bodyParser());
app.use(express.methodOverride());
app.use(express.static(path.join(__dirname, 'public')));
//development only
if ('development' == app.get('env')) {
app.use(express.errorHandler());
}
app.use(
connection(mysql,{
host: 'localhost',
user: 'root',
password : '',
port : 3306, //port mysql
database:'test-db'
},'pool')
);
//routes
//app.get('/', routes.index);
app.get('/', routes.list);
app.use(app.router);
//run script
var cp = require('child_process');
var child = cp.fork('dataGrabber/pusherMysql');
child.on('message', function(m) {
// Receive results from child process
console.log('received: ' + m);
});
http.createServer(app).listen(app.get('port'), function(){
console.log('Express server listening on port ' + app.get('port') + ' http://localhost:' + app.get('port'));
});
That`s the part where I run my script:
//run script
var cp = require('child_process');
var child = cp.fork('dataGrabber/pusherAPI');
child.on('message', function(m) {
// Receive results from child process
console.log('received: ' + m);
});
As you can see I load my script and want to receive a message from the child.
That is my pusherAPI.js script:
var mysql = require('mysql');
var Pusher = require('pusher-client');
/**
* connect with mysql db
*/
var connection = mysql.createConnection({
host : 'localhost',
user : 'root',
database : 'test-db',
port : '3306',
password : ''
});
connection.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
return;
}
console.log('connected as id ' + connection.threadId);
});
//connect with the server
var API_KEY = 'cb65d0a7a72cd94adf1f';
var pusher = new Pusher(API_KEY, {
encrypted: true
});
var channel = pusher.subscribe("ticker.160");
channel.bind("message", function(data) {
console.log(data);
this.data = data;
/**
* save data to db
*/
var trade = {
timestamp : data.trade.timestamp,
price : data.trade.topbuy.price,
};
var query = connection.query('INSERT INTO trades SET ?', trade, function(err, result) {
if (err) {
connection.rollback(function() {
throw err;
});
}
//push message back to the app.js
process.on('message', function(m) {
// Pass results back to parent process
m = "insert happened";
process.send(m);
});
connection.commit(function(err) {
if (err) {
connection.rollback(function() {
throw err;
});
}
});
});
console.log(query.sql);
});
I want to send a message back to my app.js, whenever an insertion happened to my sql db
My script starts and runs my queries. However, process.send(m); does not send anything back.
Any recommendations what I am doing wrong?
I appreciate your answer!
Update
When changing my pusherAPI.js to, I get nothing back in the console.
var channel = pusher.subscribe("ticker.160");
process.on('insert_message', function(m) {
channel.bind("message", function(data) {
console.log(data);
this.data = data;
/**
* save data to db
*/
var trade = {
timestamp : data.trade.timestamp,
price : data.trade.topbuy.price,
};
var query = connection.query('INSERT INTO trades SET ?', trade, function(err, result) {
if (err) {
connection.rollback(function() {
throw err;
});
}
//push message back to the app.js
// Pass results back to parent process
m = "insert happened";
process.send(m);
connection.commit(function(err) {
if (err) {
connection.rollback(function() {
throw err;
});
}
});
});
console.log(query.sql);
});
});
In my app.js I changed my code like that:
//run script
var cp = require('child_process');
var child = cp.fork('dataGrabber/pusherAPI');
child.on('insert_message', function(m) {
// Receive results from child process
console.log('received: ' + m);
});

Move your process.send(m); outside of the message event handler. Otherwise you're adding a new message event handler for every query and those event handlers only fire when the parent process sends it a message. Example:
var channel = pusher.subscribe("ticker.160");
channel.bind("message", function(data) {
this.data = data;
/**
* save data to db
*/
var trade = {
timestamp : data.trade.timestamp,
price : data.trade.topbuy.price,
};
var query = connection.query('INSERT INTO trades SET ?',
trade,
function(err, result) {
if (err) {
connection.rollback(function() {
throw err;
});
return;
}
//push message back to the app.js
m = "insert happened";
process.send(m);
connection.commit(function(err) {
if (err) {
connection.rollback(function() {
throw err;
});
}
});
});
});