Develop Reference

Google Cloud Function errors with ChildProcessError

This is my cloud function that is supposed to generate a watermarked image and store it in firebase storage everytime an image is uploaded.
exports.generateWatermark = functions.storage
.object()
.onFinalize(async object => {
try {
const fileBucket = object.bucket; // The Storage bucket that contains the file.
const filePath = object.name; // File path in the bucket.
const contentType = object.contentType; // File content type.
const metageneration = object.metageneration; // Number of times metadata has been generated. New objects have a value of 1.
// Exit if this is triggered on a file that is not an image.
if (!contentType.startsWith('image/')) {
return console.log('This is not an image.');
}
// Get the file name.
const fileName = path.basename(filePath);
// Exit if the image is already a watermarked image.
if (fileName.startsWith('watermark_')) {
return console.log('Already a Watermarked image.');
}
if (!filePath.startsWith('pets')) {
return console.log('Not a pet image: ', filePath);
}
// Download file from bucket.
const bucket = admin.storage().bucket(fileBucket);
const tempFilePath = path.join(os.tmpdir(), fileName);
const tempWatermarkPath = path.join(os.tmpdir(), 'watermark.png');
const metadata = {
contentType: contentType,
};
// Generate a watermarked image using Jimp
await bucket.file(filePath).download({destination: tempFilePath});
await bucket
.file('logo/cbs.png')
.download({destination: tempWatermarkPath});
console.log('Image downloaded locally to', tempFilePath, filePath);
await spawn('convert', [
tempFilePath,
'-gravity',
'NorthWest',
'-draw',
`"image Over 10,10,200,200 ${tempWatermarkPath}"`,
tempFilePath,
]);
console.log('Watermarked image created at', tempFilePath);
// We add a 'watermark_' prefix
const watermarkFileName = `watermark_${fileName}`;
const watermarkFilePath = path.join(
path.dirname(filePath),
watermarkFileName,
);
// Uploading the watermarked image.
await bucket.upload(tempFilePath, {
destination: watermarkFilePath,
metadata: metadata,
});
// Once the watermarked image has been uploaded delete the local file to free up disk space.
fs.unlinkSync(tempFilePath);
return fs.unlinkSync(tempWatermarkPath);
} catch (err) {
console.log('GENERATE WATERMARK ERROR: ', err);
throw err;
}
});
The part of the code that errors out is the imagemagick part:
await spawn('convert', [
tempFilePath,
'-gravity',
'NorthWest',
'-draw',
`"image Over 10,10,200,200 ${tempWatermarkPath}"`,
tempFilePath,
]);
This is the error that I'm getting:
Is there a way I could get more info about the error? The error is not even reaching my catch block..

childprocess.spawn uses the observer pattern.
The return value from invoking childprocess.spawn is a ChildProcess object with stdout and stderr which are EventEmitters.
You'll need an extra step to promisify the existing interface before you can await it. For example,
const spawn = (command, args) => new Promise((resolve, reject) => {
const cp = require('child_process').spawn(command, args);
let err = null, out = null;
cp.stdout.on('data', data => out += data.toString());
cp.stdout.on('error', data => err += data.toString());
cp.on('error', data => err += data.toString());
cp.on('close', code => {
(code === 0) ? resolve(out) : reject(err)
});
})
childprocess.execFile on the other hand uses callbacks. This makes it easily promisifiable uses util.promisify function. For example
const util = require('util');
const execFile = util.promisify(require('child_process').execFile);
exports.generateWatermark = functions.storage
.object()
.onFinalize(async object => {
try {
//...
await execFile('convert', [
tempFilePath,
'-gravity',
'NorthWest',
'-draw',
`"image Over 10,10,200,200 ${tempWatermarkPath}"`,
tempFilePath,
]);
//...
} catch (err) {
console.log('GENERATE WATERMARK ERROR: ', err);
throw err;
}
});

Multer doesnt upload but it wont give me any errors

I'm using multer for uploading images for creating courses in my application and my courses have images and I have a form for getting the data from user.
I use multer in middleware and add it to my route with upload.single('images') and my field which I get the image from user is named images and that means I have got the image from the user.
When I just click on the save button, I won't get any errors and my server will work like it is stuck in a middleware and when I go to my upload folder, I don't see any images loaded!
And if I load the image I will save the course in my database which is mongo. But I don't find the course saved too.
I have checked the enctype="multipart/form-data" and it was in my form.
And this is my multer middleware code
const multer = require("multer");
const mkdirp = require("mkdirp");
const fs = require("fs");
const getDirImage = () => {
let year = new Date().getFullYear();
let month = new Date().getMonth() + 1;
let day = new Date().getDay();
return `./public/uploads/images/${year}/${month}/${day}`;
};
const ImageStorage = multer.diskStorage({
destination: (req, file, cb) => {
let dir = getDirImage();
mkdirp(dir).then(made => {
console.log(`File made on ${made}`);
});
},
filename: (req, file, cb) => {
let filePath = getDirImage() + "/" + file.originalname;
if (!fs.existsSync(filePath)) cb(null, file.originalname);
else cb(null, Date.now() + "-" + file.originalname);
},
});
const uploadImage = multer({
storage: ImageStorage,
limits: {
fileSize: 1024 * 1024 * 10,
},
});
module.exports = uploadImage;
and this is the middlewares which I referenced it to my handler of route controller
router.post("/courses/create", upload.single("images"), convertFileToField.handle, courseValidator.handle(), courseController.storeCourse);
and this is the convertFileToField Code
const middleware = require('./middleware');
class ConvertFiletoField extends middleware {
handle(req, res, next) {
if (!req.file)
req.body.images = undefined;
else
req.body.images = req.file.filename;
next();
}
}
module.exports = new ConvertFiletoField();
And this is the courseValidator Middleware Code
const validator = require('./validator');
const Course = require("app/models/Course");
const path = require("path");
const { check } = require("express-validator/check");
class courseValidator extends validator {
handle() {
return [
check("title")
.not()
.isEmpty()
.withMessage("فیلد عنوان نمیتواند خالی بماند")
.custom(async (value) => {
const course = await Course.findOne({ slug: this.slug(value)});
if (course) {
throw new Error('we have this course on our site !!!!')
}
}),
check('images')
.custom(async value => {
if (! value) {
throw new Error('You need to enter a course !');
}
let fileExt = ['.png', '.jpg', 'jpeg', '.svg'];
if (! fileExt.includes(path.extname(value)))
throw new Error('the course extention is not valid !');
}),
];
}
slug(title) {
return title.replace(/([^۰-۹آ-یa-z0-9]|-)+/g, "-");
}
}
module.exports = new courseValidator();
And Finally this is the post route handler
const controller = require("app/http/controllers/controller");
const Course = require("app/models/Course");
const fs = require('fs');
const path = require("path");
const sharp = require("sharp");
class courseController extends controller {
showCourses(req, res) {
const courses = Course.find({}).sort({ createdAt: -1 });
res.render("admin/courses/index", { courses: courses });
}
createCourse(req, res) {
res.render("admin/courses/create");
}
async storeCourse(req, res) {
let status = await this.validationData(req);
if (!status) {
// For Deleting the saved image because of having validation error
if (req.file)
fs.unlink(req.file.path, (err) => {
console.log(err);
});
return this.back(req, res);
}
// Create the Course
let images = this.imageResize(req.file);
const { title, type, body, price, tags } = req.body;
const newCourse = new Course({
user: req.user._id,
title,
type,
slug: this.slug(),
body,
images: JSON.stringify(images),
price,
tags,
});
await newCourse.save();
return res.redirect("/admin/courses");
}
imageResize(image) {
let imageInfo = path.parse(image.path);
let addressImage = {};
addressImage["original"] = `${imageInfo}/${image.filename}`;
const resize = (size) => {
let imageName = `${imageInfo.name}-${size}${imageInfo.ext}`;
addressImage[size] = this.getUrlImage(`${image.destination}/${imageName}`);
sharp(image.path)
.resize(size, null)
.toFile(`${image.destination}/${imageName}`)
};
[1080, 720, 480].map(resize);
}
getUrlImage(dir) {
return dir.substr(8);
}
slug(title) {
return title.replace(/([^۰-۹آ-یa-z0-9]|-)+/g, "-");
}
}
module.exports = new courseController();
I had done everything I could and I had tried all the solutions for loading the image but I get error at my courseValidation middleware.
Please say any solution that is related to multer. I will try it out.

in the destination section of the diskStorage you must be return name of the directory in the callback function. your middleware stoped in this section because you don't call cb function.
destination: (req, file, cb) => {
let dir = getDirImage();
mkdirp(dir).then(made => {
console.log(`File made on ${made}`);
cb(made)
});
}

here middleware function need to be defined
router.post("/courses/create", multerMiddleWare,..{...});
const multerMiddleWare = (req, res, next) => {
uploadImage(req, res,
(error) => {
if (!error) return next();
return next('error');
});
};
const uploadImage = multer({
storage: ImageStorage,
limits: {
fileSize: 1024 * 1024 * 10,
},
}).single("images");

How to hash a download stream in Node js

I want to know how to hash the download stream of a file using node js
Because I wanna hash the file before I store in to mongo db in order to avoid duplicates , I am using mongo grid fs by the way. https://github.com/aheckmann/gridfs-stream
downloading file
var download = function (url, dest, callback) {
request.get(url)
.on('error', function (err) { console.log(err) })
.pipe(fs.createWriteStream(dest))
.on('close', callback);
};
final_list.forEach(function (str) {
var filename = str.split('/').pop();
console.log('Downloading ' + filename);
download(str, filename, function () { console.log('Finished Downloading' + "" + filename) });
});

function getHash(dest, filename) {
let crypto = require('crypto');
let hash = crypto.createHash('sha256').setEncoding('hex');
let fileHash = "";
let filePath = `${dest}/${filename}`
fs.createReadStream(filePath)
.pipe(hash)
.on('finish', function() {
fileHash = hash.read();
console.log(`Filehash calculated for ${filename} is ${fileHash}.`);
// insert into mongo db here
});
}

Unable to succesfully upload images to s3 and then view them

I'm trying to upload images to a s3 bucket as part of the application.
index.js
function upImg(req) {
if(req.files.img) {
var img = req.files.image;
var name = Math.round(Math.random()*10000).toString(); // Returns a random 5 digit number
if(myDB.uploadImg(img, name)) {
return name;
} else {
return "";
}
} else {
return "";
}
}
app.post('/newEV*', isLoggedIn, function(req, res) {
var myURL = req.path.replace('/newEV', '');
var imgPath = upImg(req);
fetch(myURL).then(function (events){
var myID;
var x = 0;
while(!myID) {
if(!events[x]) {
myID = x;
} else {
x++;
}
}
myDB.newEvent(myURL, req.body.name, req.body.desc, req.body.loc, imgPath, req.body.link, req.body.cap, req.body.date, req.body.time, myID, events);
res.redirect('/edit' + myURL);
});
});
myDB file
function signs3(file, name) {
devs3();
const s3 = new aws.S3();
const s3Params = {
Body: file,
Bucket: S3_BUCKET,
Key: name
};
s3.putObject(s3Params, function(err, data) {
if(err) {
throw err;
} else {
console.log("Data from putObject:" + JSON.stringify(data));
}
});
}
module.exports = {
uploadImg : function(file, name) {
var nName = "imgs/" + name;
console.log(nName);
signs3(file, nName);
return true;
}
}
I know that the signs3 function works because I can use it in other bits of my application to upload JSON files. Whenever I post to the URL, weirdly enough I can see in the console the 'data from putObject', however what I can't see is the nName. I don't understand this, as the console.log(nName) line should be run before the other one. When I go to look at bucket, the image hasn't uploaded (despite me getting an ETag from the console), and the page does not display it as there (I know this also works because it can display images already uploaded to the bucket).

You want to do something like this, soliciting events from the Request object created when you call putObject.
const req = s3.putObject( s3Params )
req.on('success', res => {
console.log ('upload complete! );
});
req.on ('error', res => {
console.error (res.error');
});
req.send();
Why does this appear to work differently for small files (JSON files) and large files (images)? Because the large files take longer to upload.

Writing a png that's generated from canvas using toDataURL then sent via XMLHttpRequest in NodeJS/Express

Been trying various methods I've found via google but none seem to work. I'm working with signature-pad, the javascript/html5 canvas solution for eSignatures. I'm trying to save the canvas data as a png to a temporary folder (TempFolder/username/signature.png). To post the the results of toDataURL I'm using XMLHttpRequest. Everything else is Node.js / Express. I left the download in to make sure the dataURL is not corrupted. Since I wasn't able to use body-parser to grab the data from passing it through XMLHttp I instead populate a hidden field with the data then submit it.
Save to Profile Function
saveToServerButton.addEventListener('click', event => {
if (signaturePad.isEmpty()) {
const warning = document.getElementById('message');
warning.value = 'Please provide a signature first.';
} else {
const image = signaturePad.toDataURL().replace(/^data:image\/png;base64,/, '');
const httpRequest = new XMLHttpRequest();
const hiddenInput = document.getElementById('base64Data');
hiddenInput.value = image;
}
});
And the route in Node
const fs = require('fs');
// POST - ESignature Processing
router.post('/eSig/save/', ensureAuthenticated, (req, res) => {
const trimmedData = req.body.base64Data;
console.log(`The trimmed data is: ${trimmedData}`);
const buffer = Buffer.from(trimmedData);
const usernameForFolder = req.user.username;
const userFolder = `${dir}/${usernameForFolder}`;
if (!fs.existsSync(userFolder)){
fs.mkdirSync(userFolder);
fs.writeFile(`${userFolder}/signature.png`, buffer, 'base64', err => {
if (err) throw err;
console.log('saved');
});
res.redirect('../../profile');
} else {
fs.writeFile(`${userFolder}/signature.png`, buffer, 'base64', err => {
if (err) throw err;
console.log('saved');
});
res.redirect('../../profile');
}
});

Figured it out. Here is the working code (buffer's need to know it's 'base64')
Save to Profile Function
saveToServerButton.addEventListener('click', event => {
if (signaturePad.isEmpty()) {
const warning = document.getElementById('message');
warning.value = 'Please provide a signature first.';
} else {
const image = signaturePad.toDataURL().replace(/\s/g, '+').replace(/^data:image\/png;base64,/, '');
const httpRequest = new XMLHttpRequest();
const hiddenInput = document.getElementById('base64Data');
hiddenInput.value = image;
}
});
Route on Node.js to save
// POST - ESignature Processing
router.post('/eSig/save/', ensureAuthenticated, (req, res) => {
const trimmedData = req.body.base64Data;
const buffer = Buffer.from(trimmedData, 'base64');
const usernameForFolder = req.user.username;
const userFolder = `${dir}/${usernameForFolder}`;
if (!fs.existsSync(userFolder)){
fs.mkdirSync(userFolder);
fs.writeFile(`${userFolder}/signature.png`, buffer, 'base64', err => {
if (err) throw err;
console.log('saved');
});
res.redirect('../../profile');
} else {
fs.writeFile(`${userFolder}/signature.png`, buffer, 'base64', err => {
if (err) throw err;
console.log('saved');
});
res.redirect('../../profile');
}
});