I've been trying for a day to write two pipe functions, one that compiles less files and another one that concats these files. I want to learn how to write transform streams/pipes for more complex plugins.
So I want to know how to read data from another pipe, and how to alter that data and send it to the next pipe. This is what I have so far:
gulp.src(sources)
.pipe(through.obj(function (chunk, enc, cb) {
var t = this;
// console.log("chunk", chunk.path);
fs.readFile(chunk.path, enc, function (err,data) {
if (err) { cb(err); }
less.render(data, {
filename : chunk.path,
sourceMap : {
sourceMapRootpath : true
}
})
.then(function (outputCss) {
// console.log("less result",outputCss);
t.push(chunk);// or this.push(outputCss) same result
cb();
});
});
}))
.pipe(through.obj(function (chunk, enc, cb) {
console.log("chunk", chunk.path); // not event getting called.
cb();
}))
I can't get the outputCSS for each file in the second pipe. How can I send it?
Well, you don't need to use fs here, you already got the stream of file (here your chunk).
Another point, you're not sending back to the pipe the files, so I guess that's why nothing is called on your second one.
const through = require('through2')
gulp.src(sources)
.pipe(through.obj((chunk, enc, cb) => {
console.log('chunk', chunk.path) // this should log now
cb(null, chunk)
}))
In ES2015:
import through from 'through2'
gulp.src(sources)
.pipe(through.obj((chunk, enc, cb) => cb(null, chunk)))
And for your specific example:
.pipe(through.obj((file, enc, cb) => {
less.render(file.contents, { filename: file.path, ... }) // add other options
.then((res) => {
file.contents = new Buffer(res.css)
cb(null, file)
})
}))
This is still pretty basic, I don't check for errors, if it's not a stream and so on, but this should give you some hint on what you've missed.
Related
My apollo-server is using graphql-upload package which includes file upload support for GraphQL endpoints. But they only documented about uploading single files. But we need multiple file upload support. Well, I get the streams as an Array. But whenever I createReadStream for each streams & pipe them to cloudinary uploader var, it just uploads the last created stream rather then uploading the each stream.
Code
// graphql reolver
const post = async (_, { post }, { isAuthenticated, user }) => {
if (!isAuthenticated) throw new AuthenticationError("User unauthorized");
const files = await Promise.all(post.files);
let file_urls = [];
const _uploadableFiles = cloudinary.uploader.upload_stream({ folder: "post_files" },
(err, result) => {
console.log("err:", err);
console.log("result:", result);
if (err) throw err;
file_urls.push({
url: result.secure_url,
public_id: result.public_id,
file_type: result.metadata,
});
return result;
}
);
files.forEach(async (file) => await file.createReadStream().pipe(_uploadableFiles));
.... other db related stuff
}
After that, I get the Secure_URL from uploaded files which is returned by cloudinary upload_stream functions callback. But it only gives me the properties of one stream which was the last of the all streams. Please help me in this case. Is there any way to pipe multiple streams?
Instead of making one const upload stream you make it into a factory function that returns an upload stream on each call for pipe'ing
Use array map so that you get an array that you can use in Promise.all
One by one each file should get uploaded to their own respective upload stream, appending the generated file url info to file_urls(on success callback), when all are done Promise.all would resolve and the code can resume to do other db related stuff
const post = async (_, { post }, { isAuthenticated, user }) => {
if (!isAuthenticated) throw new AuthenticationError("User unauthorized");
const files = await Promise.all(post.files);
let file_urls = [];
function createUploader(){
return cloudinary.uploader.upload_stream({ folder: "post_files" },
(err, result) => {
console.log("err:", err);
console.log("result:", result);
if (err) throw err;
file_urls.push({
url: result.secure_url,
public_id: result.public_id,
file_type: result.metadata,
});
return result;
}
);
}
await Promise.all( files.map(async (file) => await file.createReadStream().pipe(createUploader())) ); //map instead of forEach
//.... other db related stuff
}
In an Express.js API I'm creating a zip file that stores a collection of PDFs that is intended to be passed as a download
I have created the zipfile using the yazl package following the README file, and it's pretty good, the problem comes when I use the pipe to create the createWriteStream, because I don't know how to properly wait until is finished.
Then in my Express route I want to send the file, but this code is executed before the write stream is finished...
This is a piece of code of a Promise function named renderReports inside my repository.js file, after I write the PDFs file I use a loop to added to the yazl's zipFile, then I proceed to create the zip with the fs.createWriteStream
const renderFilePromises = renderResults.map((renderedResult, index) =>
writeFile(`./temporal/validatedPdfs/${invoices[index].id}.pdf`, renderedResult.content)
);
await Promise.all(renderFilePromises);
const zipfile = new yazl.ZipFile();
invoices.map((invoice, index) => {
zipfile.addFile(`./temporal/validatedPdfs/${invoice.id}.pdf`, `${invoice.id}.pdf`)
});
zipfile.outputStream.pipe(fs.createWriteStream("./temporal/output.zip").on('close', () => {
console.log('...Done');
}));
zipfile.end();
resolve();
And the following code is how I use the promise
app.post('/pdf-report', async (req, res, next) => {
const { invoices } = req.body;
repository.renderReports(reporter, invoices)
.then(() => {
res.sendFile('output.zip', {
root: path.resolve(__dirname, './../../temporal/'),
dotfiles: 'deny',
}, (err) => {
if (err) {
console.log(err);
res.status(err.status).end();
}
else {
console.log('Sent:', 'output.zip');
}
});
})
.catch((renderErr) => {
console.error(renderErr);
res.header('Content-Type', 'application/json');
return res.status(501).send(renderErr.message);
});
});
I hope somebody can explain how to approach this
You need to store the write stream in a variable so you can access it. Then, on this variable, wait for the stream‘s finish event. This is emitted by Node.js once the stream is done with writing.
we'e writing a serverless function in AWS Lambda that does the following:
Fetches JSON files from AWS S3 (Multiple files).
Merges JSON files into one single file.
Uploads the new file from (2) back to S3.
To handle this flow, we're trying to use the async library in Nodejs within our Lambda. Here's the code we have so far:
'use-strict';
const AWS = require('aws-sdk');
const async = require('async');
const BUCKET = 'sample-bucket';
const s3path = 'path/to/directory';
exports.handler = (event, context, callback) => {
let filepaths = []; // Stores filepaths
let all_data = []; // Array to store all data retrieved
let s3Client = new AWS.S3({ params: { Bucket: BUCKET } }); // S3 bucket config
// Runs functions in the series provided to ensure each function runs one after the other
async.series([
// Read file IDs from the event and saves in filepaths array
function(callbackSeries) {
console.log('Starting function...');
event.forEach(id => {
filepaths.push(`${id}.json`);
});
console.log('All filenames pushed.');
callbackSeries();
},
// Fetches each file from S3 using filepaths
// Then pushes objects from each file to the all_data array
function(callbackSeries) {
async.each(filepaths, (file, callback) => {
let params = {
'Bucket': BUCKET,
'Key': s3path + file
}
s3Client.getObject(params, (err, data) => {
if(err) {
throw err;
} else {
if(data.Body.toString()) {
console.log('Fetching data...');
all_data.push(JSON.parse(data.Body.toString()));
callback();
console.log('Fetched.');
}
}
});
},
(err) => {
if (err) throw err;
else callbackSeries();
});
},
// NOTHING IS BEING EXECUTED AFTER THIS
// Puts object back into bucket as one merged file
function(callbackSeries) {
console.log('Final function.')
// Concatenates multiple arrays in all_data into a single array
all_data = Array.prototype.concat(...all_data);
let params = {
'Bucket': BUCKET,
'Body': JSON.stringify(all_data),
'Key': 'obj.json'
};
s3Client.putObject(params, (err, data) => {
if(err) throw err;
else console.log('Success!');
})
}
], (err) => {
if(err) throw err;
else console.log('End.');
})
}
The first two functions in our async.series are running normally and all the console.log statements are executed. But our third function i.e.:
// Puts object back into bucket as one merged file
function(callbackSeries) {
console.log('Final function.')
// Concatenates multiple arrays in all_data into a single array
all_data = Array.prototype.concat(...all_data);
let params = {
'Bucket': BUCKET,
'Body': JSON.stringify(all_data),
'Key': 'obj.json'
};
s3Client.putObject(params, (err, data) => {
if(err) throw err;
else console.log('Success!');
})
}
is not being executed at all. We added a console.log statements but it doesn't seem to be executing at all.
I've consulted both AWS Support as well as async's documentation but can't seem to figure out the problem.
Any assistance would be highly appreciated.
Many thanks.
I asked this question for PHP a long time ago. The same applies for Node.js the code below seems a little slow when using in a loop - is there a way for writing this in pure Node.js vanilla JavaScript without plugins or React.js etc?
const dirname = 'cdn-assets/'
fs.readdir(dirname, function(err, files) {
if (err) {
// some sort of error
} else {
if (!files.length) {
// directory appears to be empty
}
}
});
Also, can I write a further conditional to check:
if(directory_has_these_files('.js, .css, .jpg, .svg, .mp4'))
So if you do this
fs.readdir('/path/to/empty_dir', (data, err) => console.log(data, '.....', err))
You'll see that the result is:
null '.....' []
So your code can be simplified as
fs.readdir(dirname, (err, files) => {
if (err && !files) {
console.error(err)
}
console.info('the files --> ', files)
let regexp = RegExp('.jpeg|.doc|.png|.zip', 'gi')
for(result in files) {
if(regexp.test(files[result])) {
console.log('I have the following', files[result])
}
}});
However....
We want this quick, modern and efficient, don't we?!
So this is even better:
fs.readdir(dirname, (err, files) => {
if (err && !files) {
console.error(err)
}
let regexp = RegExp('.jpeg|.doc|.png|.zip', 'gi');
files.filter(
file => file.match(regexp)
).map(
result => console.log('I have the following',result)
);
});
One of the advantages to using a map on a directory, is that you'll guarantee the preservation of order, it also looks cleaner.
Map is built-in iterable — Object is not, this is not to say the Map is a replacement for Object, there are use cases for both. That's another story.
I am new to gulp and JS in general. But I have this gulp file that uses protagonist to validate a file. The task was written by someone else and it works correctly. Now I need the output of Protagonist to build a file from the parsed structure but I can't understand where in the script should I get it. Also the package through makes things harder for me to understand.
What would I need to do in order to write the result of the function protagonist.parse() to a file from within gulp.
Gulp file
var through = require('through2');
var protagonist = require('protagonist');
gulp.task('protagonist', function () {
gulp.src(paths.build_blueprint)
.pipe(gulpProtagonist({type: 'ast'}))
.on('error', gutil.log);
});
function gulpProtagonist(options) {
return through.obj(function (file, enc, cb) {
protagonist.parse(file.contents.toString(), {type: options.type}, function (error, result) {
if (error) {
return cb(new PluginError('gulp-protagonist', error));
}
try {
if (result['warnings'].length > 0) {
var msgs = result['warnings'].map(buildWarningMsg);
var args = ['error'].concat(msgs.map(genError));
self.emit.apply(self, args);
}
cb(null, file);
} catch (e) {
cb(new PluginError('gulp-protagonist', e));
}
});
});
}