I am having issues with a zero byte stream. I am resizing an image and uploading it as a stream to S3. If I pipe output to response it displays correctly.
// Fetch remote file
var request = http.get('mybigfile.jpg', function(response) {
// Setup IM convert
var convert = spawn('convert', ['-', '-strip', '-thumbnail', 600, '-']);
// Pipe file stream to it
response.pipe(convert.stdin);
// Pipe result to browser - works fine
//convert.stdout.pipe(res);
// S3 requires headers
var headers = {
'content-type': response.headers['content-type'],
'x-amz-acl': 'public-read'
};
// Upload to S3
var aws = aws2js.load('s3', aws.key, aws.secret);
aws.setBucket(aws.bucket);
aws.putStream('thumb.jpg', convert.stdout, false, headers, function(err) {
if (err) {
return console.error('Error storing:', err.toString());
} else {
// No errors - this shows - but file is 0kb
console.log(path + ' uploaded to S3');
}
}
I see notes about streams not working with S3 due to content length. I am trying buffers but no success with that so far.
Well no go on streams - I guess I can use pause-stream or multipart to technically achieve this but otherwise I don't think it's possible. I ended up using a buffer.
...
// Pipe file stream to it
response.pipe(convert.stdin);
// Save to buffer
var bufs = [] ;
convert.stdout.on('data', function(chunk) {
bufs.push(chunk);
});
convert.stdout.on('end', function() {
var buffer = Buffer.concat(bufs);
// S3 requires headers
...
aws.putBuffer(path, buffer, false, headers, function(err) {
...
Related
I have this issue with node and amazon s3 when it comes to sha256 encryption. I'm reading my files from the file system using fs.createReadStream(filename).am getting this file in chunks.Then am pushing each chunk into an array. Each chunk consists of 1 *1024 *1024 bytes of data.. when the file has finished getting read, on readstream.on('end') am looping through each value in array and encrypting each chunk using sha256. in the process of looping, am also adding axios promises of each loop into an array so that when am finished looping through all the chunks and encrypting each at a time, I'm able to use promise.all to send all the requests. the result of each encrypted chunk is also sent together with the each request as an header . The challenge I've been facing and trying to solve is, whenever a request is made, it gets the calculations of the sha256 from s3 is completely different from what I have . I have tried to solve this and to understand this to no avail. below is my code, what could I be doing wrong ?
this is the error that am getting :
Corrupted chunk received:
File corrupted: Expected SHA jakoz9d12xYjzpWVJQlqYdgPxAuF+LjZ9bQRg0hzmL8=, but calculated SHA 103f77f9b006d9b5912a0da167cf4a8cec60b0be017b8262cd00deb3183f3a8b
const Encryptsha256 = function(chunksTobeHashed) {
var crypto = require('crypto');
var hash = crypto.createHash('sha256').update(chunksTobeHashed).digest('base64')
return hash;
}
const upload = async function(uploadFile) {
var folderPath = uploadFile.filePath
var chunksArray = []
var uploadFileStream = fs.createReadStream(folderPath, { highWaterMark: 1 * 1024 * 1024, encoding:"base64" })
uploadFileStream.on('data', (chunk) => {
chunksArray.push(chunk)
// console.log('chunk is ', chunk)
})
uploadFileStream.on('error', (error) => {
console.log('error is ', error)
})
// file_id: "2fe44d18-fa94b201-2fe44d18b196-f9066e05a81c"
uploadFileStream.on('end', async() => {
//code to get file id was here but removed.since it was not much neccessary to this quiz
var file_id = "2fe44d18-fa94b201-2fe44d18b196-f9066e05a81c"
let promises = [];
for (var i in chunksArray) {
var Content_SHA256 = Encryptsha256(chunksArray[i])
var payload = {
body: chunksArray[i],
}
promises.push(
axios.post(
`${baseURL}/home/url/upload/${fileId}/chunk/${i}`, payload, {
header: {
'Content-SHA256': Content_SHA256,
},
}
)
)
}
Promise.all(promises).then((response) => {
console.log('axios::', response)
})
.catch((error) => {
console.log('request error', error)
})
})
I'm using Jimp to read in a JSON string that looks like this:
As you can see the image node is a base64-encoded JPEG.
I'm able to succesfully convert it to a TIFF and save it:
Jimp.read(Buffer.from(inputImage, "base64"), function(err, image) {
image.getBuffer(Jimp.MIME_TIFF, function(error, tiff) {
context.bindings.outputBlob = tiff
...}
However, when I attempted to embed the tiff inside of a JSON object, the TIFF gets all garbled up:
const response = {
image: tiff.toString('base64'),
correlation: correlation
};
context.bindings.outputBlob = response;
Here's the full code:
const Jimp = require("jimp");
module.exports = function(context, myBlob) {
const correlation = context.bindings.inputBlob.correlation;
const inputImage = context.bindings.inputBlob.image;
const imageName = context.bindings.inputBlob.imageName;
context.log(
correlation + "Attempting to convert this image to a tiff: " + imageName
);
Jimp.read(Buffer.from(inputImage, "base64"), function(err, image) {
image.getBuffer(Jimp.MIME_TIFF, function(error, tiff) {
const response = {
image: tiff.toString('base64'),
correlation: correlation
};
context.bindings.outputBlob = response;
context.log(
correlation + "Succesfully converted " + imageName + " to tiff."
);
context.done();
});
});
};
How do we embed the tiff inside of a JSON payload?
If this output is non-negotiable, how would I render the tiff from the saved payload?
Well since you confirmed you are looking for output with context.res here is my working sample.. note that there is a maximum response size, so you can't return every image/file the way I am returning the image here
const Jimp = require('jimp')
module.exports = async function (context, req)
{
let response = {}
try
{
let url = 'https://noahwriting.com/wp-content/uploads/2018/06/APPLE-300x286.jpg'
//call function to download and resize image
response = await resizeImage(url)
}
catch (err)
{
response.type = 'application/json'
if (err.response == undefined)
{
context.log(err)
response.status = 500
response.data = err
}
else
{
response.data = err.response.data
response.status = err.response.status
context.log(response)
}
}
//response
context.res =
{
headers: { 'Content-Type': `${response.type}` },
body: response.buf
}
}
async function resizeImage(url)
{
//read image to buffer
let image = await Jimp.read(url)
//resize image
image.resize(300, Jimp.AUTO)
//save to buffer
let image_buf = await image.getBufferAsync(image.getMIME())
//image.getMIME() returns something like `image/jpeg` which is a valid Content-Type for responses.
return { 'buf': image_buf, 'type': image.getMIME() }
}
(Offtopic but I saw that you are using blob storage so..) if you plan on storing photos/files/anything in Azure Blob Storage and you want to retrieve them in some systematic way you will find out very fast that you can't query the storage and you have to deal with ugly XML. My work around to avoid this way to create a function that stores photos/files in Blob Storage but then saves the url path to the file along with the file name and any other attributes to a mongo storage. So then I can make super fast queries to retrieve an array of links, which point to the respective files.
I am trying to upload buffer data from an image into S3. It gets uploaded fine. But when I try to download/view the image in S3, it throws an error. I have tried the following -
The image is available to me in Buffer format (JSON). I cannot change this
let image = { "type": "Buffer", "data": [45, 45, 45....]
let buf = new Buffer(image )
let params = {
Bucket: "bucketName",
Key: "TestImage123haha.PNG",
Body: buf ,
ACL: 'public-read',
ContentType: 'image/jpeg'
};
s3.upload(params, function(err, data) {
if (err) {
console.log('ERROR MSG: ', err);
} else {
console.log('Successfully uploaded data' + data.Location);
}
})
After upoading the image if I try to visit the URL of the bucket where it is stored and view the image, this is what I get -
When I print the buffer data - buf in console , this is what I get -
This problem took me many days but I was able to solve it:
In your API GATE WAY of funtion lambda go to Configuration > Binary types.
Add multipart/form-data
Using Node.js, I am trying to get an image from a URL and upload that image to another service without saving image to disk. I have the following code that works when saving the file to disk and using fs to create a readablestream. But as I am doing this as a cron job on a read-only file system (webtask.io) I'd want to achieve the same result without saving the file to disk temporarily. Shouldn't that be possible?
request(image.Url)
.pipe(
fs
.createWriteStream(image.Id)
.on('finish', () => {
client.assets
.upload('image', fs.createReadStream(image.Id))
.then(imageAsset => {
resolve(imageAsset)
})
})
)
Do you have any suggestions of how to achieve this without saving the file to disk? The upload client will take the following
client.asset.upload(type: 'file' | image', body: File | Blob | Buffer | NodeStream, options = {}): Promise<AssetDocument>
Thanks!
How about passing the buffer down to the upload function? Since as per your statement it'll accept a buffer.
As a side note... This will keep it in memory for the duration of the method execution, so if you call this numerous times you might run out of resources.
request.get(url, function (res) {
var data = [];
res.on('data', function(chunk) {
data.push(chunk);
}).on('end', function() {
var buffer = Buffer.concat(data);
// Pass the buffer
client.asset.upload(type: 'buffer', body: buffer);
});
});
I tried some various libraries and it turns out that node-fetch provides a way to return a buffer. So this code works:
fetch(image.Url)
.then(res => res.buffer())
.then(buffer => client.assets
.upload('image', buffer, {filename: image.Id}))
.then(imageAsset => {
resolve(imageAsset)
})
well I know it has been a few years since the question was originally asked, but I have encountered this problem now, and since I didn't find an answer with a comprehensive example I made one myself.
i'm assuming that the file path is a valid URL and that the end of it is the file name, I need to pass an apikey to this API endpoint, and a successful upload sends me back a response with a token.
I'm using node-fetch and form-data as dependencies.
const fetch = require('node-fetch');
const FormData = require('form-data');
const secretKey = 'secretKey';
const downloadAndUploadFile = async (filePath) => {
const fileName = new URL(filePath).pathname.split("/").pop();
const endpoint = `the-upload-endpoint-url`;
const formData = new FormData();
let jsonResponse = null;
try {
const download = await fetch(filePath);
const buffer = await download.buffer();
if (!buffer) {
console.log('file not found', filePath);
return null;
}
formData.append('file', buffer, fileName);
const response = await fetch(endpoint, {
method: 'POST', body: formData, headers: {
...formData.getHeaders(),
"Authorization": `Bearer ${secretKey}`,
},
});
jsonResponse = await response.json();
} catch (error) {
console.log('error on file upload', error);
}
return jsonResponse ? jsonResponse.token : null;
}
In NodeJS, I have chunks of data from a file upload that saved the file in parts. I'd like to convert this by doing new Buffer() then upload it to Amazon s3
This would work if there was only one chunk but when there are multiple, I cannot figure out how to do new Buffer()
Currently my solution is write the chunks of data into a real file on my own server, then send the PATH of that file to Amazon s3.
How can I skip the file creation step and actually send the buffer the Amazon s3?
i guess you need to use streaming-s3
var streamingS3 = require('streaming-s3');
var uploadFile = function (fileReadStream, awsHeader, cb) {
//set options for the streaming module
var options = {
concurrentParts: 2,
waitTime: 20000,
retries: 2,
maxPartSize: 10 * 1024 * 1024
};
//call stream function to upload the file to s3
var uploader = new streamingS3(fileReadStream, aws.accessKey, aws.secretKey, awsHeader, options);
//start uploading
uploader.begin();// important if callback not provided.
// handle these functions
uploader.on('data', function (bytesRead) {
console.log(bytesRead, ' bytes read.');
});
uploader.on('part', function (number) {
console.log('Part ', number, ' uploaded.');
});
// All parts uploaded, but upload not yet acknowledged.
uploader.on('uploaded', function (stats) {
console.log('Upload stats: ', stats);
});
uploader.on('finished', function (response, stats) {
console.log(response);
cb(null, response);
});
uploader.on('error', function (err) {
console.log('Upload error: ', err);
cb(err);
});
};