Got some code that connects to an FTP server, converts the data to CSV and stores it in an array that will be later be used to update a database.
I'm dealing with many large CSV files and when accessing from FTP, they return incomplete!
I output the array length, and it's different every time (but usually close to the full length)
Removing .end() does not change the result, so it's not disconnecting early...
Downloading the CSV files manually (not an option expt. for dev) does result in the full file being read.
Any ideas appreciated, cheers :)
Here is the code
var FTPClient = require('ftp');
var c = new FTPClient();
var csv = require("csvtojson");
c.connect({
host: '****',
user: '****',
password: '****',
debug: console.log,
});
dataArray = []
c.on('ready', function() {
c.get('/file.csv', function(err, stream) {
if (err) throw err;
stream.once('close', function() {
console.log(dataArray.length)
c.end();
});
stream.on('data', function(chunk) {
csv({
noheader: false,
headers: ["Header1", "Header2", ]
})
.fromString(chunk.toString())
.subscribe(function(jsonObj) {
return new Promise(function(resolve, reject) {
dataArray.push(jsonObj)
resolve()
})
})
.on("end", function() {
console.log("done");
});
});
});
});
I now use this for streaming and parsing
const PromiseFtp = require('promise-ftp'),
ftp = new PromiseFtp();
const csv = require("csvtojson");
var jsonArray = []
var credentails = {
host: '*****',
user: '*****',
password: '*****',
}
var csvFile = {
formatA: ['filesA.csv'],
formatB: ['filesB.csv'],
formatC: ['filesC.csv']
}
async function fptUpdate(file) {
await ftp.connect(credentails).then(function() {
return ftp.get(file);
}).then(function(stream) {
return new Promise(function(resolve, reject) {
stream.pipe(csv(headers))
.on('data', (data) => {
let string = data.toString('utf8')
let json = JSON.parse(string)
jsonArray.push(json);
})
.on('end', () => {
resolve()
});
});
}).then(function() {
return ftp.end();
});
}
async function start() {
for (const file of csvFiles) {
await fptUpdate(file)
console.log(jsonArray.length)
jsonArray = []
console.log('Processed file: ' + file)
}
}
start()
Related
I would download file on local the create a stream then send to an API.
In localhost files get created via blobClient.downloadToFile(defaultFile);
But When I deploy function it can not find file to stream, so I think that the download does not happen or in bad location.
I get this error
[Error: ENOENT: no such file or directory, open 'D:\home\site\wwwroot\importPbix\exampleName.pbix'
Here's my code
const blobServiceClient = BlobServiceClient.fromConnectionString(
process.env.CONNEXION_STRING
);
const containerClient = blobServiceClient.getContainerClient(
params.containerName
);
const blobClient = containerClient.getBlobClient(process.env.FILE_LOCATION); // get file from storage
let blobData;
var defaultFile = path.join(params.baseDir, `${params.reportName}.pbix`); // use path module
let stream;
try {
blobData = await blobClient.downloadToFile(defaultFile);
console.log(blobData);
stream = fs.createReadStream(defaultFile);
} catch (error) {
params.context.log(error);
console.log(error);
}
var options = {
method: "POST",
url: `https://api.powerbi.com/v1.0/myorg/groups/${params.groupId}/imports?datasetDisplayName=${params.reportName}`,
headers: {
"Content-Type": "multipart/form-data",
Authorization: `Bearer ${params.accessToken} `,
},
formData: {
"": {
value: stream,
options: {
filename: `${params.reportName}.pbix`,
contentType: null,
},
},
},
};
//check if file keep in mem
return new Promise(function (resolve, reject) {
request(options, function (error, response) {
if (error) {
params.context.log(error);
reject(error);
} else {
params.context.log(response);
resolve(response.body);
}
fs.unlinkSync(defaultFile);
});
});
I found this post having same issue , that's why I user path module and passed __dirname to function params.baseDir.
If you want to download a file from Azure blob and read it as a stream, just try the code below, in this demo, I try to download a .txt file to a temp folder(you should create it first on Azure function)and print its content from the stream for a quick test:
module.exports = async function (context, req) {
const { BlockBlobClient } = require("#azure/storage-blob")
const fs = require('fs')
const connStr = '<connection string>'
const container = 'files'
const blobName = 'test.txt'
const tempPath = 'd:/home/temp/'
const tempFilePath = tempPath + blobName
const blobClient = new BlockBlobClient(connStr,container,blobName);
await blobClient.downloadToFile(tempFilePath).then(async function(){
context.log("download successfully")
let stream = fs.createReadStream(tempFilePath)
//Print text content,just check if stream has been readed successfully
context.log("text file content:")
context.log(await streamToString(stream))
//You can call your API here...
})
function streamToString (stream) {
const chunks = [];
return new Promise((resolve, reject) => {
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
stream.on('error', (err) => reject(err));
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
})
}
context.res = {
body: 'done'
}
}
Result
File has been downloaded:
read as stream successfully:
I am trying to upload an image from a URL to my Google Cloud Storage (Firebase). The following function shall return the file and a consecutive function will retrieve the actual Signed/Download Url to the new file. After all this I am updating a document in my Firestore Database with the new URL. That part works; the functions wait on uploading the (unfortunately incomplete) image and my document gets updated with the newly created file url. But the actual file/image is incomplete. :-(
async function saveToStorage(fileUrl) {
var storage = admin.storage();
var urlLib = require("url");
var pathLib = require("path");
//Get File Name from provided URL
var parsed = urlLib.parse(fileUrl);
var fileName = pathLib.basename(parsed.pathname);
//Create Storage Reference with new File Name
var bucket = storage.bucket('gs://myprojectname.appspot.com');
//Path Folder
var folderPath = 'data/photos/';
//Path Folder + File
var internalFilePath = folderPath + fileName ;
//Bucket File Ref
var file = bucket.file(internalFilePath);
const request = require('request');
const writeStream = file.createWriteStream({
metadata: {
contentType: 'image/jpg'
}
});
return new Promise((resolve, reject) => {
request.get(fileUrl)
.pipe(writeStream)
.on("error", (err) => {
console.error(`Error occurred`);
reject();
})
.on('finish', () => {
console.info(`Photo saved`);
resolve(file);
});
});
}
The Image that is saved/uploaded/streamed to my Cloud Storage file looks like this:
I have tried using node-fetch and request and rewrote my function in several ways, but always turn out with this result. I'm sure it has something to do with how I use my Promise, because if I omit the Promise the file actually completes but then the main code keeps executing instead of waiting for this Promise.
This has the same result (incomplete file):
return await fetch(fileUrl).then(res => {
const contentType = res.headers.get('content-type');
const writeStream = file.createWriteStream({
metadata: {
contentType
}
});
let p = new Promise((resolve, reject) => {
res.body.pipe(writeStream);
writeStream.on('finish', function() {
console.log("Stream finished")
resolve(file);
});
writeStream.on('error', function() {
reject(new Error("Whoops!"));
});
});
return p.then(
function(file) {
console.log('Photo saved');
return file},
function(error) {
console.error(error);
return;
});
});
And outright returning the stream writes a complete file, but my main code is not waiting for the file (and I need to handle the file)..
return res.body.pipe(writeStream)
.on('finish', () => {
return file;
console.log('Photo')
})
.on('error', err => {
return console.error(err);
});
Thanks for any help on this!
So this is the code that finally worked for me.
return new Promise((resolve, reject) => {
const req = request(fileUrl);
req.pause();
req.on('response', res => {
const writeStream = file.createWriteStream({
metadata: {
contentType: res.headers['content-type']
}
});
req.pipe(writeStream)
.on('finish', () => {
console.log('Photo saved');
resolve(file);
})
.on('error', err => {
writeStream.end();
console.error(err);
reject();
});
req.resume();
});
req.on('error', err => console.error(err));
});
I have a lambda function that's suppose to be writing to a database. When I run it on my local machine it works but then when I upload it to lambda and test it It doesn't put anything in the database. The role I have the function using has full access to DynamoDB and its the exact same code that works fine when I run it from my laptop. Any idea why that would be the case?
Here's my lambda. The dao class contains the code that actually accesses dynamo. I'm just trying to upload some constant strings right now.
const DAO = require('./PostStatusDAO.js');
exports.handler = async (event, context, callback) => {
var dao = new DAO();
dao.post("this is a test", "#jordan", "#matt", "none");
const response = {
statusCode: 200,
body: {
result: "good"
}
};
return response;
};
const AWS = require('aws-sdk');
const ddb = new AWS.DynamoDB.DocumentClient({region: 'us-west-2'});
class PostStatusDAO {
post(in_text, in_user, in_author, in_attachment) {
var params = {
Item: {
user: String(in_user),
timestamp: Date.now(),
author: String(in_author),
text: String(in_text),
attachment: String(in_attachment),
},
TableName: 'Feed',
};
console.log(params);
var result = ddb.put(params, (err, data) => {
console.log("callback");
if(err) {
console.log("Error: ", err);
} else {
console.log("Data: ", data);
}
});
// console.log(result);
}
}
module.exports = PostStatusDAO;
To see the reason why your function is failing you have to either run it synchronously or return the promise back to the caller/runtime like this:
const DAO = require('./PostStatusDAO.js');
exports.handler = async(event, context, callback) => {
var dao = new DAO();
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
dao.post("this is a test", "#jordan", "#matt", "none", function(err, data) {
if (err) {
console.log("Error: ", err);
reject(err);
}
else {
console.log("Data: ", data);
resolve(data);
}
})
})
};
const AWS = require('aws-sdk');
const ddb = new AWS.DynamoDB.DocumentClient({region: 'us-west-2'});
class PostStatusDAO {
async post(in_text, in_user, in_author, in_attachment, callback) {
var params = {
Item: {
user: String(in_user),
timestamp: Date.now(),
author: String(in_author),
text: String(in_text),
attachment: String(in_attachment),
},
TableName: 'Feed',
};
console.log(params);
return ddb.put(params, callback).promise();
}
}
module.exports = PostStatusDAO;
I'm handling with an webapplication that is not mine and now I've got to send a hundred e-mails.
Unfortunately, the code is not documented and not so well written, that means i have to go testing it to discover what I am able to do and what I'm not. but I don't know how to access this function that is on the code via node. Is it actually possible to do it? Here's the code:
router.post('/aprovadosemail', miPermiso("3"), (req, res) => {
var templatesDir = path.resolve(__dirname, '..', 'templates');
var emailTemplates = require('email-templates');
// Prepare nodemailer transport object
emailTemplates(templatesDir, function(err, template) {
if (err) {
console.log(err);
} else {
var users = [];
projetoSchema.find({"aprovado":true, "categoria":"Fundamental II (6º ao 9º anos)"}, function (err, docs) {
if (err) throw err;
//console.log(docs);
docs.forEach(function(usr) {
let url = "http://www.movaci.com.b/projetos/confirma/"+usr._id+"/2456";
let url2 = "http://www.movaci.com.br/projetos/confirma/"+usr._id+"/9877";
users.push({'email': usr.email, 'projeto': usr.nomeProjeto, 'url': url, 'url2': url2});
});
for (var i = 0; i < users.length; i++) {
console.log(users[i]);
}
const transporter = nodemailer.createTransport(smtpTransport({
host: 'smtp.zoho.com',
port: 587,
auth: {
user: "generic#mail.com",
pass: "genericpassword"
},
getSocket: true
}));
var Render = function(locals) {
this.locals = locals;
this.send = function(err, html, text) {
if (err) {
console.log(err);
} else {
transporter.sendMail({
from: 'no-reply4#movaci.com.br',
to: locals.email,
subject: 'MOVACI - Projeto aprovado!',
html: html,
text: text
}, function(err, responseStatus) {
if (err) {
console.log(err);
} else {
console.log(responseStatus.message);
}
});
}
};
this.batch = function(batch) {
batch(this.locals, templatesDir, this.send);
};
};
// Load the template and send the emails
template('rateada', true, function(err, batch) {
for(var user in users) {
var render = new Render(users[user]);
render.batch(batch);
};
});
res.send('ok');
});
};
});
});
Seems like previous dev did not knew email-templates package deeply (at least have not read how it works).
So in fact it has send method, You can create an email object from email-templates and pass necessary defaults, then You call .send method of it by passing dynamical parts - it just simply merge additional params passed in send arguments, sends mail using nodemailer inside of promise which it returns.
If it's interesting for You - read source code of it: https://github.com/niftylettuce/email-templates/blob/master/index.js
I tried to simplify it modular parts using promises.
I've not debugged it, but You may check my solution and fix it as You wish.
Have 2 files (to routing from handler separately, it may have variables that may conflict and etc):
1) methods/users/aprovadosEmail.js:
const
Email = require('email-templates'),
const
emailTemplatesDir = path.resolve(__dirname + '/../../templates'),
smtpTransportConfig = {
host: 'smtp.zoho.com',
port: 587,
secure: false,
auth: {
user: "no-reply4#movaci.com.br",
pass: "some-password-here"
}
},
createEmail = (template, subject) => {
return new Email({
views: {
root: emailTemplatesDir,
},
transport: smtpTransportConfig,
template,
message: {
from: 'no-reply4#movaci.com.br',
subject
}
});
},
getApprovedUsers = () => {
return new Promise((resolve, reject) => {
const criteria = {
aprovado: true,
categoria:"Fundamental II (6º ao 9º anos)"
};
projetoSchema.find(
criteria,
(error, docs) => {
if(error) return reject(error);
const users = docs.map(doc => {
return {
email: doc.email,
projeto: doc.nomeProjeto,
url: "http://www.movaci.com.b/projetos/confirma/"+doc._id+"/2456",
url2: "http://www.movaci.com.br/projetos/confirma/"+doc._id+"/9877"
};
});
resolve(users);
});
});
},
sendMailToUser = (mail, user) => {
return mail.send({
message: {
to: user.email
},
locals: user
});
},
broadcastMailToUsers = (mail, users) => {
return Promise
.all(
users
.map(user => sendMailToUser(mail, user))
);
};
module.exports = (req, res) => {
const mail = createEmail('rateada', 'MOVACI - Projeto aprovado!'); // mail object
getApprovedUsers()
.then(users => broadcastMailToUsers(mail, users))
.then(result => {
console.log('Result of broadcast:', result);
res.send('ok');
})
.catch(error => {
res.status(500).send(error);
});
};
2) current routes file where routing part that uses module file:
router.post(
'/aprovadosemail',
miPermiso("3"),
require(__dirname+'/../methods/users/aprovadosEmail')
);
How would you upload a video to twitter using the POST media/upload (chunked) endpoint with node?
This goes through all of the steps outlined in the link above: INIT, APPEND, FINALIZE and STATUS
var bufferLength, filePath, finished, fs, oauthCredentials, offset, request, segment_index, theBuffer;
request = require('request');
fs = require('fs');
filePath = '/thevideo.mp4';
bufferLength = 1000000;
theBuffer = new Buffer(bufferLength);
offset = 0;
segment_index = 0;
finished = 0;
oauthCredentials = {
consumer_key: '',
consumer_secret: '',
token: '',
token_secret: ''
};
fs.stat(filePath, function(err, stats) {
var formData, normalAppendCallback, options;
formData = {
command: "INIT",
media_type: 'video/mp4',
total_bytes: stats.size
};
options = {
url: 'https://upload.twitter.com/1.1/media/upload.json',
oauth: oauthCredentials,
formData: formData
};
normalAppendCallback = function(media_id) {
return function(err, response, body) {
finished++;
if (finished === segment_index) {
options.formData = {
command: 'FINALIZE',
media_id: media_id
};
request.post(options, function(err, response, body) {
console.log('FINALIZED',response.statusCode,body);
delete options.formData;
//Note: This is not working as expected yet.
options.qs = {
command: 'STATUS',
media_id: media_id
};
request.get(options, function(err, response, body) {
console.log('STATUS: ', response.statusCode, body);
});
});
}
};
};
request.post(options, function(err, response, body) {
var media_id;
media_id = JSON.parse(body).media_id_string;
fs.open(filePath, 'r', function(err, fd) {
var bytesRead, data;
while (offset < stats.size) {
bytesRead = fs.readSync(fd, theBuffer, 0, bufferLength, null);
data = bytesRead < bufferLength ? theBuffer.slice(0, bytesRead) : theBuffer;
options.formData = {
command: "APPEND",
media_id: media_id,
segment_index: segment_index,
media_data: data.toString('base64')
};
request.post(options, normalAppendCallback(media_id));
offset += bufferLength;
segment_index++
}
});
});
});
Please try this
const splitFile = require('split-file')
const Twitter = require('twitter')
const fs = require('fs-extra')
const Promise = require('bluebird')
const pathToMovie = __dirname + '/test/152.mp4';
const mediaType = 'video/mp4' // `'video/mp4'` is also supported
let Names
const mediaSize = require('fs').statSync(pathToMovie).size
/* Twitter support Maximum 15MB video files. So we need to split this
file in to three files */
splitFile.splitFile(pathToMovie, 3)
.then((names) => {
Names = names
return init()
})
.catch((err) => {
console.log('Error: ', err)
})
const client = new Twitter({
consumer_key: '<your consumer_key >',
consumer_secret: '<your consumer_secret >',
access_token_key: '<your access_token_key >',
access_token_secret: '<access_token_secret>'
});
const init = () => {
initTweetUpload(mediaSize, mediaType) // Declare that you wish to upload some media
.then(appendTweetUpload) // Send the data for the media
.then(appendTweetUpload) // Send the data for the media
.then(appendTweetUpload) // Send the data for the media
.then(finalizeTweetUpload) // Declare that you are done uploading chunks
// eslint-disable-next-line promise/always-return
.then((data) => {
const status = {
media_ids: data,
status: 'NodeJS Media Upload',
}
client.post('statuses/update', status, (error, tweet, response) => {
console.log(error)
console.log(tweet)
})
}).catch((err) => {
console.log('Error: ', err)
})
}
const initTweetUpload = (mediaSize, mediaType) => makePost('media/upload',
{
command: 'INIT',
total_bytes: mediaSize,
media_type: mediaType,
}).then((data) => data.media_id_string)
let i = 0
const appendTweetUpload = (mediaId) => {
const p = Names.shift()
/* mediaData is the raw binary file content being uploaded ,It must be
<= 5 MB */
const mediaData = fs.readFileSync(p)
return makePost('media/upload', {
command: 'APPEND',
media_id: mediaId,
media: mediaData,
segment_index: i++,
}).then((data) => mediaId)
}
const finalizeTweetUpload = (mediaId) => makePost('media/upload', {
command: 'FINALIZE',
media_id: mediaId,
}).then((data) => mediaId)
const makePost = (endpoint, params) =>
// params.media_category = 'tweet_video';
new Promise((resolve, reject) => {
client.post(endpoint, params, (error, data, response) => {
if (error) {
reject(error)
} else {
resolve(data)
}
})
})
dependencies
1. https://www.npmjs.com/package/twitter
2. https://www.npmjs.com/package/split-file