Ok, I do not understand what is going here, works locally but not on my server.
I have a angular controller that post to my node server.
each time I try and run the function that triggers the post I get
POST http://www.mysite.co.uk/mm3/back-end/savephotos 404 (Not Found)
Im honestly lost, ive rewritten the post 5 times I cant find the problem.
If anyone can see where ive gone wrong please help.
angular controller
mm3\js\controller.js
//all photos've been pushed now sending it to back end
$timeout(function () {
$http.post('back-end/savephoto', $scope.photosToPhp).then(function (success) {
$scope.generating = false;
$scope.generateBtn = 'Generate';
//creating mock up gallery
for (var x = 0; x < success.data.photos; x++) {
var file = '/mm3/tmp/' + success.data.folder + "/out" + x + ".png";
$scope.gallery.push(file);
}
$scope.photosToPhp = [];
}, function (error) {
});
}, 800);
then my node back-end
UPDATED:
So I have added a few console logs in my function to see where its going wrong and where it is getting to.
I keep getting:
test 1 function started error saving photo
mm3\back-end\controller.js
app.post('/mm3/back-end/savePhoto', function (req, res) {
console.log('test 1 function started');
var folder = Math.random().toString(36).substr(2, 20);
var photos = req.body;
var counts = 0;
var callback = function(counts){
if(counts < photos.length){
saveBase64(photos[counts],folder,counts,callback);
console.log('test 2 save photo');
}else{
var counts = 0;
var response = {"folder":folder, "photos": photos.length};
console.log('test 3 save photo else');
res.send(response)
}
};
saveBase64(photos[counts],folder,counts,callback);
});
app.post('/mm3/downloadZip', function(req, res){
var photos = req.body;
var out = photos[0];
var test = out.split('/');
var loc = test.pop();
var end = test.join('/');
console.log('test 3 function Generate zip file');
console.log(end);
var outName = '/' + end +'/mm3/MockUp.zip';
var output = fs.createWriteStream(outName);
var archive = archiver('zip', {store: true });
var zip = function(photos, f){
for(var t = 0; t < photos.length; t++){
var file = 'mockUp'+ t +'.jpg';
var from = '/var/www/html' + photos[t];
archive.file( from, { name: file });
}
f();
};
output.on('close', function() {
var photos = req.body;
var out = photos[0];
var test = out.split('/');
var loc = test.pop();
var end = test.join('/');
res.send(end + '/MockUp.zip');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
archive.on('error', function(err) {
throw err;
});
archive.pipe(output);
zip(photos, f);
function f(){
archive.finalize();
}
});
function saveBase64(photo,folder, counts, callback){
var result = photo.split(',')[1];
var path = '/mm3/tmp/' + folder;
var filename = path + "/out"+ counts + ".png";
mkdirp( path, function() {
fs.writeFile(filename, result, 'base64', function(error){
if (error) {
console.log('error saving photo');
}else{
console.log('photo saved');
counts ++;
callback(counts);
}
});
});
}
I think this is the problem:
app.post('back-end/savephoto', function (req, res) {
// skipped some lines
});
change it to
app.post('/back-end/savephoto', function (req, res) {
// skipped some lines
});
In Angular, the below:
$http.post('back-end/savephoto......
Becomes:
$http.post('/back-end/savephoto.....
In Node, the below:
app.post('back-end/savephoto.....
Becomes:
app.post('back-end/savephoto....
Then, you need to add a console.log under the Node route to see if it even is executed. This will narrow it down. Also, you can remove the $http.post call outside of the timeout to eliminate the obvious.
Let me know how you get on.
Shayan
Querying a database for a list of servers to perform a command on. The array is populated properly and echos out as planned, but none of the connections occur. I tried both passing the array directly into rexec and looping through a forEachAsync. Neither process the server list properly. Am I referencing the array elements improperly?
Mind the syntax errors at the end, I was just trying to include both methods I tried.
#!
var mysql = require('mysql');
var resultset = require('node-array');
var rexec = require('remote-exec');
var fs = require('fs');
var _ = require('lodash');
//var streamBuffers = require('stream-buffers');
var moment = require('moment');
var util = require('util');
var now = moment().format('YYYYMMDD_HHmmss');
var logStdout = process.stdout;
var errStderr = process.stderr;
console.log = function () {
logStdout.write(util.format.apply(null, arguments) + '\n');
}
console.error = function () {
errStderr.write(util.format.apply(null, arguments) + '\n');
}
var connection = mysql.createConnection({
host : 'abc',
user : 'user',
password : '******',
database : 'db'
});
var ssh_options = {
port: 22,
username: 'e109gh',
privateKey: fs.readFileSync('R:/nodeJS/sshkey.priv'),
stdout: fs.createWriteStream('./out.txt'),
stderr: fs.createWriteStream('./err.txt')
}
var my_conn_options = _.clone(ssh_options);
var cmds = ['hostname -i'];
connection.query('SELECT name FROM server', function(err, rows) {
rows.forEachAsync(function(element, index, array) {
console.log(element.name);
rexec(element.name,cmds,my_conn_options,function(err){
if (err) {
now = moment().format('YYYY-MM-DD HH:mm:ss');
console.error(err);
} else {
console.log("it worked for "+element.name);
}
});
});
});
// var buffer = new streamBuffers.WritableStreamBuffer();
connection.end(function(err) {});
// my_conn_options.stdout = buffer;
//
// rexec(rows,cmds,my_conn_options,function(err){
// if (err) {
// now = moment().format('YYYY-MM-DD HH:mm:ss');
// console.error(err);
// } else {
// console.log()
// }
// });
//
//});
I have a file which is tab separated. It has thousands of data. How can I use nodeJs to read the file, line by line, parse them and create an object and insert them in a mongo DB.
I am just learning node and mongo. I come from different background. So how can this be done.
Finally the Mongo DB has to be populated with proper data.
I searched in net but I could not find the complete solution.
Thanks.
I had an issue with the answer by Juvenik. My problem was that the database would not be populated by the time readline had completed. The lines were being read synchronously, but the DB insertion was asynchronous.
Instead, I found a simpler solution with the line-reader package. It reads the lines and waits for a callback before continuing.
var MongoClient = require('mongodb').MongoClient
var dbName = 'yourDbName'
var url = 'mongodb://localhost:27017/' + dbName
var collectionName = 'yourCollectionName'
var filename = 'yourFileName.txt'
var printLine = 1000
MongoClient.connect(url, function(err, db) {
if (err) {
console.error('Problem connecting to database')
} else {
console.log('Connected correctly to server.')
var lineReader = require('line-reader')
var collection = db.collection(collectionName)
var lineNum = -1
var headers = []
lineReader.eachLine(filename, function(line, last, cb) {
lineNum++
try {
var split = line.split('\t')
var object = {}
if (lineNum > 0) {
for (var i = 0; i < split.length; i += 1) {
object[headers[i]] = split[i]
}
collection.insert(object, function (insertErr, insertObj) {
if (insertErr) console.error(insertErr)
if (lineNum % printLine === 0) console.log('Line ' + lineNum)
if (last) {
console.log('Done with ' + filename + ' (' + lineNum + ' records)')
process.exit(0)
} else {
cb()
}
})
} else {
headers = line.split('\t')
cb()
}
} catch (lineError) {
console.error(lineError)
}
})
}
})
I came across similar problem. This approach worked for me.
Have a look, it might be helpful.
var mongoDb = require('mongodb');
var mongoClient = mongoDb.MongoClient;
var dbname = 'YOUR_DB_NAME';
var collectionName = 'YOUR_COLLECTION_NAME';
var url = 'mongodb://localhost:27017/'+dbname;
var filename = 'FIle_Name.txt';
console.log('***************Process started');
mongoClient.connect(url,function(err,db){
if(err){
console.log('error on connection '+err);
}
else{
console.log('***************Successfully connected to mongodb');
var collection = db.collection(collectionName);
var fs = require('fs');
var readline = require('readline');
var stream = require('stream');
var instream = fs.createReadStream(filename);
var outstream = new stream;
var rl = readline.createInterface(instream,outstream);
console.log('***************Parsing, please wait ...');
rl.on('line',function(line){
try{
var arr = line.split('\t');
var object = {};
//Parse them here
//Example
object['name'] = arr[0]; //Just an example
var res = collection.insert(object);
}
catch (err){
console.log(err);
}
});
rl.on('close',function(){
db.close();
console.log('***************completed');
});
}
});
I am a learner too. If someone can make it better, it will be good.
Here is a more performant (inserting batches of objects) and updated version (using async and latest mongo driver) of frank-0's answer
const lineReader = require('line-reader');
async function readFileAndInsertInMongo(file) {
let total = 0;
return new Promise((resolve, reject) => {
let buffer = [];
lineReader.eachLine(file, (line, last, cb) => {
// prepare your object based on the line content
let insertObject = {'some_content': 'some_value'};
if (total % 10000 === 0 || last) {
collection.insertMany(buffer, function(err, res){
if (last) {
if (err) {
reject(err);
} else {
resolve(res);
}
} else {
buffer = [];
return cb();
}
});
} else {
buffer.push(insertObject);
return cb();
}
});
});
}
This really is the best solution I have found to parse huge files and insert them in the database without exploding Node's memory. Hope this can help ;)
I am trying the learn younode workshop, the make it modular step to be precise. I wrote the code in the link and am getting the "error in the title". I have checked brackets and parenthesis but can not seem to find where i got it wrong. Any help would be appreciated (i just started with node).
My code is this and you can find it also at this link:
http://pastebin.com/G8x2GH7h
module.exports = function (directoryPath, extention, function(error, arrayOfNames) {
var fs = require('fs');
var path = require('path');
var FileNamesArray = [];
fs.readdir(directoryPath,function (error,list){
list.forEach(function (file) {
if (path.extname(file) === '.' + extention) {
FileNamesArray.push(file);
}
});
});
return FileNamesArray;
}){
return ArrayOfNames;
}
Your problem is that you are declaring function inside of function declaration.
It should be like this
exports.functionName = function() {
// your code
};
And then In you app.js you can use
var abc = require('path to file name');
abc.functionName();
example
// square.js
var PI = Math.PI;
exports.area = function (r) {
return PI * r * r;
};
exports.circumference = function (r) {
return 2 * PI * r;
};
// app.js
var square = require("./square.js");
var mySquare = square(2);
console.log('The area of my square is ' + mySquare.area());
Remove the anonymous function inside your function and hopefully it should work for you.
You are missing the title of you function in the export section, should be in this way:
module.exports = {
getFileNamesList : function (directoryPath, extention, function(error, arrayOfNames) {
var fs = require('fs');
var path = require('path');
var FileNamesArray = [];
fs.readdir(directoryPath,function (error,list){
list.forEach(function (file) {
if (path.extname(file) === '.' + extention) {
FileNamesArray.push(file);
}
});
});
return FileNamesArray;
}){
return ArrayOfNames;
}
}
I would like to read a very, very large file into a JavaScript array in node.js.
So, if the file is like this:
first line
two
three
...
...
I would have the array:
['first line','two','three', ... , ... ]
The function would look like this:
var array = load(filename);
Therefore the idea of loading it all as a string and then splitting it is not acceptable.
Synchronous:
var fs = require('fs');
var array = fs.readFileSync('file.txt').toString().split("\n");
for(i in array) {
console.log(array[i]);
}
Asynchronous:
var fs = require('fs');
fs.readFile('file.txt', function(err, data) {
if(err) throw err;
var array = data.toString().split("\n");
for(i in array) {
console.log(array[i]);
}
});
If you can fit the final data into an array then wouldn't you also be able to fit it in a string and split it, as has been suggested?
In any case if you would like to process the file one line at a time you can also try something like this:
var fs = require('fs');
function readLines(input, func) {
var remaining = '';
input.on('data', function(data) {
remaining += data;
var index = remaining.indexOf('\n');
while (index > -1) {
var line = remaining.substring(0, index);
remaining = remaining.substring(index + 1);
func(line);
index = remaining.indexOf('\n');
}
});
input.on('end', function() {
if (remaining.length > 0) {
func(remaining);
}
});
}
function func(data) {
console.log('Line: ' + data);
}
var input = fs.createReadStream('lines.txt');
readLines(input, func);
EDIT: (in response to comment by phopkins) I think (at least in newer versions) substring does not copy data but creates a special SlicedString object (from a quick glance at the v8 source code). In any case here is a modification that avoids the mentioned substring (tested on a file several megabytes worth of "All work and no play makes Jack a dull boy"):
function readLines(input, func) {
var remaining = '';
input.on('data', function(data) {
remaining += data;
var index = remaining.indexOf('\n');
var last = 0;
while (index > -1) {
var line = remaining.substring(last, index);
last = index + 1;
func(line);
index = remaining.indexOf('\n', last);
}
remaining = remaining.substring(last);
});
input.on('end', function() {
if (remaining.length > 0) {
func(remaining);
}
});
}
Using the Node.js readline module.
var fs = require('fs');
var readline = require('readline');
var filename = process.argv[2];
readline.createInterface({
input: fs.createReadStream(filename),
terminal: false
}).on('line', function(line) {
console.log('Line: ' + line);
});
js:
var array = fs.readFileSync('file.txt', 'utf8').split('\n');
ts:
var array = fs.readFileSync('file.txt', 'utf8').toString().split('\n');
Essentially this will do the job: .replace(/\r\n/g,'\n').split('\n').
This works on Mac, Linux & Windows.
Code Snippets
Synchronous:
const { readFileSync } = require('fs');
const array = readFileSync('file.txt').toString().replace(/\r\n/g,'\n').split('\n');
for(let i of array) {
console.log(i);
}
Asynchronous:
With the fs.promises API that provides an alternative set of asynchronous file system methods that return Promise objects rather than using callbacks. (No need to promisify, you can use async-await with this too, available on and after Node.js version 10.0.0)
const { readFile } = require('fs').promises;
readFile('file.txt', function(err, data) {
if(err) throw err;
const arr = data.toString().replace(/\r\n/g,'\n').split('\n');
for(let i of arr) {
console.log(i);
}
});
More about \r & \n here: \r\n, \r and \n what is the difference between them?
use readline (documentation). here's an example reading a css file, parsing for icons and writing them to json
var results = [];
var rl = require('readline').createInterface({
input: require('fs').createReadStream('./assets/stylesheets/_icons.scss')
});
// for every new line, if it matches the regex, add it to an array
// this is ugly regex :)
rl.on('line', function (line) {
var re = /\.icon-icon.*:/;
var match;
if ((match = re.exec(line)) !== null) {
results.push(match[0].replace(".",'').replace(":",''));
}
});
// readline emits a close event when the file is read.
rl.on('close', function(){
var outputFilename = './icons.json';
fs.writeFile(outputFilename, JSON.stringify(results, null, 2), function(err) {
if(err) {
console.log(err);
} else {
console.log("JSON saved to " + outputFilename);
}
});
});
file.lines with my JFile package
Pseudo
var JFile=require('jfile');
var myF=new JFile("./data.txt");
myF.lines // ["first line","second line"] ....
Don't forget before :
npm install jfile --save
With a BufferedReader, but the function should be asynchronous:
var load = function (file, cb){
var lines = [];
new BufferedReader (file, { encoding: "utf8" })
.on ("error", function (error){
cb (error, null);
})
.on ("line", function (line){
lines.push (line);
})
.on ("end", function (){
cb (null, lines);
})
.read ();
};
load ("file", function (error, lines){
if (error) return console.log (error);
console.log (lines);
});
To read a big file into array you can read line by line or chunk by chunk.
line by line refer to my answer here
var fs = require('fs'),
es = require('event-stream'),
var lines = [];
var s = fs.createReadStream('filepath')
.pipe(es.split())
.pipe(es.mapSync(function(line) {
//pause the readstream
s.pause();
lines.push(line);
s.resume();
})
.on('error', function(err) {
console.log('Error:', err);
})
.on('end', function() {
console.log('Finish reading.');
console.log(lines);
})
);
chunk by chunk refer to this article
var offset = 0;
var chunkSize = 2048;
var chunkBuffer = new Buffer(chunkSize);
var fp = fs.openSync('filepath', 'r');
var bytesRead = 0;
while(bytesRead = fs.readSync(fp, chunkBuffer, 0, chunkSize, offset)) {
offset += bytesRead;
var str = chunkBuffer.slice(0, bytesRead).toString();
var arr = str.split('\n');
if(bytesRead = chunkSize) {
// the last item of the arr may be not a full line, leave it to the next chunk
offset -= arr.pop().length;
}
lines.push(arr);
}
console.log(lines);
This is a variation on the answer above by #mtomis.
It creates a stream of lines. It emits 'data' and 'end' events, allowing you to handle the end of the stream.
var events = require('events');
var LineStream = function (input) {
var remaining = '';
input.on('data', function (data) {
remaining += data;
var index = remaining.indexOf('\n');
var last = 0;
while (index > -1) {
var line = remaining.substring(last, index);
last = index + 1;
this.emit('data', line);
index = remaining.indexOf('\n', last);
}
remaining = remaining.substring(last);
}.bind(this));
input.on('end', function() {
if (remaining.length > 0) {
this.emit('data', remaining);
}
this.emit('end');
}.bind(this));
}
LineStream.prototype = new events.EventEmitter;
Use it as a wrapper:
var lineInput = new LineStream(input);
lineInput.on('data', function (line) {
// handle line
});
lineInput.on('end', function() {
// wrap it up
});
i just want to add #finbarr great answer, a little fix in the asynchronous example:
Asynchronous:
var fs = require('fs');
fs.readFile('file.txt', function(err, data) {
if(err) throw err;
var array = data.toString().split("\n");
for(i in array) {
console.log(array[i]);
}
done();
});
#MadPhysicist, done() is what releases the async. call.
Using Node.js v8 or later has a new feature that converts normal function into an async function.
util.promisify
It's an awesome feature. Here's the example of parsing 10000 numbers from the txt file into an array, counting inversions using merge sort on the numbers.
// read from txt file
const util = require('util');
const fs = require('fs')
fs.readFileAsync = util.promisify(fs.readFile);
let result = []
const parseTxt = async (csvFile) => {
let fields, obj
const data = await fs.readFileAsync(csvFile)
const str = data.toString()
const lines = str.split('\r\n')
// const lines = str
console.log("lines", lines)
// console.log("str", str)
lines.map(line => {
if(!line) {return null}
result.push(Number(line))
})
console.log("result",result)
return result
}
parseTxt('./count-inversion.txt').then(() => {
console.log(mergeSort({arr: result, count: 0}))
})
I had the same problem, and I have solved it with the module line-by-line
https://www.npmjs.com/package/line-by-line
At least for me works like a charm, both in synchronous and asynchronous mode.
Also, the problem with lines terminating not terminating \n can be solved with the option:
{ encoding: 'utf8', skipEmptyLines: false }
Synchronous processing of lines:
var LineByLineReader = require('line-by-line'),
lr = new LineByLineReader('big_file.txt');
lr.on('error', function (err) {
// 'err' contains error object
});
lr.on('line', function (line) {
// 'line' contains the current line without the trailing newline character.
});
lr.on('end', function () {
// All lines are read, file is closed now.
});
Another answer using an npm package. The nexline package allows one to asynchronously read a file line-by-line:
"use strict";
import fs from 'fs';
import nexline from 'nexline';
const lines = [];
const reader = nexline({
input: fs.createReadStream(`path/to/file.ext`)
});
while(true) {
const line = await reader.next();
if(line === null) break; // line is null if we reach the end
if(line.length === 0) continue; // Ignore empty lines
// Process the line here - below is just an example
lines.push(line);
}
This approach will work even if your text file is larger than the maximum allowed string length, thereby avoiding the Error: Cannot create a string longer than 0x1fffffe8 characters error.
To put each line as an item inside an array, a new function was added in Node.js v18.11.0 to read files line by line
filehandle.readLines([options])
This is how you use this with a text file you want to read a file and put each line in an array
import { open } from 'node:fs/promises';
const arr = [];
myFilereader();
async function myFileReader() {
const file = await open('./TextFileName.txt');
for await (const line of file.readLines()) {
arr.push(line);
}
console.log(arr)
}
To understand more read Node.js documentation here is the link for file system readlines():
https://nodejs.org/api/fs.html#filehandlereadlinesoptions