I'm trying to upload a file to S3 after creation however the file doesn't seem to be closing with the fs.createWriteStream. Hence I keep getting a 0 byte file being uploaded.
function createManifest(manfile) {
console.log(['a'].toString(), ['a', 'b'].toString());
var arrayLength = files.length;
var lastItem = arrayLength - 1;
console.log( chalk.blue("The last item value is:",lastItem))
console.log( chalk.yellow("The arrayLength value is:",arrayLength))
var logStream = fs.createWriteStream("manny_temp.json", {'flags': 'a'});
// use {'flags': 'a'} to append and {'flags': 'w'} to erase and write a new file
// logStream.on('open', function(fd) {
logStream.write('{'+ "\r\n");
logStream.write("\"entries\": [" + "\r\n");
for (var i = 0; i < arrayLength; i++) {
console.log( chalk.inverse(files[i]))
console.log( chalk.blue("The i value is:",i))
if ( i == lastItem) {
logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}" + "\r\n");
} else {
logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}," + "\r\n");
}
}
logStream.write(' ]' + "\r\n");
logStream.write('}');
// }).on('end', function() {
logStream.end();
//fs.renameSync(logStream.path, manfile.toString());
return callback(filepath);
// logStream.close();
//fs.renameSync(logStream.path, "manny.json");
// });
}
I've tried a multitude of ways to get the file to close so that the next function can upload the file upon creation, and even added a sleep, but it always seems to leave a hanging inode.
Using the fs.write seems to write only one line vs writing all the lines in the array / streaming data.
Does anyone have a suggestion?
You need to listen for the close event on the logStream before calling the callback. That is what signals that the backing file descriptor has been closed (no more writing will take place). I should note, this is different than listening for the finish event since finish merely indicates the stream is closed, but not necessarily that the file descriptor is closed, so it is technically less safe to rely on if you're going to do something with the file soon after).
Replace this:
return callback(filepath);
with this:
logStream.on('close', function() {
callback(filepath);
});
It is possible to improve on this further, by instead using the (err, result)-style callbacks common in the node ecosystem:
logStream.on('close', function() {
callback(null, filepath);
});
logStream.on('error', function(err) {
callback(err);
});
That way you can capture any errors that may result from opening or writing to the file.
I'm also assuming you do have callback defined in a parent scope somewhere.
Your question is asked without the basic research though, Here is an attempt to make it more readable code and executable at glance. Hope you will find an answer with below code.
If your question is still the same, re-write your question in a better way.
var chalk = require('chalk');
var fs = require('fs');
function createManifest(manfile) {
var files = ['/test.json'];
console.log(['a'].toString(), ['a', 'b'].toString());
var arrayLength = files.length;
var lastItem = arrayLength - 1;
console.log( chalk.blue("The last item value is:",lastItem))
console.log( chalk.yellow("The arrayLength value is:",arrayLength))
var logStream = fs.createWriteStream("manny_temp.json", {'flags': 'a'});
logStream.write('{'+ "\r\n");
logStream.write("\"entries\": [" + "\r\n");
for (var i = 0; i < arrayLength; i++) {
console.log( chalk.inverse(files[i]))
console.log( chalk.blue("The i value is:",i))
if ( i == lastItem) {
logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}" + "\r\n");
} else {
logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}," + "\r\n");
}
}
logStream.write(' ]' + "\r\n");
logStream.write('}');
logStream.end();
logStream.on('finish', function(){
fs.renameSync("manny_temp.json", "manny.json");
});
};
createManifest();
mscdex is nearly right. You need to listen for the 'finish' event, not the 'close' event because that is emitted by readable streams but this is a writable stream.
And you can pass your handler to 'end' instead. See https://nodejs.org/dist/latest-v4.x/docs/api/stream.html#stream_event_finish
logStream.end(function() {
fs.renameSync("manny_temp.json", "manny.json");
callback(filepath);
});
I tried to edit mscdex's answer to this effect but this was rejected for some reason.
Edit: You'd need to define or pass in 'callback', as well as 'filepath'.
Related
This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 5 years ago.
I am doing simple app in javascript. I have "main_script" where I invoke everything. There is global variable "feeds" which is an array, like this:
var feeds = [];
Then after that I use function, that loads JSON file from multipe URLs (also array):
feeds = LoadJsonFeeds(urls); // Load feeds
console.log("main_code feeds.length: " + feeds.length);
That console log I mention later. Ok and now he is my LoadJsonFeeds (in different .js file, just a function):
function LoadJsonFeeds(urls) {
var feeds_tmp = [];
// URLs can be more - for example 50 feeds from url[0] and 20 from url[1]
for(var u = 0; u < url.length; u++) {
$.getJSON(url[u], function(data) {
var allFeeds = data.Result.Items; // allFeeds without check if they are ok
for(var i = 0; i < allFeeds.length; i++) {
// Is feed ok?
if (allFeeds[i].Text != null)
{
// Some more checking, but lets say ok for this
feeds_tmp.push(allFeeds[i]);
}
// This I mention later
console.log("LoadJson feeds.length: " + feeds.length);
}
});
}
console.log("LoadJson return"); // Mention later
return feeds_tmp;
}
And here is the problem I am struggling with. When I look at the console, here what I see:
LoadJson return
main_code feeds.length: 0
LoadJson feeds.length: 1
LoadJson feeds.length: 2
LoadJson feeds.length: 3
etc...
I just don't see the logic behind it! How can it first returned the function with nothing, then the main_script continues. After that, the function ALTER one by one the global variable "feeds". I suspect the anonymous function, but don't know what to do with it.
What am I trying to achive? Simple, I wanted to have function, that load JSON files from URLs. For example url[0] has 50 feeds, url[1] has 20. If everything is ok then it should return array of 70 feeds. I use this for the first time in main_script, and then in interval for update, which I call every few seconds. In this function I check, which feed is new and put it somewhere else:
function UpdateFeeds(url) {
console.log("updating...");
var feeds_tmp = LoadJsonFeeds(url);
console.log("Update feeds_tmp.length: " + feeds_tmp.length); // This is 0
for(var f_tmp = 0; f_tmp < feeds_tmp.length; f_tmp++) { // This does not happen because feeds_tmp.length = 0
for(var f = 0; f < feeds.length; f++) {
// Check what feed is new and put it somewhere else (the new one)
}
}
}
feeds = feeds_tmp; // Make all new feeds the global variable
}
But since the returned array is 0, that forloop does not happen. But it will still alter the global variable "feeds" anyway. For the main function it does not matter. In global variable the datas are in it, but I really need to find a new ones and do some work with it. But since it does not work that way, I am pretty lost.
What am I missing and how to fix this? Thank you!
Your console.log("LoadJson feeds.length: " + feeds.length); called later because its a asynchronous call , you can update this function as
function LoadJsonFeeds(urls,callback) {
var feeds_tmp = [];
// URLs can be more - for example 50 feeds from url[0] and 20 from url[1]
for(var u = 0; u < url.length; u++) {
$.getJSON(url[u], function(data) {
var allFeeds = data.Result.Items; // allFeeds without check if they are ok
for(var i = 0; i < allFeeds.length; i++) {
// Is feed ok?
if (allFeeds[i].Text != null)
{
// Some more checking, but lets say ok for this
feeds_tmp.push(allFeeds[i]);
}
// This I mention later
console.log("LoadJson feeds.length: " + feeds.length);
}
if(u==url.length.1) // to make sure all URL loaded
callback(feeds_tmp)
});
}
}
And call your function as
feeds = LoadJsonFeeds(urls,function(feeds){
console.log("main_code feeds.length: " + feeds.length);
}); // Load feeds
I am using CasperJS to test a website. A part of the test is resource check.
What I want to do:
Pass array or object array to CasperJS and iterate through them.
The first step is one array then object array. both have same issue.
Node.js code:
require('child_process').exec('/usr/local/bin/casperjs script.js [url,regex]' , function(err, stdout, stderr) {
err && console.log(err);
stderr && console.log(stderr.toString());
stdout && console.log(stdout.toString());
})
Casperjs script:
var casper = require('casper').create(),
a = casper.cli.args[0],
// we need something here to string to js array
w=a[0],
r=a[1];
casper.start(w, function() {
if (this.resourceExists(r)) {
console.log("PASS\t" +r+ "\t"+ w);
} else {
console.log("FAIL\t" +r+ "\t"+ w);
}
});
casper.run();
The problem is CasperJS takes args as string.
When you call it like that:
'/usr/local/bin/casperjs script.js "[\''+yourURL+'\',\''+yourRegex+'\']"'
You could simply use
a = JSON.parse(casper.cli.args[0]),
w = a[0],
r = new RegExp(a[1]);
If casper.cli.args[0] is actually JSON, then it can be parsed as such. resourceExists() takes regular expressions only as RegExp objects.
A better way if the data that you pass gets too long, then you should write the data to a temporary file with node.js' fs module and read it with PhantomJS' fs module, parsing it along the way.
Better approach to problem:
https://groups.google.com/forum/#!topic/casperjs/bhA81OyHA7s
Passing variables as options. Works like a charm
My Final nodejs:
var exec = require('child_process'),
array = [
{url:"",regex:""}
];
for (var i = 0; i < array.length; i++) {
var url = array[i]["url"];
var regex = array[i]["regex"];
exec.exec('/usr/local/bin/casperjs casper2.js --url="'+url+'" --regex="'+regex+'" ' , function(err, stdout, stderr) {
err && console.log(err);
stderr && console.log(stderr.toString());
stdout && console.log(stdout.toString());
});
};
In CasperJS
w=casper.cli.get("url"),
reg= casper.cli.get("regex"),
rpart = reg.split("/"),
r=new RegExp(rpart[1],rpart[2]);
casper.start(w, function() {
if (this.resourceExists(r)) {
console.log("PASS\t" +r+ "\t"+ w);
} else {
console.log("FAIL\t" +r+ "\t"+ w);
}
});
casper.run();
I got a main process and this fork a child. This child do some calculations. At some point of the code in the child I want to ask the parent for some data. This request is highly dynamic and at this point i want to wait for a answer/response of this request. But how do I wait for process.send() or is it possible to add a Callback function to .send()?
I tried to break down my Problem to a simple example.
The highly dynamic value is in my example the randomval in the worker.
And i know that the assignment
var c = process.send({msg:'get_c',randomval:Math.floor((Math.random()*10)+1)});
can't work. But i no other idea how to describe the Problem.
main.js
var childProcessCalc = require('child_process').fork(__dirname + '/worker');
childProcessCalc.send({msgtype:'docalc'});
childProcessCalc.on('message',function(msg){
if(msg.msg === 'pi')
{
console.log("Pi"+msg.pi+" by c="+msg.c);
}
else if(msg.msg === 'get_c')
{
console.log('child wants c');
childProcessCalc.send({msgtype:'your_c',c:1000000*msg.randomval});
}
});
childProcessCalc.on('exit',function(){
console.log('main:the childProzess has exit!')
});
worker.js
process.on('message', function(msg){
if(msg.msgtype == 'docalc') {
//Here is my Problem, how to wait for the message thats the response for
//exactly this send / randomval, or how to add a callback to send
var c = process.send({msg:'get_c',randomval:Math.floor((Math.random()*10)+1)});
var Pi=0;
var n=1;
for (var i=0;i<=c;i++)
{
Pi=Pi+(4/n)-(4/(n+2))
n=n+4
}
process.send({msg:'pi',pi:Pi,c:c})
}
else if(msg.msgtype === 'your_c')
{
console.log('parent hase sendc='+msg.c);
}
});
I have a solution to my problem and it works well for me, but because im very new at nodejs i still not now if this is the best way. Its feel like a overhead.
In a few words what i have done:
I added a object that stores a random callback identifier with the callback function that has to be called if we got a response for the given callback identifier.
When i now call send() from worker i send the identifier to the main process and the main process send this identifier back when he has finished. So i can lookup in my callback var (dynamicMassages ) for the callbackfnc to call and execute it.
main2.js
var childProcessCalc = require('child_process').fork(__dirname + '/worker2');
childProcessCalc.send({msgtype:'docalc'});
childProcessCalc.send({msgtype:'docalc'});
childProcessCalc.on('message',function(msg){
if(msg.msg === 'pi')
{
console.log("Pi"+msg.pi+" by c="+msg.c);
}
else if(msg.msg === 'get_c')
{
console.log('child wants c');
childProcessCalc.send({msgtype:'your_c',callbackid:msg.callbackid, c:1000000*msg.randomval});
}
});
childProcessCalc.on('exit',function(){
console.log('main:the childProzess has exit!')
});
worker2.js
var dynamicMassages = {};
process.on('message', function(msg){
var getRandomId = function(){
var text = "";
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for( var i=0; i < 5; i++ )
{
text += possible.charAt(Math.floor(Math.random() * possible.length));
}
if(dynamicMassages[text] === undefined)
{
return text;
}
else
{
return getRandomId();
}
};
if(msg.msgtype == 'docalc') {
var randomId = getRandomId();
var callbackFnc = function(c){
var Pi=0;
var n=1;
for (var i=0;i<=c;i++)
{
Pi=Pi+(4/n)-(4/(n+2))
n=n+4
}
console.log("callbackFnc For:"+randomId);
process.send({msg:'pi',pi:Pi,c:c})
delete dynamicMassages[randomId];
};
dynamicMassages[randomId] = callbackFnc;//callbackFnc;
process.send({msg:'get_c',callbackid: randomId, randomval:Math.floor((Math.random()*10)+1)});
}
else if(msg.msgtype === 'your_c')
{
console.log('parent hase sendc='+msg.c+' for callbackId '+msg.callbackid);
if(msg.callbackid !== undefined)
{
dynamicMassages[msg.callbackid](msg.c);
}
}
});
Please leave a comment if you would to it the same way.
I'd suggest that you go with a message bus, either a full-blown advanced solution such as RabbitMQ, or a little smaller solution, such as axon.
Basically, what you want to do is inter-process communication, and I'd try to stick to established protocols and standards as much as possible, and avoid rolling your very own solution. As RabbitMQ builds on top of AMQP, I guess you can call it standard.
I'm using Javascript to write an application that will be used with Phonegap to make an Android application. I'm using the Phonegap File API to read directories and files. The relevant code is shown below:
document.addEventListener("deviceready", onDeviceReady, false);
// PhoneGap is ready
//
function onDeviceReady() {
window.requestFileSystem(LocalFileSystem.PERSISTENT, 0, onFileSystemSuccess, fail);
}
function onFileSystemSuccess(fileSystem) {
fileSystem.root.getDirectory("/sdcard", {create: false, exclusive: false}, getDirSuccess, fail);
}
function getDirSuccess(dirEntry) {
// Get a directory reader
var directoryReader = dirEntry.createReader();
// Get a list of all the entries in the directory
directoryReader.readEntries(readerSuccess,fail);
}
var numDirs = 0;
var numFiles = 0;
function readerSuccess(entries) {
var i;
for (i=0; i<entries.length; i++)
{
if(entries[i].isFile === true)
{
numFiles++;
entries[i].file(fileSuccess,fail);
}
else if (entries[i].isDirectory === true)
{
numDirs++;
getDirSuccess(entries[i]);
}
}
}
So as of now, the program works fine. The reader will read the contents of the /sdcard directory..if it encounters a file, it will call fileSuccess (which I've excluded in the code for brevity), and if it encounters another directory, it will call getDirSuccess again. My question is this: How can I know when the entire /sdcard directory is read? I can't think of a good way of accomplishing this without going through the /sdcard directory more than one time. Any ideas are appreciated, and thank you in advance!
+1 on a good question since I have to do this anyway myself. I would use the old setTimeout trick. Once the cancel doesn't occur anymore, you know you are done and can fire your event, but just ensure its only fired once.
Here's what I mean and I've named the variables long simply to be more readable (not my style)...
// create timeout var outside your "readerSuccess" function scope
var readerTimeout = null, millisecondsBetweenReadSuccess = 100;
function readerSuccess(entries) {
var i = 0, len = entries.length;
for (; i < len; i++) {
if (entries[i].isFile) {
numFiles++;
entries[i].file(fileSuccess,fail);
} else if (entries[i].isDirectory) {
numDirs++;
getDirSuccess(entries[i]);
}
if (readerTimeout) {
window.clearTimeout(readerTimeout);
}
}
if (readerTimeout) {
window.clearTimeout(readerTimeout);
}
readerTimeout = window.setTimeout(weAreDone, millisecondsBetweenReadSuccess);
}
// additional event to call when totally done
function weAreDone() {
// do something
}
So the logic in this is you keep cancelling the "weAreDone" function from being called as you are reading through stuff. Not sure if this is the best way or more efficient but it would not result in more than one loop given the appropriate "millisecondsBetweenReadSuccess".
Instead of using a setTimeout, which can fail if you have a very slow device, you can use a counter to see how many callbacks still need to be called. If the counter reaches zero, you're all done :)
This is the recursive code:
var fileSync = new function(){
this.filesystem = null;
this.getFileSystem = function(callback){
var rfs = window.requestFileSystem || window.webkitRequestFileSystem;
rfs(
1// '1' means PERSISTENT
, 0// '0' is about max. storage size: 0==we don't know yet
, function(filesystem){
fileSync.filesystem = filesystem;
callback(filesystem);
}
, function(e){
alert('An error occured while requesting the fileSystem:\n\n'+ e.message);
}
);
}
this.readFilesFromReader = function(reader, callback, recurse, recurseFinishedCallback, recurseCounter)
{
if (recurse && !recurseCounter)
recurseCounter = [1];
reader.readEntries(function(res){
callback(res);
if (recurse)
{
for (var i=0; i<res.length; i++) {
/* only handle directories */
if (res[i].isDirectory == true)
{
recurseCounter[0]++;
fileSync.readFilesFromReader(res[i].createReader(), callback, recurse, recurseFinishedCallback, recurseCounter);
}
}
}
/* W3C specs say: Continue calling readEntries() until an empty array is returned.
* You have to do this because the API might not return all entries in a single call.
* But... Phonegap doesn't seem to accommodate this, and instead always returns the same dir-entries... OMG, an infinite loop is created :-/
*/
//if (res.length)
// fileSync.readFilesFromReader(reader, callback, recurse, recurseFinishedCallback, recurseCounter);
//else
if (recurse && --recurseCounter[0] == 0)
{
recurseFinishedCallback();
}
}
, function(e){
fileSync.onError(e);
if (recurse && --recurseCounter[0] == 0)
recurseFinishedCallback();
});
};
this.onError = function(e){
utils.log('onError in fileSync: ' + JSON.stringify(e));
if (utils.isDebugEnvironment())
alert('onError in fileSync: '+JSON.stringify(e));
}
}
var utils = new function(){
this.log = function(){
for (var i=0;i<arguments.length;i++)
console.log(arguments[i]);
}
this.isDebugEnvironment = function(){ return true }// simplified
}
Example code to test this:
var myFiles = [];
var directoryCount = 0;
window.onerror = function(){ alert('window.onerror=\n\n' + arguments.join('\n')) }
var gotFilesCallback = function(entries)
{
for (var i=0;i<entries.length;i++)
{
if (entries[i].isFile == true)
myFiles.push(entries[i].fullPath)
else
++directoryCount;
}
}
var allDoneCallback = function(){
alert('All files and directories were read.\nWe found '+myFiles.length+' files and '+directoryCount+' directories, shown on-screen now...');
var div = document.createElement('div');
div.innerHTML = '<div style="border: 1px solid red; position: absolute;top:10px;left:10%;width:80%; background: #eee;">'
+ '<b>Filesystem root:</b><i>' + fileSync.filesystem.root.fullPath + '</i><br><br>'
+ myFiles.join('<br>').split(fileSync.filesystem.root.fullPath).join('')
+ '</div>';
document.body.appendChild(div);
}
/* on-device-ready / on-load, get the filesystem, and start reading files */
var docReadyEvent = window.cordova ? 'deviceready':'load';
document.addEventListener(docReadyEvent, function()
{
fileSync.getFileSystem(function(filesystem){
var rootDirReader = filesystem.root.createReader();
fileSync.readFilesFromReader(rootDirReader, gotFilesCallback, true, allDoneCallback);
})
}, false);
I'm trying to us Async.js to process an array of items. Is there some cute way to get this to work properly? If you're smarter than I, you'd except that because of the deferred execution, http://3 gets printed three times.
jsFiddle link
var a_servers = ['http://1', 'http://2', 'http://3'];
var a_actions = [];
for (var i = 0; i < a_servers.length; i += 1)
{
var server = a_servers[i];
a_actions.push(function(callback)
{
document.write(server + '<br/>');
callback(false, server );
});
}
async.series(a_actions, function(err, a_servers)
{
document.write('processed ' + a_servers.length + ' servers<br>');
console.info(a_servers);
});
You have a classic closure scope issue. You need to pass the server variable from the outer scope to the inner scope to get the desired behavior. You can use an IIFE for this.
for (var i = 0; i < a_servers.length; i += 1)
{
var server = a_servers[i];
a_actions.push((function(server) {
return function(callback) {
document.write(server + '<br/>');
callback(false, server );
})(server));
}