Right now I'm using this script in PHP. I pass it the image and size (large/medium/small) and if it's on my server it returns the link, otherwise it copies it from a remote server then returns the local link.
function getImage ($img, $size) {
if (#filesize("./images/".$size."/".$img.".jpg")) {
return './images/'.$size.'/'.$img.'.jpg';
} else {
copy('http://www.othersite.com/images/'.$size.'/'.$img.'.jpg', './images/'.$size.'/'.$img.'.jpg');
return './images/'.$size.'/'.$img.'.jpg';
}
}
It works fine, but I'm trying to do the same thing in Node.js and I can't seem to figure it out. The filesystem seems to be unable to interact with any remote servers so I'm wondering if I'm just messing something up, or if it can't be done natively and a module will be required.
Anyone know of a way in Node.js?
You should check out http.Client and http.ClientResponse. Using those you can make a request to the remote server and write out the response to a local file using fs.WriteStream.
Something like this:
var http = require('http');
var fs = require('fs');
var google = http.createClient(80, 'www.google.com');
var request = google.request('GET', '/',
{'host': 'www.google.com'});
request.end();
out = fs.createWriteStream('out');
request.on('response', function (response) {
response.setEncoding('utf8');
response.on('data', function (chunk) {
out.write(chunk);
});
});
I haven't tested that, and I'm not sure it'll work out of the box. But I hope it'll guide you to what you need.
To give a more updated version (as the most recent answer is 4 years old, and http.createClient is now deprecated), here is a solution using the request method:
var fs = require('fs');
var request = require('request');
function getImage (img, size, filesize) {
var imgPath = size + '/' + img + '.jpg';
if (filesize) {
return './images/' + imgPath;
} else {
request('http://www.othersite.com/images/' + imgPath).pipe(fs.createWriteStream('./images/' + imgPath))
return './images/' + imgPath;
}
}
If you can't use remote user's password for some reasons and need to use the identity key (RSA) for authentication, then programmatically executing the scp with child_process is good to go
const { exec } = require('child_process');
exec(`scp -i /path/to/key username#example.com:/remote/path/to/file /local/path`,
(error, stdout, stderr) => {
if (error) {
console.log(`There was an error ${error}`);
}
console.log(`The stdout is ${stdout}`);
console.log(`The stderr is ${stderr}`);
});
Related
I try to get text content from the webpage. For example Google.com
I write at console:
$ ('#SIvCob').innerText
and get:
"Google offered in: русский"
This is the text, what I find out. Now I want to save it to file (.txt).
Two moments: there is no only one item, that I search, actually 7-10. And, there is a refresh every second! I go to write a cycle.
I know about copy() function and about right click on the console and "Save As," but I need a CODE, which will do it automatically.
Thanks in advance.
The browser has no API to write to the file system since that would be a security risk. But you can use Nodejs and their File System API to write you text file.
You will also need to use the HTTP API to get the web content. And you will also need to parse your HTML, you can do it with fast-html-parser or any other module of your choice. (high5, htmlparser, htmlparser2, htmlparser2-dom, hubbub, libxmljs, ms/file, parse5, ...)
var http = require('http');
var fs = require('fs');
var parser = require('node-html-parser');
var options = {
host: 'www.google.com',
port: 80,
path: '/index.html'
};
var file = '/path/to/myFile.txt';
http.get(options, function(res) {
res.setEncoding('utf8');
var body = '';
res.on('data', function (chunk) {body += chunk});
res.on('end', function () {
var dom = parser.parse(body);
var text = dom.querySelector('#SIvCob').text;
fs.writeFile(file, text, function (err) {
if (err) throw err;
console.log('The file has been saved!');
});
});
});
I'm certain I'm missing something obvious, but the gist of the problem is I'm receiving a PNG from a Mapbox call with the intent of writing it to the file system and serving it to the client. I've successfully relayed the call, received a response of raw data and written a file. The problem is that my file ends up truncated no matter what path I take, and I've exhausted the answers I've found skirting the subject. I've dumped the raw response to the log, and it's robust, but any file I make tends to be about a chunk's worth of unreadable data.
Here's the code I've got at present for the file making. I tried this buffer move as a last ditch after several failed and comparably fruitless iterations. Any help would be greatly appreciated.
module.exports = function(req, res, cb) {
var cartography = function() {
return https.get({
hostname: 'api.mapbox.com',
path: '/v4/mapbox.wheatpaste/' + req.body[0] + ',' + req.body[1] + ',6/750x350.png?access_token=' + process.env.MAPBOX_API
}, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var mapPath = 'map' + req.body[0] + req.body[1] + '.png';
var map = new Buffer(body, 'base64');
fs.writeFile(__dirname + '/client/images/maps/' + mapPath, map, 'base64', function(err) {
if (err) throw err;
cb(mapPath);
})
})
});
};
cartography();
};
It is possible to rewrite your code in more compact subroutine:
const fs = require('fs');
const https = require('https');
https.get(url, (response)=> { //request itself
if(response) {
let imageName = 'image.png'; // for this purpose I usually use crypto
response.pipe( //pipe response to a write stream (file)
fs.createWriteStream( //create write stream
'./public/' + imageName //create a file with name image.png
)
);
return imageName; //if public folder is set as default in app.js
} else {
return false;
}
})
You could get original name and extension from url, but it safer to generate a new name with crypto and get file extension like i said from url or with read-chunk and file-type modules.
So I'am using a api wrapper package which again uses request for the api requests. Which works fine in most setups. But I want to use that package in a node-webkit environment and use a XHR in place of the request module. It would work with the API and works if I rewrite the module. But I don't wanna do that because of the update comfort. So forking is not an option for me. Is it possible to replace one function in a module without replacing the module.
var request = require('request');
var makeRequest = function(path, args, secure, callback, encoding) {
var maxlen = 2048;
var path = buildUrl(path, args);
if (path.length > maxlen) {
throw new Error("Request too long for google to handle (2048 characters).");
}
var options = {
uri: (secure ? 'https' : 'http') + '://some.api.com' + path
};
if (encoding) options.encoding = encoding;
if (config('proxy')) options.proxy = config('proxy');
if (typeof callback === 'function') {
request(options, function (error, res, data) {
if (error) {
return callback(error);
}
if (res.statusCode === 200) {
return callback(null, data);
}
return callback(new Error("Response status code: " + res.statusCode), data);
});
}
return options.uri;
};
module.exports = makeRequest;
So now i want to replace the request function oder the whole makeRequest function without changing the makeRequest. So basicly I want to overwrite the function.
edit: Add code Example.
take a look at rewire or proxyquire, that could solve your problems.
i dont see any other solution if the module you use uses makeRequest only internally, and even then this only works if makeRequest is required within the module (file) you require.
but keep in mind, this is probably bad, and should usually only be used for testing.
I've been trying this for a while, but I keep getting the error:
Error: Command failed: Invalid Parameter - /images
I installed ImageMagick and the gm package, so that's definitely not the problem.
gm(imageLocation)
.resize(100) // use your own width and height
.write('here.jpg', function (err) {
if (!err) console.log(' hooray! ');
else console.log(err);
});
imageLocation being ./images/3.jpg. Why does this error keep happening? I looked at the documentation
I'm on a Windows 32 bit machine. My server is supposed to get an image from a folder, resize it, and then display it. It seems like I have to write the resized photo and then display that, but the writing process always errors out and the image ends up being empty.
If there's a way to skip the writing part and just displaying the photo directly, that would be awesome too.
Thanks!
URL Query I used: http://localhost:8123/images/3.jpg
Complete code:
var querystring = require('querystring'); //used for parsing parts of urls
url = require('url');
http = require('http');
fs = require('fs');
gm = require('gm').subClass({ imageMagick: true });;
var server = http.createServer();
server.on('request', function(request, response){
var parsed_url = url.parse(request.url, true); //true gets the query as well
imageLocation = '.' + parsed_url.pathname;
gm(imageLocation)
.resize(100) // use your own width and height
.write('here.jpg', function (err) {
if (!err) console.log(' hooray! ');
else console.log(err);
});
if (getImage('here.jpg', response)){
//image is displayed
}
else{
respond404(parsed_url.pathname, response);
}
})
function respond404(path, response){
respond(404, "The requested path " + path + " was not found", response)
}
function getImage(location, response)
{
try{
var img = fs.readFileSync(location);
response.writeHead(200, {'Content-Type':'image/jpg'}); //parse this end
response.end(img, 'binary');
return true;
}catch(e){
return false;
}
}
server.listen(8123);
The answer Svbaker put can be used in Linux (maybe Mac as well?)
For Windows I got it to work by opening the command line in administrator mode and starting my server there.
I was able to get your code to work by changing how you required gm as follows:
var gm = require('gm');
I also had to remember to execute node with the correct permissions in my case:
sudo node server.js
Technically this is my first try in nodejs and frankly I am not sure if I am doing it right. I am creating a local server that will stream the output from a distant server. However, when I run my code and I enter a URL in the browser, the program fails with the following message:
events.js:45
throw arguments[1]; // Unhandled 'error' event
^
Error: ENOTFOUND, Domain name not found
at IOWatcher.callback (dns.js:74:15)
The URL I used was: 127.0.0.1:9000/http://www.yahoo.fr. And in the browser I had the following message:
No data received
Unable to load the webpage because the server sent no data.
Here are some suggestions:
Reload this web page later.
Error 324 (net::ERR_EMPTY_RESPONSE): The server closed the connection without sending any data.
Any help is greatly appreciated.
Here is the code:
var base, dest, node_client,
count = 0,
url = require('url'),
util = require('util'),
http = require('http'),
http_client = require('http'),
request = require('request'),
events = require('events'),
httpProxy = require('./lib/node-http-proxy'),
data_emitter = new events.EventEmitter();
httpProxy.createServer(9000, 'localhost').listen(8000);
http.createServer(function (req, res) {
if(!count)
{
base = url.parse(req.url).pathname;
node_client = http_client.createClient(80, base);
count++;
} else {
dest = req.url.substr(1, req.url.length -1);
}
request = node_client.request("GET", dest, {"host": base});
request.addListener("response", function (response) {
var body = "";
response.addListener("data", function (data) {
body +=data;
});
response.addListener("end", function () {
var out = JSON.parse(body);
if(out.length > 0) {
data_emitter.emit("out", out);
}
});
});
// request.close();
var listener = data_emitter.addListener("data", function(out) {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write(JSON.stringify(out));
res.close();
});
}).listen(9000);
Wild guess : your browser automatically requests 127.0.0.1:9000/favicon.ico and your program then tries to resolve favicon.ico which obviously fails and makes your program crash before it can send any data for the real request.
Why such tangled code?
This is a scenario where it makes sense to avoid nested callbacks, and use named functions. If you refactor the code, then people are more likely to be help you.
Can you do console.log(out) in your listener callback? Let us know if Node.js has any response data to return.
Well, for any newbie like me in this area, here is how I solved it. It's not clean and can be implemented in better way. Feel free to change, give suggestions.
Code:
var url = require('url'),
http = require('http'),
request = require('request'),
httpProxy = require('./lib/node-http-proxy'),
des = '',
util = require('util'),
colors = require('colors'),
is_host = true;
httpProxy.createServer(9000, 'localhost').listen(8000);
var server = http.createServer(function (req, res) {
var pathname = '';
if(is_host) {
dest = req.url.substr(0, req.url.length -1);
pathname = dest;
is_host = false;
} else {
pathname = req.url.substr(0, req.url.length);
if(pathname.charAt(0) == "/") {
console.log('new request');
console.log(pathname);
pathname = dest + pathname;
}
}
console.log(pathname);
request.get({uri: pathname}, function (err, response, html) {
res.end(html);
});
console.log('fetched from ' + pathname);
});
server.listen(9000);