I'm trying to scrape a web-page for some data and I managed to post a request and got the right data. The problem is that I get something like :
"Kannst du bitte noch einmal ... erzýhlen, wie du wýhrend der Safari einen Lýwen verjagt hast?"
normally erzählen - während, so Ä,Ö,ß,Ü are not showing correctly.
here is my code:
var querystring = require('querystring');
var iconv = require('iconv-lite')
var request = require('request');
var fs = require('fs');
var writer = fs.createWriteStream('outputBodyutf8String.html');
var form = {
id:'2974',
opt1:'',
opt2:'30',
ref:'A1',
tid:'157',
tid2:'',
fnum:'2'
};
var formData = querystring.stringify(form);
var contentLength = formData.length;
request({
headers: {
'Content-Length': contentLength,
'Content-Type': 'application/x-www-form-urlencoded'
},
uri: 'xxxxxx.php',
body: formData,
method: 'POST'
}, function (err, res, body) {
var utf8String = iconv.decode(body,"ISO-8859-1");
console.log(utf8String);
writer.write(utf8String);
});
how to get the HTML body in with the correct letters?
How do I find out the correct encoding of a response?
I went to the website you are attempting to scrape, and found this:
And another character encoding declaration here:
This website defined two different charater encodings! Which do I use?
Well, this doesn't apply to you.
When reading an HTML file from a local machine, then the charset or content-type defined in the meta tags will be used for encoding.
Since you are retrieving this document, over HTTP, the files will be encoded according to the response header.
Here's the reponse header I received after visiting the website.
As you can see, they don't have a defined character set. It should be located in the Content-Type property. Like this:
Since they don't have any indicated charset in the response header, then, according to this post, it should use the meta declaration.
But wait, there was two meta charset declarations.
Since the compiler reads the file top to bottom, the second declared charset should be used.
Conclusion: They use UTF-8
Also, I don't think you need the conversion. I may be wrong, but you should just be able to access the response.
request({
headers: {
'Content-Length': contentLength,
'Content-Type': 'application/x-www-form-urlencoded'
},
uri: 'xxxxxx.php',
body: formData,
method: 'POST'
}, function (err, res, body) {
console.log(body);
writer.write(body);
});
Edit: I don't believe the error is on their side. I believe it's on your side. Give this a try:
Remove the writer:
var writer = fs.createWriteStream('outputBodyutf8String.html');
And in the request callback, replace everything with this:
function (err, res, body) {
console.log(body);
fs.writeFile('outputBodyutf8String.html', body, 'utf8', function(error) {
if(error)
console.log('Error Occured', error);
);
}
All the code should look like this:
var querystring = require('querystring');
var iconv = require('iconv-lite')
var request = require('request');
var fs = require('fs');
var form = {
id:'2974',
opt1:'',
opt2:'30',
ref:'A1',
tid:'157',
tid2:'',
fnum:'2'
};
var formData = querystring.stringify(form);
var contentLength = formData.length;
request({
headers: {
'Content-Length': contentLength,
'Content-Type': 'application/x-www-form-urlencoded'
},
uri: 'xxxxxxx.php',
body: formData,
method: 'POST'
}, function (err, res, body) {
console.log(body);
fs.writeFile('outputBodyutf8String.html', body, 'utf8', function(error) {
if(error)
console.log('Error Occured', error);
);
}
Related
I've been trying to send a JSON data using fetch but the backend receives an empty object.
In my Client JS I have
const user = "company1";
const username = "muneeb";
const data = {user, username};
fetch("http://127.0.0.1:3000/users/api/login", {
method: 'POST',
body: JSON.stringify(data)
}).then((response) => {
console.log(response);
});
The server side has:
router.post('/users/api/login', async (req, res, next) => {
try {
// console.log(request.body);
const request = JSON.stringify(req.body);
let imageTitles = [];
console.log(request);
*its random from here on out but you get the idea*
await components.getImages(imageTitles);
const finalKey = imageTitles.join("");
let images = await components.output(req.body.user ,req.body.username);
res.send(components.jsonConverter(imageTitles, images)); //---Top priority
db.setPassword(req.body.user, req.body.username , finalKey);
} catch (err) {
console.log(err);
res.send(err).sendStatus(500);
};
})
A few things I have already tried :
It works perfectly in Insomnia(postman).
express.json() is present , it helped me go from undefined to blank JSON.
I have enabled cors settings.
That's it for now.
The body parser express.json will only be applied for requests with Content-Type: application/json. You have to add the content type to your fetch call:
fetch("http://127.0.0.1:3000/users/api/login", {
method: 'POST',
body: JSON.stringify(data),
headers: {
'Content-Type': 'application/json'
}
})
I would like to access the content of a file I uploaded to GitHub via Node.js.
The GitHub repository is private, so I have generated an access token at https://github.com/settings/tokens
Unfortunately, I keep getting a 404 – Not found error. What am I doing wrong?
const request = require('request');
const URL = 'https://raw.githubusercontent.com/myuser/myrepo/master/myfile.js';
const TOKEN = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx';
var options = {
url: URL,
headers: {
'Authorization': TOKEN
}
};
function callback(error, response, body) {
console.log(response.statusCode);
console.error(error);
console.log(body);
}
request(options, callback);
Thanks to the comment of #bhavesh27 I figured, I was missing a "token " in my header.
const request = require('request');
const URL = 'https://raw.githubusercontent.com/myuser/myrepo/master/myfile.js';
const TOKEN = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx';
var options = {
url: URL,
headers: {
'Authorization': 'token ' + TOKEN
}
};
function callback(error, response, body) {
console.log(response.statusCode);
console.error(error);
console.log(body);
}
request(options, callback);
I am trying to do query to find a account using rest services of the target application name hexion.
When I am running it is giving Invalid uri error.
The url that I tested in postman is like below
https://ekaa-dev1.fa.us6.oraclecloud.com/crmRestApi/resources/11.13.18.05/accounts?q=OrganizationName = Hexion
and in postman I am getting response too.
But I feel somewhere in my code I am doing some syntax error but not able to find that
//nodejs v4.2.6
console.log("Hello, World!");
var Request = require("request");
var serviceUserName="msonawane#gmail.com";
var password="Welcome01";
var personalDataURL="https://ekaa-dev1.fa.us6.oraclecloud.com/crmRestApi/resources/11.13.18.05/accounts";
var option1 = {
uri: personalDataURL,
qs: {
q:{OrganizationName:"Hexion"}
},
headers: {
"Authorization" : auth,
"Content-Type": 'application/json',
"Accept":'application/json'
}
};
var auth = `Basic ` + new Buffer(serviceUserName+`:`+password).toString(`base64`);
Request.get(option1, { json: true },
(error, response, body) => {
console.log(response);
//console.log(response.url);
if (error) { return console.log(body,error); }
console.log(body.url);
console.log(body.explanation);
});
I expect it to return response after successful get
Please let me know error, I have changed the auth credentials so once you find anything to be corrected let me for the above code, I will try with right credentials and update you
request.get method expects first parameter as url, but you are passing options1 obj, it couldn't find url hence it is giving error "Invalid uri /".
You can append query parameter to url OR use querystring npm
var personalDataURL= "https://ekaa-dev1.fa.us6.oraclecloud.com/crmRestApi/resources/11.13.18.05/accounts?q=OrganizationName=Hexion"
request({
headers: {
"Authorization" : auth,
"Content-Type": 'application/json',
"Accept":'application/json'
},
uri: personalDataURL,
method: 'GET'
}, function (err, res, body) {
//it works!
});
For more details, refer request
From my API(nodejs), I'm accessing a third-party API (using http) to download files.
The service returns a Base64 string, chopped into smaller pieces, to be able to handle larger files.
Is it possible to do multiple http-requests (loop ?) to the third-party service, send each piece in response, to the browser until there is no longer any response from the third-party service?
The reason i want to do this, is because I don't want to consume to much memory on the node server.
I will put the pieces back together in the browser.
Any suggestions on how to do this?
See my current code below.
var request = require('request');
router.post('/getfiledata', function(req, res) {
var fileid = req.body.fileid;
var token = req.headers.authorization;
getFileData(req, res, dbconfig, fileid, token, function(err, chunkOfFile) {
if (err) {
res.status(500).send({
status: 500,
message: err
});
return;
}
res.send(chunkOfFile);
});
});
function getFileData(req, res, dbconfig, fileid, token, next) {
var url ="http://*ip*/service/rest/getfiledata";
var reqbody = {
fileId: fileid
};
var options = {
url: url,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': token
},
json: true,
body: reqbody
};
/*SOME LOOP HERE TO GET EACH CHUNK AND SEND TO BROWSER*/
request(options, function(err, resp, body) {
if (err) {
console.log(err);
next(err, undefined);
return;
} else {
next(undefined, body)
};
});
};
I think you need Socket.io to push chunks to the browser.
Server :
socket.send("chunk", chunkOfFile)
Client :
let fullString = ""
socket.on("chunk", chunkOfFile => fullString += chunkOfFile )
Something like that
The request library you are using allows for streaming of data from one source to another. Check out the documentation on github.
Here is an example from that page:
request
.get(source)
.on('response', function(response) {
console.log(response.statusCode) // 200
console.log(response.headers['content-type']) // 'image/png'
})
.pipe(request.put(destination))
You may choose to use the http module from Nodejs, as it implements the EventEmitter class too.
I ended up doing a recursive loop from the client. Sending http-requests to my API(node) until the response no longer returns any base64 data chunks.
Thank you guys!
I have tried to get all facebook pages list. But I got an error.
Error is : request is not defined
Code :
var url = 'https://graph.facebook.com/me/accounts';
var accessToken = req.user.facebookAccessToken;
var params = {
access_token: accessToken,
};
request.get({ url: url, qs: params}, function(err, resp, pages) {
// console.log(resp);
pages = JSON.parse(pages);
})
The error you're getting isn't related to Facebook.
"request is not defined" means you're trying to do something with the variable request (call its .get() function in this case) but the variable request hasn't been set anywhere.
Your tags indicate this is in Node. Do you have this line already?
var request = require('request');
If not, add that before the code in your question. That loads the 'request' module you're trying to use.
You got the issue on request ,,
install npm request
require('request').get({
uri: url,
headers: {'content-type': 'application/x-www-form-urlencoded'},
body: require('querystring').stringify(params)
}, function (err, resp, body) {
})