I need to make a crawler. For http request i used to do this.
var http=require('http');
var options={
host:'http://www.example.com',
path:'/foo/example'
};
callback=function(response){
var str='';
response.on('data',function(chunk){
str+=chunk;
});
response.on('end', function () {
console.log(str);
});
}
http.request(options, callback).end();
but I have to make a crawler for https://example.com/foo/example
If I am using the same for https://example.com/foo/example it is giving this error
events.js:72
throw er; // Unhandled 'error' event
^
Error: getaddrinfo ENOTFOUND
at errnoException (dns.js:37:11)
at Object.onanswer [as oncomplete] (dns.js:124:16)
I'd recommend this excellent HTTP Request module: http://unirest.io/nodejs.html
You can install it with:
npm install -g unirest
Here's some example Node code with Unirest:
var url = 'https://somewhere.com/';
unirest.get(url)
.end(function(response) {
var body = response.body;
// TODO: parse the body
done();
});
...so to get the HTML at www.purple.com you'd do this:
#!/usr/bin/env node
function getHTML(url, next) {
var unirest = require('unirest');
unirest.get(url)
.end(function(response) {
var body = response.body;
if (next) next(body);
});
}
getHTML('http://purple.com/', function(html) {
console.log(html);
});
Related
I am attempting to grab data from an API from openWeatherAPI with a correct api key and query (I checked with Postman to ensure the call is correct), but ran into a syntax error. When I try to call the on() function inside of my https.get callback function, I am met with the following error in my terminal:
response.on("data", (data) => {
^
TypeError: Cannot read properties of undefined (reading 'on')
at ClientRequest.<anonymous> (C:file-path\api-prac\app.js:16:18)
at Object.onceWrapper (node:events:628:26)
at ClientRequest.emit (node:events:513:28)
at HTTPParser.parserOnIncomingClient [as onIncoming] (node:_http_client:693:27)
at HTTPParser.parserOnHeadersComplete (node:_http_common:128:17)
at TLSSocket.socketOnData (node:_http_client:534:22)
at TLSSocket.emit (node:events:513:28)
at addChunk (node:internal/streams/readable:315:12)
at readableAddChunk (node:internal/streams/readable:289:9)
at TLSSocket.Readable.push (node:internal/streams/readable:228:10)
My code:
const express = require("express");
const https = require("https");
const app = express()
// what should happen when user tries to go to home page
app.get("/", function(req, res) {
const url = "https://api.openweathermap.org/data/2.5/weather?q=London&appid=my-api-key";
https.get(url, function(req, response) {
console.log("blah blah repsonse");
response.on("data", (data) => {
console.log(data);
// const weatherDatta = JSON.parse(data)
/* extra code will be put here to send a response */
})
});
res.send("server is up");
}
app.listen(3000, function() {
console.log("app running on server 3000");
})
I tried looking at the documentation shown on the https://nodejs.org/api/https.html website, but was unable to find anything that helped outside of what I was already doing with my code.
The arguments for your https.get() callback are wrong. It should be this:
https.get(url, function(response) {
response.on('data', ...);
});
There is no second argument so when you try to make one, it's undefined and does not work.
Code example in the doc here.
Note also that there is no guarantee that you get the entire response in the first data event. The response may arrive in chunks so if you're trying to get the whole response, you should be accumulating all the data events and then processing them all in the end event. And, you should be handling errors in multiple places:
https.get(url, function(response) {
let result = "";
response.on('data', data => {
result += data.toString();
}).on('end', () => {
try {
let weatherData = JSON.parse(result);
// use the weatherData here
} catch(e) {
console.log(e);
// handle JSON parsing error here
}
}).on('error', err => {
console.log(err);
// handle http request error here
});
});
Note, using an http request library such as got() or node-fetch() or even fetch() which is built-in to the newest versions of nodejs will make this code much simpler because they will retrieve the entire response for you and are promise based which makes a number of things including error handling much simpler.
Note how much simpler this is with the got() library.
got(url).json().then(weatherData => {
// use weatherData here
}).catch(err => {
console.log(err);
// handler error here
});
I have a Heroku server with node.js and express that pings a website's API every second. This works fine for hours at a time, but every once in a while I'll get this error:
2018-01-10T02:19:28.579566+00:00 app[web.1]: events.js:141
2018-01-10T02:19:28.579578+00:00 app[web.1]: throw er; // Unhandled 'error' event
2018-01-10T02:19:28.579579+00:00 app[web.1]: ^
2018-01-10T02:19:28.579581+00:00 app[web.1]:
2018-01-10T02:19:28.579582+00:00 app[web.1]: Error: connect ETIMEDOUT 45.60.11.241:443
2018-01-10T02:19:28.579583+00:00 app[web.1]: at Object.exports._errnoException (util.js:907:11)
2018-01-10T02:19:28.579584+00:00 app[web.1]: at exports._exceptionWithHostPort (util.js:930:20)
2018-01-10T02:19:28.579585+00:00 app[web.1]: at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1078:14)
2018-01-10T02:19:28.684990+00:00 heroku[web.1]: Process exited with status 1
Sometimes the error is ETIMEDOUT but sometimes it's other things (can't remember right now).
Some other post I read made me think maybe this is a problem?
app.listen(app.get('port'), function() {
console.log("Node app is running at localhost:" + app.get('port'))
})
Or maybe it's the part inside the API call loop?
try
{
async.series([
function(callback) {
apiQuery( callback, method, params);
},
], function(error, results) {
console.log(results)
});
}
catch(e)
{
console.log("something went wrong!")
}
Not sure why the try catch isn't catching the error if it's this.
Maybe it's how I'm starting the loop?
runLoop()
//start looping the api data pulls
function runLoop() {
setInterval(apiLoop, 1000)
}
Would it be better to have apiLoop call itself in the callback function? Or would that create nested functions that keep using increasingly larger memory?
Here is the api call code:
function apiQuery( callback2, method, params )
{
if ( ! params ) params = [];
var host_name = 'www.host.com';
var url = '/Api/' + method;
if ( params ) url += "/" + params.join('/');
var options = {
host: host_name,
path: url,
};
callback = function(response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
return callback2(null, str);
});
}
https.request(options, callback).end();
}
Maybe ETIMEOUT is caused by the website's server, anyway you can catch the error
const req = https.request(options, callback)
req.on('error', (e) => {
console.error(e);
});
req.end();
I tried to build a chat box server by node.js. When the browser requestes the page, it workes well at first. But when I refresh the page, the Server crashes.
Below is the error message:
events.js:183
throw er; // Unhandled 'error' event
^
Error: read ECONNRESET
at _errnoException (util.js:1022:11)
at TCP.onread (net.js:615:25)
I used the node --inspect index.js, but could not find the point.
Below is the code of index.js:
const http = require('http');
const fs = require('fs');
const extract = require('./extract');
const wss = require('./websockets-server');
var handleError = function (err,res) {
res.writeHead(404);
res.end();
}
var server = http.createServer(function (req, res) {
console.log("Responding to a request.");
var filePath = extract(req.url);
console.log("filePath:"+filePath);
fs.readFile(filePath,function (err,data) {
if(err){
handleError(err,res);
return;
}else {
res.end(data);
}
})
})
server.listen(3000);
When I comment the 4th line, the import of websockets-server. Server works well when I refresh the page. Maybe it's about the websocket while it works without websocket.
Below is code of websockets-server.js:
const WebSocket = require('ws');
var WebSocketServer = WebSocket.Server;
var port = 3001;
var ws = new WebSocketServer({
port:port
});
var message = [];
console.log('websockets server started');
ws.on('connection', function (socket) {
console.log('client connection established');
message.forEach(function (msg) {
socket.send(msg);
})
socket.on('message', function (data) {
console.log('message received: ' + data);
message.push(data);
ws.clients.forEach(function (clientSocket) {
clientSocket.send(data);
});
});
});
Does the problem is about the websocket? Whether should I do process when the client shutdown the connection with the server while refreshing the page.
extract.js below:
const path = require('path');
var extractFilePath = function (url) {
var filePath;
var fileName = 'index.html';
if(url.length > 1){
fileName = url.substring(1);
}
console.log('The fileName is: ' + fileName);
filePath = path.resolve(__dirname, 'app', fileName);
return filePath;
}
module.exports = extractFilePath;
I guess that you maybe execute var ws = new WebSocket("ws://localhost:3001"); in html file. I haven't figured out exact reason about your error as I'm not proficient in WebSocket. But there is a solution:
window.onbeforeunload = function () {
ws.close();
}
close connection before reload, then the error will not reappear.
You need to add an error listener on the socket. Error listener only on the websocket instance does not help in this case.
socket.on('error', function(e){
console.log(e);
});
The ECONNRESET error means that the other side (browser) closed the connection abruptly. On browser refresh, browser simple killed the connection with the websocket server.
To solve this, you have to listen for the error event on the websocket server instance.
// listen for "error" event so that the whole app doesn't crash
wss.on("error", function(error){
console.log(error);
}
I was having the same problem, but it resolved after this command:
npm install #ionic/app-scripts#nightly --save-dev
I'm getting this error and unable to fix this.
events.js:72
throw er; // Unhandled 'error' event
^
Error: connect ECONNREFUSED
at errnoException (net.js:901:11)
at Object.afterConnect [as oncomplete] (net.js:892:19)
var http = require('http');
var options = {
host: 'localhost',
port: '8081',
path: '/index.htm'
};
var callback = function(response){
var body = '';
response.on('data', function(data) {
body += data;
});
response.on('end', function() {
console.log(body);
});
}
var req = http.request(options, callback);
req.end();
var app = require('http').createServer(handler);
var Rcon = require('rcon');
var url = require('url');
app.listen(7777);
console.log('Server started.');
function handler (req, res) {
console.log('New connection!');
res.writeHead(200);
var urlParts = url.parse(req.url, true);
var server = urlParts.query;
var conn = new Rcon(server.ip, server.port, server.password);
conn.on('auth', function() {
conn.send(server.command);
console.log('Sent command!');
}).on('response', function(data) {
res.end(data);
console.log('Response: '+data);
}).on('error', function(data) {
res.end('error');
console.log('Error: '+data);
}).on('err', function(data) {
res.end('error');
console.log('Error: '+data);
});
conn.connect();
}
All of that works perfectly on localhost but when I upload to a remote server and run it, it responds with error in browser and with this in the console:
Server started. New connection! Error: TypeError: Object 0�P�q has no
method 'writeInt32LE' New connection! Error: Error: EINVAL, Invalid
argument
First guess: rcon module contains compiled extensions and you checked the files of node_modules/rcon into git and then tried to run them on a different CPU architecture. You need to run npm rebuild on the remote server for a quick fix and to get your node_modules folder out of your git repository as the correct long-term fix.