I am attempting to grab data from an API from openWeatherAPI with a correct api key and query (I checked with Postman to ensure the call is correct), but ran into a syntax error. When I try to call the on() function inside of my https.get callback function, I am met with the following error in my terminal:
response.on("data", (data) => {
^
TypeError: Cannot read properties of undefined (reading 'on')
at ClientRequest.<anonymous> (C:file-path\api-prac\app.js:16:18)
at Object.onceWrapper (node:events:628:26)
at ClientRequest.emit (node:events:513:28)
at HTTPParser.parserOnIncomingClient [as onIncoming] (node:_http_client:693:27)
at HTTPParser.parserOnHeadersComplete (node:_http_common:128:17)
at TLSSocket.socketOnData (node:_http_client:534:22)
at TLSSocket.emit (node:events:513:28)
at addChunk (node:internal/streams/readable:315:12)
at readableAddChunk (node:internal/streams/readable:289:9)
at TLSSocket.Readable.push (node:internal/streams/readable:228:10)
My code:
const express = require("express");
const https = require("https");
const app = express()
// what should happen when user tries to go to home page
app.get("/", function(req, res) {
const url = "https://api.openweathermap.org/data/2.5/weather?q=London&appid=my-api-key";
https.get(url, function(req, response) {
console.log("blah blah repsonse");
response.on("data", (data) => {
console.log(data);
// const weatherDatta = JSON.parse(data)
/* extra code will be put here to send a response */
})
});
res.send("server is up");
}
app.listen(3000, function() {
console.log("app running on server 3000");
})
I tried looking at the documentation shown on the https://nodejs.org/api/https.html website, but was unable to find anything that helped outside of what I was already doing with my code.
The arguments for your https.get() callback are wrong. It should be this:
https.get(url, function(response) {
response.on('data', ...);
});
There is no second argument so when you try to make one, it's undefined and does not work.
Code example in the doc here.
Note also that there is no guarantee that you get the entire response in the first data event. The response may arrive in chunks so if you're trying to get the whole response, you should be accumulating all the data events and then processing them all in the end event. And, you should be handling errors in multiple places:
https.get(url, function(response) {
let result = "";
response.on('data', data => {
result += data.toString();
}).on('end', () => {
try {
let weatherData = JSON.parse(result);
// use the weatherData here
} catch(e) {
console.log(e);
// handle JSON parsing error here
}
}).on('error', err => {
console.log(err);
// handle http request error here
});
});
Note, using an http request library such as got() or node-fetch() or even fetch() which is built-in to the newest versions of nodejs will make this code much simpler because they will retrieve the entire response for you and are promise based which makes a number of things including error handling much simpler.
Note how much simpler this is with the got() library.
got(url).json().then(weatherData => {
// use weatherData here
}).catch(err => {
console.log(err);
// handler error here
});
Related
i am using openweathermap API. When i try to parse response data to json it gives me an error.
i dont know what to do about this can anyone help??
CODE:
const express = require("express");
const https = require("https");
const app = express();
app.get("/", function (req, res) {
const url = "https://api.openweathermap.org/data/2.5/forecast?appid=faa02526fbe231fa9d2dc1aa991a26f2&q=London";
https.get(url, function (response) {
console.log(response.statusCode);
response.on("data", function (data) {
JSON.parse(data);
});
});
res.send("upp and running");
});
app.listen(3000, function () {
console.log("up and running");
});
console output:
up and running
200
undefined:1
SyntaxError: Unexpected end of JSON input
at JSON.parse (<anonymous>)
at IncomingMessage.<anonymous> (E:\webdevel\WeatherProject\app.js:14:18)
at IncomingMessage.emit (events.js:375:28)
at IncomingMessage.Readable.read (internal/streams/readable.js:500:10)
at flow (internal/streams/readable.js:982:34)
at resume_ (internal/streams/readable.js:963:3)
at processTicksAndRejections (internal/process/task_queues.js:82:21)
[nodemon] app crashed - waiting for file changes before starting...
Your problem is that you think that .on("data", handler) will only be called once with the full reply. That's not how the API works, check the documentation.
Basically, the response body is streamed in. The data event can be emitted many times, each with a chunk of the body response. Therefore, buffer all the data and wait for the response to finish:
https.get(url, function (response) {
console.log(response.statusCode);
let resultData = '';
response.on('data', data => resultData += data);
response.on('end', () => {
// Now we got the whole response body
JSON.parse(resultData);
});
});
PROBLEM
I am attempting to get multiple files from the AWS S3 bucket. I am using the AWS SDK v3, which makes use of the GetObjectCommand() API. This command returns a ReadableStream which I am piping into the res object so a single image loads successfully when I make a 'GET' request to my single image endpoint. However, when I create a loop and make multiple calls and attempt to pipe the data to res, the whole thing crashes and throws the error:
events.js:292
throw er; // Unhandled 'error' event
^
Error [ERR_STREAM_WRITE_AFTER_END]: write after end
at writeAfterEnd (_http_outgoing.js:668:15)
at write_ (_http_outgoing.js:680:5)
at ServerResponse.write (_http_outgoing.js:661:15)
at IncomingMessage.ondata (internal/streams/readable.js:719:22)
at IncomingMessage.emit (events.js:315:20)
at IncomingMessage.EventEmitter.emit (domain.js:467:12)
at addChunk (internal/streams/readable.js:309:12)
at readableAddChunk (internal/streams/readable.js:284:9)
at IncomingMessage.Readable.push (internal/streams/readable.js:223:10)
at HTTPParser.parserOnBody (_http_common.js:139:24)
at TLSSocket.socketOnData (_http_client.js:509:22)
at TLSSocket.emit (events.js:315:20)
at TLSSocket.EventEmitter.emit (domain.js:467:12)
at addChunk (internal/streams/readable.js:309:12)
at readableAddChunk (internal/streams/readable.js:284:9)
at TLSSocket.Readable.push (internal/streams/readable.js:223:10)
at TLSWrap.onStreamRead (internal/stream_base_commons.js:188:23)
Emitted 'error' event on ServerResponse instance at:
at ServerResponse.onerror (internal/streams/readable.js:760:14)
at ServerResponse.emit (events.js:315:20)
at ServerResponse.EventEmitter.emit (domain.js:467:12)
at writeAfterEndNT (_http_outgoing.js:727:7)
at processTicksAndRejections (internal/process/task_queues.js:81:21) {
code: 'ERR_STREAM_WRITE_AFTER_END'
}
WHAT I HAVE TRIED
Since I am using async/await, I tried to have an async callback in the forEach loop which makes a call for every single key in the imageKeys array I send in the 'GET' request, but this doesn't work. I'm not sure how to solve this issue of being able to pipe the multiple data streams to res without it closing. Would be great to get some help! NOTE: getFile() is a custom function imported in filesController.js.
EXPRESS CONTROLLERS (filesController.js)
For single file (working):
files.get("/image/:key", async (req, res) => {
try {
const key = req.params.key;
// data returned is a ReadableStream. check out Node.js documentation for Readable Streams
const data = await getFile(carfixBucket, key);
// a readable stream can pipe data to a writable stream 'res'.
// I.E you READ (copy data from) a readable stream and you WRITE (copy data to) a writable stream.
// a
data.pipe(res);
// res.status(200).send(data)
} catch (err) {
res.status(500).send(err);
}
});
For multiple files(not working):
files.get("/images", async (req, res) => {
try {
const { imageKeys } = req.body;
console.log(imageKeys)
imageKeys.forEach( async (key) => {
const data = await getFile(carfixBucket, key)
data.pipe(res)
})
// res.status(200).send("Success")
} catch (err) {
res.status(500).send(err);
}
})
Custom Function
getFile is a function I created and exported which accepts the name of the S3 bucket and the key(name) of the file stored in the bucket. It returns a ReadableStream.
const getFile = async (bucketName, key) => {
const bucketParams = {
Bucket: bucketName,
Key: key,
};
try {
// the response returned from Bucket contains a ReadableStream on 'Body' field. to convert to a string if necessary
const response = await S3.send(new GetObjectCommand(bucketParams));
return response.Body;
} catch (err) {
return err;
}
};
I'm using plain node.js script to make a POST request and the link that I make request to takes around 1 to 1.2 mins to respond and in that time I get this error Error: socket hang up.
I get the socket hang up error around the same time as I would have gotten the response from the server which is around 1 to 1.2 mins
I've tried setting connection: "keep-alive" in headers and setting timeout to 200000ms in options but no luck. Any help?
BTW the request works fine if i use axios
Here's my script
const fs = require("fs")
const path = require("path")
const https = require("https")
const file = fs.createWriteStream(path.join(__dirname, "data.csv"))
const options = {
hostname: "example.com",
path: "/example",
method: "POST",
headers: {
<<headers>>
}
<<some other options>>
}
const req = https.request(options, response => {
response.on("data", function (chunk) {
console.log("started...")
chunk.pipe(file)
})
response.on("end", function () {
file.on("finish", function () {
file.close()
})
})
})
req.on("error", e => {
console.error(e)
})
req.end()
This is the full error:
Error: socket hang up
at connResetException (internal/errors.js:612:14)
at TLSSocket.socketOnEnd (_http_client.js:493:23)
at TLSSocket.emit (events.js:326:22)
at endReadableNT (_stream_readable.js:1308:12)
at processTicksAndRejections (internal/process/task_queues.js:80:21) {
code: 'ECONNRESET'
}
Found the mistake i was making
First instead of doing chunk.pipe(file) inside response.on('data')
i directly did response.pipe(file) and it worked i don't know why exactly, i'm guessing it's because response is the stream and chunk is just the buffer, if anybody knows then please tell me.
Second i was setting the request body inside options, i had to set the request body parameter like this req.write(formData) where formData is a string that i want to send as request body.
I'm a new developer learning how to work with API's and I've run into this error a few times now that keeps crashing Node:
SyntaxError: Unexpected end of JSON input
at JSON.parse (<anonymous>)
at IncomingMessage.<anonymous> (/Users/SA/Desktop/pokeApp/app.js:13:34)
at IncomingMessage.emit (events.js:314:20)
at IncomingMessage.Readable.read (_stream_readable.js:513:10)
at flow (_stream_readable.js:986:34)
at resume_ (_stream_readable.js:967:3)
at processTicksAndRejections (internal/process/task_queues.js:80:21)
I'm not really sure how to resolve it or what the problem is exactly... my JS code looks like this:
const express = require("express");
const https = require("https");
const app = express();
app.get("/", (req,res) => {
const url = "https://pokeapi.co/api/v2/pokemon/1/";
https.get(url, function(response){
console.log(response.statusCode);
response.on("data", (data) =>{
const pokemon = JSON.parse(data);
console.log(pokemon);
})
})
res.send("server running");
})
app.listen(3000, () => {
console.log("Port 3000");
})
This is basically the same setup I used for a weatherAPI and had no issues.
I also checked the JSON with JSON lint to see if there were any problems and it came back okay.
You need to wait for the full response to come in - it's a large file, so it may not come in all at once:
https.get(url, function(response){
let result = '';
response.on("data", (data) =>{
result += data;
});
response.on('end', () => {
const pokemon = JSON.parse(result);
console.log(pokemon);
});
})
I'm trying to get json data from the iex api. I am using the inline editor for googles dialogflow and when trying to get the json from the api, I keep getting the error:
Error: Parse Error
at Error (native)
at Socket.socketOnData (_http_client.js:363:20)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at readableAddChunk (_stream_readable.js:176:18)
at Socket.Readable.push (_stream_readable.js:134:10)
at TCP.onread (net.js:559:20)
The console log shows that I'm requesting the correct path to get the json request (in this case I wanted the microsoft json info
API Request: api.iextrading.com/1.0/stock/MSFT/company
I am not sure why the json is not getting read in correctly but I think the error is occurring because the body var of my code is not receiving information from http request. I'm just not sure what is wrong with my code.
Here is my code:
'use strict';
const http = require('http');
const functions = require('firebase-functions');
const host = 'api.iextrading.com';
exports.dialogflowFirebaseFulfillment = functions.https.onRequest((req, res) => {
// Get the company
let company = req.body.queryResult.parameters['company_name']; // city is a required param
// Call the iex API
callCompanyApi(company).then((output) => {
res.json({ 'fulfillmentText': output });
}).catch(() => {
res.json({ 'fulfillmentText': `I don't know this company`});
});
});
function callCompanyApi (company) {
return new Promise((resolve, reject) => {
// Create the path for the HTTP request to get the company
let path = '/1.0/stock/' + company + '/company';
console.log('API Request: ' + host + path);
// Make the HTTP request to get the company info
http.get({host: host, path: path}, (res) => {
let body = ''; // var to store the response chunks
res.on('data', (d) => { body += d; });// store each response chunk
res.on('end', () => {
// After all the data has been received parse the JSON for desired data
console.log(body);
let response = JSON.parse(body);
let description = response['description'];
// Create response
let output = `${description}`
// Resolve the promise with the output text
console.log(output);
resolve(output);
});
res.on('error', (error) => {
console.log(`Error calling the iex API: ${error}`)
reject();
});
});
});
}
If you are using the Inline Dialogflow Editor, then you're running on Cloud Functions for Firebase (or Firebase Cloud Functions). By default, there is a restriction on the base plan that you cannot make network calls outside of Google's network.
To get around this, you can upgrade your Firebase plan to a subscription such as the Blaze Plan. This does require a credit card on file, however the base level of usage should be part of the free tier.
You can also run your webhook anywhere else, as long as there is a web server with a valid SSL certificate that can handle HTTPS requests. If you want to run it locally, you can even use something like ngrok.