From the example code on the gtfs-realtime-bindings library's documentaion,
I tried to put similar logic inside my router but was not able to take the result out of request()
route.get('/', (req, res) => {
const data = [];
request(requestSettings, function (error, response, body) {
if (!error && response.statusCode == 200) {
const feed = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(body);
feed.entity.forEach(function(entity) {
if (entity.tripUpdate) {
data.push(entity.tripUpdate);
}
});
}
res.status(200).json( data );
}); // empty array returns
Then I tried to use axios, but decode() raise "Error: illegal buffer"
const realtimeRequest = () => {
axios.get('https://opendata.hamilton.ca/GTFS-RT/GTFS_TripUpdates.pb')
.then(function (body) {
const feed = GtfsRealtimeBindings.transit_realtime.FeedMessage.decode(body.data);
// body.data looks like this '\x03436\x12\x041219 ����\x06'
return feed;
}).catch(error => console.log(error));
}
route.get('/', (req, res) => {
realtimeRequest()
.then(() => {
res.status(200).json( data );
})
}
/**
Error: illegal buffer
at create_typed_array (/node_modules/protobufjs/src/reader.js:47:15)
at create_buffer (/node_modules/protobufjs/src/reader.js:69:19)
at Function.create_buffer_setup (/node_modules/protobufjs/src/reader.js:70:11)
at Function.decode (/node_modules/gtfs-realtime-bindings/gtfs-realtime.js:134:34) **/
For the axios request, try adding responseType: 'arraybuffer'.
This worked for me for the same issue stated for "axios, but decode() raise 'Error: illegal buffer'"
Related
The problem:
I have a function that maps over countries and regions and creates an array of urls, then makes a GET request to each one. I want to save the responses in a single json file, and I want this function to handle that as well.
Expected results:
I expected to be able to run the function as needed (like when source data is updated), and get a new or updated local json file with all the data objects in one array.
Actual results:
A file with only one record, an array with the last response object.
What I've tried:
I tried using fs.writeFile and fs.readFile. I did not get any errors, but the resulting file had only one record, even though console showed all the requests being made. It seemed that each response was being written over the previous.
Minimum reproducable (node.js) example:
const fs = require('fs')
// subset of countries and codes for demo purposes
const countryDirList = [
'antarctica',
'central-asia',
]
const fbCountryCodes = [
{ "region": "antarctica", "codes": ["ay", "bv"] },
{ "region": "central-asia", "codes": ["kg", "kz"] },
]
const callingUrlsInSequence = async () => {
fs.writeFile('./test.json', '[]', function (err) {
if (err) throw err
console.log('File - test.json - was created successfully.')
})
try {
const urlConstructor = countryDirList.map(async (country) => {
console.log('countries mapped', country)
return fbCountryCodes.filter(async (f) => {
if (country === f.region) {
const urls = f.codes.map(async (c) => {
const response = await axios({
method: 'get',
url: `https://raw.githubusercontent.com/factbook/factbook.json/master/${country}/${c}.json`,
responseType: 'json',
headers: {
'Content-Type': 'application/json',
},
})
fs.readFile('./test.json', function (err, data) {
let json = JSON.parse(data)
json.push(response.data)
setTimeout(() => {
fs.writeFile('./test.json', JSON.stringify(json), function (err) {
if (err) throw err
console.log('The "data to append" was appended to file!')
})
}, 1000)
})
return response.data
})
const dataArr = await Promise.all(urls)
dataArr.map((item) =>
console.log(
'dataArr',
item.Government['Country name']['conventional short form']
)
)
}
})
})
} catch (err) {
console.log('axios error: ', err)
}
}
callingUrlsInSequence()
I'm re-writing this question now because it kept getting downvoted, and I could see that it was not very concise.
I can also see now, that obviously, the fs.readFile inside the fs.writeFile is not going to work in the code I provided, but I'm leaving it there in case it might help someone else, combined with the solution I provided in response to my own question.
I ended up learning how to solve this problem with both node-fetch and axios. They are not exactly the same.
For both:
First, check for existence of destination file, and create one if it's not already there.
const createNew = () => {
try {
if (existsSync('./data.json')) {
console.log('file exists')
return
} else {
writeFile('./data.json', '[]', (error, data) => {
if (error) {
console.log('fs.writeFile - create new - error: ', error)
return
}
})
}
} catch (err) {
console.log('fs.existsSync error: ', err)
}
}
createNew()
Then make the array of urls:
const countryDirList = [...countries]
const fbCountryCodes = [...codes]
const urls = []
// maybe a reducer function would be better, but my map + filter game is much stronger X-D
const makeUrls = (countriesArr, codesArr) =>
countriesArr.map((country) => {
return codesArr.filter((f) => {
if (country === f.region) {
return f.codes.map((code) => {
return urls.push(
`https://raw.githubusercontent.com/factbook/factbook.json/master/${country}/${code}.json`
)
})
}
})
})
makeUrls(countryDirList, fbCountryCodes)
Next, make the requests.
Axios:
fs.readFile('./data.json', (error, data) => {
if (error) {
console.log(error)
return
}
Promise.all(
urls.map(async (url) => {
let response
try {
response = await axios.get(url)
} catch (err) {
console.log('axios error: ', err)
return err
}
return response
})
)
.then((res) => {
const responses = res.map((r) => r.data)
fs.writeFile('./data.json', JSON.stringify(responses, null, 2), (err) => {
if (err) {
console.log('Failed to write data')
return
}
console.log('Updated data file successfully')
})
})
.catch((err) => {
console.log('axios error: ', err)
})
})
Node-fetch:
//same basic structure, readFile with fetch and write file inside
fs.readFile('./data2.json', (error, data) => {
if (error) {
console.log(error)
return
}
async function fetchAll() {
const results = await Promise.all(
urls.map((url) => fetch(url).then((r) => r.json()))
)
fs.writeFile('./data2.json', JSON.stringify(results, null, 2), (err) => {
if (err) {
console.log('Failed to write data')
return
}
console.log('Updated data file successfully')
})
}
fetchAll()
})
Both methods produce exactly the same output: a json file containing a single array with however many response objects in it.
I am trying to make a get route for this API:
https://api.nasa.gov/mars-photos/api/v1/rovers/opportunity/photos?sol=1000&api_key=92Ll6nGuQhfGjZnT2gxaUgiBhlCJ9K1zi2Fv5ONn
And although the syntax for the get route, the API still doesn't work in postman nor in client-side.
Here's the get route code:
app.get('/roverInfo/:rover_name', async (req, res) => {
const { rover_name } = req.params
try {
let images = await fetch(`https://api.nasa.gov/mars-photos/api/v1/rovers/${rover_name}/photos?sol=1000&api_key=${process.env.API_KEY}`).then((res) => res.json())
res.send({ images })
} catch (err) {
console.log('error:', err)
}
})
sandbox here
and here's the client-side request:
const showRovers = async (rovers) => {
try {
await fetch(`https://localhost:3000/roverInfo/:rover_name`)
.then((res) => {
return res.json()
})
.then((rovers) => updateStore(store, { rovers }), console.log(rovers))
} catch (error) {
console.log('errors:', error)
}
}
and here's the error I am getting:
Failed to load resource: net::ERR_SSL_PROTOCOL_ERROR
ADVISE: Don't mix await/async with .then, use either one
app.get("/roverInfo/:rover_name", async (req, res) => {
const { rover_name } = req.params;
try {
const res = await fetch(
`https://api.nasa.gov/mars-photos/api/v1/rovers/${rover_name}/photos?sol=1000&api_key=${process.env.API_KEY}`
) // removed .then
const images = await res.json(); // await response to json
res.send({ images });
} catch (err) {
console.log("error:", err);
}
});
02. should be http instead of https
03. need to pass rover name to param instead of using :rover_name
let getRovers = showRovers('opportunity');
const showRovers = async (roverName) => {
try {
console.log("roverName", roverName)
// use http here
await fetch(`http://localhost:3000/roverInfo/${roverName}`)
.then((res) => {
return res.json();
})
.then((rovers) => updateStore(store, { rovers }));
} catch (error) {
console.log("errors:", error);
}
};
I'm getting error cannot set headers on express js, I think the problem is have to write setHeader, i was set but stil can't, this my code:
router.get('/cek', (req, res) => {
const child = execFile(commandd, ['-c', 'config', 'GSM.Radio.C0']);
child.stdout.on('data',
function (data) {
value = (JSON.stringify(data));
x = value.split('.');
y = JSON.stringify(x[2])
result = y.replace(/\D/g, "");
res.setHeader('Content-Type', 'text/html');
res.send(result);
}
);
child.stderr.on('data',
function (data) {
console.log('err data: ' + data);
}
);
});
I tired to fixing this error for two days, but still cannot, anybody can help?
As stated by Frederico Ibba, this is usually caused after res.send is sent and there is still data being processed... Your workaround for this may simply be to receive all the data before sending it out using res.send. You can try this.
async function executeCommand() {
return new Promise((resolve, reject) => {
const child = execFile(commandd, ['-c', 'config', 'GSM.Radio.C0']);
child.stdout.on('data',
function (data) {
value = (JSON.stringify(data));
x = value.split('.');
y = JSON.stringify(x[2])
result = y.replace(/\D/g, "");
resolve(result);
}
);
child.stderr.on('data',
function (err) { // Renamed data for err for clarification
reject(err);
}
);
});
}
router.get('/url', async (req, res) => {
try {
const result = await executeCommand();
res.setHeader('Content-Type', 'text/html');
res.send(result);
} catch(error) {
// There was an error. I'm throwing a 500
res.sendStatus(500);
}
});
Note that this will be effective only if you are confident that the data is being fired once, as indicated by skirtle
I am working on a react app and am trying to find a way to pass a variable I define in my front-end (Question.js) to my back-end (server.js) so that I can issue different queries. I have the code
//Question.js
state = {
data: null
};
componentDidMount() {
this.callBackendAPI()
.then(res => this.setState({ data: res.express }))
.catch(err => console.log(err));
}
callBackendAPI = async () => {
const response = await fetch('/express_backend');
const body = await response.json();
if (response.status !== 200) {
throw Error(body.message)
}
return body;
};
//server.js
con.connect(function (err) {
if (err) throw err;
con.query("SELECT question FROM s1questions WHERE ID = 1", function (err, result, fields) {
if (err) throw err;
app.get('/express_backend', (req, res) => {
var x = JSON.stringify(result[0].question);
res.send({ express: `${x}` });
});
});
});
Your sever should probably split your database connection from your route handler definitions. Also, you could use query parameters to access questions based on their id in the database.
// question.js
callBackendAPI = async () => {
const response = await fetch(`/express_backend?questionId=1`);
const body = await response.json();
if (response.status !== 200) {
throw Error(body.message)
}
return body;
};
// server.js
app.get('/express_backend', (req, res) => {
const { questionId } = req.query;
// query database for question by id
});
I'm accessing a Mongo database through NodeJS and Express as below:
var MongoClient = require('mongodb').MongoClient;
...
app.get("/app/visits", function (req, res, next) {
console.log("get visits");
MongoClient.connect('mongodb://localhost:27017/db', function (err, db) {
if (!err) { console.log("We are connected"); }
visits = db.collection('visits', function (err, collection) { });
visits.find().toArray(function (err, user) {
this.user = JSON.stringify(user);
if (err) { throw err; } else console.dir(this.user);
});
res.send(this.user);
});
});
In the browser this works fine. If I change res.send(this.user); to res.status(301).send(this.user); the status is also changed.
But the problem, Angular 2 with native script code returns the error:
getActualVisits()
{
return this.http.get("http://localhost:1234/app/visits").map(response => response.json())
}
I have no idea WHY after 7 hours of trying repair that.
Method getActualVisits() is calling from:
getActualSpecialization() {
let v = this.getActualVisits();
...
}
You need to call .subscribe after .map in order to observe the values that are returned.
getActualVisits() {
return this.http.get("http://localhost:1234/app/visits")
.map(response => response.json())
.subscribe(
data => this.actualVisits = data,
err => this.logError(err),
() => console.log('get actual visits complete')
);
}
See the following docs for more information https://auth0.com/blog/2015/10/15/angular-2-series-part-3-using-http/