Develop Reference

NODE.JS How do I save JSON data without filling up my storage [closed]

Closed. This question needs details or clarity. It is not currently accepting answers.
Want to improve this question? Add details and clarify the problem by editing this post.
Closed 1 year ago.
The community reviewed whether to reopen this question 1 year ago and left it closed:
Original close reason(s) were not resolved
Improve this question
The title really explains it.
I made a discord bot, and I added a ranking system that had its memory in my filesystem. However, if too many people join, my storage would just get filled up. Perhaps there is a way for me to access the node.js server? Maybe localhost? I just want anything that saves data for free, is not managed by anyone other than me, and doesn't take up my storage.

config
For this answer we'll establish a simple config object to store any values -
// config.json
{"counter":0}
server
We will create a simple server using http.createServer. We will use the request method and request URL to look up a handler or respond with 404 when no handler is found -
// server.js
import { createServer } from "http"
import { readFile, writeFile } from "fs/promises"
const server = createServer(async (req, res) => {
const handler = routes?.[req.method.toLowerCase()]?.[req.url]
if (handler == null) {
res.writeHead(404, {'Content-Type': 'text/plain'})
res.end(`No route for ${req.method} ${req.url}`)
}
else {
await handler(req, res)
res.end()
}
})
server.listen(8000)
Next we define the routes to /getConfig and /saveConfig -
// server.js (continued)
const routes = {
get: {
"/getConfig": async (req, res) => {
res.writeHead(200, {'content-type': 'application/json'})
res.write(await readFile("./config.json"))
}
},
post: {
"/saveConfig": async (req, res) => {
await writeFile("./config.json", await readBody(req))
res.writeHead(204)
},
"/reset": async (req, res) => {
await writeFile("./config.json", JSON.stringify({ counter: 0 }))
res.writeHead(204)
}
}
}
This depends on a reusable helper, readBody -
// server.js (continued)
function readBody(req) {
return new Promise((resolve, reject) => {
const body = []
req.on('data', chunk => body.push(Buffer.from(chunk)))
req.on('end', _ => resolve(Buffer.concat(body).toString()))
req.on('error', reject)
})
}
client
In this case your bot is the http client. The node docs for http.get include this long-winded example, but don't let it worry you -
// example from node docs
http.get('http://localhost:8000/', (res) => {
const { statusCode } = res;
const contentType = res.headers['content-type'];
let error;
// Any 2xx status code signals a successful response but
// here we're only checking for 200.
if (statusCode !== 200) {
error = new Error('Request Failed.\n' +
`Status Code: ${statusCode}`);
} else if (!/^application\/json/.test(contentType)) {
error = new Error('Invalid content-type.\n' +
`Expected application/json but received ${contentType}`);
}
if (error) {
console.error(error.message);
// Consume response data to free up memory
res.resume();
return;
}
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => { rawData += chunk; });
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
console.log(parsedData);
} catch (e) {
console.error(e.message);
}
});
}).on('error', (e) => {
console.error(`Got error: ${e.message}`);
});
You're not expected to copy this verbatim. Imagine writing that much code each time you wanted to fetch some JSON. You can think of the http module as a low-level API that enables you to design higher-level functions -
// client.js
import * as http from "http"
function request (href, { body = "", ...options } = {}) {
return new Promise((resolve, reject) =>
http.request(href, options, res => {
const data = []
res.on('data', chunk => data.push(chunk))
res.on('end', _ => resolve({
status: res.statusCode,
headers: res.headers,
data: Buffer.concat(data).toString()
}))
})
.on('error', reject)
.end(body)
)
}
Above our request function resolves a { status, headers, data } object, and we can write specialized forms get and getJson that make it even easier to intereact with -
// client.js (continued)
async function get (href) {
const { status, headers, data } = await request(href)
if (status < 200 || status >= 300)
throw Error(status)
return { status, headers, data }
}
async function getJson (href) {
const { headers, data } = await get(href)
if (!headers['content-type'].startsWith("application/json"))
throw Error(`expected application/json but received ${headers['content-type']}`)
return JSON.parse(data)
}
We can do the same for post -
// client.js (continued)
async function post (href, body = "") {
const { status, headers, data } = await request(href, { body, method: "POST" })
if (status < 200 || status >= 300)
throw Error(status)
return { status, headers, data }
}
Finally here is our bot code. It reads the config via get, updates the config via post, and re-reads it via get to return the confirmed result -
// client.js (continued)
async function bot() {
const config = await getJson("http://localhost:8000/getConfig")
await post("http://localhost:8000/saveConfig", JSON.stringify({counter: config.counter + 1}))
return getJson("http://localhost:8000/getConfig")
}
bot().then(console.log, console.error)
run
Start the server in your terminal -
$ node ./server.js
In a separate terminal, run the client a few times -
$ node ./client.js
{ counter: 1 }
$ node ./client.js
{ counter: 2 }
$ node ./client.js
{ counter: 3 }
node modules
Above we took a sort of DIY approach to the problem. But this kind of problem has been solved many ways before. There are popular libraries like express and koajs that would make much of this a lot easier. Now that you know the purpose they serve, give 'em a try!

Just use a database, mongoDB atlas will work well in your case because it is cloud based and very easy to set up. You can follow this tutorial to connect your discord bot with mongoDB atlas.

Reading a stream over HTTP with Javascript

I am trying to build a web app to stream music. I use MongoDB to store the audio, a Node API to connect to the database and a Vuejs frontend. Below is the endpoint which streams the music, based on this article: https://medium.com/#richard534/uploading-streaming-audio-using-nodejs-express-mongodb-gridfs-b031a0bcb20f
trackRoute.get('/:trackID', (req, res) => {
try {
var trackID = new ObjectID(req.params.trackID);
} catch (err) {
return res.status(400).json({ message: "Invalid trackID in URL parameter. Must be a single String of 12 bytes or a string of 24 hex characters" });
}
res.set('content-type', 'audio/mp3');
res.set('accept-ranges', 'bytes');
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'tracks'
});
let downloadStream = bucket.openDownloadStream(trackID);
downloadStream.on('data', (chunk) => {
res.write(chunk);
});
downloadStream.on('error', () => {
res.sendStatus(404);
});
downloadStream.on('end', () => {
res.end();
});
});
I tested it with Postman and it works there. I am trying to read the stream in my Vuejs application. I'm just not sure how to do it. I tried the following to test it:
const url = 'http://localhost:4343/api/track/6061c90b2658b9001e65311d';
http.get(url, function (res) {
res.on('data', function (buf) {
console.log(buf);
});
res.on('end', function () {
console.log('ended');
});
})
This does not work however. How should I go about reading it in the frontend?

Returning a json response from node.js to the front-end.

So, I am trying to fetch my back end code and receive the json response from the API I am querying on the back-end. It is returning an object but, my json is either buried in it or not there. I can log the data in node, I just can't send it back. I have tried res-send, res-json and res-end. I'm not sure where I'm going wrong. Thank you for any help!
web app console response
Response {type: "basic", url: "http://localhost:3003/father", redirected: false, status: 200, ok: true, …}
Front-end javascript
document.querySelector('.sub').addEventListener('click', function (e) {
e.preventDefault()
const wordSearched = document.querySelector('.word').value;
fetch(`/${wordSearched}`)
.then(function(answer) {
console.log(answer);
const number = answer.total_results;
console.log(number);
const tag = document.querySelector(".tagg").innerHTML = `The word
${wordSearched} was used ${number} times!`;
return tag
})
.catch(function(error) {
console.log('😮 There has been a problem with the fetch operation: ',
error.message);
})});
Back-end node.js
const express = require('express');
const app = express();
const morgan = require('morgan');
const fetch = require('node-fetch');
app.use(express.static('\public'));
app.use(morgan('short'));
app.get('/:wordSearched', (req, res) => {
let word = req.params.wordSearched;
console.log(word);
fetch(`https://api.esv.org/v3/passage/search/?q=${word}`, {
headers: {
'Authorization': 'Token kkkdddd88383' // API Token
}
})
.then(function(response) {
return response.json()
})
.then(function(myJson) {
const number = myJson.total_results;
console.log(number);
res.send(myJson) //cant send number??
})
.catch(function(error) {
console.log('😮 There has been a problem with the fetch operation: ', error.message);
})
});
//local host
app.listen(3003, () => {
console.log('server is up on 3003');
});

So I figured out my issue after reading the fetch documentation at https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch. I had to call:
.then(function(response) {
return response.json()
})
On my front-end as well. I had previously only called it on the back-end thinking is was json I was sending. Unless I made the .json() call on each end I was not going to get my answer through.

Firebase Cloud Function never failing on 404 API call

I have a function running on the creation of a document.
When I send this information to an external API Firebase returns on 'ok' message before the API call is complete.
const functions = require('firebase-functions');
const request = require('request');
const admin = require('firebase-admin');
const rp = require('request-promise');
const port = '****';
const ip = '***.***.***.***';
admin.initializeApp(functions.config().firebase);
exports.sendUser = functions.firestore
.document('user/{userId}')
.onCreate((snap, context) => {
const data = snap.data();
const options = {
method: 'POST',
uri: 'http://' + ip + ':' + port + '/user',
body: data,
json: true,
};
rp(options)
.then(function (parsedBody) {
console.log('TEN ', parsedBody);
return parsedBody;
})
.catch(function (err) {
console.log('ERR ', err);
return err;
});
});
As you can see from my function it is not doing anything special apart from sending the data to an external source.
The API look like the following:-
app.post('/user', function (req, res) {
fs.exists(path, function(exists) {
if (exists === true) {
console.log('Currently Printing Different User Info');
fs.unlinkSync(path);
res.status(404).json({errorCode: 404, errorMessage: 'Currently Printing Different User.'});
return;
} else {
fs.writeFile(path, '', () => { console.log('File Created'); });
fs.unlinkSync(path);
res.status(200).json({statusCode: 200, statusMessage: 'Here we go'});
return;
}
});
})
How can I get Firebase to recognise the returned 404 as a failed call, and also wait until the call is complete before returning ok or failed.
The API is behaving correctly with Postman but not when data is posted via Firebase.
Has anyone encountered this before, or can anybody see what I am doing wrong?
The data is being parse over to the serve but only once Firebase has returned with 'ok' even if I purposely trigger a fail.
I need this in place to be able to use the Firebase Cloud Function retry function.
Images can be seen # https://imgur.com/a/1qYxrci

The Cloud Function returns the result before the call is complete because you don't return the Promise returned by the request-promise call.
Changing your code as follows should do the trick (at least for this problem):
exports.sendUser = functions.firestore
.document('user/{userId}')
.onCreate((snap, context) => {
const data = snap.data();
const options = {
method: 'POST',
uri: 'http://' + ip + ':' + port + '/user',
body: data,
json: true,
};
return rp(options) // <-- See the change here
.then(function (parsedBody) {
console.log('TEN ', parsedBody);
return parsedBody;
})
.catch(function (err) {
console.log('ERR ', err);
return err;
});
});
I would suggest you watch the official Video Series (https://firebase.google.com/docs/functions/video-series/) which explain very well this point about returning Promises for background functions (in particular the ones titled "Learn JavaScript Promises").

Write json file to Firebase with Node.js?

I have a third party API that I need to call every 5 seconds. I get JSON as response, and I'd like to write the JSON content to a Firebase node using Node.js. Based on Firebase examples I could import data with this code:
var usersRef = ref.child("users");
usersRef.set({
alanisawesome: {
date_of_birth: "June 23, 1912",
full_name: "Alan Turing"
},
gracehop: {
date_of_birth: "December 9, 1906",
full_name: "Grace Hopper"
}
});
Curl examples worked too. But what I really wanted to do is to import a third party API response directly to my Firebase database using the API endpoint. How can i do it with Node.js?

First, you need to make a request to the api endpoint and receive the data.
Then, you can send that json data to firebase
var request = require('request');
var usersRef = ref.child("users");
request('<your_endpoint>', function (error, response, body) {
if (!error && response.statusCode == 200) {
var asJson = JSON.parse(body)
usersRef.set(asJson)
}
})

I ran into a lot of little "gotchas" implementing the sample node.js code from the Firebase docs. Below is a fully working code set correcting all the issues which will run as a Google Cloud Platform function (node.js v8 - async/await will not work in v6):
const admin = require('firebase-admin');
// You need this library in order to use firebase in functions
const functions = require('firebase-functions');
/**
* Responds to any HTTP request.
*
* #param {!express:Request} req HTTP request context.
* #param {!express:Response} res HTTP response context.
*/
exports.uploadFile = async (req, res) => {
// Check if firebase is already initialized, per: https://maxrohde.com/2016/09/21/test-if-firebase-is-initialized-on-node-js-lambda/
if (admin.apps.length === 0) {
admin.initializeApp(functions.config().firebase);
}
var db = admin.firestore();
var message = '';
createUsers(db);
message = await getUsers(db);
res.status(200).send('Database content:\n' + message);
};
// Write data in a function so you can wait for all the Promises to complete and return per: https://github.com/firebase/functions-samples/issues/78
function createUsers(db) {
var docRef = db.collection('users').doc('alovelace');
var setAda = docRef.set({
first: 'Ada',
last: 'Lovelace',
born: 1815
})
.catch((err) => {
console.log('Error writing document', err);
});
var aTuringRef = db.collection('users').doc('aturing');
var setAlan = aTuringRef.set({
'first': 'Alan',
'middle': 'Mathison',
'last': 'Turing',
'born': 1912
})
.catch((err) => {
console.log('Error writing document', err);
});
return Promise.all([setAda, setAlan]);
}
async function getUsers(db) {
var message = '';
await db.collection('users').get()
.then((snapshot) => {
snapshot.forEach((doc) => {
// You need to stringify doc.data() if you want to render it outside of a console.log()
message += '\n' + doc.id + '=>' + JSON.stringify(doc.data());
});
})
.catch((err) => {
console.log('Error getting documents', err);
});
return message;
}