I have a node.js / next.js api built that essentially does a bunch of stuff after the user submits text into a form on the front end. One of the things it does is write stage completion messages periodically to a JSON file to signify the completion of certain stages.
my api looks something like this
import dbConnect from '../../../lib/dbConnect'
import Demo from '../../../models/Demo'
import fs from 'fs'
import shell from 'shelljs';
export default async function handler(req, res) {
const {
method,
body,
} = req
await dbConnect()
switch (method) {
case 'GET':
try {
const demos = await Demo.find({})
res.status(200).json({ success: true, data: demos })
} catch (error) {
res.status(400).json({ success: false })
}
break
case 'POST':
try {
const initialjson = '[]'
const timestamp = Date.now();
// stage 1
if (shell.exec('./initial_checks.sh').code !== 0) {
shell.echo('Sorry stage failed');
shell.exit(1);
};
const objSuccess1 = JSON.parse(initialjson);
objSuccess1.push("Stage 1 complete", + timestamp);
const finalJSONSuccess1 = JSON.stringify(objSuccess1);
fs.writeFileSync('success-stage.json', finalJSONSuccess1);
// stage 2
if (shell.exec('./secondary_checks.sh').code !== 0) {
shell.echo('Sorry stage failed');
shell.exit(1);
};
const objSuccess2 = JSON.parse(initialjson);
objSuccess2.push("Stage 2 complete", + timestamp);
const finalJSONSuccess2 = JSON.stringify(objSuccess2);
fs.writeFileSync('success-stage.json', finalJSONSuccess2);
const demo = await Demo.create(
req.body
)
res.status(201).json({ success: true, data: demo })
} catch (error) {
res.status(400).json({ success: false })
}
break
default:
res.status(400).json({ success: false })
break
}
}
I am using socket.io, my server.js file is
server.js
const app = require("express")();
const server = require("http").Server(app);
const io = require("socket.io")(server);
const next = require("next");
const dev = process.env.NODE_ENV !== "production";
const nextApp = next({ dev });
const nextHandler = nextApp.getRequestHandler();
let port = 3000;
const fs = require('fs')
const data = fs.readFileSync('success-stage.json', 'utf8')
io.on("connect", (socket) => {
socket.emit("now", {
message: data
});
});
nextApp.prepare().then(() => {
app.all("*", (req, res) => {
return nextHandler(req, res);
});
server.listen(port, (err) => {
if (err) throw err;
console.log("> Ready on port: " + port);
});
});
and here is the pages/index.js file
import { useEffect, useRef, useState } from "react";
import io from "socket.io-client";
export default function IndexPage() {
const socket = useRef();
const [hello, setHello] = useState();
useEffect(() => {
socket.current = io();
socket.current.on("now", (data) => {
setHello(data.message);
});
}, []);
return <h1>{hello}</h1>;
}
so at this point we are seeing the 2nd message from my JSON file match what is rendered on the frontend when I build my application. It looks like this
["Stage 2 complete",1664289144513]
I am wondering how I can stream this data onto the front end for clients without having to refresh the page? I need it to show the current stage's success message... There are 5 total stages, so i guess i am looking for a way to either stream data or maybe to revalidate the browser window like every second without having to refresh... is this possible?
Any help would be greatly appreciated... Thanks in advance for your time everyone...
You've already got a solution implemented that can handle this. What you're describing is exactly what sockets are for -- bidirectional communication between the client and server without refreshing the page.
Just create a new socket listener on the frontend for a new topic, maybe "stageStatus", and then emit messages to that topic on the backend at various stages in the process. That's it!
Related
(1/9/2023) Update : SvelteKit now supports server only load functions and Form actions to send requests to the server.
I want to call my database, but I don't want it be able to get accessed by end users by them going to the API endpoint that I set up. I was wondering how I would be able to just call my database from a file in the lib folder and just returning the data there. When I try it I get the error global not defined:
lib/db.js:
import dotenv from "dotenv";
dotenv.config();
import { MongoClient } from "mongodb";
const uri = process.env["MONGODB_URI"];
const options = {
useUnifiedTopology: true,
useNewUrlParser: true,
};
let client;
let clientPromise;
if (!uri) {
throw new Error("Please add your Mongo URI to .env.local");
}
if (process.env["NODE_ENV"] === "development") {
if (!global._mongoClientPromise) {
client = new MongoClient(uri, options);
global._mongoClientPromise = client.connect();
}
clientPromise = global._mongoClientPromise;
} else {
client = new MongoClient(uri, options);
clientPromise = client.connect();
}
export default clientPromise;
routes/items/index.js:
import clientPromise from "$lib/db";
export async function get() {
const client = await clientPromise;
const db = client.db();
const data = await db.collection("items").find({}).toArray();
const items = data.map(({ name }) => ({ name }));
if (items) {
return {
body: {
items,
},
};
}
}
My attempt:
lib/stores/items.js
import clientPromise from "$lib/db";
import { writable } from "svelte/store";
export const items= writable([]);
const fetchItems = async () => {
const client = await clientPromise;
const db = client.db();
const data = await db.collection("items").find({}).toArray();
const items = data.map(({ name }) => ({ name }));
substances.set(items);
};
fetchItems();
Trying the above code in various places always yields a global not defined error in the client.
I found one question from someone with the same problem, but I couldn't figure out how to create a helper file.
Protecting API is done on back-end side. Usually it either server (like NodeJS) or tools Nginx/Apache (proxy, etc.). You're basically looking for Content-Security-Policy topic, which is vaporous but not related to SvelteKit.
Btw, calling DB directly from the Front-end wouldn't be secure and is not possible.
To get data from any database, you should create enpoint
For user authentication, you can create handle hook:
export async function handle({ request, resolve }) {
let user = await authenticate(request)
request.locals.user = user
request.locals.isAuthenticated = !!user
if (request.path.startsWith('/api')) {
if (!user) {
return {
status: 401,
body: JSON.stringify({
error: {
message: 'Unauthorized'
}
})
}
}
const response = await resolve(request)
return response
}
I have the following files:
My routes - where the orders_count route lives:
routes/index.js
const express = require('express');
const router = express.Router();
const transactionsController = require('../controllers/transactionsController');
const ordersController = require('../controllers/ordersController');
const ordersCountController = require('../controllers/ordersCountController');
router.get('/transactions', transactionsController);
router.get('/orders', ordersController);
router.get('/orders_count', ordersCountController);
module.exports = router;
I then have my orders count controller living in the controllers directory:
controllers/ordersCountController.js
const ordersCountService = require('../services/ordersCountService');
const ordersCountController = (req, res) => {
ordersCountService((error, data) => {
if (error) {
return res.send({ error });
}
res.send({ data })
});
};
module.exports = ordersCountController;
My controller then calls my order count service which fetches data from another API.
services/ordersService.js
const fetch = require('node-fetch');
// connect to api and make initial call
const ordersCountService = (req, res) => {
const url = ...;
const settings = { method: 'Get'};
fetch(url, settings)
.then(res => {
if (res.ok) {
res.json().then((data) => {
return data;
});
} else {
throw 'Unable to retrieve data';
}
}).catch(error => {
console.log(error);
});
}
module.exports = ordersCountService;
I'm trying to return the JSON response. I initially had it setup with requests but looking at the NPM site, it appears that it's depreciated so have been digging through how to use node-fetch.
I have tried both 'return data' and res.send({data}), but neither are solving the problem.
I am still new to this so I am likely missing something very obvious, but how come I am not sending the JSON back through so that it displays at the /api/orders_count endpoint?
I keep thinking I messed something up in my controller but have been looking at it for so long and can't seem to figure it out.
Any help would be greatly appreciated and if there is anything I can add for clarity, please don't hesitate to ask.
Best.
please learn promises and await syntax. life will be easier.
never throw a string. always prefer a real error object, like that : throw new Error('xxx'); that way you will always get a stack. its way easier to debug.
avoid the callback hell : http://callbackhell.com/
you need to decide if you want to catch the error in the controller or in the service. no need to do in both.
in the controller you call the service that way :
ordersCountService((error, data) => {
but you declare it like that :
const ordersCountService = (req, res) => {
which is not compatible. it should look like this if you work with callback style :
const ordersCountService = (callback) => {
...
if (error) return callback(error)
...
callback(null, gooddata);
here is an example to flatten your ordersCountService function to await syntax, which allows the "return data" you were trying to do :
const fetch = require('node-fetch');
// connect to api and make initial call
const ordersCountService = async (req, res) => {
const url = ...;
const settings = { method: 'Get'};
try {
const res = await fetch(url, settings);
if (!res.ok) throw new Error('Unable to retrieve data');
return await res.json();
} catch(error) {
console.log(error);
}
}
module.exports = ordersCountService;
in fact i would prefer to error handle in the controller. then this woud be sufficient as a service
const fetch = require('node-fetch');
// connect to api and make initial call
const ordersCountService = async () => {
const url = ...;
const settings = { method: 'Get'};
const res = await fetch(url, settings);
if (!res.ok) throw new Error('Unable to retrieve data');
return await res.json();
}
module.exports = ordersCountService;
then you can call this funtion like this :
try {
const data = await ordersCountService(req, res);
} catch(err) {
console.log(err);
}
//or
ordersCountService(req, res).then((data) => console.log(data)).catch((err) => console.error(err));
the problem
I created a Shopify node.js app using the Shopify CLI and I want to display a simple bar under the header using a script tag. I used the script tag API to add a script tag
"script_tags": [
{
"id": 174240039086,
"src": "https://xxxxx.ngrok.io/script_tag",
}
]
And I also added a <div id="script-app"></div> into the theme, under the header.
Here is my script_tag.js file, located in /pages/script_tag.js
import ReactDOM from 'react-dom';
class TestScriptTag extends React.Component {
constructor() {
super();
}
render() {
return (
<div>
<p>this is a bar</p>
</div>
);
}
}
ReactDOM.render(<TestScriptTag />, document.getElementById('script-app'));
export default TestScriptTag;
Lastly, here is my server.js (most of it is what came with the CLI):
import "#babel/polyfill";
import dotenv from "dotenv";
import "isomorphic-fetch";
import createShopifyAuth, { verifyRequest } from "#shopify/koa-shopify-auth";
import Shopify, { ApiVersion } from "#shopify/shopify-api";
import Koa from "koa";
import next from "next";
import Router from "koa-router";
import { flushSync } from "react-dom";
const fs = require('fs');
dotenv.config();
const port = parseInt(process.env.PORT, 10) || 8083;
const dev = process.env.NODE_ENV !== "production";
const app = next({
dev,
});
const handle = app.getRequestHandler();
Shopify.Context.initialize({
API_KEY: process.env.SHOPIFY_API_KEY,
API_SECRET_KEY: process.env.SHOPIFY_API_SECRET,
SCOPES: process.env.SCOPES.split(","),
HOST_NAME: process.env.HOST.replace(/https:\/\//, ""),
API_VERSION: ApiVersion.October20,
IS_EMBEDDED_APP: false,
// This should be replaced with your preferred storage strategy
SESSION_STORAGE: new Shopify.Session.MemorySessionStorage(),
});
// Storing the currently active shops in memory will force them to re-login when your server restarts. You should
// persist this object in your app.
const ACTIVE_SHOPIFY_SHOPS = {};
app.prepare().then(async () => {
const server = new Koa();
const router = new Router();
server.keys = [Shopify.Context.API_SECRET_KEY];
server.use(
createShopifyAuth({
async afterAuth(ctx) {
console.log("here")
// Access token and shop available in ctx.state.shopify
const { shop, accessToken, scope } = ctx.state.shopify;
const host = ctx.query.host;
ACTIVE_SHOPIFY_SHOPS[shop] = scope;
const response = await Shopify.Webhooks.Registry.register({
shop,
accessToken,
path: "/webhooks",
topic: "APP_UNINSTALLED",
webhookHandler: async (topic, shop, body) =>
delete ACTIVE_SHOPIFY_SHOPS[shop],
});
if (!response.success) {
console.log(
`Failed to register APP_UNINSTALLED webhook: ${response.result}`
);
}
// Redirect to app with shop parameter upon auth
ctx.redirect(`/?shop=${shop}&host=${host}`);
},
})
);
const handleRequest = async (ctx) => {
await handle(ctx.req, ctx.res);
ctx.respond = false;
ctx.res.statusCode = 200;
};
router.get("/", async (ctx) => {
const shop = ctx.query.shop;
// This shop hasn't been seen yet, go through OAuth to create a session
if (ACTIVE_SHOPIFY_SHOPS[shop] === undefined) {
ctx.redirect(`/auth?shop=${shop}`);
} else {
await handleRequest(ctx);
}
});
router.get("/script_tag", (ctx) => {
handleRequest(ctx);
});
router.get("(/_next/static/.*)", handleRequest); // Static content is clear
router.get("/_next/webpack-hmr", handleRequest); // Webpack content is clear
router.get("(.*)", verifyRequest(), handleRequest); // Everything else must have sessions
server.use(router.allowedMethods());
server.use(router.routes());
server.listen(port, () => {
console.log(`> Ready on http://localhost:${port}`);
});
});
I am getting the error: document not defined.
What I've tried
I thought this is due to server side rendering, so I thought I could get around it by doing this:
if (typeof window !== "undefined") {
ReactDOM.render(<TestScriptTag />, document.getElementById('script-app'));
}
But still nothing renders and I get this when I inspect the shop page.
I've also tried changing the routing to this:
router.get("/script_tag", (ctx) => {
ctx.type = "module";
ctx.body = fs.createReadStream('./pages/script_tag.js')
});
But then I get an error about the import statement in script_tag.js - SyntaxError: Unexpected identifier '{'. import call expects exactly one argument.
I'm not sure what the proper way is to serve the javascript file I want to inject into the header. I feel like I'm missing something stupid. Please help!!
Good evening all!
I have been stuck on this issue for a while and I can't seem to solve it through sheer Googling and so I am reaching out to you all.
Context:
I am writing a small application that handles all the calendars and basic project information for all the interns at our company because my boss is constantly asking me what they're up to and I wanted to give him something that he could look at, so I decided to solve it with code whilst also learning a new framework in the process(Express).
Right now I have my routes all set up, I have my controllers all set up, and I have my DB cursor all set up. When I make the call to the route I have defined, it runs the getAllUsers() controller function and inside that controller function it makes a call to the database using the getAllUsers() function on the DB cursor, I want the code to wait for the DB cursor to return its result before continuing but it isn't and I can't work out why. The DB cursor code does work because it fetches the data and logs it out fine.
Any help would be greatly appreciated, I have put the three bits of code in question below, let me know if you need me to show more.
p.s ignore the 'here1', 'here2', etc calls, this is how I have been working out what's happening at any point in time.
routes.ts
import express from 'express';
import controllers from './controller.js';
export default (app: express.Application) => {
// Users
app.route('/users').get(controllers.getAllUsers)
app.route('/users').post(controllers.postNewUser)
app.route('/users').delete(controllers.deleteUser)
app.route('/user/:emailAddress').get(controllers.getUser)
app.route('/user/:emailAddress').put(controllers.updateUser)
}
controllers.ts
import express from 'express';
import dbcursor from '../services/dbcursor.js';
// Interfaces
import { Project, User } from '../services/interfaces.js'
const controllers = {
// Users
getAllUsers: async (req: express.Request, res: express.Response) => {
try {
const dbRes = await dbcursor.getAllUsers();
console.log('here 3', dbRes)
res.status(200).json({
message: 'Users fetched succesfully!',
dbRes: dbRes
});
} catch (err) {
res.status(400).json({
message: 'Failed to get users.',
dbRes: err
});
}
},
}
dbcursor.ts
import dotenv from 'dotenv';
import mongodb from 'mongodb'
dotenv.config();
// Interfaces
import { User, Project } from './interfaces'
// DB Client Creation
const { MongoClient } = mongodb;
const uri = process.env.DB_URI || ''
const client = new MongoClient(uri, { useNewUrlParser: true, useUnifiedTopology: true });
const dbcursor = {
// Users
getAllUsers: async () => {
let dbRes;
try {
await client.connect(async err => {
if (err) throw err;
console.log("here 1", dbRes)
const collection = client.db("InternManager").collection("Users");
dbRes = await collection.find().toArray()
console.log("here 2", dbRes)
return dbRes;
});
} catch(err: any) {
return err;
}
},
}
It's generally not a good idea to mix callbacks and promises. Try not passing a callback to the client.connect method, and you should be able to await the promise as expected
getAllUsers: async () => {
let dbRes;
try {
await client.connect();
console.log("here 1", dbRes)
const collection = client.db("InternManager").collection("Users");
dbRes = await collection.find().toArray()
console.log("here 2", dbRes)
return dbRes;
} catch(err: any) {
throw err; // If you're just catching and throwing the error, then it would be okay to just ignore it
}
},
I would like to create a local Javascript module I can "require" in other files to handle all MongoDB CRUD operations.
I wrote something as:
-- dbConn.js file --
require('dotenv').config()
const MongoClient = require('mongodb').MongoClient
const ObjectID = require('mongodb').ObjectID
let _connection
const connectDB = async () => {
try {
const client = await MongoClient.connect(process.env.MONGO_DB_URI, {
useNewUrlParser: true,
useUnifiedTopology: true
})
console.log('Connected to MongoDB')
return client
} catch (err) {
console.log(error)
}
}
exports.findOne = async () => {
let client = await connectDB()
if (!client) {
return;
}
try {
const db = client.db("Test_DB");
const collection = db.collection('IoT_data_Coll');
const query = {}
let res = await collection.findOne(query);
return res;
} catch (err) {
console.log(err);
} finally {
client.close();
}
}
exports.findAll = async () => {
let client = await connectDB()
if (!client) {
return;
}
try {
const db = client.db("Test_DB");
const collection = db.collection('IoT_data_Coll');
const query = {}
let res = await collection.find(query).toArray();
return res;
} catch (err) {
console.log(err);
} finally {
client.close();
}
}
Then in another file (not necessary inside Express app), say
-- app.js ---
const findAll = require('./dbConn').findAll
const findOne = require('./dbConn').findOne
findAll().then(res => JSON.stringify(console.log(res)))
findOne().then(res => JSON.stringify(console.log(res)))
I wonder if it is correct?
I have to close the connection after each method/CRUD operation?
I was trying to use IIF instead of ".then", as:
(async () => {
console.log(await findOne())
})()
But I receive a weird error saying that findAll is not a function.
What's wrong with it?
Thanks.
It really depends on your use case which isn’t clear If you are using Express or just stand alone and how frequent are you planning to run app.js
Either way your code is expensive, each time you reference dbCon.js you are opening a new connection to the database.
So you can fix app.js by only requiring dbCon.js once and use it..
The best practice is to ofcourse use connection pooling https://www.compose.com/articles/connection-pooling-with-mongodb/