I am using axios to practice Web Scraping by making a Web Viewer, and I noticed that the CSS Wasn't Loading.
I used this code:
console.log("Tribble-Webviewer is starting!")
const express = require('express')
const app = express()
const port = 3000
const publicDir = app.use(express.static('public'))
var cheerio = require('cheerio'); // Basically jQuery for node.js
const axios = require('axios').default;
const rp = require('request-promise');
const url = 'https://pointless.com/';
app.get('/', (req, res) => {
app.use('/static', express.static('public'))
})
app.get('/test', (req, res) => {
axios.get(url)
.then(({ data }) => res.send(data))
})
app.listen(port, () => {
console.log(`Tribble-Pro is listening on port ${port}`)
})
If you load the /test page, the CSS does not show.
Example of the CSS not loading below:
Image
I used this async function:
async function getCssTest() {
try {
const response = await axios.get(urlplusstyle);
res.send(response)
} catch (error) {
console.error(error);
}
}
Related
I want to put the epic games output in the express js get. I have 0 experience with js, so please no hate if it doesn't make sense. XD
const EpicGamesAPI = require('epicgames-status');
const express = require('express')
const app = express()
const port = 3000
EpicGamesAPI().then(callback => {
const epicgames = console.log(callback);
})
app.get('/', (req, res) => {
res.send(epicgames)
})
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
})
epicgames in res.send(epicgames) is not in the same scope as const epicgames = console.log(callback);. You can place the EpicGamesAPI callback in the get callback:
app.get('/', (req, res) => {
EpicGamesAPI().then(callback => {
const epicgames = console.log(callback);
res.send(epicgames);
})
})
When you make the get request, the EpicGamesAPI will be called. You might want to send callback to res.send, rather than a console.log.
I want a server side generated page in next.js to be served as a file. So I wanted to grab the rendered content inside a custom server.js file:
const express = require('express');
const next = require('next');
const port = parseInt(process.env.PORT, 10) || 3000;
const dev = process.env.NODE_ENV !== 'production';
const app = next({dev});
const handle = app.getRequestHandler();
app.prepare().then(() => {
const server = express();
server.get('/', async (req, res) => {
const nextResponse = await app.renderToHTML(req, res, '/', req.query);
console.log('nextResponse', nextResponse);
console.log('res.body', res.body);
});
server.get('*', (req, res) => {
return handle(req, res);
});
server.listen(port, (err) => {
if (err) throw err;
console.log(`> Ready on http://localhost:${port}`);
});
});
Oddly enough every console.log returns null or undefined.
I thought that renderToHTML would just return the rendered HTML string. Is there any way to do this?
This one is a bit tricky but achievable.
The idea is to override res.end function in order to catch rendered HTML there. The tricky part is that Next.js gzips and streams response using the compression library that's overriding res.end somewhere in the middle of the handle function.
The compression library is initialized using the handleCompression function of the Next.js's Server object (which is accessible using the app.getServer()), so that function needs to get overridden too.
So it should be looking something like this:
const { parse } = require('url');
const next = require('next');
const express = require('express');
const dev = process.env.NODE_ENV !== 'production';
const app = next({ dev });
const port = process.env.PORT || 3000;
const handle = app.getRequestHandler();
app.prepare()
.then(() => {
const server = express();
server.get('*', async (req, res) => {
const parsedUrl = parse(req.url, true);
const nextServer = await app.getServer();
const _handleCompression = nodeServer.handleCompression.bind(nodeServer);
nextServer.handleCompression = (req, res) => {
_handleCompression(req, res);
const _resEnd = res.end.bind(res)
res.end = function (payload) {
console.log('Rendered HTML: ', payload);
return _resEnd(payload);
}
}
return handle(req, res, parsedUrl);
});
server.listen(port, err => {
if (err) throw err;
console.log('> Ready on http://localhost:' + port);
});
});
After you get rendered HTML you don't have to use the saved _resEnd function. Feel free to manipulate, serve as a file, or whatever you want with it.
[`const express = require('express');
const app = express();
const https = require('https');
const url = "https://api.thevirustracker.com/free-api?countryTimeline=US";
app.get("/", (req ,res) => {
res.send("Server is Running")
https.get(url, (response) => {
response.on("data", (data) => {
const TimelineData = JSON.parse(data);
console.log(TimelineData);
})
})
})
app.listen(3000, ()=>console.log("Server is Running 0n 5000"));`]1
const express = require('express');
const app = express();
const https = require('https');
const url = "https://api.thevirustracker.com/free-api?countryTimeline=US";
app.get("/", (req ,res) => {
res.send("Server is Running")
https.get(url, (response) => {
response.on("data", (data) => {
const TimelineData = JSON.parse(data);
console.log(TimelineData);
})
})
})
app.listen(3000, ()=>console.log("Server is Running 0n 5000"));
To deliver large data in an effective manner API send data in chunk/stream format. and to receive each chunk it triggers the 'data' event and in your case, it might be possible that API sends data in chunk format. and it will not send you complete data in a single event.
Let's assume the complete response of your API is :
{ name: 'bella', age: 34, count: 40138 }
And API send it in 2 chunks :
Chunk1: { name: 'bella', age: 34, count: 4013
Chunk2: 8 }
In that case Json.Parse() on Chunk1 or Chunk2 will not work and threw an exception.
To deal with this problem you need to listen to the 'end' event and capture data from the'data' and parse it in the 'end' event.
Use the below code:
const express = require('express');
const app = express();
const https = require('https');
const url = "https://archive.org/advancedsearch.php?q=subject:google+sheets&output=json";
app.get("/", (req, res) => {
res.send("Server is Running")
https.get(url, (response) => {
var responseData = '';
response.on("data", (dataChunk) => {
responseData += dataChunk;
})
response.on('end', () => {
const TimelineData = JSON.parse(responseData);
console.log(TimelineData);
});
}).on('error', (e) => {
console.error(e);
});
})
app.listen(5000, () => console.log("Server is Running 0n 5000"));
The "data" event can be fired multiple times: https://nodejs.org/api/http.html#http_class_http_clientrequest
You have to listen for the "end" event and concat all chunks from the "data" event togehter for the full body response.
const express = require('express');
const app = express();
const https = require('https');
const url = "https://api.thevirustracker.com/free-api?countryTimeline=US";
app.get("/", (req, res) => {
res.send("Server is Running")
https.get(url, (response) => {
const chunks = [];
response.on("data", (data) => {
chunks.push(data);
})
response.on("end", () => {
let size = chunks.reduce((prev, cur) => {
return prev + cur.length;
}, 0);
let data = Buffer.concat(chunks, size).toString();
console.log(JSON.parse(data))
});
})
})
app.listen(3000, () => console.log("Server is Running 0n 5000"));
why are you using https?
replace https with http and run it again.
const express = require('express');
const app = express();
const http = require('http');
const url = "https://api.thevirustracker.com/free-api?countryTimeline=US";
app.get("/", (req ,res) => {
res.send("Server is Running")
http.get(url, (response) => {
response.on("data", (data) => {
const TimelineData = JSON.parse(data);
console.log(TimelineData);
})
})
})
const express = require('express')
const app = express()
const port = 3000
app.post('/', (req, res) => {
res.send('Hello World!")
})
app.listen(port, () => {
console.log('server running')
})
When you run the program in nodejs, open the brower and type http://localhost:3000. The output will be....
Listen for 'end ' the problem will be resolved
Try importing all the dependencies. Importing is better than requiring because you can selectively load only the pieces you need. Also in package.json file add "type":"module" before scripts. The days of const something= require('something') are a thing of the past now because of new ESM modules.
import express from 'express';
import https from 'https';
const app=express();
const port=3000;
In package.json file
"name": "restApiWithNode",
"version": "1.0.0",
"description": "",
"main": "index.js",
"type": "module",
Read this article for clarity https://formidable.com/blog/2021/node-esm-and-exports/
I am trying to test my app with supertest. When I run nodemon index.js, and go to the address to make a GET request, I get a working response with correct data.
Supertest is called with app.js, which it's contents are down below as well.
But when I do that, response body returns []. Why might that be? I'm new to all this, so if anything must be added, please say so.
index.js:
const app = require('./app') // the actual Express app
const http = require('http')
const config = require('./utils/config')
const server = http.createServer(app)
server.listen(config.PORT, () => {
console.log('Server running')
})
app.js:
const config = require('./utils/config')
const express = require('express')
const bodyParser = require('body-parser')
const app = express()
const cors = require('cors')
const blogsRouter = require('./controllers/blogs')
const middleware = require('./utils/middleware')
const mongoose = require('mongoose')
console.log('connecting to', config.MONGODB_URI)
mongoose.connect(config.MONGODB_URI, { useNewUrlParser: true })
.then(() => {
console.log('connected to MongoDB')
})
.catch((error) => {
console.log('error connection to MongoDB:', error.message)
})
app.use(cors())
app.use(express.static('build'))
app.use(bodyParser.json())
app.use(middleware.requestLogger)
app.use('/api/blogs', blogsRouter)
app.use(middleware.unknownEndpoint)
app.use(middleware.errorHandler)
module.exports = app
Here is the part that I run test:
blog_api.test.js:
const mongoose = require('mongoose')
const supertest = require('supertest')
const app = require('../app')
const api = supertest(app)
test('blogs are returned as json', async () => {
await api
.get('/api/blogs')
.expect(200)
.expect('Content-Type', /application\/json/)
})
test('there are one blog', async () => {
const response = await api.get('/api/blogs')
//response doesn't return anything yet works fine in server?
expect(response.body.length).toBe(1)
})
afterAll(() => {
mongoose.connection.close()
})
first test works fine but second one fails because response.body returns [].
Just for good measure, here is the code where I actually define what the GET request should do:
in my controller:
blogsRouter.get('/', (request, response) => {
Blog.find({}).then(notes => {
let resJson = response.json(notes.map(note => note.toJSON()))
return resJson
})
})
I have this app.js file:
let express = require('express')
let app = express()
let Clarifai = require('clarifai')
app.use(express.urlencoded({extended: true}))
app.use(express.static('./public'))
let link = app.post('/route', (req, res) => {
let linkString = req.body.link
res.send(JSON.stringify(linkString))
})
app.listen(3000)
const capp = new Clarifai.App({
apiKey: 'MyAPIKeyIsHere'
});
predict = capp.models.initModel({id: Clarifai.FOOD_MODEL, version: "aa7f35c01e0642fda5cf400f543e7c40"})
.then(generalModel => {
return generalModel.predict(link)
})
.then(response => {
var concepts = response['outputs'][0]['data']['concepts']
console.log(concepts)
})
console.log('Express app running on port 3000')
console.log(link)
I am trying to return a string from the app.post method but it returns a JSON file. How should I do it exactly?
You can explicitly set the content type to text/html, before sending the data.
res.set('Content-Type', 'text/html');
res.send(JSON.stringify(linkString));
Are you sure that req.body.link is a string? If yes you could just pass linkString variable in send:
let link = app.post('/route', (req, res) => {
let linkString = req.body.link
res.send(linkString)
})