I have been working on a project which is to create a node.js server and I have to read some text from the file. Then I want to use a Python API to consume these text and get the result back. I have been able to hit these API and get the response successfully.
What have I done till now -
1. Created a node server
2. read all the file and get the text content from them
3. create a flask server with appropriate api
4. hit these API to get the response
app.js
var bodyParser = require('body-parser');
var request = require('request-promise');
var app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.get('/', function(req, res){
res.send('Node server')
})
app.get('/postdatatoFlask', function (req, res) {
var fs = require('fs');
categories = ['business', 'entertainment']
// answers SHOULD CONTAIN ALL THE RESPONSES FROM THE FLASK API
var answers = []
//iterating over different file
categories.forEach(function(category){
for(var i=100;i<=100;i++){
//creating the path for the new file to be read
var path = 'dataset/'+category+'/'+i.toString()+'.txt';
//textContent contains the data read from file
var textContent = fs.readFileSync(path, 'utf8')
//creating the object
var data = { content : textContent }
//posting the data to the flask server on route postdata
var options = {
method: 'POST',
uri: 'http://127.0.0.1:5000/postdata',
body: data,
json: true
};
// returndata is the variable contains the data return by flask API
var returndata
var sendrequest = request(options)
.then(function (parsedBody) {
// parsedBody contains the data sent back from the Flask server
returndata = parsedBody;
answers.push(returndata)
console.log(returndata) // do something with this data, here I'm assigning it to a variable.
})
.catch(function (err) {
console.log(err);
});
}
})
console.log('Showing the responses!!!')
console.log(answers)
console.log('Stoping the showcase!!!!')
res.send('done');
});
app.listen(3000);
compute.py
import json
import time
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
from io import StringIO
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
import pandas as pd
app = Flask(__name__)
#app.route('/')
def index():
return "Flask server"
#app.route('/postdata', methods = ['POST'])
def postdata():
data = request.get_json()
# print(data['content'])
textContent = data['content']
print(textContent)
print('-- --- -- --- -- -- -- -- -- --')
return json.dumps({"newdata":textContent})
if __name__ == "__main__":
app.run(port=5000)
Problem
1. In app.js I want to store all the responses in answer array but I am unable to populate it.
2. why output from console.log(returndata) print after console.log(answer)
With answers.push(returndata), you are pushing an array inside another array. If you want to concat the content of returndata to answers, use answers.push(...returndata)
Javascript has an async, non blocking IO runtime, so here:
var sendrequest = request(options)
.then(function (parsedBody) {
// parsedBody contains the data sent back from the Flask server
returndata = parsedBody;
answers.push(returndata)
console.log(returndata) // do something with this data, here I'm assigning it to a variable.
})
The callback inside then is put to the callstack after those:
console.log('Showing the responses!!!')
console.log(answers)
console.log('Stoping the showcase!!!!')
var options = {
method: 'POST',
uri: 'http://127.0.0.1:5000/postdata',
body: data,
json: true
};
// returndata is the variable contains the data return by flask API
try {
// temp.push(await request(options));
answers.push(await request(options));
} catch(err) {
throw err;
}
Related
My frontend code:
const user = useSelector(selectUser)
function getWatchLater(name){
axios.get('http://localhost:5000/watchlater', {user:name})
.then((response)=>{
// console.log(response.data)
setWatchLater(response.data)
})
}
The user variable holds the username and the function sends that username to the backend to get the data. Don't worry, the user variable does hold the username, i have checked it thoroughly.
My backend code:
const mysql = require('mysql2')
const express = require('express')
const cors = require('cors');
const app = express()
app.use(cors());
app.use(express.json())
app.get("/watchlater", (request, response)=>{
const user = request.body.user;
//console.log(user);
});
So basically, it will get the username and run the query. The problem is it does not get the username at all from the frontend. I tried console logging the variable user but to no avail. It returns empty.
The second argument in the axios.get() function is expecting a config object. Checkout the axios documentations on instance method and request config.
In short, in the frontend part, pass your payload into the data field of the config object, as shown in the example below.
const config = {
headers: { Authorization: token },
data: { user:name }
}
const response = await axios.get(`${url}`, config)
You need to send parameter using params object of config in case of get request. Your frontend request should change to this,
const user = useSelector(selectUser)
function getWatchLater(name){
axios.get('http://localhost:5000/watchlater', { params: { user: name }
}).then((response)=>{
// console.log(response.data)
setWatchLater(response.data)
})
}
In your express endpoint you should receive it as,
app.get("/watchlater", (request, response)=>{
const user = request.params.user;
//console.log(user);
});
Below is my project folder structure :
app
Controller
testController.js
Service
testInfoService.js
testMoreDataService.js
config
node-modules
public
index.js
routes
routes.js
Here some code :
routes.js
router.get('/config', (req, res) => {
require(controllerDirectoryPath+"/testController.js").index(req, res)
})
testController.js
const testInfoService = require('somePath/Service/testInfoService')
const index = async (req, res) =>{
console.log('Request object : ');
console.log(req);
console.log('Response object : ');
console.log(res);
//getting more info from service
testMoreInfo = await testInfoService.getMoreInformation(args)
return res.status(200).send(testMoreInfo);
}
module.exports = {index:index}
testInfoService.js
const testMoreDataService = require("somePath/testMoreDataService")
const getMoreInformation = async (args)=> {
...some code here
response = await testMoreDataService.getMoreData(args)
return response
}
module.exports = {getMoreInformation:getMoreInformation}
testMoreDataService.js
const getMoreData = async (args)=> {
**//here I want to use Request and Response object**
...some code here
return response
}
module.exports = {getMoreData:getMoreData}
Is there any way I can access req and res object in my service?
I am using node : 12.16.2 with Express.
Main file is public/index.js
Also let me know is this the correct way I am following for any application(REST-API)?
The structure is correct,
Route -> Controller -> Service
You don't have to propagate the actual req, res object all the way down to all the services.
In your controller, you would fetch the following,
Query parameter. Eg. req.query.limit
Path variable. Eg.req.param.id
Request body. Eg. req.body/req.body.name
Once you get the required information in the controller, you could construct an object or pass the relevant information to your service.
Do all the business logic in your service and return the final information to the controller.
The controller would then send this response back to the client.
I created a python script that parses a website(IMDB) and organizes it into a dataframe.
I also have a node.js app that allows me to find a variable (movie ID based on movie name in the code called pyvar) that I would include in the python script. So how can I include this variable that I get after running javascript app into python script, run the script and then send the result back to the node.js app? (that would be dataframe converted to lets say json)
Node.js app
var express = require("express")
var app = express()
var request = require("request")
app.set("view engine", "ejs")
app.get("/", function(req, res){
res.render("search")
})
app.get("/results", function(req, res){
var query = req.query.search
var url = "http://www.omdbapi.com/?s=" + query + "&apikey=thewdb"
request(url, function(error, response, body){
if(!error && response.statusCode == 200){
var data = JSON.parse(body)
res.render("results", {data: data})
var pyvar = data["Search"][0]["imdbID"]
}
})
})
app.listen(process.env.PORT, process.env.IP, function(){
console.log("Movie App has started!!!");
})
The python script in a nutshell looks like this:
url = 'website.org/' + pyvar + '/blah'
parse(url)
return dataframe
After that I would send the dataframe in some form back to the node.js app and display the results or even better if it would allow me to download the dataframe converted to xlsx but it might be too complicated.
You can use child_process spawn to execute your python script, as Felix Kling suggest in his comment, and return it result to your nodejs app. Then you could use a package like node-xlsx to transform the data to an Excel file.
Something like that:
app.js
// ...
const { spawn } = require('child_process');
const xlsx = require('node-xlsx');
// ...
app.get("/results", (req, res) => {
let query = req.query.search;
let url = "http://www.omdbapi.com/?s=" + query + "&apikey=thewdb";
request(url, (error, response, body) => {
if (!error && response.statusCode == 200) {
let data = JSON.parse(body);
let pyvar = data["Search"][0]["imdbID"];
// Call the python script
let pythonScript = spawn('./script.py', [pyvar]);
pythonScript.stdout.on('data', data => {
// Here transform the datatable to xls sheet
let xlsx = xlsx.build([{ name: "myXlsxSheet", data: data.toString() }])
// And send the file
res.end(new Buffer(xlsx, 'binary'));
});
}
})
})
// ...
script.py
#!/usr/bin/python
import sys
import pandas
pyvar = sys.argv[1]
# Here the script that parse the website
url = 'website.org/' + pyvar + '/blah'
data = parse(url)
print pandas.DataFrame(data)
I'm struggling with what feels like the final step in passing some data from a model file back into a controller using Node Request.
I've successfully set up a callback from my model file which uses request to load JSON from an external source.
My controller can access this, but I think I still need some kind of nested second callback in the final step as I want the variable pageJSON to contain the JSON object and can't quite figure out how.
Think I've hit a bit of a brick wall with this and some fresh eyes on the problem would be appreciated! It feels like I'm missing something really simple at this point (I hope!)
My model file:
module.exports = function (config, callback) {
const request = require('request');
const options = {
'url' : config.urls.page,
'json' : true,
'auth': {
'user': config.auth.username,
'pass': config.auth.password
}
};
request(options, (error, response, body) => {
if (error) {
console.log(error);
}
callback(body);
});
}
My controller file:
const express = require('express');
const router = express.Router();
const app = express();
const config = require('../config');
const page = require('../models/page');
let pageJSON = page(config, (json) => {
console.log(json); // This shows the JSON structure in console
return json;
});
console.log(pageJSON); // Undefined
// Manipulate JSON and pass request view accordingly using Express
You will have to deal with json manipulation within your controller callback (or call another callback from it):
let pageJSON = page(config, (json) => {
console.log(json); // This shows the JSON structure in console
processJSON(json);
});
pageJSON is undefined because nothing is returned from your model.
Title says it all. I'm new to this so I'm sure it must be a simple mistake.
Here's the controller
$scope.removeProduct = function(product){
console.log(product._id);
var inData = new Object();
inData._id = product._id;
console.log(inData);
$http({ url:"/api/deleteprod/", inData, method: "POST"
}).then(function () {
console.log("got here");
var index = $scope.vehicles.indexOf(product);
$scope.vehicles.splice(index, 1);
})
};
and here's the server side.
module.exports = function(app, mongoose, config) {
app.post('/api/deleteprod', function(req, res){
console.log("in app post",req);
var MongoClient = mongodb.MongoClient;
var url='mongodb://localhost:27017/seedsdev';
});
};
Obviously what I want is to pass the _id to the server so I can work with it, but when I output req it's about 50 pages long and has none of the info I wanted. Before it's passed the object can be seen to be fine fia console.log.
What's the rookie mistake I'm making?
When calling $http, you pass the post data with a data property. Currently you're passing an inData property. Change to this:
$http({ url:"/api/deleteprod/", data: inData, method: "POST" }).then(...)
Update:
On the server side, you'll need to make sure you have a JSON parsing middleware, like that from body-parser:
app.use(require('body-parser').json())
Once you are parsing the body using body-parser, you'll have a req.body property with the parsed JSON.
What you are missing are two below things.
1) Data in post request as suggested by #Jacob
2) A parser of Post param body-parser. //npm install body-parser --save
This will help you to parse the POST data in node js.
So code would look like
$scope.removeProduct = function(product){
console.log(product._id);
var inData = new Object();
inData._id = product._id;
console.log(inData);
$http({ url:"/api/deleteprod/", data: inData, method: "POST"
}).then(function () {
console.log("got here");
var index = $scope.vehicles.indexOf(product);
$scope.vehicles.splice(index, 1);
})
};
IN Backend
var bodyParser = require('body-parser');
app.use(bodyParser.json()); // support json encoded bodies
app.use(bodyParser.urlencoded({ extended: true })); // support encoded bodies
module.exports = function(app, mongoose, config) {
app.post('/api/deleteprod', function(req, res){
console.log("in app post",req.body._id);
var MongoClient = mongodb.MongoClient;
var url='mongodb://localhost:27017/seedsdev';
});
};