I want to set up a server that scrapes data off an existing API, cleans it up and outputs it to my own server where I can manipulate the data further.
So far I've got all the pieces working.
My server runs.
I can scrape the data into the correct JSON format
I have a function that runs when a POST https request is called that writes JSON to my database.
Now I need to connect the two.
There are 3 main components:
the server.js file
The Post controller & schema file. This contains the function that handles the http request and constructs the object and writes to the database.
I have a separate file that contains a bunch of functions that scrape the data and prepare it in the proper JSON format.
My problem is getting scraping file to run. At the moment to test it I just run Node filename.js it worked fine when I was outputting the json to console.log. However now I've written in the http request. the problem is, when I run the server to open the http endpoint, I can't run the file.js that contains the scraper.
Essentially I want the scraper to run on a schedule, and send the data to the endpoint I've set up on the server.
Is there a way to replicate calling node filename.js in code? As per my understanding when you run node filename.js it just runs down the file, executing code as it finds it. Is there a way to do this? Or do I need to call each function 1 by 1 from my server file? Or do I just encapsulate the whole js file in a master function (my worry would then be that variables become siloed)?
I've posted the code below for reference:
server.js
require("dotenv").config(); // ALLOWS ENVIRONMENT VARIABLES TO BE SET ON PROCESS.ENV SHOULD BE AT TOP
const express = require("express");
const app = express();
const postRoutes = require("./routes/postRoutes.js");
const path = require('path');
// Middleware
app.use(express.json()); // parse json bodies in the request object
// Redirect requests to endpoint starting with /posts to postRoutes.js
app.use("/posts", postRoutes);
// Global Error Handler. IMPORTANT function params MUST start with err
app.use((err, req, res, next) => {
console.log(err.stack);
console.log(err.name);
console.log(err.code);
res.status(500).json({
message: "Something went rely wrong",
});
});
// Listen on pc port
const PORT = process.env.PORT || 3000;
app.listen(PORT, () => console.log(`Server running on PORT ${PORT}`));
const battlesGetData = require(path.resolve(__dirname, "./battlesgetData.js")); // is this right?!?
battlesGetData.js
const fetch = require("node-fetch");
const postController = require("./controllers/postControllers");
//const fs = require('fs');
const distinct = (value, index, self) => {
return self.indexOf(value) === index;
}
async function getBattleHistory(player = '', data = {}) {
const battleHistory = await fetch('https://api2.splinterlands.com/battle/history?player=' + player)
.then((response) => {
if (!response.ok) {
throw new Error('Network response was not ok');
}
return response;
})
.then((battleHistory) => {
return battleHistory.json();
})
.catch((error) => {
console.error('There has been a problem with your fetch operation:', error);
});
return battleHistory.battles;
}
const extractGeneralInfo = (x) => {
return {
created_date: x.created_date ? x.created_date : '',
match_type: x.match_type ? x.match_type : '',
mana_cap: x.mana_cap ? x.mana_cap : '',
ruleset: x.ruleset ? x.ruleset : '',
inactive: x.inactive ? x.inactive : ''
}
}
const extractMonster = (team) => {
const monster1 = team.monsters[0];
const monster2 = team.monsters[1];
const monster3 = team.monsters[2];
const monster4 = team.monsters[3];
const monster5 = team.monsters[4];
const monster6 = team.monsters[5];
return {
summoner_id: team.summoner.card_detail_id,
summoner_level: team.summoner.level,
monster_1_id: monster1 ? monster1.card_detail_id : '',
monster_1_level: monster1 ? monster1.level : '',
monster_1_abilities: monster1 ? monster1.abilities : '',
monster_2_id: monster2 ? monster2.card_detail_id : '',
monster_2_level: monster2 ? monster2.level : '',
monster_2_abilities: monster2 ? monster2.abilities : '',
monster_3_id: monster3 ? monster3.card_detail_id : '',
monster_3_level: monster3 ? monster3.level : '',
monster_3_abilities: monster3 ? monster3.abilities : '',
monster_4_id: monster4 ? monster4.card_detail_id : '',
monster_4_level: monster4 ? monster4.level : '',
monster_4_abilities: monster4 ? monster4.abilities : '',
monster_5_id: monster5 ? monster5.card_detail_id : '',
monster_5_level: monster5 ? monster5.level : '',
monster_5_abilities: monster5 ? monster5.abilities : '',
monster_6_id: monster6 ? monster6.card_detail_id : '',
monster_6_level: monster6 ? monster6.level : '',
monster_6_abilities: monster6 ? monster6.abilities : ''
}
}
let battlesList = [];
usersToGrab = ["rus48-bot", "sbalani"]
const battles = usersToGrab.map(user =>
getBattleHistory(user)
.then(battles => battles.map(
battle => {
const details = JSON.parse(battle.details);
if (details.type != 'Surrender') {
if (battle.winner && battle.winner == battle.player_1) {
const monstersDetails = extractMonster(details.team1)
const info = extractGeneralInfo(battle)
return {
...monstersDetails,
...info,
battle_queue_id: battle.battle_queue_id_1,
player_rating_initial: battle.player_1_rating_initial,
player_rating_final: battle.player_1_rating_final,
winner: battle.player_1,
}
} else if (battle.winner && battle.winner == battle.player_2) {
const monstersDetails = extractMonster(details.team2)
const info = extractGeneralInfo(battle)
return {
...monstersDetails,
...info,
battle_queue_id: battle.battle_queue_id_2,
player_rating_initial: battle.player_2_rating_initial,
player_rating_final: battle.player_2_rating_final,
winner: battle.player_2,
}
}
}
})
).then(x => battlesList = [...battlesList, ...x])
)
Promise.all(battles).then(() => {
const cleanBattleList = battlesList.filter(x => x != undefined)
fetch("http://localhost:3000/posts/", {
method: "post",
body: cleanBattleList,
headers: {"Content-Type": "application/json"}
})
.then(json => console.log(json))
.catch(err => console.log(err))
/* fs.writeFile(`data/history.json`, JSON.stringify(cleanBattleList), function (err) {
if (err) {
console.log(err);
}
}); */
});
This is the POST function that gets called
exports.createNewPost = async (req, res, next) => {
/*
let { summoner_id, summoner_level,
monster_1_id, monster_1_level, monster_1_abilities,
monster_2_id, monster_2_level, monster_2_abilities,
monster_3_id, monster_3_level, monster_3_abilities,
monster_4_id, monster_4_level, monster_4_abilities,
monster_5_id, monster_5_level, monster_5_abilities,
monster_6_id, monster_6_level, monster_6_abilities,
created_date, match_type, mana_cap, ruleset, inactive,
battle_queue_id, player_rating_initial, player_rating_final, winner
} = req.body; // using postman this is what allows us to post JSON
let post = new PostBattle(summoner_id, summoner_level,
monster_1_id, monster_1_level, monster_1_abilities,
monster_2_id, monster_2_level, monster_2_abilities,
monster_3_id, monster_3_level, monster_3_abilities,
monster_4_id, monster_4_level, monster_4_abilities,
monster_5_id, monster_5_level, monster_5_abilities,
monster_6_id, monster_6_level, monster_6_abilities,
created_date, match_type, mana_cap, ruleset, inactive,
battle_queue_id, player_rating_initial, player_rating_final, winner); // the title & body defined in the previous line taken from the JSON are now deposited here.
*/
let json = req.body;
for (var obj in json) {
console.log(obj + ": " + json[obj]);
let post = new PostBattle(json[obj].summoner_id, json[obj].summoner_level,
json[obj].monster_1_id, json[obj].monster_1_level, json[obj].monster_1_abilities,
json[obj].monster_2_id, json[obj].monster_2_level, json[obj].monster_2_abilities,
json[obj].monster_3_id, json[obj].monster_3_level, json[obj].monster_3_abilities,
json[obj].monster_4_id, json[obj].monster_4_level, json[obj].monster_4_abilities,
json[obj].monster_5_id, json[obj].monster_5_level, json[obj].monster_5_abilities,
json[obj].monster_6_id, json[obj].monster_6_level, json[obj].monster_6_abilities,
json[obj].created_date, json[obj].match_type, json[obj].mana_cap, json[obj].ruleset, json[obj].inactive,
json[obj].battle_queue_id, json[obj].player_rating_initial, json[obj].player_rating_final, json[obj].winner);
console.log(post);
//let post = new PostBattle(json);
post = await post.save();
console.log("a post is happening");
}
Related
Trying to implement Twitter API to post tweets with multiple images. I am posting requests from the admin dashboard with an AD id(not the Twitter ad) , fetching the images URL from our database and using the URLs to write image files in the upload directory. Then using the Twitter-api-2 package to post a request to Twitter API to get the mediaIdS and post the tweet.
Problem: When I write files to local upload folders, the async function also get executed, therefore cannot find the media files in the local folder, leading to an error.
const router = require('express').Router()
const { parse } = require('dotenv');
const { link } = require('joi');
const { TwitterApi } = require('twitter-api-v2')
const { FetchSingleAdBasics } = require('../helpers/fetch-single-ad-basics');
const request = require('request');
const fs = require('fs');
const path = require('path');
const https = require('https')
function saveImagesToUploads(url, path){
const fullUrl = url
const localPath = fs.createWriteStream(path)
const request = https.get(fullUrl, function(response){
console.log(response)
response.pipe(localPath)
})
}
var jsonPath1 = path.join(__dirname,'../..','uploads/0.png');
var jsonPath2 = path.join(__dirname,'../..','uploads/1.png');
var jsonPath3= path.join(__dirname,'../..','uploads/2.png');
var jsonPath4 = path.join(__dirname,'../..','uploads/3.png');
router.post('/twitter-post', async(req, res) => {
const {adId} = req.body
const imagesArr = []
const imageIdsArr = []
const {text} = req.body
const AD = adId && await FetchSingleAdBasics(adId);
const PostMessage = `${AD?.year} ${AD?.make} ${AD?.model} ${AD?.trim}\r\n${AD?.engineSize}L Engine\r\nPrice: AED${AD?.addetails[0].price}\r\nMileage: ${AD?.mileage} - ${AD?.mileageUnit}\r\nMechanical Condition: ${AD?.addetails[0].mechanicalCondition}\r\nAvailable in: ${AD?.adcontacts[0]?.location}\r\nCheckout full details at: https://ottobay.com/cars/uc/${AD?.id}`
if (!AD)
return res
.status(422)
.json({ message: "failed", error: "Ad Not Found" })
try {
imagesArr.push(await AD?.adimages[0]?.LeftSideView)
imagesArr.push(await AD?.adimages[0]?.LeftFront)
imagesArr.push(await AD?.adimages[0]?.Front)
imagesArr.push(await AD?.adimages[0]?.FrontRight)
// the following function must await for this to finish
imagesArr?.map((item,index) => {
saveImagesToUploads(item, "./uploads/" + `${index}`+ '.png')
})
const filesArr = [jsonPath1,jsonPath3,jsonPath4,jsonPath2]
console.log(filesArr)
console.log(filesArr?.length)
const idsArray = []
// this function get executed without waiting for previous function, leading to error
// this function does not apply filesArr?.length === 4 condition
filesArr?.length === 4 && await Promise.all(filesArr?.length === 4 && filesArr?.map(async (item) => {
try {
const mediaId = await client.v1.uploadMedia(item,{ mimeType : 'png' })
idsArray.push(mediaId)
return imageIdsArr;
} catch(err) {
console.log(err)
throw err;
}
}));
const response = idsArray?.length === 4 && await client.v1.tweetThread([{ status: PostMessage, media_ids: idsArray }]);
// remove files after successfull tweet
await fs.promises.readdir(jsonUploadPath).then((f) => Promise.all(f.map(e => fs.promises.unlink(`${jsonUploadPath}${e}`))))
res.json({status: 'success', response})
} catch (error) {
res.json({status: 'failed',error})
// console.log("tweets error", error.data.errors);
}
})
module.exports = router
I am desperately looking a way to run my node app with my adzuna api keys, my app must be able to fetch data (json) from adzuna:
JobSearch.js is my main js file
configureFormListener() {
this.searchForm.addEventListener('submit', (event) => {
event.preventDefault();
this.startLoading();
this.resultsContainer.innerHTML = '';
const { search, location } = extractFormData(this.searchForm);
fetch(`http://localhost:3000/?search=${search}&location=${location}&country=${this.countryCode}`)
.then(response => response.json())
.then(({ results }) => {
this.stopLoading();
return results
.map(job => jobTemplate(job, this.currencySymbol))
.join('');
})
.then(jobs => this.resultsContainer.innerHTML = jobs)
.catch(() => this.stopLoading());
});
}
I understood that server.js complete config.js
const server = createServer((req, res) => {
const requestURL = url.parse(req.url);
const decodedParams = decodeParams(new URLSearchParams(requestURL.search));
const { search, location, country = 'gb' } = decodedParams;
const targetURL = `${config.BASE_URL}/${country.toLowerCase()}/${config.BASE_PARAMS}&app_id=${config.APP_ID}&app_key=${config.API_KEY}&what=${search}&where=${location}`;
if (req.method === 'GET') {
console.log(chalk.green(`Proxy GET request to : ${targetURL}`));
axios.get(targetURL)
.then(response => {
res.writeHead(200, headers);
res.end(JSON.stringify(response.data));
})
.catch(response => {
console.log(chalk.red(response));
res.writeHead(500, headers);
res.end(JSON.stringify(response));
});
}
});
I am a beginner on nodeJS so i suppose that need separated the property & api keys for the encapsulation & safety
config.js :
require('dotenv').config()
module.exports = {
APP_ID: process.env.APP_ID,
API_KEY: process.env.API_KEY,
BASE_URL: 'https://api.adzuna.com/v1/api/jobs',
BASE_PARAMS: 'search/1?&results_per_page=20&content-type=application/json',
};
.env :
APP_ID="000000000000000"
API_KEY="000000000000000
...But i dont know is a localhost port error or maybe it dosent happens to take the api keys values, because when I run the search request http://localhost:3000/?search=... it made me net::ERR_CONNECTION_REFUSED and the accessKeyId section's browser is always empty
What's wrong with my code ?
So I am an extremely beginner programmer trying to understand how to call/get data from Google Distance Matrix API in purely Javascript.
Below is my codes
https = require('https')
function getDistance(app){
console.log('testing1');
let defaultPath = 'maps.googleapis.com/maps/api/distancematrix/json?units=metric';
let APIKey = '&key=<API KEY HERE>';
let origin = 'Tampines Avenue 1, Temasek Polytechnic';
let destination = 'Tampines Central 5, Tampines Mall';
let newPath = defaultPath+ '&origins=' + origin + '&destinations=' +destination + APIKey;
console.log('https://'+ newPath); //this prints the correct path
https.get('https://'+ newPath, (res) =>{ //i assume the problem begins here?
console.log('testing2')//doesnt print
let body = ''; //no idea what this does. Copied from a school lab sheet
res.on('data', (d) => {
console.log('testing3') //this doesn't print
let response = JSON.parse(body);
let distance = response.rows[0].elements.distance.text //are these correct?
let travelTime = response.rows[0].elements.duration.text
console.log(distance) //doesnt print
console.log(response.rows[0]) //doesnt print
app.add('distance between is ' + distance + '. Time taken is ' + travelTime);
console.log(response);
});
});
}
I'm pretty sure the 'https://'+ newPath is correct as it is printed in the console.log
And throwing the link into a browser
I get this as result
so can someone please explain to me what i'm doing wrong?
Oh and also, dont know if this is necessary but im doing this in dialogflow.cloud.google.com as a chatbot for my assignment
This is the error I get
Error: No responses defined for platform: undefined at
WebhookClient.send_
(/srv/node_modules/dialogflow-fulfillment/src/dialogflow-fulfillment.js:428:13)
at promise.then
(/srv/node_modules/dialogflow-fulfillment/src/dialogflow-fulfillment.js:246:38)
at at process._tickDomainCallback
(internal/process/next_tick.js:229:7)
I found a similar problem on GitHub: https://github.com/dialogflow/dialogflow-fulfillment-nodejs/issues/22
The solution was
Okay, so here's what I did to make this work properly.
I used request-promise-native instead of http to make the AJAX Call.
const rp = require('request-promise-native');
I returned a promise from the handler of the promise that rp returns.
return rp(options).then(data => { // Extract relevant details from data. // Add it to the agent. agent.add('Here's the data: ', JSON.stringify(data)); return Promise.resolve(agent); });
The full code is
'use strict';
const express = require('express');
const bodyParser = require('body-parser');
const rp = require('request-promise-native');
const { WebhookClient } = require('dialogflow-fulfillment');
const { Card, Suggestion } = require('dialogflow-fulfillment');
const { Carousel } = require('actions-on-google');
process.env.DEBUG = 'dialogflow:debug'; // enables lib debugging statements
const imageUrl = 'https://developers.google.com/actions/images/badges/XPM_BADGING_GoogleAssistant_VER.png';
const imageUrl2 = 'https://lh3.googleusercontent.com/Nu3a6F80WfixUqf_ec_vgXy_c0-0r4VLJRXjVFF_X_CIilEu8B9fT35qyTEj_PEsKw';
const linkUrl = 'https://assistant.google.com/';
const API_KEY = 'YOUR-API-KEY-HERE';
const server = express();
server.use(
bodyParser.urlencoded({
extended: true
})
);
server.use(bodyParser.json());
server.post('/dialog-flow-fulfillment', (request, response) => {
const agent = new WebhookClient({ request, response });
function googleAssistantOther(agent) {
let conv = agent.conv();
conv.ask(`Sure! Details about ${agent.parameters.movie} coming your way!`);
return getMovieDataFromOMDb(agent.parameters.movie).then(movie => {
conv.ask(`Okay! So there you go.`);
conv.ask(new Card({
title: `${movie.Title}(${movie.Year})`,
imageUrl: movie.Poster,
text: `${movie.Rated} | ${movie.Runtime} | ${movie.Genre} | ${movie.Released} (${movie.Country})`,
buttonText: 'Website',
buttonUrl: movie.Website || `https://www.imdb.com/title/${movie.imdbID}`
}));
conv.ask(new Suggestion(`More details`));
conv.ask(new Suggestion(`Another movie`));
agent.add(conv);
return Promise.resolve(agent);
});
}
function welcome(agent) {
agent.add(`Welcome to my agent!`);
}
function fallback(agent) {
agent.add(`I didn't understand`);
agent.add(`I'm sorry, can you try again?`);
}
function getMovieDetailsOther(agent) {
return getMovieDataFromOMDb(agent.parameters.movie).then(movie => {
// const responseDataToSend = `${movie.Title} is a ${
// movie.Actors
// } starer ${movie.Genre} movie, released in ${
// movie.Year
// }. It was directed by ${movie.Director}`;
// console.log(`Generated response as ${responseDataToSend}`);
// agent.add(responseDataToSend);
agent.add(`Okay! So there you go.`);
agent.add(new Card({
title: `${movie.Title}(${movie.Year})`,
imageUrl: movie.Poster,
text: `${movie.Rated} | ${movie.Runtime} | ${movie.Genre} | ${movie.Released} (${movie.Country})`,
buttonText: 'Website',
buttonUrl: movie.Website || `https://www.imdb.com/title/${movie.imdbID}`
}));
agent.add(new Suggestion(`More details`));
agent.add(new Suggestion(`Another movie`));
return Promise.resolve(agent);
}, error => {
console.log(`Got an error as ${error}`);
agent.add(`Sorry bout that! An error prevented getting data for: ${agent.parameters.movie || 'the requested movie'}`
);
})
.catch(function (err) {
console.log(`Caught an err as ${err}`);
agent.add(err);
});
// agent.add(`This message is from Dialogflow's Cloud Functions for Firebase editor!`);
// const newCard = new Card({
// title: `Title: this is a card title`,
// imageUrl: imageUrl,
// text: `This is the body text of a card. You can even use line\n breaks and emoji! 💁`,
// buttonText: 'This is a button',
// buttonUrl: linkUrl
// });
// // newCard.setPlatform('facebook');
// agent.add(newCard);
// agent.add(new Suggestion(`Quick Reply`));
// agent.add(new Suggestion(`Suggestion`));
// agent.setContext({ name: 'weather', lifespan: 2, parameters: { city: 'Rome' }});
}
function moreDetailsOther(agent) {
return getMovieDataFromOMDb(agent.parameters.movie).then(movie => {
agent.add(`Okay! I've got you covered on this too.`);
agent.add(`So the ${movie.Actors} starer ${movie.Type} is produced by ${movie.Production}, is directed by ${movie.Director}`);
agent.add(`It ${movie.Awards}. It's available in ${movie.Language}`);
agent.add(`Written by ${movie.Writer}, it plots ${movie.Plot}`);
agent.add(new Suggestion(`Stats on the movie`));
agent.add(new Suggestion(`Another movie`));
return Promise.resolve(agent);
}, error => {
console.log(`Got an error as ${error}`);
agent.add(`Sorry bout that! An error prevented getting data for: ${agent.parameters.movie || 'the requested movie'}`
);
})
.catch(function (err) {
console.log(`Caught an err as ${err}`);
agent.add(err);
});
}
function movieStatsOther(agent) {
return getMovieDataFromOMDb(agent.parameters.movie).then(movie => {
let ratingDetails = `${movie.Title} scored `;
movie.Ratings.forEach(rating => {
ratingDetails += `${rating.Value} on ${rating.Source} `
});
agent.add(`Sure! Here are the stats.`);
agent.add(ratingDetails);
agent.add(new Suggestion(`Another movie`));
return Promise.resolve(agent);
}, error => {
console.log(`Got an error as ${error}`);
agent.add(`Sorry bout that! An error prevented getting data for: ${agent.parameters.movie || 'the requested movie'}`
);
})
.catch(function (err) {
console.log(`Caught an err as ${err}`);
agent.add(err);
});
}
function getMovieDataFromOMDb(movieName) {
const movieToSearch = movieName || 'The Godfather';
const options = {
uri: 'https://www.omdbapi.com/',
json: true,
qs: {
t: movieToSearch,
apikey: API_KEY
}
};
return rp(options);
}
// Run the proper handler based on the matched Dialogflow intent
let intentMap = new Map();
intentMap.set('Default Welcome Intent', welcome);
intentMap.set('Default Fallback Intent', fallback);
if (agent.requestSource === agent.ACTIONS_ON_GOOGLE) {
intentMap.set(null, googleAssistantOther);
// intentMap.set('More Details', googleAssistantMoreDetails);
// intentMap.set('Movie Stats', googleAssistantMovieStats);
} else {
intentMap.set('Get Movie Details', getMovieDetailsOther);
intentMap.set('More Details', moreDetailsOther);
intentMap.set('Movie Stats', movieStatsOther);
}
agent.handleRequest(intentMap);
});
server.listen(process.env.PORT || 8000, () => {
console.log('Server is up and running...');
});
Codepen: https://codepen.io/siddajmera/pen/eraNLW?editors=0010
You don't show all your code, but it looks like getDistance() is your Intent Handler function that you've registered in code that you're not showing.
If so, and if you're making an asynchronous call to an API, you need to return a Promise to indicate that you're waiting for the HTTP call to complete before you send the result.
Without the Promise, the function completes right after it makes the call with http.get(), but without anything being set for the response with app.add().
While it is possible to turn an event-based call (what you're doing now) into a Promise, it isn't that easy if you're not familiar with it, and there are better solutions.
Using a package such as request-promise (and more likely request-promise-native - it uses the same syntax, but request-promise has the documentation) is far easier. With this, you would return the Promise that is generated by the http call, and in the then() clause of it, you would make your calls to app.add().
I haven't tested it, but it might look something like this:
const rp = require('request-promise-native');
function getDistance(app){
console.log('testing1');
let defaultPath = 'maps.googleapis.com/maps/api/distancematrix/json?units=metric';
let APIKey = '&key=<API KEY HERE>';
let origin = 'Tampines Avenue 1, Temasek Polytechnic';
let destination = 'Tampines Central 5, Tampines Mall';
let newPath = defaultPath+ '&origins=' + origin + '&destinations=' +destination + APIKey;
let url = 'https://'+newPath;
console.log(url);
rp.get(url)
.then( response => {
console.log(response);
let distance = response.rows[0].elements.distance.text
let travelTime = response.rows[0].elements.duration.text
app.add('distance between is ' + distance + '. Time taken is ' + travelTime);
})
.catch( err => {
console.log( err );
app.add('Something went wrong.');
});
};
Currently I have axios and cheerio return data from a webpage. I then setup express to setup a few views. I double checked my index.hbs and it include {{data}} inside the body. This should allow the page to render the text from the index render data: dealss . Am I missing anything ? The dealss obj holds 4 different objects that I can access.
getdeals(result => console.log(result.totaldeals[0].date))
This returns [ 09/04/2019/ ] in the console.
const path = require('path')
const express = require('express')
const hbs = require('hbs')
const axios = require('axios');
const cheerio = require('cheerio');
const app = express()
// Define paths for express config
const publicDirPath = path.join(__dirname, '../public')
const viewsPath = path.join(__dirname, '../templates/views')
const partialsPath = path.join(__dirname, '../templates/partials')
// Setup handlebars engine and views location
app.set('view engine', 'hbs')
app.set('views', viewsPath)
hbs.registerPartials(partialsPath)
// Setup static directory to serve
app.use(express.static(publicDirPath))
// Views
app.get('', (req, res) => {
res.render('index', {
title: 'ClearVision',
data: dealss,
name: 'Chris'
})
})
app.get('/about', (req, res) => {
res.render('about', {
title: 'ClearVision - About Us',
header: 'About Us',
name: 'Chris'
})
})
app.get('/help', (req, res) => {
res.render('help', {
title: 'ClearVision - Help',
helptext: 'Please contact x for help.',
name: 'Chris'
})
})
app.get('/weather', (req, res) => {
res.send({
forecast: 'It is sunny.',
location: 'x, Ca'
})
})
app.listen(1337, () => {
console.log('Server is currently running on port 1337.')
})
const url = 'https://abcdef.com/';
axios.defaults.withCredentials = true
// Get the deals
const getdeals = (callback) => {
axios(url, {
headers: {
Cookie: "x=xx;"
}
})
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
// Deals Page
const statsTable = $('tbody > tr');
const totaldeals = [];
// Loop Table for data in each row
statsTable.each(function () {
const nwline = "\n"
let date = $(this).find('td:nth-child(1)').text()
let bodydeals = $(this).find('td:nth-child(2)').text()
let newdeal = $(this).find('td:nth-child(3)').text()
let revdeal = $(this).find('td:nth-child(4)').text()
let monthlydealrev = $(this).find('td:nth-child(5)').text()
// Clear /n
if (date.includes(nwline)) {
date = date.toString().replace("\n", ""),
date = date.toString().replace("\n", "")
}
// Clear /n
if (bodydeals.includes(nwline)) {
bodydeals = bodydeals.toString().replace("\n", ""),
bodydeals = bodydeals.toString().replace("\n", ""),
bodydeals = bodydeals.toString().replace("\n", "")
}
// Clear /n
if (newdeal.includes(nwline)) {
newdeal = newdeal.toString().replace("\n", ""),
newdeal = newdeal.toString().replace("\n", ""),
newdeal = newdeal.toString().replace("\n", "")
}
// Clear /n
if (revdeal.includes(nwline)) {
revdeal = revdeal.toString().replace("\n", ""),
revdeal = revdeal.toString().replace("\n", ""),
revdeal = revdeal.toString().replace("\n", "")
}
// Clear /n (lookup jquery table functions)
if (monthlydealrev.includes(nwline)) {
monthlydealrev = monthlydealrev.toString().replace("\n", ""),
monthlydealrev = monthlydealrev.toString().replace("\n", ""),
monthlydealrev = monthlydealrev.toString().replace("\n", "")
}
totaldeals.push({
date,
bodydeals,
newdeal,
revdeal,
monthlydealrev
})
})
callback({
totaldeals
})
//console.log(totaldeals[1].date)
})
.catch(console.error);
}
function newFunction() {[getdeals(result => console.log(result.totaldeals))]}
I added a data: dealss under the res.render for the index. I also checked the index.hbs which has {{data}} in there. Shouldn't this just add the text to the screen?
Any ideas on how to print it to the view?
You just need to pass it as a variable to your hbs file:
app.get('', (req, res) => {
getdeals(result => {
res.render('index', {
title: 'ClearVision',
data: result, // or result.totaldeals depending
name: 'Chris' // on what you really mean
})
});
})
Improvements
If you rewrite getdeals() to return a Promise instead of accepting a callback you can use async/await:
const getdeals = () => {
// NOTE THIS CHANGE, return axios promise:
return axios(url, {
/* ... */
})
.then(response => {
/* .. */
statsTable.each(function () {
/* .. */
})
return totaldeals; // NOTE we return the result instead
// of calling a callback. This will
// return a resolved Promise
})
// Don't catch here, your request will hang if an error occurs
}
Now with the change above (return axios and return the result) we can rewrite the route as:
app.get('', async (req, res, next) => { // must have async keyword!
try {
let result = await getdeals();
res.render('index', {
title: 'ClearVision',
data: result, // or result.totaldeals depending
name: 'Chris' // on what you really mean
})
}
catch (err) {
console.log(err);
next(err); // this will close the request socket
}
})
I'm trying to import csv file into neo4j using node.js. I have to insert data into multiple collection/table, so I have to insert data using node.js script. But my problem is, I can't prevent data duplication when inserting csv data.
Sample CSV data:
name
-------------
Afghanistan
Afghanistan
Aland
Albania
Albania
Bangladesh
Bangladesh
index.js
cp = require('child_process');
child = cp.fork(__dirname + "/background-import-csv-file.js");
child.on('message', function(msg) {
console.log("background-insert-process said : ", msg);
});
file = path.resolve(__dirname, `./file/simplemaps.csv`);
child.send(file);
In background-import-csv-file.js, I have write code in two different way.
First Promise based (background-import-csv-file.js) :
cp = require('child_process');
csv = require('fast-csv');
Q = require('q');
DB = require("./common/driver");
Country = require('./collection/country');
process.on("message", (file) => {
stream = fs.createReadStream(file);
csv
.fromStream(stream, { headers: true })
.on("data", function(data) {
let countryData = { "name": data.name };
neo = new DB();
country = new Country(neo);
country.insert(countryData)
.then(resp => process.send(resp.msg) )
.catch(err => process.send(err) )
})
.on("end", () => process.send("file read complete") );
});
./collection/country.js:
Q = require('q');
Country = function Country(neo) {
this.country = "Country"; this.neo = neo;
};
Country.prototype.find = function find(filters) {
query = `MATCH (a:Country { name: '${filters.name}' } ) RETURN {country:properties(a)}`;
return this.neo.run(query, filters).then(resp => resp);
}
Country.prototype.create = function create(data) {
query = `CREATE (ax:Country { name: '${data.name}' } ) RETURN ax `;
return this.neo.run(query, {}).then(resp => resp[0].properties).catch(err => err)
}
Country.prototype.insert = function insert(country) {
filter = { name: country.name };
return Q(this.find(filter))
.then(resp => resp.length > 0 ? Q.resolve({ msg: `country: [${country.name}] is already exist` }) : Q.resolve(this.create(country)) )
.then(resp => resp)
.catch(e => Q.reject(e));
}
module.exports = Country;
./common/driver.js
neo4j = require('neo4j-driver').v1;
function DB() {
this.driver = neo4j.driver(); this.session = this.driver.session();
}
DB.prototype.run = function run(query, data) {
return this.session.run(query, data)
.then(response => response.records.map(
record => record._fields[0] ?
record._fields.length ? record._fields[0] : {} : {}
) ).catch(err => new Error(err) );
}
module.exports = DB;
When I run index.js in terminal, In database, I have 2 Afghanistan, 1 Aland, 2 Albania and 2 Bangladesh. But I need 1 Afghanistan, 1 Aland, 1 Albania and 1 Bangladesh in my database. When I analyze code, than found that before inserting data, I'm checking data ( Country.prototype.find = function find(filters)) if it is already exist or not, but it always return empty result. That why it insert multiple data. If I run index.js again, then no new data is inserted into database. To solve this problem, I've tried following CQL :
MERGE (c:Country { name: '${data.name}' } ) RETURN c
It is inserted unique data, but It kill so much time. Then I have written the following code:
Event-driven (background-import-csv-file.js) :
process.on("message", (file) => {
stream = fs.createReadStream(file);
csv
.fromStream(stream, { headers: true })
.on("data", function(data) {
countryData = { "name": data.name };
neo = new DB();
country = new Country(neo);
country.find(countryData);
country.on('find', resp => resp.length > 0 ? Q.resolve({ msg: `country: [${country.name}] is already exist` }) : Q.resolve(country.create(countryData)) );
country.on('create', resp => console.log(resp) );
})
.on("end", () => process.send("file read complete") );
});
./collection/country.js:
EventEmitter = require('events').EventEmitter;
util = require('util');
Country = function Country(neo) {
this.neo = neo; EventEmitter.call(this);
};
util.inherits(Country, EventEmitter);
Country.prototype.find = function find(filters) {
query = `MATCH (a:Country { name: '${filters.name}' } ) RETURN {country:properties(a)}`;
return this.neo.run(query, {}).then(resp => this.emit('find', resp));
}
Country.prototype.create = function create(data) {
query = `CREATE (ax:Country { name: '${data.name}' } ) RETURN ax `;
return this.neo.run(query, {}).then(resp => this.emit('create', resp[0].properties)).catch(err => err)
}
And this time, it shows same result. What am I missing? Any suggestion will be very helpfull.
NB: I'm using fast-csv for csv parsing and Q for promise.
Actually I can imagine the following solutions:
Modify the CSV file itself with programming language(like node.js) to remove the duplicate lines with the same name.
Add the neo4j unique constrains
CREATE CONSTRAINT ON (c:Country) ASSERT c.name IS UNIQUE
Involve the middleware, like a queue which to prevent the duplicate items, for this, you need to define you own message structure and duplicate arithmetic.
above.
My problem was, in csv file parsing, it was so fast (event-driven) it wasn't wait to finish insert data into database. So I have to pause file parsing and then resume it.
I solve my problem using the following code:
Promise based (background-import-csv-file.js) :
cp = require('child_process');
csv = require('fast-csv');
Q = require('q');
DB = require("./common/driver");
Country = require('./collection/country');
process.on("message", (file) => {
stream = fs.createReadStream(file);
csvstream = csv
.fromStream(stream, { headers: true })
.on("data", function(data) {
csvstream.pause(); // pause the csv file parsing
countryData = { "name": data.name };
neo = new DB();
country = new Country(neo);
country.insert(countryData)
.then(resp => {
process.send(resp.msg);
neo.close();
return csvstream.resume(); // after completing db process, resume
})
.catch(err => {
process.send(err);
return csvstream.resume(); // if failed, then resume
});
})
.on("end", () => process.send("file read complete") );
});