Develop Reference

Programmatically flatten yaml with typescript

Example input (generally, any yaml file with any content)
some:
nested:
entry: paris
another:
nested:
entry: tel-aviv
Desired output:
Map(2) {
'.some.nested.entry' => 'paris',
'.another.nested.entry' => 'tel-aviv'
}
I tried to use recursion, but failed to define the base case (the function atom):
import { readFileSync } from 'fs'
import { parse, stringify } from 'yaml'
const yamlData = parse(readFileSync('input.yaml').toString())
// An ugly hack to demonstrate what I want to achieve
function atom(yamlData): boolean {
const stringified = stringify(yamlData)
const distance = stringified.search('entry')
const atomicity = distance === 0;
return stringified.search('entry') < 3 // obviously not the way ...
}
function flatten(yamlData, prefix: string): Map<string, any>
{
const flattened = new Map<string, any>();
for (const key in yamlData) {
if (atom(yamlData[key])) {
const abskey = prefix + "." + key;
const value = stringify(yamlData[key]);
const a = value.split(": ")[0].trim() // <--- ugly ...
const b = value.split(": ")[1].trim() // <--- ugly ...
flattened.set(abskey + "." + a, b)
continue;
}
const flattenedTag = flatten(yamlData[key], prefix + "." + key)
for (const [keyTag, value] of flattenedTag.entries()) {
flattened.set(keyTag, value)
}
}
return flattened
}
const flattened = flatten(yamlData, "");
console.log(flattened)

Result won't update VAR

I am trying to run a query, inside AXIOS which gets data from a 3rd party URL. Then uses some of that data to search our mongoDB database.
However it seems it won't update var total = 0
While the query below does function correctly, the return result won't allow me to set that it to the query.
Promise.all(arr.forEach( async (id,index) => {
//(CODE REMOVED JUST TO GET THIS FUNCTION TO WORK)
const search = await geoLocation.find({
'location': {
'$geoWithin': {
'$box': [
[-35.2418503, -13.5076852], [112.8656697, 129.0020486]
]
}
}}).toArray();
total = search.length;
}));
See the full code below
var array = [];
var pointarray = []
var total = 0;
areas.forEach((id,index) => {
if(id.type == "Point"){
pointarray[index] = "N"+id.id;
}else{
array[index] = "R"+id.id;
}
});
var arraySearch = "https://nominatim.openstreetmap.org/lookup?osm_ids="+array.toString()+"&polygon_geojson=1&bbox=1&format=json";
var pointSearch = "https://nominatim.openstreetmap.org/lookup?osm_ids="+pointarray.toString()+"&polygon_geojson=1&bbox=0&format=json"
const requestOne = axios.get(arraySearch);
const requestTwo = axios.get(pointSearch);
axios.all([requestOne, requestTwo])
.then(axios.spread((...responses) => {
const responseOne = responses[0]
const responseTwo = responses[1]
/*
process the responses and return in an array accordingly.
*/
return [
responseOne.data,
responseTwo.data,
];
}))
.then(arr => {
Promise.all(arr.forEach( async (id,index) => {
//const middleIndex = id[index].boundingbox.length / 2;
//const firstHalf = id[index].boundingbox.splice(0, middleIndex);
//const secondHalf = id[index].boundingbox.splice(-middleIndex);
//res.send(secondHalf[0]);
const query = [{
$match: {
location: {
$geoWithin: {$box:[[Number(firstHalf[0]),Number(firstHalf[1])],[Number(secondHalf[0]),Number(secondHalf[1])]]
}
}
}
},{
$count: 'id'
}]
const search = await geoLocation.find({
'location': {
'$geoWithin': {
'$box': [
[-35.2418503, -13.5076852], [112.8656697, 129.0020486]
]
}
}}).toArray();
total = search.length;
// total = search.length;
// const search = geoLocation.aggregate(query).toArray.length;
}));
})
.catch(errors => {
console.log("ERRORS", errors);
})
.then(function () {
res.send(total);
});

How to recursively crawl a page, scrape links, follow links, then scrape again and export in node js?

So far I am able to get my code to export the links of the first page but am having trouble with inserting the subsequent pages links. I just want it to go one deep in terms of search, just follow the links one time and scrape links and insert into my objects array then export to json file. Can someone please tell me what my function is missing?
const cheerio = require('cheerio');
const fs = require('fs');
const chalk = require('chalk');
const axios = require('axios');
const START_URL = 'https://www.reddit.com/r/Frontend/';
const outputFile = 'data.json';
var results = {};
results.sites = [];
var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + '//' + url.hostname;
var MAX_PAGES_TO_VISIT = 1;
pagesToVisit.push(START_URL);
const crawl = () => {
if (numPagesVisited >= MAX_PAGES_TO_VISIT) {
console.log('Reached max limit of number of pages to visit.');
return;
}
var nextPage = pagesToVisit.pop();
if (nextPage in pagesVisited) {
// We've already visited this page, so repeat the crawl
crawl();
} else {
// New page we haven't visited
collectURLS(nextPage, crawl);
}
};
const collectURLS = async (url, callback) => {
pagesVisited[url] = true;
numPagesVisited++;
const { data } = await axios.get(START_URL);
const $ = cheerio.load(data);
if (results === null) {
$("a[href^='http']").each((i, elem) => {
const link = $(elem).attr('href');
pagesToVisit.push(baseUrl + $(this).attr('href'));
if (results === null) {
var obj = {
url: link,
links: [],
};
results.sites.push(obj);
callback();
}
});
} else if (results !== null) {
$("a[href^='http']").each((i, elem) => {
const link = $(elem).attr('href');
results.sites.links.push(link);
});
}
exportResults();
};
//Export results object to json file
const exportResults = () => {
fs.writeFile(outputFile, JSON.stringify(results, null, 4), function (err) {
if (err) throw err;
console.log('complete');
});
console.log(
chalk.black.bgWhite(
`\n ${chalk.underline.bold(
results.length
)} Results exported successfully to ${chalk.underline.bold(outputFile)}\n`
)
);
};
crawl();
I want my output to look like this
{
"sites": [
{
"url1": "holds this pages link and was scraped from original URL",
"links": [where this URLs scraped links will go]
}
{
"url2": "holds this pages link and was scraped from original URL",
"links": [where this URLs scraped links will go]
}
.
.
.
]
}

I would start with this because it's much simpler to follow:
const crawl = async () => {
var nextPage
while (nextPage = pagesToVisit.pop()) {
await collectURLS(nextPage)
}
exportResults()
}
Also make sure not to use this in arrow functions unless you understand the difference from normal functions.

error "r" is not defined when I try to log it into the console

When trying to test my code using the variable r. Which I defined with window.r , It gives me an error variable not defined.
When I type r.parseIngredients() into the command line, the following error occurs:
VM3439:1 Uncaught ReferenceError: r is not defined
at :1:1
index.js file:
import Search from './models/Search';
import Recipe from './models/Recipe';
import * as searchView from './views/searchView';
import { elements, renderLoader, clearLoader } from './views/base';
// const search = new Search('pizza');
// console.log(search);
// search.getResults();
/** GLobal state of the app
* - Search object
* - Current Recipe Object
* - Shopping list object
* - Liked recipes
*/
const state = {};
/*
-- Search Controller
*/
const controlSearch = async () => {
// 1) Get query from view
// const query = searchView.getInput();
const query = 'pizza';
if (query) {
// 2) New seach object and add to state
state.search = new Search(query);
// 3) Prepare UI for results
searchView.clearInput();
searchView.clearResults();
renderLoader(elements.searchRes);
try {
// 4) Search for recipes
await state.search.getResults();
// 5) Render results on UI
clearLoader();
searchView.renderResults(state.search.results);
} catch (err) {
alert('Something wrong with the search...');
clearLoader();
}
}
}
elements.searchForm.addEventListener('submit', e => {
e.preventDefault();
controlSearch();
});
// Testing
window.addEventListener('load', e => {
e.preventDefault();
controlSearch();
});
elements.searchResPages.addEventListener('click', e => {
const btn = e.target.closest('.btn-inline');
if (btn) {
const goToPage = parseInt(btn.dataset.goto, 10);
searchView.clearResults();
searchView.renderResults(state.search.results, goToPage);
}
});
/*
Recipe Controller
*/
const controlRecipe = async () => {
// Get ID from url
const id = window.location.hash.replace('#', '');
console.log(id);
if (id) {
// Prepare UI for changes
// Create new recipe object
state.recipe = new Recipe(id);
// Testing
window.r = state.recipe;
try {
// get recipe data
await state.recipe.getRecipe();
// Calculate servings and time
state.recipe.calcTime();
state.recipe.calcServings();
// Render recipe
console.log(state.recipe);
} catch (err) {
alert('Error processing recipe!');
}
}
};
// window.addEventListener('hashchange', controlRecipe);
// window.addEventListener('load', controlRecipe);
['hashchange', 'load'].forEach(event => window.addEventListener(event, controlRecipe));
Recipe.js file:
import axios from 'axios';
import { key, proxy } from '../config.js'
export default class Recipe {
constructor(id) {
this.id = id;
}
async getRecipe() {
try {
const res = await axios(`${proxy}http://food2fork.com/api/get?key=${key}&rId=${this.id}`);
this.title = res.data.recipe.title;
this.author = res.data.recipe.publisher;
this.img = res.data.recipe.image_url;
this.url = res.data.recipe.source_url;
this.ingredients = res.data.recipe.ingredients;
console.log(res);
} catch (error) {
console.log(error);
alert('Something went wrong')
}
}
calcTime() {
// Assuming that we need 15 min for each 3 ingredients
const numIng = this.ingredients. length;
const periods = Math.ceil(numIng / 3);
this.time = periods * 15;
}
calcServings() {
this.servings = 4;
}
parseIngredients() {
const unitsLong = ['tablespoons', 'tablespoon', 'ounces', 'ounce', 'teaspoons', 'teaspoon', 'cups', 'pounds'];
const unitsShort = ['tbsp', 'tbsp', 'oz', 'oz', 'tsp', 'tsp', 'cup', 'pound'];
const newIngredients = this.ingredients.map(el => {
// 1) Uniform units
let ingredient = el.toLowerCase();
unitsLong.forEach((unit, i) => {
ingredient = ingredient.replace(unit, unitsShort[i]);
});
// 2) Remove parentheses
ingredient = ingredient.replace(/ *\([^)]*\) */g, ' ')
// 3) Parse ingredients into count, unit and ingredient
const arrIng = ingredient.split(' ');
const unitIndex = arrIng.findIndex(el2 => unitsLong.includes(el2));
let objIng;
if (unitIndex > -1) {
// There is a unit
// Ex. 4 1/2 cups, arrCount is [4, 1/2]
// 4 cups, arrCount is [4]
const arrCount = arrIng.slice(0, unitIndex);
let count;
if (arrCount.length === 1) {
count = arrIng[0];
} else {
count = eval(arrIng.slice(0, unitIndex).join('+'));
}
} else if (parseInt(arrIng[0], 10)) {
// There is no unit, but the first element is a number
objIng = {
count: parseInt(arrIng[0], 10),
unit: '',
ingredient: arrIng.slice(1).join(' ')
}
} else if (unitIndex === -1) {
// There is no unit and No number in the 1st position
objIng = {
count: 1,
unit: '',
ingredient
}
}
return objIng;
});
this.ingredients = newIngredients;
}
}
r.parseIngredients() should display the ingredients units abbreviated. ex: 4 tablespoons should be 4 tbsp.

Why does my code using insertMany() skips some of the records and insert same records multiple times?

I have 9577 unique records in a csv file.
This code inserts 9800 records and insert not all records, but duplicates of some of them. Any idea why it does not inserts the unique 9577 records and also duplicates of some of them? Below I also insert the remain part of the code so you get the whole picture
function bulkImportToMongo(arrayToImport, mongooseModel) {
const Model = require(`../../../models/${mongooseModel}`);
let batchCount = Math.ceil(arrayToImport.length / 100);
console.log(arrayToImport.length);
let ops = [];
for (let i = 0; i < batchCount; i++) {
// console.log(i);
let batch = arrayToImport.slice(i, i + 100);
console.log(batch.length);
ops.push(Model.insertMany(batch));
}
return ops;
return Promise.all(ops).then(results => {
// results is an array of results for each batch
console.log("results: ", results);
});
}
I parse the csv file
const Promise = require("bluebird");
const csv = require("fast-csv");
const path = require("path");
const fs = Promise.promisifyAll(require("fs"));
const promiseCSV = Promise.method((filePath, options) => {
return new Promise((resolve, reject) => {
var records = [];
csv
.fromPath(filePath, options)
.on("data", record => {
records.push(record);
})
.on("end", () => {
// console.log(records);
resolve(records);
});
});
});
And here is the script that connects it all together:
const path = require("path");
const promiseCSV = require("./helpers/ImportCSVFiles");
const {
connectToMongo,
bulkImportToMongo
} = require("./helpers/mongoOperations");
const filePath = path.join(__dirname, "../../data/parts.csv");
const options = {
delimiter: ";",
noheader: true,
headers: [
"facility",
"partNumber",
"partName",
"partDescription",
"netWeight",
"customsTariff"
]
};
connectToMongo("autoMDM");
promiseCSV(filePath, options).then(records => {
bulkImportToMongo(records, "parts.js");
});

//It looks like your issue is simply i++. Perhaps you meant i += 100?
for (let i = 0; i < batchCount; i+=100 /* NOT i++ */) {
//...
}

I solved it.
I hope this helps other... :-)
I had two errors, in the function promiseCSV (changed to parseCSV) and second I had bad logic in bulkImportToMongo.
Complete solution:
I parsed and imported 602.198 objects and here is how long time it took using node --max_old_space_size=8000 on a MacBook Pro with 8gb of ram.
console
➜ database git:(master) ✗ node --max_old_space_size=8000 partImport.js
Connected to db!
Time to parse file: : 5209.325ms
Disconnected from db!
Time to import parsed objects to db: : 153606.545ms
➜ database git:(master) ✗
parseCSV.js
const csv = require("fast-csv");
function promiseCSV(filePath, options) {
return new Promise((resolve, reject) => {
console.time("Time to parse file");
var records = [];
csv
.fromPath(filePath, options)
.on("data", record => {
records.push(record);
})
.on("end", () => {
console.timeEnd("Time to parse file");
resolve(records);
});
});
}
module.exports = promiseCSV;
mongodb.js
const mongoose = require("mongoose");
mongoose.Promise = global.Promise;
function connectToMongo(databaseName) {
mongoose.connect(`mongodb://localhost:27017/${databaseName}`, {
keepAlive: true,
reconnectTries: Number.MAX_VALUE,
useMongoClient: true
});
console.log("Connected to db!");
}
function disconnectFromMongo() {
mongoose.disconnect();
console.log("Disconnected from db!");
}
function bulkImportToMongo(arrayToImport, mongooseModel) {
const Model = require(`../../../models/${mongooseModel}`);
const batchSize = 100;
let batchCount = Math.ceil(arrayToImport.length / batchSize);
let recordsLeft = arrayToImport.length;
let ops = [];
let counter = 0;
for (let i = 0; i < batchCount; i++) {
let batch = arrayToImport.slice(counter, counter + batchSize);
counter += batchSize;
ops.push(Model.insertMany(batch));
}
return Promise.all(ops);
}
module.exports.bulkImportToMongo = bulkImportToMongo;
module.exports.connectToMongo = connectToMongo;
module.exports.disconnectFromMongo = disconnectFromMongo;
partImport.js
const path = require("path");
const parseCSV = require("./helpers/parseCSV");
const {
connectToMongo,
disconnectFromMongo,
bulkImportToMongo
} = require("./helpers/mongodb");
const filePath = path.join(__dirname, "../../data/parts.csv");
const options = {
delimiter: ";",
noheader: true,
headers: [
"facility",
"partNumber",
"partName",
"partDescription",
"netWeight",
"customsTariff"
]
};
connectToMongo("autoMDM");
parseCSV(filePath, options)
.then(records => {
console.time("Time to import parsed objects to db");
return bulkImportToMongo(records, "parts.js");
})
/* .then(result =>
console.log("Total batches inserted: ", result, result.length)
) */
.then(() => {
disconnectFromMongo();
console.timeEnd("Time to import parsed objects to db");
})
.catch(error => console.log(error));