I'm downloading posts from subreddit
const electron = require('electron');
const _ = require('lodash');
const getsub = (sub, epoch) => {
let res = {};
const sub_url = `https://api.pushshift.io/reddit/search/submission/?subreddit=${sub}&limit=1000&sort=desc&before=${epoch}`;
return new Promise((resolve, reject) => {
axios
.get(sub_url)
.then(response => {
let d = response.data.data;
let e = d[d.length - 1].created_utc;
res.data = d;
res.epoch = e;
return resolve(res);
})
.catch(error => {
return reject(error.message);
});
});
};
The function in question
async function getSubreddit(sub) {
const utc = new Date().toJSON().slice(0, 10).replace(/-/g, "/");
const endEpoch = new Date(utc).getTime() / 1000;;
var dataArray = [];
let epoch;
const resp = await getsub(sub, endEpoch);
const jsondat = resp.data;
epoch = resp.epoch;
var result = _.map(dataArray, function(o) {
return _.pick(o, ['title', 'url', 'subreddit', 'created_utc', 'domain', 'full_link', 'created_utc'])
})
console.log(jsondat.length);
// console.log({result,epoch});
// downloadStuff(sub,result)
}
while jsondat shows a length of 1000. But after I parse it with lodash, the result shows length as 0.
I'm confused because this code works in regular javascript page.
EDIT: This is a silly error on my part. Here is the corrected code
async function getSubreddit(sub) {
var utc = new Date().toJSON().slice(0, 10).replace(/-/g, "/");
var endEpoch = new Date(utc).getTime() / 1000;;
var dataArray = [];
const resp = await getsub(sub, endEpoch);
const jsondat = resp.data;
dataArray = jsondat.concat(dataArray);
epoch = resp.epoch;
var result = _.map(dataArray, function(o) {
return _.pick(o, ['title', 'author','url', 'subreddit', 'created_utc', 'domain', 'full_link', 'created_utc'])
})
console.log({result,epoch});
downloadStuff(sub,result)
}
Related
Here is my code (it takes a csv file as input, does some API calls, and outputs the results as another csv file):
let allRows = [];
async function fileToLines(file) {
return new Promise((resolve, reject) => {
reader = new FileReader();
reader.onload = function (e) {
parsedLines = e.target.result.split(/\r|\n|\r\n/);
resolve(parsedLines);
};
reader.readAsText(file);
});
}
document
.getElementById('fileInput')
.addEventListener('change', async function (e) {
var file = e.target.files[0];
if (file != undefined) {
fileToLines(file).then(async id => {
console.log(id)
console.log(parsedLines)
console.log(typeof id);
var idInt = id.map(Number);
var idFiltered = id.filter(function (v) { return v !== '' });
console.log(idFiltered)
if (file != undefined) {
allRows = [];
}
for (let id of idFiltered) {
const row = await getRelease(id);
allRows.push(row);
}
download();
});
}
});
function debounce(inner, ms = 0) {
let timer = null;
let resolves = [];
return function (...args) {
// Run the function after a certain amount of time
clearTimeout(timer);
timer = setTimeout(() => {
/* Get the result of the inner function, then apply it to the resolve function of
each promise that has been created since the last time the inner function was run */
let result = inner(...args);
resolves.forEach(r => r(result));
resolves = [];
}, ms);
return new Promise(r => resolves.push(r));
};
}
const throttleFetch = debounce(fetch, 2500);
function getRelease(idFiltered) {
return throttleFetch(`https://api.discogs.com/releases/${idFiltered}`, {
headers: {
'User-Agent': '***/0.1',
'Authorization': 'key=***, secret=***',
},
}).then(response => response.json())
.then(data => {
if (data.message === 'Release not found.') {
return { error: `Release with ID ${idFiltered} does not exist` };
} else {
const id = data.id;
const delimiter = document.getElementById("delimiter").value || "|";
const artists = data.artists ? data.artists.map(artist => artist.name) : [];
const barcode = data.identifiers.filter(id => id.type === 'Barcode')
.map(barcode => barcode.value);
var formattedBarcode = barcode.join(delimiter);
const country = data.country || 'Unknown';
const genres = data.genres || [];
const formattedGenres = genres.join(delimiter);
const labels = data.labels ? data.labels.map(label => label.name) : [];
const formattedLabels = labels.join(delimiter);
const catno = data.labels ? data.labels.map(catno => catno.catno) : [];
const formattedCatNo = catno.join(delimiter);
const styles = data.styles || [];
const formattedStyles = styles.join(delimiter);
const tracklist = data.tracklist ? data.tracklist
.map(track => track.title) : [];
const formattedTracklist = tracklist.join(delimiter);
const year = data.year || 'Unknown';
const format = data.formats ? data.formats.map(format => format.name) : [];
const qty = data.formats ? data.formats.map(format => format.qty) : [];
const descriptions = data.formats ? data.formats
.map(descriptions => descriptions.descriptions) : [];
const preformattedDescriptions = descriptions.toString()
.replace('"', '""').replace(/,/g, ', ');
const formattedDescriptions = '"' + preformattedDescriptions + '"';
console.log(idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
)
return [idFiltered,
artists,
format,
qty,
formattedDescriptions,
formattedLabels,
formattedCatNo,
country,
year,
formattedGenres,
formattedStyles,
formattedBarcode,
formattedTracklist
];
}
});
}
function download() {
const ROW_NAMES = [
"release_id",
"artist",
"format",
"qty",
"format descriptions",
"label",
"catno",
"country",
"year",
"genres",
"styles",
"barcode",
"tracklist"
];
var csvContent = "data:text/csv;charset=utf-8,"
+ ROW_NAMES + "\n" + allRows.map(e => e.join(",")).join("\n");
var encodedUri = encodeURI(csvContent);
var link = document.createElement("a");
link.setAttribute("href", encodedUri);
link.setAttribute("download", "my_data.csv");
document.body.appendChild(link); // Required for Firefox
link.click();
}
The app passes tests for 5, 10, 20, or 30 lines of input, but when I try 40, I get this error:
Uncaught (in promise) TypeError: e.join is not a function
csvContent file:///D:/DOWNLOADS/CODE/***/csv.js:151
download file:///D:/DOWNLOADS/CODE/***/csv.js:151
<anonymous> file:///D:/DOWNLOADS/CODE///***/csv.js:38
async* file:///D:/DOWNLOADS/CODE/***/csv.js:20
I really don't know why that would be. Any ideas please? TIA.
Apparently I need to add more details but IDK what?
I have some code that when you console.log it, it looks like the image below:
The code I am running is as follows:
onClick={() => {
const stream = fetch(
'https://lichess.org/api/games/user/neio',
{ headers: { Accept: 'application/x-ndjson' } }
);
const onMessage = obj => {
console.log('test', obj);
};
const onComplete = () =>
console.log('The stream has completed');
stream.then(readStream(onMessage)).then(onComplete);
}}
export const readStream = processLine => response => {
const stream = response.body.getReader();
const matcher = /\r?\n/;
const decoder = new TextDecoder();
let buf = '';
const loop = () =>
stream.read().then(({ done, value }) => {
if (done) {
if (buf.length > 0) processLine(JSON.parse(buf));
} else {
const chunk = decoder.decode(value, {
stream: true,
});
buf += chunk;
const parts = buf.split(matcher);
buf = parts.pop();
for (const i of parts) processLine(JSON.parse(i));
return loop();
}
});
return loop();
};
export default readStream;
What I am trying to do is build a parent object that contains all these individual rows of data.
I'm new at promises and fetch etc. So currently, I have no idea on how to build this parent object that contains each individual row.
Any suggestions?
Can't you have a global array and add items to it like:
var arrCollection = [];
...
const onMessage = obj => {
arrCollection.push(obj);
};
You can have an object with those items doing like:
var objCollection = { items: arrCollection };
I'm trying to get the amount received from the request by going to a list and then trying to export it to another file, but I'm having trouble doing this because I tried several ways to pass the list as return and it didn't work.
const moment = require("moment");
const sslChecker = require("ssl-checker");
const express = require("express");
//const dominios = require('./server');
var dominios = [
"api-prd.koerich.com.br",
"api-prd.reservapto.com.br",
"c.btg360.com.br",
"compassouol.com",
"gmfy.compasso.com.br",
"webqplus.cbn.com.bo",
"webqplus.cerchile.cl",
"webqplus.cervepar.com.py",
"webqplus.fnc.com.uy",
"webqplus.quilmes.com.ar",
"www.efurukawa.com",
];
var get_domain = dominios;
for (var i = 0; i < get_domain.length; i++) {
sslChecker(get_domain[i])
.then((certdata) => {
var startdate = new Date(certdata.validFrom);
var enddate = new Date(certdata.validTo);
var certstart = moment(startdate);
var certend = moment(enddate);
var ssldata = [
{
domain: get_domain,
issued: certstart.format("LLLL"),
expires: certend.format("LLLL"),
daysleft: certdata.daysRemaining,
},
];
var data = JSON.parse(certdata.daysRemaining);
var nomes = (certdata.validFor[0]);
var lista = [];
lista.push(data);
lista.push(nomes);
console.log(lista); //get this values
})
.catch((err) => {
console.log(err);
});
}
//module.exports = lista; //export here
console.log:
async function getList() {
mondongo = [];
var dominios = [
"api-prd.koerich.com.br",
"api-prd.reservapto.com.br",
"c.btg360.com.br",
"compassouol.com",
"gmfy.compasso.com.br",
"webqplus.cbn.com.bo",
"webqplus.cerchile.cl",
"webqplus.cervepar.com.py",
"webqplus.fnc.com.uy",
"webqplus.quilmes.com.ar",
"www.efurukawa.com",
]
try {
for (var i = 2; i < domains.length; i++) {
mondongo.push(
sslChecker(domains[i])
.then((certdata) => {
var startdate = new Date(certdata.validFrom);
var enddate = new Date(certdata.validTo);
var certstart = moment(startdate);
var certend = moment(enddate);
var ssldata = [
{
domain: domains,
issued: certstart.format("LLLL"),
expires: certend.format("LLLL"),
daysleft: certdata.daysRemaining,
},
];
var data = certdata.daysRemaining;
var nome = certdata.validFor[0];
var lista = [];
lista.push(nome);
lista.push(data);
return lista;
})
.catch((err) => {
//console.log(err);
})
);
}
}
catch (error) {
console.log(error);
}
var arroz = await Promise.all(mondongo);
return arroz;
}
return:
create a function and use the async await methods.
I have 9577 unique records in a csv file.
This code inserts 9800 records and insert not all records, but duplicates of some of them. Any idea why it does not inserts the unique 9577 records and also duplicates of some of them? Below I also insert the remain part of the code so you get the whole picture
function bulkImportToMongo(arrayToImport, mongooseModel) {
const Model = require(`../../../models/${mongooseModel}`);
let batchCount = Math.ceil(arrayToImport.length / 100);
console.log(arrayToImport.length);
let ops = [];
for (let i = 0; i < batchCount; i++) {
// console.log(i);
let batch = arrayToImport.slice(i, i + 100);
console.log(batch.length);
ops.push(Model.insertMany(batch));
}
return ops;
return Promise.all(ops).then(results => {
// results is an array of results for each batch
console.log("results: ", results);
});
}
I parse the csv file
const Promise = require("bluebird");
const csv = require("fast-csv");
const path = require("path");
const fs = Promise.promisifyAll(require("fs"));
const promiseCSV = Promise.method((filePath, options) => {
return new Promise((resolve, reject) => {
var records = [];
csv
.fromPath(filePath, options)
.on("data", record => {
records.push(record);
})
.on("end", () => {
// console.log(records);
resolve(records);
});
});
});
And here is the script that connects it all together:
const path = require("path");
const promiseCSV = require("./helpers/ImportCSVFiles");
const {
connectToMongo,
bulkImportToMongo
} = require("./helpers/mongoOperations");
const filePath = path.join(__dirname, "../../data/parts.csv");
const options = {
delimiter: ";",
noheader: true,
headers: [
"facility",
"partNumber",
"partName",
"partDescription",
"netWeight",
"customsTariff"
]
};
connectToMongo("autoMDM");
promiseCSV(filePath, options).then(records => {
bulkImportToMongo(records, "parts.js");
});
//It looks like your issue is simply i++. Perhaps you meant i += 100?
for (let i = 0; i < batchCount; i+=100 /* NOT i++ */) {
//...
}
I solved it.
I hope this helps other... :-)
I had two errors, in the function promiseCSV (changed to parseCSV) and second I had bad logic in bulkImportToMongo.
Complete solution:
I parsed and imported 602.198 objects and here is how long time it took using node --max_old_space_size=8000 on a MacBook Pro with 8gb of ram.
console
➜ database git:(master) ✗ node --max_old_space_size=8000 partImport.js
Connected to db!
Time to parse file: : 5209.325ms
Disconnected from db!
Time to import parsed objects to db: : 153606.545ms
➜ database git:(master) ✗
parseCSV.js
const csv = require("fast-csv");
function promiseCSV(filePath, options) {
return new Promise((resolve, reject) => {
console.time("Time to parse file");
var records = [];
csv
.fromPath(filePath, options)
.on("data", record => {
records.push(record);
})
.on("end", () => {
console.timeEnd("Time to parse file");
resolve(records);
});
});
}
module.exports = promiseCSV;
mongodb.js
const mongoose = require("mongoose");
mongoose.Promise = global.Promise;
function connectToMongo(databaseName) {
mongoose.connect(`mongodb://localhost:27017/${databaseName}`, {
keepAlive: true,
reconnectTries: Number.MAX_VALUE,
useMongoClient: true
});
console.log("Connected to db!");
}
function disconnectFromMongo() {
mongoose.disconnect();
console.log("Disconnected from db!");
}
function bulkImportToMongo(arrayToImport, mongooseModel) {
const Model = require(`../../../models/${mongooseModel}`);
const batchSize = 100;
let batchCount = Math.ceil(arrayToImport.length / batchSize);
let recordsLeft = arrayToImport.length;
let ops = [];
let counter = 0;
for (let i = 0; i < batchCount; i++) {
let batch = arrayToImport.slice(counter, counter + batchSize);
counter += batchSize;
ops.push(Model.insertMany(batch));
}
return Promise.all(ops);
}
module.exports.bulkImportToMongo = bulkImportToMongo;
module.exports.connectToMongo = connectToMongo;
module.exports.disconnectFromMongo = disconnectFromMongo;
partImport.js
const path = require("path");
const parseCSV = require("./helpers/parseCSV");
const {
connectToMongo,
disconnectFromMongo,
bulkImportToMongo
} = require("./helpers/mongodb");
const filePath = path.join(__dirname, "../../data/parts.csv");
const options = {
delimiter: ";",
noheader: true,
headers: [
"facility",
"partNumber",
"partName",
"partDescription",
"netWeight",
"customsTariff"
]
};
connectToMongo("autoMDM");
parseCSV(filePath, options)
.then(records => {
console.time("Time to import parsed objects to db");
return bulkImportToMongo(records, "parts.js");
})
/* .then(result =>
console.log("Total batches inserted: ", result, result.length)
) */
.then(() => {
disconnectFromMongo();
console.timeEnd("Time to import parsed objects to db");
})
.catch(error => console.log(error));
Can someone provide an example on how to use the beforeEach? http://www.node-tap.org/api/
Ideally, an example of the promise version, but a callback version example would also be nice.
Here is a test I created which works fine:
'use strict';
const t = require('tap');
const tp = require('tapromise');
const app = require('../../../server/server');
const Team = app.models.Team;
t.test('crupdate', t => {
t = tp(t);
const existingId = '123';
const existingData = {externalId: existingId, botId: 'b123'};
const existingTeam = Team.create(existingData);
return existingTeam.then(() => {
stubCreate();
const newId = 'not 123'
const newData = {externalId: newId, whatever: 'value'};
const newResult = Team.crupdate({externalId: newId}, newData);
const existingResult = Team.crupdate({externalId: existingId}, existingData);
return Promise.all([
t.equal(newResult, newData, 'Creates new Team when the external ID is different'),
t.match(existingResult, existingTeam, 'Finds existing Team when the external ID exists')
]);
});
})
.then(() => {
process.exit();
})
.catch(t.threw);
function stubCreate() {
Team.create = data => Promise.resolve(data);
}
Before I do anything, I want to persist existingTeam. After it's saved, I want to stub Team.create. After these two things, I want to start actually testing. I think it would be cleaner if instead of using a Promise.all or perhaps duplicating the test code, I could use beforeEach.
How would I convert this to use beforeEach? Or what is an example of its usage?
Simple, just return promise from callback function
const t = require('tap');
const tp = require('tapromise');
const app = require('../../../server/server');
const Team = app.models.Team;
const existingId = '123';
const existingData = {
externalId: existingId,
botId: 'b123'
};
t.beforeEach(() => {
return Team.create(existingData).then(() => stubCreate());
});
t.test('crupdate', t => {
t = tp(t);
const newId = 'not 123'
const newData = {
externalId: newId,
whatever: 'value'
};
const newResult = Team.crupdate({
externalId: newId
}, newData);
const existingResult = Team.crupdate({
externalId: existingId
}, existingData);
return Promise.all([
t.equal(newResult, newData, 'Creates new Team when the external ID is different'),
t.match(existingResult, existingTeam, 'Finds existing Team when the external ID exists')
]);
}).then(() => {
process.exit();
}).catch(t.threw);
function stubCreate() {
Team.create = data => Promise.resolve(data);
}