I am trying to store data from a csv file in an array using createReadStream in node.js but for some reason the array stays empty and no data is stored in it.
When i console log the values where i'am trying to push the data it logs it just fine.
I've tried both csv-parser and csv-parse for this but none of them seem to work.
const fs = require("fs");
const csv = require('csv-parser');
function parseCSV () {
let finaldata = []
fs.createReadStream('data.csv')
.pipe(csv())
.on('data', function (row){
finaldata.push({Userid: row[0], Timestamp: row[1]})
}).on('end', function(){
console.log(finaldata)
})
}
parseCSV()
I fixed this using the following code below.
I kept experimenting to land on something that finally worked.
I have no idea why this fix works but i would love an explanation.
The change i made here is to call the Object constructor instead of using {} so finaldata.push(Object(row)) instead of finaldata.push({Userid: row[0], Timestamp: row[1]})
const fs = require("fs");
const csv = require('csv-parser');
const parseCSV = () => {
let finaldata = []
fs.createReadStream('data.csv')
.pipe(csv())
.on('data', (row) => {
finaldata.push(Object(row))
}).on('end', () => {
console.log(finaldata)
})
}
parseCSV()
Related
I have a Next.js application here which needs to read a CSV file from a URL in the same repo in multiple places, but I cannot seem to be able to retrieve this data. You can find the relevant file in my repo here.
Note, the URL I'm trying to pull data from is this: https://raw.githubusercontent.com/ivan-rivera/balderdash-next/main/public/test_rare_words.csv
Here is what I've tried so far:
Approach 1: importing the data
let vocab = {};
...
async function buildVocab() {
const words = await import(VOCAB_URL); // this works when I point to a folder in my directory, but it does not work when I deploy this app. If I point at the URL address, I get an error saying that it cannot find the module
for (let i = 0; i < words.length; i++) {
vocab[words[i].word] = words[i].definition;
}
}
Approach 2: papaparse
const papa = require("papaparse");
let vocab = {};
...
export async function buildVocab() {
await papa.parse(
VOCAB_URL,
{
header: true,
download: true,
delimiter: ",",
step: function (row) {
console.log("Row:", row.data); // this prints data correctly
},
complete: function (results) {
console.log(results); // this returns an object with several attributes among which is "data" and "errors" and both are empty
},
}
);
// this does not work because `complete` does not return anything
vocab = Object.assign({}, ...raw.map((e) => ({ [e.word]: e.definition })));
console.log(vocab);
}
Approach 3: needle
const csvParser = require("csv-parser");
const needle = require("needle");
let vocab = {};
...
let result = [];
needle
.get(VOCAB_URL)
.pipe(csvParser())
.on("data", (data) => {
result.push(data);
});
vocab = Object.assign({}, ...result.map((e) => ({ [e.word]: e.definition })));
// This approach also returns nothing, however, I noticed that if I force it to sleep, then I do get the results I want:
setTimeout(() => {
console.log(result);
}, 1000); // now this prints the data I'm looking for
What I cannot figure out is how to force this function to wait for needle to retrieve the data. I've declared it as an async function and I'm calling it with await buildVocab() but it doesn't help.
Any ideas how I can fix this? Sorry, I'm a JS beginner, so it's probably something fundamental that I'm missing :(
After spending hours on this, I think I finally found a solution:
let vocab = {};
export async function buildVocab() {
await fetch(VOCAB_URL)
.then((resp) => resp.text())
.then((text) => {
papa.parse(text, { header: true }).data.forEach((row) => {
vocab[row.word] = row.definition;
});
});
}
The only oddity that I still can't work out is this: I'm calling my buildVocab function inside another async function and I noticed that if I do not include a console.log statement in that function, then the vocab still does not get populated in time. Here is the function:
export async function sampleWord() {
await buildVocab();
const keys = Object.keys(vocab);
const index = Math.floor(Math.random() * keys.length);
console.log(`selected word: ${keys[index]}`); // this is important!
return keys[index];
}
I am trying javascript for the first time and I am having this trouble with the example:
https://www.twilio.com/blog/web-scraping-and-parsing-html-with-node-js-and-cheerio
It is a web scrapper example that uses got and cheerio, both of which I have installed. But when i run the sample code it gives me 'cannot convert undefined or null to object error.
Why is that? I didn't change anything from the example at all.
the code in question:
const $ = cheerio.load(response.body);
$('a').each((i, link) => {
const href = link.attribs.href;
console.log(href);
});
}).catch(err => {
console.log(err);
});
How does your index.js file look like? I did the tutorial and my code is working. Maybe you are miss typed the url?
Here is my index.js
const fs = require("fs");
const cheerio = require("cheerio");
const got = require("got");
const vgmUrl = "https://www.vgmusic.com/music/console/nintendo/nes";
got(vgmUrl)
.then((response) => {
const $ = cheerio.load(response.body);
$("a").each((i, link) => {
const href = link.attribs.href;
console.log(href);
});
})
.catch((err) => {
console.log(err);
});
I'm trying to use stream-json to read a zip, unzip it, and then write it to file. I don't think I understand how to use the library.
Based on the link above, they have this example:
const {chain} = require('stream-chain');
const {parser} = require('stream-json');
const {pick} = require('stream-json/filters/Pick');
const {ignore} = require('stream-json/filters/Ignore');
const {streamValues} = require('stream-json/streamers/StreamValues');
const fs = require('fs');
const zlib = require('zlib');
const pipeline = chain([
fs.createReadStream('sample.json.gz'),
zlib.createGunzip(),
parser(),
pick({filter: 'data'}),
ignore({filter: /\b_meta\b/i}),
streamValues(),
data => {
const value = data.value;
// keep data only for the accounting department
return value && value.department === 'accounting' ? data : null;
}
]);
let counter = 0;
pipeline.on('data', () => ++counter);
pipeline.on('end', () =>
console.log(`The accounting department has ${counter} employees.`));
However I don't want to count anything, I just want to write to file. Here is what I have that works:
function unzipJson() {
const zipPath = Path.resolve(__dirname, 'resources', 'AllPrintings.json.zip');
const jsonPath = Path.resolve(__dirname, 'resources', 'AllPrintings.json');
console.info('Attempting to read zip');
return new Promise((resolve, reject) => {
let error = null;
Fs.readFile(zipPath, (err, data) => {
error = err;
if (!err) {
const zip = new JSZip();
zip.loadAsync(data).then((contents) => {
Object.keys(contents.files).forEach((filename) => {
console.info(`Writing ${filename} to disk...`);
zip.file(filename).async('nodebuffer').then((content) => {
Fs.writeFileSync(jsonPath, content);
}).catch((writeErr) => { error = writeErr; });
});
}).catch((zipErr) => { error = zipErr; });
resolve();
} else if (error) {
console.log(error);
reject(error);
}
});
});
}
However I can't easily add any processing to this, so I wanted to replace it with stream-json. This is my partial attempt, as I don't know how to finish:
function unzipJson() {
const zipPath = Path.resolve(__dirname, 'resources', 'myfile.json.zip');
const jsonPath = Path.resolve(__dirname, 'resources', 'myfile.json');
console.info('Attempting to read zip');
const pipeline = chain([
Fs.createReadStream(zipPath),
zlib.createGunzip(),
parser(),
Fs.createWriteStream(jsonPath),
]);
// use the chain, and save the result to a file
pipeline.on(/*what goes here?*/)
Later on I intend to add extra processing of the json file(s), but I want to learn the basics before I start throwing in extra functionality.
I can't produce a minimal example unfortunately, as I don't know what goes into the pipeline.on function. I'm trying to understand what I should do, not what I've done wrong.
I also looked at the related stream-chain, which has an example that ends like so:
// use the chain, and save the result to a file
dataSource.pipe(chain).pipe(fs.createWriteStream('output.txt.gz'));`
But at no point does the documentation explain where dataSource comes from, and I think my chain creates it's own by reading the zip from file?
How am I supposed to use these streaming libraries to write to file?
I don't want to count anything, I just want to write to file
In that case, you'll need to convert the token/JSON data stream back into a text stream that you can write to a file. You can use the library's Stringer for that. Its documentation also contains an example that seems to be more in line with what you want to do:
chain([
fs.createReadStream('data.json.gz'),
zlib.createGunzip(),
parser(),
pick({filter: 'data'}), // omit this if you don't want to do any processing
stringer(),
zlib.Gzip(), // omit this if you want to write an unzipped result
fs.createWriteStream('edited.json.gz')
]);
I'm a total JS noob. I have read a JSON and filtered out specific items from it and saved it to a variable named mapped. How do I export this JSON to a properly formatted CSV file?
let json = require('./users.json')
let filtered = json.Users.filter((a)=>{
return new Date(a.UserCreateDate) > new Date('2020-05-11T00:00:00.000000+05:30')
})
let mapped=filtered.map((a)=>{
let email
a.Attributes.forEach(element => {
if(element.Name=='email'){
email = element.Value
}
});
return {
name: a.Username,
email: email,
UserCreateDate: a.UserCreateDate,
UserStatus: a.UserStatus
}
})
console.log(JSON.stringify(mapped, null, 4), mapped.length)
Although there are quite a few answers to this topic, I haven't been able to successfully implement any of those.
I assume you wanna use the keys as the header in CSV file.
What you need is to open a write stream, the format of the CSV file is pure strings with commas separating the values.
// ...
let mapped=filtered.map((a)=>{
//...
})
// ...
const fs = require('fs');
let writeStream = fs.createWriteStream('./output.csv');
writeStream.on('open', () => {
// get the header
const header = Object.keys(mapped[0]).toString().concat('\n');
writeStream.write(header);
mapped.forEach((obj) => {
const values = Object.values(obj).toString().concat('\n');
writeStream.write(values);
});
writeStream.end();
});
I'm trying to query a MongoDB database and then display it to the frontend code/page.
Here's what I got so far. Note, it does sucessfully console.log() the search results on the backend just not on the frontend.
Backend File
export async function searching() {
let output = "";
const mongo = require('mongodb').MongoClient
const url = "mongodb+srv://[protected..]";
await mongo.connect(url, {useNewUrlParser: true,useUnifiedTopology: true}, (err, client) => {
const db = client.db('domains')
const collection = db.collection('domains')
collection.find().toArray((err_again, items) => {
output = items
console.log(output)
return output
})
})
}
Frontend
export async function button2_click(event) {
let output = await searching()
console.log(output)
}
Note I'm doing this in Wix code so some of the synctax front-end syntax might be different.
The "console.log(output)" gets an undefined response.
When I console log the backend file the "console.log(output)" successfully outputs the array, but it's now showing on the frontend console.
Please help I've been spending hours on this with no luck. THANK YOU!
I was able to figure this out so I figured I would post the answer here:
export async function findRecord(database, sub_db, query) {
let output = "";
const mongo = require('mongodb').MongoClient
const url = "mongodb+srv://...";
const client = await mongo.connect(url, {useNewUrlParser: true,useUnifiedTopology: true});
const db = client.db(database)
const collection = db.collection(sub_db)
const items = await collection.find(query).toArray();
client.close()
return items;
}