Develop Reference

How to write file using fs.createWriteStream

am trying to build a web scraper that downloads all the pdfs in a website. i've written all the logic necessary to do this but for some reason it downloads an empty pdf file which is not suppose to be so, the problem seems to be coming from the downloadFile function when i try to pipe the data which for some reason seems not to be working because i get an empty pdf file after the function is ran. i'll would appreciate it if someone can help me out with this problem, thanks.
here's a sample of my code:
app.js
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");
const axiosInstance = require("./getAxios");
const axios = axiosInstance();
const Surl = "https://www.health.gov.ng/";
// linkList sample: "https://www.health.gov.ng/index.php?option=com_content&view=article&id=143&Itemid=512";
let = connectionFailCount = 0;
let linkList = [];
let dlinkList = [];
const getWebsiteLinks = async (Surl) => {
try {
console.log(`Crawling all links from: ${Surl}`);
const response = await axios.get(Surl);
const $ = cheerio.load(response.data);
const ranges = $("a").each(function (idx, el) {
if ($(el).attr("href")) {
return $(el).attr("href");
}
});
for (let index = 0; index < ranges.length; index++) {
let raw_links = $("a")[index].attribs.href;
if (raw_links.startsWith("/")) {
linkList.push(Surl + raw_links);
}
}
if (linkList.length > 0) {
console.log(`Finished crawling links: Found ${linkList.length} links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
return;
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
getWebsiteLinks(Surl);
console.log(`Connection error. \n
Reconnecting to server....`);
} else if (connectionFailCount === 5) {
console.error(`Can not connect to server. Try again later.`);
}
}
};
const downloadLinks = async (linkList) => {
try {
console.log("Crawling links to find pdf links. this may take a while...");
for (const link of linkList) {
const response = await axios.get(link);
// Skip where there's delayed server response
if (response.code === "ECONNRESET") continue;
const $ = cheerio.load(response.data);
$("a").each(function (idx, el) {
if ($(el)?.attr("href")?.endsWith(".pdf")) {
let addr = $(el).attr("href");
let dlink = Surl + addr;
dlinkList.push({
pathName: addr,
url: dlink,
});
}
});
}
console.log(dlinkList);
if (dlinkList.length > 0) {
console.log(`Crawling Finish: Found ${dlinkList.length} pdf links`);
console.log(
"--------------------------------------------------------\n\n"
);
}
} catch (error) {
if (connectionFailCount === 0) {
connectionFailCount += 1;
console.log(`Connection error. \n
Reconnecting to server: ${connectionFailCount} count`);
downloadLinks(linkList);
}
if (connectionFailCount === 3) {
console.error(`Can not connect to server. Try again later.`);
return;
}
// console.error("downloadLinksError: ", error);
}
};
const downloadFiles = async (dlinkList) => {
console.log("Creating directory to save PDF files");
const appRoot = path.dirname(path.resolve(__dirname));
// Had to change and restructure code due to error
const folderName = `PDF/${Surl.split("/").pop()}`;
const subFolderName = Surl.split("/").pop();
try {
if (!fs.existsSync(path.join(appRoot, folderName))) {
fs.mkdirSync(path.join(appRoot, "PDF"));
fs.mkdirSync(path.join(`${appRoot}/PDF`, subFolderName));
}
dlinkList.forEach(async (link) => {
let name = link.pathName;
let url = link.url;
let file = fs
.createWriteStream(
`${appRoot}/${folderName}/${name.split("/").pop()}`,
"utf-8"
)
.on("error", (err) => {
console.error("createWriteStreamError: ", err);
});
try {
console.log("Downloading PDF file...");
const { data } = await axios({
url,
method: "GET",
responseType: "stream",
});
if (data) {
console.log("PDF file Downloaded");
data.pipe(file);
}
} catch (error) {
console.error(error);
}
});
return;
} catch (error) {
console.error("downloadFilesError: ", error);
}
};
(async () => {
await getWebsiteLinks(Surl);
await downloadLinks(linkList);
await downloadFiles(dlinkList);
})();
getAxios.js
const axios = require("axios");
const https = require("https");
module.exports = function () {
const domain = "https://www.health.gov.ng/";
let instance;
if (!instance) {
//create axios instance
instance = axios.create({
baseURL: domain,
timeout: 60000, // Increase time out incase of network delay or delayed server response
maxContentLength: 500 * 1000 * 1000, // Increase maximum response ata length
httpsAgent: new https.Agent({ keepAlive: true }),
headers: { "Content-Type": "application/xml" },
});
}
return instance;
};

GraphQL file upload plain JS "Must provide query string."

I am uploading files to GraphQL API with plain JS. I've been doing this from the same origin for months now and now am trying to implement the exact same thing externally with NodeJS.
My code looks something like this:
const FormData = require('form-data');
const fs = require('fs')
const axios = require('axios')
const payload = generateRequest(input)
axios.post(apiBaseUrl + "/graphql", payload, {
headers: {...payload.getHeaders()}
}).then(response => {
let res = response.data
if (res.data.triggerPcWorkflow.status === 200) {
console.log("success!")
} else {
console.error(res.data.triggerPcWorkflow.message)
}
})
.catch(err => {
if (err.response) {
console.log(err.response.data);
console.log(err.response.status);
console.log(err.response.headers);
}
})
With the generateRequest function generating the multipart payload (https://github.com/jaydenseric/graphql-multipart-request-spec).
I have two identical versions of the Backend running on localhost:5000 and mycooldomain.com. When setting apiBaseUrl to http://localhost:5000 everything works flawlessly. However just by changing the URL to https://www.mycooldmain.com I get a 400 error thrown at me with { errors: [ { message: 'Must provide query string.' } ] }
BTW: A simple query works with both URLs...
Here is my generateRequest function:
const mutation = `
mutation MyMutation($xyz: String) {
doSomething(myInput: $xyz) {
status
message
}
}
`
let sendData = new FormData();
const fileNull = [];
// map
files = generateRequestInput.files
let map = '{'
for (let i = 0; i < files.length; i++) {
fileNull.push(null);
map += `"${i}": ["variables.files.${i}"], `
}
map = map.substring(0, map.length-2);
map += '}'
// operations
const operations = JSON.stringify({
query: mutation,
variables: {
"xyz": "Hello"
}
});
// build payload
sendData.append("operations", operations)
sendData.append("map", map)
for (let i = 0; i < files.length; i++) {
sendData.append(i, files[i]);
}
return sendData
I know that map looks a bit ugly but thats not the point (unless it is).
Has anyone had a similar problem or knows what my error is here?
Thanks!

I skipped on the axios dependency and implemented the request with FormData directly.
The code below works.
function makeRequest(formData, options) {
formData.submit(options, (err, res) => {
if (err) {
console.error(err.message)
return
} else {
if (res.statusCode < 200 || res.statusCode > 299) {
console.error(`HTTP status code ${res.statusCode}`)
}
const body = []
res.on('data', (chunk) => body.push(chunk))
res.on('end', () => {
const resString = Buffer.concat(body).toString()
console.log(resString)
})
}
})
}
const options = {
host: 'mycooldomain.com',
port: 443,
path: '/graphql',
method: 'POST',
protocol: 'https:'
}
makeRequest(payload, options)

How to pass object from one file to another in javascript?

I have node js server file (index.js) and client file (orderlist.js)
In index.js i am getting promise object , like that
function returnOrderArray() {
var i = 0;
const promise = new Promise((resolve, reject) => {
connection.query('SELECT * FROM orders', function(error, results) {
while (i < results.length) {
order.id[i] = results[i].id;
order.wavetype[i] = results[i].wavetype;
order.color[i] = results[i].color;
order.thick[i] = results[i].thick;
order.readydate[i] = results[i].readydate;
order.createdate[i] = results[i].createdate;
order.manager[i] = results[i].manager;
i++;
}
resolve(order);
// console.log(order);
});
});
return promise;
}
then i want to pass it to other js file.
I tried to do that with module.exports
app.get('/orderlist', checkUserSession, async function(request, response) {
returnOrderArray().catch(error => console.log(error)).then((() => {
module.exports.order = order;
response.render("orderlist.ejs", { username: request.session.username });
})).catch(error => console.log(error));
});
and then import it in orderlist.js
var ind = require('../../index')
function asd() {
alert(ind.order);
}
but it seems not to work.
What am i doing wrong , and what's the best way to pass objects to other files in js?
oh , and file architecture
filearch

You need to export your module like so: module.exports = returnOrderArray
try this,
orderlist.js
const returnOrderArray = () => {...some code..}
module.exports = returnOrderArray
index.js
const returnOrderArray = require('./path/to/orderlist.js')
const run = async() => {
const orderlist = await returnOrderArray() // use await or then as you prefer
}
run()
async_await link if curious!
Hope this will work :)

How to read User ID and Password from CSV file into a Array using Java Script

I have a CSV file as below - accounts.csv
User_Name,Password
TestAccount1,Passw0rd
TestAccount2,Passw0rd
TestAccount3,Passw0rd
TestAccount4,Passw0rd
Trying to write a JavaScript code to read them into a array variable and use it further in script like
UserName[1] and Password [1] or may be like UserName and Password
Tried using the below code:
const fs = require('fs');
var parse = require("csv-parse"); //installed csc-parse
var csvFile = "accounts.csv";
class User {
constructor(user_id, user_password) {
this.user_id = user_id;
this.user_password = user_password;
}
}
const processData = (err, data) => {
if (err) {
console.log(`An error was encountered: ${err}`);
return;
}
//skipping has heading row
data.shift();
const userList = data.map(row => new User(...row));
}
fs.createReadStream(csvFile)
.pipe(parse({ delimiter: ',' }, processData));

Try the below snippet :
const fs = require('fs');
var parse = require("csv-parse"); //installed csv-parse
var csvFile = "accounts.csv";
const userList = [];
const processData = (data) => {
if (data != undefined) {
//console.log(data);
userList.push({ "username" : data[0], "password" : data[1]});
}
}
fs.createReadStream(csvFile)
.pipe(parse({
delimiter: ',',
from_line: 2 // to skip the header row
}, processData))
.on('data', processData)
.on('end', () => {
console.log('CSV file successfully processed');
console.log(userList);
});

Why does my code using insertMany() skips some of the records and insert same records multiple times?

I have 9577 unique records in a csv file.
This code inserts 9800 records and insert not all records, but duplicates of some of them. Any idea why it does not inserts the unique 9577 records and also duplicates of some of them? Below I also insert the remain part of the code so you get the whole picture
function bulkImportToMongo(arrayToImport, mongooseModel) {
const Model = require(`../../../models/${mongooseModel}`);
let batchCount = Math.ceil(arrayToImport.length / 100);
console.log(arrayToImport.length);
let ops = [];
for (let i = 0; i < batchCount; i++) {
// console.log(i);
let batch = arrayToImport.slice(i, i + 100);
console.log(batch.length);
ops.push(Model.insertMany(batch));
}
return ops;
return Promise.all(ops).then(results => {
// results is an array of results for each batch
console.log("results: ", results);
});
}
I parse the csv file
const Promise = require("bluebird");
const csv = require("fast-csv");
const path = require("path");
const fs = Promise.promisifyAll(require("fs"));
const promiseCSV = Promise.method((filePath, options) => {
return new Promise((resolve, reject) => {
var records = [];
csv
.fromPath(filePath, options)
.on("data", record => {
records.push(record);
})
.on("end", () => {
// console.log(records);
resolve(records);
});
});
});
And here is the script that connects it all together:
const path = require("path");
const promiseCSV = require("./helpers/ImportCSVFiles");
const {
connectToMongo,
bulkImportToMongo
} = require("./helpers/mongoOperations");
const filePath = path.join(__dirname, "../../data/parts.csv");
const options = {
delimiter: ";",
noheader: true,
headers: [
"facility",
"partNumber",
"partName",
"partDescription",
"netWeight",
"customsTariff"
]
};
connectToMongo("autoMDM");
promiseCSV(filePath, options).then(records => {
bulkImportToMongo(records, "parts.js");
});

//It looks like your issue is simply i++. Perhaps you meant i += 100?
for (let i = 0; i < batchCount; i+=100 /* NOT i++ */) {
//...
}

I solved it.
I hope this helps other... :-)
I had two errors, in the function promiseCSV (changed to parseCSV) and second I had bad logic in bulkImportToMongo.
Complete solution:
I parsed and imported 602.198 objects and here is how long time it took using node --max_old_space_size=8000 on a MacBook Pro with 8gb of ram.
console
➜ database git:(master) ✗ node --max_old_space_size=8000 partImport.js
Connected to db!
Time to parse file: : 5209.325ms
Disconnected from db!
Time to import parsed objects to db: : 153606.545ms
➜ database git:(master) ✗
parseCSV.js
const csv = require("fast-csv");
function promiseCSV(filePath, options) {
return new Promise((resolve, reject) => {
console.time("Time to parse file");
var records = [];
csv
.fromPath(filePath, options)
.on("data", record => {
records.push(record);
})
.on("end", () => {
console.timeEnd("Time to parse file");
resolve(records);
});
});
}
module.exports = promiseCSV;
mongodb.js
const mongoose = require("mongoose");
mongoose.Promise = global.Promise;
function connectToMongo(databaseName) {
mongoose.connect(`mongodb://localhost:27017/${databaseName}`, {
keepAlive: true,
reconnectTries: Number.MAX_VALUE,
useMongoClient: true
});
console.log("Connected to db!");
}
function disconnectFromMongo() {
mongoose.disconnect();
console.log("Disconnected from db!");
}
function bulkImportToMongo(arrayToImport, mongooseModel) {
const Model = require(`../../../models/${mongooseModel}`);
const batchSize = 100;
let batchCount = Math.ceil(arrayToImport.length / batchSize);
let recordsLeft = arrayToImport.length;
let ops = [];
let counter = 0;
for (let i = 0; i < batchCount; i++) {
let batch = arrayToImport.slice(counter, counter + batchSize);
counter += batchSize;
ops.push(Model.insertMany(batch));
}
return Promise.all(ops);
}
module.exports.bulkImportToMongo = bulkImportToMongo;
module.exports.connectToMongo = connectToMongo;
module.exports.disconnectFromMongo = disconnectFromMongo;
partImport.js
const path = require("path");
const parseCSV = require("./helpers/parseCSV");
const {
connectToMongo,
disconnectFromMongo,
bulkImportToMongo
} = require("./helpers/mongodb");
const filePath = path.join(__dirname, "../../data/parts.csv");
const options = {
delimiter: ";",
noheader: true,
headers: [
"facility",
"partNumber",
"partName",
"partDescription",
"netWeight",
"customsTariff"
]
};
connectToMongo("autoMDM");
parseCSV(filePath, options)
.then(records => {
console.time("Time to import parsed objects to db");
return bulkImportToMongo(records, "parts.js");
})
/* .then(result =>
console.log("Total batches inserted: ", result, result.length)
) */
.then(() => {
disconnectFromMongo();
console.timeEnd("Time to import parsed objects to db");
})
.catch(error => console.log(error));