I'm using elasticsearch search engine with my react app, I was reading one file at the backend as you see in the code and it work perfectly, but now I want to read three different JSON files to three different indexes using the "fs" package and bulk request, can you please help me?
the code:
// Start reading the json file
fs.readFile("DocRes.json", { encoding: "utf-8" }, function (err, data) {
if (err) {
throw err;
}
// Build up a giant bulk request for elasticsearch.
bulk_request = data.split("\n").reduce(function (bulk_request, line) {
var obj, ncar;
try {
obj = JSON.parse(line);
} catch (e) {
console.log("Done reading 1");
return bulk_request;
}
// Rework the data slightly
ncar = {
id: obj.id,
name: obj.name,
summary: obj.summary,
image: obj.image,
approvetool: obj.approvetool,
num: obj.num,
date: obj.date,
};
bulk_request.push({
index: { _index: "ncar_index", _type: "ncar", _id: ncar.id },
});
bulk_request.push(ncar);
return bulk_request;
}, []);
// A little voodoo to simulate synchronous insert
var busy = false;
var callback = function (err, resp) {
if (err) {
console.log(err);
}
busy = false;
};
// Recursively whittle away at bulk_request, 1000 at a time.
var perhaps_insert = function () {
if (!busy) {
busy = true;
client.bulk(
{
body: bulk_request.slice(0, 1000),
},
callback
);
bulk_request = bulk_request.slice(1000);
console.log(bulk_request.length);
}
if (bulk_request.length > 0) {
setTimeout(perhaps_insert, 100);
} else {
console.log("Inserted all records.");
}
};
perhaps_insert();
});
You can create multiple promises for each file read and feed it to the elastic search bulk_request.
const fsPromises = require('fs').promises,
files = ['filename1', 'filename1'],
response = [];
const fetchFile = async (filename) => {
return new Promise((resolve, reject) => {
const path = path.join(__dirname, filename);
try {
const data = await fsPromises.readFile(path)); // make sure path is correct
resolve(data);
} catch (e) {
reject(e)
}
});
files.forEach((fileName) => results.push(fetchFile()));
Promise.all(results).then(data => console.log(data)).catch(e => console.log(e));
}
Once you get data from all the promises pass it to the elastic search.
Related
I am fairly new to node.js and i am wondering how to (or even if) i can read and write to a JSON file. I am trying to create an accessible punishment history.
Ideally i would want to be able to create something along the lines of this:
{
"punishments": {
"users": {
"<example user who has a punishment history>": {
"punishment-1567346": {
"punishment-id": "1567346",
"punishment-type": "mute",
"punishment-reason": "<reason>"
},
"punishment-1567347": {
"punishment-id": "1567347",
"punishment-type": "ban",
"punishment-reason": "<reason>"
}
}
}
}
}
Then i would have a way to access the formatted punishment history. I genuinely have no clue where to start.
You can use a NodeJS built-in library called fs to do read/write operations.
Step #1 - Import fs
const fs = require('fs');
Step #2 - Read the file
let rawdata = fs.readFileSync('punishmenthistory.json');
let punishments= JSON.parse(rawdata);
console.log(punishments);
Now you can use the punishments variable to check the data inside the JSON File. Also, you can change the data but it only resides inside the variable for now.
Step #3 - Write to the File
let data = JSON.stringify(punishments);
fs.writeFileSync('punishmenthistory.json', data);
Full code:
const fs = require('fs');
let rawdata = fs.readFileSync('punishmenthistory.json');
let punishments= JSON.parse(rawdata);
console.log(punishments);
let data = JSON.stringify(punishments);
fs.writeFileSync('punishmenthistory.json', data);
References:
https://stackabuse.com/reading-and-writing-json-files-with-node-js/
Use NodeJS File System https://nodejs.org/dist/latest-v14.x/docs/api/fs.html.
Here I have used writeFileSync API to write to file and readFileSync to read from file. Also, when writing don't forget to JSON.stringify(data) because you are writing the data to a JSON file.
const fs = require("fs");
const path = require("path");
// Write Data
const data = {
"punishments": {
"users": {
"<example user who has a punishment history>": {
"punishment-1567346": {
"punishment-id": "1567346",
"punishment-type": "mute",
"punishment-reason": "<reason>"
},
"punishment-1567347": {
"punishment-id": "1567347",
"punishment-type": "ban",
"punishment-reason": "<reason>"
}
}
}
}
};
fs.writeFileSync(path.join(__dirname, "outputfilepath", "outputfile.json"), JSON.stringify(data), "utf8");
// Read data
const rData = fs.readFileSync(path.join(__dirname, "outputfilepath", "outputfile.json"), "utf8");
const jsonData = JSON.parse(rData);
Here is the working example,
https://repl.it/repls/OutrageousInbornBruteforceprogramming#index.js
you can do something like this for reading:
const fs = require('fs')
function jsonReader(filePath, cb) {
fs.readFile(filePath, (err, fileData) => {
if (err) {
return cb && cb(err)
}
try {
const object = JSON.parse(fileData)
return cb && cb(null, object)
} catch(err) {
return cb && cb(err)
}
})
}
jsonReader('./customer.json', (err, customer) => {
if (err) {
console.log(err)
return
}
console.log(customer.address) // => "Infinity Loop Drive"
})
and like this for writing:
const fs = require('fs')
const customer = {
name: "Newbie Co.",
order_count: 0,
address: "Po Box City",
}
const jsonString = JSON.stringify(customer)
fs.writeFile('./newCustomer.json', jsonString, err => {
if (err) {
console.log('Error writing file', err)
} else {
console.log('Successfully wrote file')
}
})
I am trying to get a list of object name from s3 bucket using min.io javascript API (https://docs.min.io/docs/javascript-client-api-reference#listObjectsV2). The API returns a stream. However, I always get an empty list.
The example of the dataStream is:
{
name: 'sample-mp4-file-1.mp4',
lastModified: 2020-10-14T02:35:38.308Z,
etag: '5021b3b7c402468d5b018a8b4a2b448a',
size: 10546620
}
{
name: 'sample-mp4-file-2.mp4',
lastModified: 2020-10-14T15:54:44.672Z,
etag: '5021b3b7c402468d5b018a8b4a2b448a',
size: 10546620
}
My function
public async listFiles(
bucketName: string,
prefix?: string
): Promise<string[]> {
const objectsList = [];
await minioClient.listObjectsV2(bucketName, "", true, "", function(
err,
dataStream
) {
if (err) {
console.log("Error listFiles: ", err);
return;
}
console.log("Succesfully get data");
dataStream.on("data", function(obj) {
objectsList.push(obj.name);
});
dataStream.on("error", function(e) {
console.log(e);
});
dataStream.on("end", function(e) {
console.log("Total number of objects: ", objectsList.length);
});
});
return objectsList;
}
Expected output is a list object name, [sample-mp4-file-1.mp4, sample-mp4-file-2.mp4]
According to the documentation, listObjectsV2() is returning a stream, not a promise. Therefore, await is returning immediately, before objectsList will contain anything.
The API you're using has to support Promises if you want to await them.
You could work around this by doing something like this:
const objectsList = await new Promise((resolve, reject) => {
const objectsListTemp = [];
const stream = minioClient.listObjectsV2(bucketName, '', true, '');
stream.on('data', obj => objectsListTemp.push(obj.name));
stream.on('error', reject);
stream.on('end', () => {
resolve(objectsListTemp);
});
});
I've created an API which calls get cloudWatch AWS API and gives back datapoints that can be graphed on my app. I have separate routes for each package (as shown in the routing code below). This API uses REST MVC Method.
So a couple things I'm doing with my function.
Reading in EC2 Instance data from a SQLite3 database to grab
information about a running instance (IP, instance_id,
instance_launchtime) so that I can put it in the parameters required
for the getMetricStatistics API from the AWS SDK.
This data from step1 is then put into an array of parameters (3 that respond with 3 different metric datapoints). This loops through each parameter, inserting it into the getMetricStatistics API (ONE BY ONE SINCE getMetricStatistics doesn't accept multiple metrics at once) to grab data points for that instance and push them to an array.
For the database is async I believe, that is why I've attached a promise to it. When I load in the endpoint into my browser, it just keeps loading and won't show any data. When I do refresh the page, however, it shows all the results correctly...
This is my controller for the API:
// Return results sent from Cloud Watch API
const InsightModel = require('../models/insight.model.js');
const cloudWatch = InsightModel.cloudWatch;
const CWParams = InsightModel.CWParams;
const packageById = InsightModel.packageById;
let cpuUtilParam;
let cpuCBParam;
let cpuCUParam;
let insightParams = [];
let metricResults = [];
exports.getAnalytics = (req, res) => {
const currentDate = new Date().toISOString();
let promise1 = new Promise((resolve, reject) => {
packageById(req.params.packageKey, (err, data) => {
if (err) {
reject(
res.status(500).send({
message:
err.message ||
'Error while getting the insight configuration data.',
})
);
} else {
cpuUtilParam = new CWParams(
currentDate,
'CPUUtilization',
'AWS/EC2',
data[0].launch_time,
data[0].instance_id
);
cpuCBParam = new CWParams(
currentDate,
'CPUCreditBalance',
'AWS/EC2',
data[0].launch_time,
data[0].instance_id
);
cpuCUParam = new CWParams(
currentDate,
'CPUCreditUsage',
'AWS/EC2',
data[0].launch_time,
data[0].instance_id
);
insightParams = [cpuUtilParam, cpuCBParam, cpuCUParam];
resolve(insightParams);
}
});
})
let promise2 = new Promise((resolve, reject) => {
insightParams.forEach(metric => {
cloudWatch.getMetricStatistics(metric, function(err, data) {
if (err) {
reject(
res.status(500).send({
messaage:
err.message ||
'Error occured while running cloudWatch getMetricStatistcs API: ',
})
);
} else {
metricResults.push(data);
if (metricResults.length === insightParams.length)
resolve(metricResults);
}
});
});
});
Promise.all([promise1, promise2])
.then(metricResults => {
res.send(metricResults);
console.log('AWS CW API successful');
})
.catch(err => {
res.status(500).send({
messaage:
err.message ||
'Error occured while reading in a promise from cloudWatch getMetricStatistcs API: ',
})
});
metricResults = [];
};
The model for the API:
// Call AWS Cost Explorer API
const AWS = require('aws-sdk');
const config = require('./AWSconfig');
const database = require('./db');
const insightdb = database.insightdb;
AWS.config.update({
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
region: config.region,
});
//Linking AWS CloudWatch Service
var cloudWatch = new AWS.CloudWatch();
const packageById = (packageId, callback) => {
insightdb.all(
'SELECT * FROM ec2Instance WHERE package_id == ?',
packageId,
(err, rows) => {
if (err) {
callback(err, null);
} else {
callback(null, rows);
}
}
);
};
// Parameter class to feed into the CloudWatch getMetricStatistics function
const CWParams = function(reqDate, metricName,service,launchTime,instanceId) {
(this.EndTime = reqDate) /* required */,
(this.MetricName = metricName) /* required */,
(this.Namespace = service) /* required */,
(this.Period = 3600) /* required */,
(this.StartTime = launchTime) /* ${createDate}`, required */,
(this.Dimensions = [
{
Name: 'InstanceId' /* required */,
Value: instanceId /* required */,
},
]),
(this.Statistics = ['Maximum']);
};
//Exports variables to the controller (so they can be re-used)
module.exports = { cloudWatch, CWParams, packageById };
The route for the API:
module.exports = app => {
const insight = require('../controllers/insight.controller.js');
app.get('/insights/aws/:packageKey', insight.getAnalytics);
};
As it stands, in the second Promise constructor, insightParams is guaranteed not to have been composed yet because insightParams = [.....] is in a callback that is called asynchronously. Therefore, the program flow needs to ensure all the "promise2" stuff happens only after "promise1" is fulfilled.
Things become a lot simpler in the higher level code if asynchronous functions are "promisified" at the lowest possible level. So do two things in the model:
Promisify cloudWatch.getMetricStatistics()
Write packageById() to return Promise rather than accepting a callback.
The model thus becomes:
const AWS = require('aws-sdk'); // no change
const config = require('./AWSconfig'); // no change
const database = require('./db'); // no change
const insightdb = database.insightdb; // no change
AWS.config.update({
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
region: config.region
}); // no change
var cloudWatch = new AWS.CloudWatch(); // no change
// Promisify cloudWatch.getMetricStatistics() as cloudWatch.getMetricStatisticsAsync().
cloudWatch.getMetricStatisticsAsync = (metric) => {
return new Promise((resolve, reject) => {
cloudWatch.getMetricStatistics(metric, function(err, data) {
if (err) {
if(!err.message) { // Probably not necessary but here goes ...
err.message = 'Error occured while running cloudWatch getMetricStatistcs API: ';
}
reject(err); // (very necessary)
} else {
resolve(data);
}
});
});
};
// Ensure that packageById() returns Promise rather than accepting a callback.
const packageById = (packageId) => {
return new Promise((resolve, reject) => {
insightdb.all('SELECT * FROM ec2Instance WHERE package_id == ?', packageId, (err, rows) => {
if (err) {
reject(err);
} else {
resolve(rows);
}
});
});
};
Now getAnalytics() can be written like this:
exports.getAnalytics = (req, res) => {
packageById(req.params.packageKey)
.then(data => {
const currentDate = new Date().toISOString();
let insightParams = [
new CWParams(currentDate, 'CPUUtilization', 'AWS/EC2', data[0].launch_time, data[0].instance_id),
new CWParams(currentDate, 'CPUCreditBalance', 'AWS/EC2', data[0].launch_time, data[0].instance_id),
new CWParams(currentDate, 'CPUCreditUsage', 'AWS/EC2', data[0].launch_time, data[0].instance_id)
];
// Composition of `insightParams` is synchronous so you can continue
// with the `cloudWatch.getMetricStatisticsAsync()` stuff inside the same .then().
return Promise.all(insightParams.map(metric => cloudWatch.getMetricStatisticsAsync(metric))); // Simple because of the Promisification above.
}, err => {
// This callback handles error from packageById() only,
// and is probably unnecessary but here goes ...
if(!err.message) {
err.message = 'Error while getting the insight configuration data.';
}
throw err;
})
.then(metricResults => {
res.send(metricResults);
console.log('AWS CW API successful');
})
.catch(err => {
// Any async error arising above will drop through to here.
res.status(500).send({
'message': err.message
}));
});
};
Note that multiple catches each with res.status(500).send() are not necessary. Error propagation down the Promise chain allows a single, terminal .catch()
What do i want?
good question, isn't it? Well...
I am working on a application to calculate budgets with electron-vue.
In my App i try to save the users in a JSON file to create a opportunity to hold them after a application restart.
the JSON File should look like this:
{
"deniz": {
"salary": 1234,
},
"hüseyin": {
"salary": 4321,
}
}
What do i get?
I am getting this instead:
{
"deniz": {
"salary": 1234
}
}{
"hüseyin": {
"salary": 4321
}
}
Problem is, its a wrong JSON format. I am creating a whole new obj inside a obj.
How am i doing this?
I created a userDataControllerMixin.js to separate the logic from the component it self.
I have two InputFields in my component, 1.userName and 2.userSalary to collect the user data.
Inside my userDataControllerMixin.js:
export const userDataControllerMixin = {
data() {
return {
userDataAbsPath: 'src/data/userData.json',
};
},
mounted() {
this.getUsers();
},
methods: {
// FETCH THE userData.json
getUsers() {
const fs = require('fs');
const loadJSON = fs.readFile('src/data/userData.json', 'utf8', (err, data) => {
if (err) {
console.log(`failed to read file: ${err}`);
}
// console.log(data);
});
return loadJSON;
},
// USING THIS CONSTRUCTOR TO BUILD A JSON FORMAT
User(user, salary) {
this[user] = {
salary: Number(salary),
};
return user;
},
// GET INPUT FROM USERS INPUTBOX
getInput(inputName, inputSalary) {
const userName = this.inputName;
const userSalary = this.inputSalary;
const user = new this.User(userName, userSalary);
console.log(user);
this.createOrLoadJSON(user);
},
// CREATES A JSON WITH DATA FROM THE USERS
createOrLoadJSON(data) {
const fs = require('fs');
const json = JSON.stringify(data, null, 4);
if (fs.existsSync(this.userDataAbsPath)) {
console.log('file exists!');
fs.appendFileSync(this.userDataAbsPath, json);
} else {
console.log('file not exists!');
fs.writeFile(this.userDataAbsPath, json, (error) => {
if (error !== null) {
console.log(error);
}
});
}
this.postUsers();
},
// PRINTS DATA FROM userData.json TO DOM
postUsers() {
},
},
};
How can i fix this?
Problem is, appendFile method is no concat method. It just add some text after another.
You must concat your json with Object.assign first.
createOrLoadJSON(data) {
const fs = require('fs');
if (fs.existsSync(this.userDataAbsPath)) {
console.log('file exists!');
const existingJSON = fs.readFileSync(this.userDataAbsPath, "utf8"); // read file and return encoded value
const newJSON = Object.assign(JSON.parse(existingJSON), data); // concatenate file value and new data
fs.writeFile(this.userDataAbsPath, JSON.stringify(newJSON, null, 4)); // rewrite file
} else {
console.log('file not exists!');
fs.writeFile(this.userDataAbsPath, JSON.stringify(data, null, 4), (error) => { // if file does not exist stringify data here
if (error !== null) {
console.log(error);
}
});
}
this.postUsers();
},
Working example:
// proper concat with Object.assign
var assign = {
foo: 'bar'
};
var assign2 = {
bar: 'baz'
};
var assign3 = Object.assign(assign, assign2);
console.log('Object assign: ', assign3);
// appendFile look more like this
var append = {
foo: 'bar'
};
var append2 = {
bar: 'baz'
};
var append3 = JSON.stringify(append) + JSON.stringify(append2);
console.log('fs.appendFile: ', append3);
I am writing an ember includedCommand for fetching and updating the app/index.html file - which uses NodeJS https and fs module to replace the indexFile by calling a function BuildIndexFile, where I am facing a weird issue -
When I perform command ember server --update-index - I can see the BuildIndexFile is being called and the https request is made to the remote server which downloads the file and gets written by fs.writeFileSync in app/index.html.
But when I perform ember update-index which is an included command, I can see BuildIndexFile has been called, and it reaches till console.log('Fetching index.html'); and I believe it is calling https.request... but it closes from there, I have no idea why the call didn't go through, when I debugged using node --inspect-brk ./node_modules/.bin/ember update-index I can see the https is available on the file, but not executing.
I am attaching my sample code available as a in-repo-addon available at lib/hello/index.js -
/* eslint-env node */
'use strict';
const parseArgs = require('minimist');
const watchman = require('fb-watchman');
let client = new watchman.Client();
client.capabilityCheck({optional: [], required: ['relative_root']}, function (error, response) {
if (error) {
console.log(error);
}
console.log('Watchman', response);
});
const ServeCommand = require('ember-cli/lib/commands/serve');
const ARGS = parseArgs(process.argv.slice(2));
const fs = require('fs');
const https = require('https');
module.exports = {
name: 'hello',
isDevelopingAddon() {
return true;
},
includedCommands: function() {
var self = this;
return {
hello: ServeCommand.extend({
name: 'hello',
description: 'A test command that says hello',
availableOptions: ServeCommand.prototype.availableOptions.concat([{
name: 'updateindex',
type: String
}]),
run: function(commandOptions, rawArgs) {
console.log(commandOptions, rawArgs);
if (commandOptions['updateindex']) {
console.log('Update Index')
}
const sampleHelloPromise = sampleHello();
const servePromise = this._super.run.apply(this, arguments);
return Promise.all([sampleHelloPromise, servePromise]);
}
}),
updateIndex: {
name: 'update-index',
description: 'Update Index File',
availableOptions: [{
name: 'index-file',
type: String
}],
run: function(commandOptions, rawArgs) {
BuildIndexFile(self.project.root, 'https://yahoo.com', {});
}
}
}
},
preBuild: function(result) {
let self = this;
if (ARGS['update-index']) {
BuildIndexFile(self.project.root, 'https://google.com', {}).then(function() {
delete ARGS['update-index'];
})
.catch(function(e) {
console.log(e);
});;
}
}
};
async function sampleHello() {
return await new Promise(resolve => {
setTimeout(() => resolve('hello'), 2000);
})
}
const BuildIndexFile = (rootPath, target, headers) => {
try {
debugger;
const indexFile = `${rootPath}/app/index.html`;
let noIndexFile = !fs.existsSync(indexFile);
return new Promise(function (resolve, reject) {
let options = {
hostname: target.replace(/^http(?:s):\/\//i, ''),
port: 443,
method: 'GET'
};
let dataContent = '';
console.log('Fetching index.html');
var request = https.request(options, function(response) {
response.on('data', function(d) {
dataContent += d;
});
response.on('end', function() {
fs.writeFileSync(indexFile, dataContent);
return resolve();
});
});
request.on('error', function(e) {
console.log(e);
return reject(`Error: Creating Index File`);
});
request.end();
});
} catch(e) {
throw e;
}
}