Develop Reference

nodejs - streaming csv to string variable

I have a code that accepts a list of nested objects, each of which should be converted to a log row.
The code goes through a loop on each object, then an inner loop on each property, and extracts its property (there are hundreds of properties), then puts all the information of a row - as a map of the object's name and its value, into a variable called returnVar.
We use the 'fast-csv' library, with WriteStream that is named csvStream. also with a fs.createWriteStream pipe.
Finally, we loop over each object and write it with csvStream.write(),
that will insert the properties name in the first line of the file, and the logs (in the same order) in the other lines.
I need to change the code so that instead of doing pipe to file stream, it will print to a string type variable.
This is the code:
let Promise = require('bluebird');
let csv = require('fast-csv');
let fs = Promise.promisifyAll(require('fs'));
...
return new Promise(function (resolve, reject) {
var csvStream = csv.createWriteStream({ headers: propNames })
.transform(function (item) { // every item is a nested object that contains data for a log line
var returnVar = {}; // every returnVar will represents a map of property and value, that will be transform to a log line
for (var prop in item) {
if (item.hasOwnProperty(prop)) {
if (propNames.indexOf(prop) >= 0) {
if (typeof item[prop] === 'object') {
returnVar[prop] = JSON.stringify(item[prop]);
}
else {
returnVar[prop] = item[prop];
}
}
//the object might be a complex item that contains some properties that we want to export...
else if (typeof item[prop] === 'object') {
var nestedItem = item[prop];
for (var nestedProp in nestedItem) {
if (propNames.indexOf(prop + "_" + nestedProp) >= 0) {
returnVar[prop + "_" + nestedProp] = nestedItem[nestedProp];
}
}
}
}
}
return returnVar; // return log line
});
// create file path
var fileId = "Report_" + cryptoService.generateRandomPassword(16) + ".csv";
var filePath = tempPath + fileId;
getOrCreateTempDirectory().then(function () {
var writableStream = fs.createWriteStream(filePath);
writableStream.on("finish", function () {
resolve({
fileId: fileId
});
});
csvStream.pipe(writableStream);
_.each(results.records, function (result) {
// write line to file
csvStream.write(result._source);
});
csvStream.end();
});
});

https://c2fo.io/fast-csv/docs/formatting/methods#writetobuffer
https://c2fo.io/fast-csv/docs/formatting/methods#writetostring
Change
csvStream.write(result._source);
to
csvStream.writeToString(result._source).then(data => console.log(data));
Promise.all(_.map(results.records, result => csvStream.writeToString(result._source)))
.then(rows=>console.log(rows))
// rows should be an array of strings representing all the results
You can also use async/await

JavaScript - Dynamically generate a file path from an object

I've looked for hours and still I couldn't find anything about it so I'm asking here. I have a small PHP script which generates a JSON tree of files and folders, and then fetches it to client side as a JavaScript variable. The outcome usually looks like this:
{
"folder_name" : {
"another_folder": {
"third_folder": {
0: "some_file.txt"
1: "another_file.png"
2: "third_file.pdf"
}
}
}
}
What I'm wanted to achive would look something like this:
generatePath("some_file.txt")
Which would return:
"folder_name/another_folder/third_folder/some_file.txt"
So my question is how can I create a path to any of those files just from an object in JavaScript? Is there even a way to do anything like this?
[Edit]: Sadly I don't have any code to show anymore...

So assuming the input is a real JSON and it will parse into JS object here is some approach:
const data = {
"folder_name" : {
"another_folder": {
"third_folder": {
"0": "some_file.txt",
"1": "another_file.png",
"2": "third_file.pdf"
}
}
},
"folder_name2" : {
"0": "up_file.txt",
"another_folder2": {
"third_folder2": ["some_file2.txt", "another_file2.png", "third_file2.pdf"]
}
}
}
function generatePath(value, currentPath = '', currentObject = data) {
for (const property in currentObject) {
if (currentObject[property] === value) {
return `${currentPath}/${value}`;
}
if (typeof currentObject[property] === 'object') {
const result = generatePath(value, `${currentPath && currentPath + '/'}${property}`, currentObject[property]);
if (result) {
return result;
}
}
}
}
document.write(
generatePath("some_file.txt"),
'<br>',
generatePath("some_file2.txt"),
'<br>',
generatePath("up_file.txt"),
'<br>',
generatePath("no_file.txt")
);

You can create recursive function for this using for...in loop that will store previous path elements in one array.
const data = {
"folder_name": {
"another_folder": {
"third_folder": {
0: "some_file.txt",
1: "another_file.png",
2: "third_file.pdf"
}
}
}
}
function generatePath(data, file) {
let result
(function getPath(obj, file, prev = []) {
for (let i in obj) {
if (typeof obj[i] == 'object') {
getPath(obj[i], file, prev.concat(i).slice())
}
if (file == obj[i]) {
result = prev.concat(obj[i]).join('/')
}
}
})(data, file)
return result;
}
console.log(generatePath(data, "third_file.pdf"))
console.log(generatePath(data, "some_file.txt"))

Javascript - Split and return array in multiple arrays or string

I have a json like this:
{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"}]}}
... and I dynamically fetch the values and create a variable this:
var existingParams = [
"name",
"updated"].filter(field => getBody.search[field]);
var sqlVal = existingParams.map(field => {
if (field === 'name') {
function getValues(item, index) {
var getVal = [item.tag];
return "%" + getVal + "%";
}
console.log(name.map(getValues));
return name.map(getValues);
} else {
return getBody.search[field];
}
})
For the above example I get for sqlVal:
console.log(sqlVal);
[ [ '%Peter%' ], '2018-11-07' ]
... which is fine.
BUT, if I have two values:
{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"}]}}
... I'm getting this structure:
[ [ '%Peter%', '%Jack%' ], '2018-11-07' ]
... but what I need is sth like:
[ '%Peter%', '%Jack%', '2018-11-07' ]
... or:
[ ['%Peter%'], ['%Jack%'], '2018-11-07' ]
And in case of further e.g. 3 names:
{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"},{"tag":"Maria"}]}}
... I need sth like:
[ '%Peter%', '%Jack%', '%Maria%', '2018-11-07' ]
... or:
[ ['%Peter%'], ['%Jack%'], ['%Maria%'], '2018-11-07' ]
... and so on
How do I need to adjust the above query to get this?

If I understand your question correctly, then this problem can be solved via the Array#reduce() method.
The general idea with this approach is to transform your input object to an array - the reduce operation can be used to do this, with the special-case rule of "flattening" the nested value on the name key into the final result:
var input = {"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"}]}}
var result = Object
.entries(input.search)
.reduce(function(result, entry) {
const key = entry[0]
const value = entry[1]
if(key === 'name') {
// When the 'name' key is encountered, handle the value
// differently, by addting the items of this value array
// to the result
value.forEach(function(item) {
result.push('%' + item.tag + '%')
})
}
else {
// Append values for other keys directly to the result
result.push(value)
}
return result
}, [])
console.log(result )

You could simply use Object.values + reduce for something like this:
const json = { "search": { "updated": "2018-11-07", "name": [{ "tag": "Peter" }, { "tag": "Jack" }, { "tag": "Maria" }] } }
const result = Object.values(json.search).reduce((r,c) =>
(Array.isArray(c) ? r.push(...c.map(({tag}) => `%${tag}%`)) : r.push(c), r),[])
console.log(result)
If the order is important (names first then date) you could use reverse:
const json = { "search": { "updated": "2018-11-07", "name": [{ "tag": "Peter" }, { "tag": "Jack" }, { "tag": "Maria" }] } }
const result = Object.values(json.search).reverse().reduce((r,c) =>
(Array.isArray(c) ? r.push(...c.map(({tag}) => `%${tag}%`)) : r.push(c), r),[])
console.log(result)

First of all you did not provide a Minimal, Complete, and Verifiable example so it is quite hard for me to figure out where you are running into issues. For example, you are referencing existingParam but nowhere are they defined. This is key to understanding the problem because all of the code that you posted is heavily invested in the values and format of this value.
Second, how are you parsing the JSON? With the standard JSON#parse function you would get back an object with the same structure as your provided JSON. However, you are either not using this or you are mutating the object after it was parsed into a new format. Either way, the object that JSON#parse returns for the provided JSON is not an array and therefor you cannot use Array#map on it.
For the sake of being productive though I am going to try and explain how to do things.
JSON:
let data1 = '{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"}]}}',
data2 = '{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"}]}} ',
data3 = '{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"},{"tag":"Maria"}]}}';
Now that we have our JSON data we need to parse it and store it as a JSON object. To do so I am going to create a function; this way the data can be passed to the same function and handled the same way but the implementation will stay the same. Also, since we are only looking at the values in the search property we are going to go ahead and jump right into it.
Parse the JSON:
function parseResponse (response) {
let parsedResponse = JSON.parse(response);
parsedResponse = parsedResponse['search'];
}
Now that we have our function that takes our response and parses it we can then begin to sort through it to find and isolate the parts that we want. In this case we will add some code to loop through our properties and find the updated and name properties.
function parseResponse (response) {
let parsedResponse = JSON.parse(response);
parsedResponse = parsedResponse['search'];
for (let prop in parsedResponse) {
if (prop === 'updated') {
// do stuff with 'updated'
}
if (prop === 'name') {
// do stuff with 'name'
}
}
}
Because we want to return a result we are going to add a variable updated and names which will hold the values that we pull out of the string until we are ready to return them. Now that we have our loop and our temporary variables we can go ahead and pull the updated value out of our data and place it in the updated variable.
function parseResponse (response) {
let parsedResponse = JSON.parse(response),
updated = '',
names = [];
parsedResponse = parsedResponse['search'];
for (let prop in parsedResponse) {
if (prop === 'updated') {
updated = parsedResponse[prop];
}
if (prop === 'name') {
// do stuff with 'name'
}
}
}
With our updated value squared away we can jump into our names. Since you listed the format ['%name%', '%name%', '%name%'] first I am going to go ahead and show you how to do it this way. Here we are going to grab the property name, iterate through the names, grab the tag property, and then add the %s before pushing it to our names temporary variable.
function parseResponse (response) {
let parsedResponse = JSON.parse(response),
updated = '',
names = [];
parsedResponse = parsedResponse['search'];
for (let prop in parsedResponse) {
if (prop === 'updated') {
updated = parsedResponse[prop];
}
if (prop === 'name') {
for (let i = 0; i < parsedResponse[prop].length; i++) {
let name = parsedResponse[prop][i].tag;
name = '%' + name + '%';
names.push(name);
}
}
}
}
With everything in place all that is left is to assemble the result. To do so we are going to flatten the array of names, add them to the array, and then add the updated value to the end before returning it. To flatten the array we are going to use the spread operator.
function parseResponse (response) {
let parsedResponse = JSON.parse(response),
updated = '',
names = [];
parsedResponse = parsedResponse['search'];
for (let prop in parsedResponse) {
if (prop === 'updated') {
updated = parsedResponse[prop];
}
if (prop === 'name') {
for (let i = 0; i < parsedResponse[prop].length; i++) {
let name = parsedResponse[prop][i].tag;
name = '%' + name + '%';
names.push(name);
}
}
}
return [...names, updated];
}
With all of that set we can go ahead and call parseResponse() with data1, data2, or data3 and get back a response that looks like so:
let data1 = '{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"}]}}',
data2 = '{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"}]}} ',
data3 = '{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"},{"tag":"Maria"}]}}';
function parseResponse (response) {
let parsedResponse = JSON.parse(response),
updated = '',
names = [];
parsedResponse = parsedResponse['search'];
for (let prop in parsedResponse) {
if (prop === 'updated') {
updated = parsedResponse[prop];
}
if (prop === 'name') {
for (let i = 0; i < parsedResponse[prop].length; i++) {
let name = parsedResponse[prop][i].tag;
name = '%' + name + '%';
names.push(name);
}
}
}
return [...names, updated];
}
console.log(parseResponse(data1));
console.log(parseResponse(data2));
console.log(parseResponse(data3));

Spread operator can be used to flatten the result :
var obj = {"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"},{"tag":"Maria"}]}}
var arr = [...obj.search.name.map(n => `%${n.tag}%`), obj.search.updated]
console.log( arr )
Another alternative could be to extract during parsing :
var arr = [], json = '{"search":{"updated":"2018-11-07","name":[{"tag":"Peter"},{"tag":"Jack"},{"tag":"Maria"}]}}'
JSON.parse(json, (k, v) => v.trim && arr.push(k === 'tag' ? `%${v}%` : v))
console.log( arr )

Use node module in browser

I'm using the very simple Refify npm module to handle circular structure JSON. It stringifies a circular structure JSON object in Node.js to then send to the client. My Angular frontend receives the stringified JSON and needs to call the parse method of refify to convert it back to a usable object.
How do I include the refify node module in my Angular frontend so I can reference refify?
Backend usage looks like so:
var refify = require("refify");
app.get("/api/entries, function(req, res){
var circularJSON = //a circular JSON object
res.send(refify.stringify(circularJSON));
});
The frontend reference would look like this:
$http.get("/api/entries").success(function(data){
$scope.entries = refify.parse(data);
});

You can either use browserify as a build step, or you could use wzrd.in CDN, which is a CDN for npm modules.
browserify as a build step
Use node-style require() to organize your browser code and load modules installed by npm. Browserify will recursively analyze all the require() calls in your app in order to build a bundle you can serve up to the browser in a single <script> tag. For more information, and examples, click here.
wzrd.in CDN
<script src="https://wzrd.in/standalone/refify#latest"></script>
<script>
window.refify // You can use refify now!
</script>
You can go to https://wzrd.in/standalone/refify#latest, copy the code, and paste it into your own file if you want. See jsfiddle here.

Here is the forked version of Refify you can use in node.js as well as browsers.
Forked Refify
You can simply download the index.js and include it in your AngularJS application. and use it.
see the below code, I have added the whole forked index.js file in snippet and example at the end.
(function(obj) {
if (typeof exports === 'undefined') {
obj.refify = refify;
} else {
module.exports = refify;
}
function refify(obj) {
var objs = [];
var paths = []
var keyStack = [];
var objStack = [];
return walk(obj);
function walk(it) {
if (typeof it !== 'object') {
return it;
}
objs.push(it);
paths.push(keyStack.slice())
objStack.push(it)
var copy = initCopy(it);
for (var k in it) {
keyStack.push(k);
var v = it[k];
var i = objs.indexOf(v);
if (i == -1) {
copy[k] = walk(v)
} else {
var $ref = '#/' + paths[i].join('/');
copy[k] = {
$ref: $ref
};
}
keyStack.pop();
}
objStack.pop();
return copy;
}
}
refify.parse = function(it) {
if (typeof it !== 'object') it = JSON.parse(it);
var keyStack = [];
var copy = initCopy(it);
walk(it);
return copy;
function walk(obj) {
if (typeof obj !== 'object') {
set(copy, keyStack.slice(), obj);
return;
}
for (var k in obj) {
keyStack.push(k);
var current = obj[k];
var objPath = parseRef(current);
while (objPath) {
current = get(copy, objPath);
objPath = parseRef(current);
}
if (current === obj[k]) {
// We did *not* follow a reference
set(copy, keyStack.slice(), initCopy(current));
walk(current);
} else {
// We *did* follow a reference
set(copy, keyStack.slice(), current);
}
keyStack.pop();
}
}
}
refify.stringify = function(obj, replacer, spaces) {
return JSON.stringify(refify(obj), replacer, spaces)
}
function parseRef(value) {
if (typeof value !== 'object') return false;
if (!value.$ref) return false;
var path = value.$ref == '#/' ? [] : value.$ref.split('/').slice(1);
return path
}
function get(obj, path) {
if (!path.length) return obj;
if (typeof obj !== 'object') return;
var next = obj[path.shift()];
return get(next, path);
}
refify.set = set;
function set(obj, path, value) {
if (path.length === 0) throw new Error("Cannot replace root object");
var key = path.shift();
if (!path.length) {
obj[key] = value;
return;
}
switch (typeof obj[key]) {
case 'undefined':
obj[key] = isNaN(parseInt(key, 10)) ? {} : [];
break;
case 'object':
break;
default:
throw new Error("Tried to set property " + key + " of non-object " + obj[key]);
}
set(obj[key], path, value);
}
function initCopy(obj) {
if (typeof obj !== 'object') return obj;
return Array.isArray(obj) ? [] : {}
}
}(this));
// Example with forked version
var obj = {
inside: {
name: 'Stackoverflow',
id: '98776'
}
};
obj.inside.parent = obj;
var refifyObject= refify(obj);
document.getElementById("out").innerHTML = JSON.stringify(refifyObject);
<div id="out"></div>

if you want use module in browser, you can use Commonjs and AMD
for example :
requirejs.org
browserify
commonjs.org
systemjs
you can convert refify to module ( browserify ,requirejs,commonjs,...) and use.
Useful Links :
create module in RequireJS
writing modular js/

Nicer way to get nested object attributes

Often in a response from a remote API call, I receive nested objects:
var response = {
data : {
users : [
{
name : 'Mr. White'
}
]
}
}
I want to check whether the first user's name is 'Mr. White', and would naturally want to write something like.
var existed = response.data.users[0].name === 'Mr. White'
However I cannot be sure if all the objects are present, so to avoid exceptions instead I end up writing:
var existed = response && response.data && response.data.users && response.data.users[0].name === 'Mr. White'
Is there a nicer way to do this? Another ugly option that comes to mind is:
var existed = false;
try {
var existed = response.data.users[0].name === 'Mr. White';
} catch(e) { }
In addition to vanilla javascript, I usually have underscore.js and jquery available too.
Edit:
Oops, noticed I asked a dupe of javascript test for existence of nested object key.
An interesting option based on those answers is:
var existed = (((response || {}).data || {}).users || [{}])[0].name === 'Mr. White';

You could hide this naughty try/catch block inside a function like this one :
function resolve(root, path){
try {
return (new Function(
'root', 'return root.' + path + ';'
))(root);
} catch (e) {}
}
var tree = { level1: [{ key: 'value' }] };
resolve(tree, 'level1[0].key'); // "value"
resolve(tree, 'level1[1].key'); // undefined
More on this : https://stackoverflow.com/a/18381564/1636522

I would use the try catch approach but wrap it in a function to hide the ugliness.

Instead of a try/catch, this should be done via checking whether each level in the object is defined or not.
go for
if(typeof(response)!="undefined"
&& typeof(response.data)!="undefined"
&& typeof(response.data.users)!="undefined"
&& typeof(response.data.users[0])!="undefined"
&& typeof(response.data.users[0].name)!="undefined"
) {
//executes only if response.data.users[0].name is existing
}

Here is a function which I used in one of my projects http://jsfiddle.net/JBBAJ/
var object = {
data: {
users: [
{
firstName: "White"
},
{
firstName: "Black"
}
]
}
}
var read = function(path, obj) {
var path = path.split(".");
var item = path.shift();
if(item.indexOf("]") == item.length-1) {
// array
item = item.split("[");
var arrayName = item.shift();
var arrayIndex = parseInt(item.shift().replace("]", ""));
var arr = obj[arrayName || ""];
if(arr && arr[arrayIndex]) {
return read(path.join("."), arr[arrayIndex]);
} else {
return null;
}
} else {
// object
if(obj[item]) {
if(path.length === 0) {
return obj[item];
} else {
return read(path.join("."), obj[item]);
}
} else {
return null;
}
}
}
console.log(read("data.users[0].firstName", object)); // White
console.log(read("data.users[1].firstName", object)); // Black
console.log(read("data.test.users[0]", object)); // null
The idea is to pass your path as a string along with your object. The idea was to prevent the throwing of an exception and receive just null as result of the path is wrong. The good thing is that the function works with every path and you don't need to write long if statements.