This question already has answers here:
How to filter object array based on attributes?
(21 answers)
Closed 8 years ago.
I have seen very old answers to this question and many of the technologies used 2 years back have changed.
What I have is JSON files sent by a database over to my server, and what I would like to know is how to filter that data.
I am running a server with node.js, and what I would like to do is something like:
var results = QueryLibrary.load(jsondata);
var filtered = results.query('select where user = "user1"');
How can I do something like this in javascript running in node?
underscore has a where function that does just this
var _ = require("underscore");
var json = '[{"user": "a", "age": 20}, {"user": "b", "age": 30}, {"user": "c", "age": 40}]';
var users = JSON.parse(json);
var filtered = _.where(users, {user: "a"});
// => [{user: "a", age: 20}]
Another utility library, Lo-Dash, has a where function that operates identically.
You can add underscore to your project using
$ npm install --save underscore
or lodash
$ npm install --save lodash
If you only care about the where function, lodash offers it as a separate module
// only install lodash.where
$ npm install --save lodash.where
To use it in your project
var where = require("lodash.where");
// ...
var filtered = where(users, {"user": "a"});
Even if you use a library to do this, a better approach is probably to setup a chain of streams that handles all of your data processing in smaller modules.
Without knowing what you actually want to do, I've created this as an example. For the purposes of this code, maybe think of a debug logging stream or something.
json-parser.js
input: string (JSON)
output: object
var Transform = require("stream").Transform;
function JsonParser() {
Transform.call(this, {objectMode: true});
this._transform = function _transform(json, enc, done) {
try {
this.push(JSON.parse(json));
}
catch (e) {
return done(e);
}
done();
}
}
JsonParser.prototype = Object.create(Transform.prototype, {
constructor: {
value: JsonParser
}
});
module.exports = JsonParser;
obj-filter.js
input: object
output: object (result of where(data, filters))
var Transform = require("stream").Transform;
var where = require("lodash.where");
function ObjFilter(filters) {
Transform.call(this, {objectMode: true});
this._transform = function _transform(obj, enc, done) {
this.push(where(obj, filters));
done();
}
}
ObjFilter.prototype = Object.create(Transform.prototype, {
constructor: {
value: ObjFilter
}
});
module.exports = ObjFilter;
stringifier.js
input: object
output: string (JSON)
var Transform = require("stream").Transform;
function Stringifier() {
Transform.call(this, {objectMode: true});
this._transform = function _transform(obj, enc, done) {
this.push(JSON.stringify(obj));
done();
}
}
Stringifier.prototype = Object.create(Transform.prototype, {
constructor: {
value: Stringifier
}
});
module.exports = Stringifier;
app.js
// modules
var JsonParser = require("json-parser");
var ObjFilter = require("obj-filter");
var Stringifier = require("stringifier");
// run
var parser = new JsonParser();
// setup stream chain
parser.pipe(new ObjFilter({"user": "a"}))
.pipe(new Stringifier())
.pipe(process.stdout);
// send example json in
parser.write('[{"user": "a", "age": 20}, {"user": "b", "age": 30}, {"user": "c", "age": 40}]');
// output
// => [{"user":"a","age":20}]
Here, I made a Stringifier stream that converts objects back into JSON so that we can see them dumped into the console, though you could easily create any streams you needed to handle the operations that your app requires. Your stream end points will likely not end up in writing to the console.
As a last note, you would probably create a database stream that accepts some sort of query options and emits json. You would pipe that stream directly into parser.
Anyway, I hope this gives you a better idea of how to process data in node.js.
You can use any of the normal array/object built-in functions that javascript has, normally that kind of query would be made at the time of retrieving your data from the database, not after.
something like
for(i=0;i<objIdsArray.length;i++) {
for(j=0;j<mockJSON.length;j++) {
if(mockJSON[j]["id"] === parseInt(objIdsArray[i])) {
mockJSON.splice(j, 1); // to delete it, could be any other instruction
}
}
}
Related
A perplexing question. I am writing a node based command line tool that has a source json file, and then will take a changes JSON file (i.e. add, update, delete, etc) and I need to basically make the operation into a new JSON file and output to a new file. Not as simple as it sounds. With no command line arguments, etc. you would need to have a directive field in the JSON like so?
The app would work like this:
$ myapp source.json changes.json newfile.json
{
"action": "addRecord",
"payload": {
"id": "1",
"name": "James Hetfield",
"guitar": "More Beer"
}
},
"action": "deleteRecord",
"payload": {
"id": "3",
"name": "Dave Mustaine",
"guitar": "Ole Faithful"
}
}
My JSON structure is probably wrong as well, but wondering how you would use JSON.parse, JSON.stringify to read the file in with the fs library and actually make actions happen when identifying action and then execute a CRUD like statement with payload
Here is what I have so far:
#!/usr/bin/env node
// using fs for parsing for small example
const fs = require('fs');
// We do not simply require the file due to scaling.
// Large file will cause event locking. To handle
// even greater files we would use Node streams.
const file = './changes.json';
// target filename for output
const target = 'output.json';
fs.readFile(file, 'utf8', function(err, data) {
if (err) {
console.log('Error' + err);
return;
}
let obj = JSON.parse(data, (key, value) => {
if (key === 'action') {
if (value === 'addSong') {
return value = "Song Added";
}
if (value === "addPlaylist") {
return value = "Playlist added";
}
if (value === "deletePlaylist") {
return value = "Playlist deleted";
}
}
return value;
});
console.dir(obj);
});
fs seems to just read through the file and pick up the last value read. Not good. Wondering how to do the compare and possibly structure the JSON to make the action happen and then append? or transform the JSON into a new file with the update. Stuck.
Basically, this shouldn´t be a very difficult question, but I´ve tried for like 2 or 3 hours and couldnt reach my goal, especially for such an "easy" question. Im using Node.js and my goal is, to load data from a Json file into a variable, add some new data to this and store my data into the same json file again.
Herefor, my json file looks like this:
[
{
"name": "Max",
"date": "1.1.2020"
}, {
"name": "Henry",
"date": "2.2.2020"
}
]
Here´s my code:
const fs = require('fs');
const filename = './jsonFile.json';
const data = loadJSON();
// console.log(data) keeps saying undefined (whithout using .toString in loadJSON)
function loadJSON() {
JSON.parse(fs.readFileSync(filename).toString); // toString doesnt work
}
function saveJSON(data) {
fs.writeFileSync(filename, JSON.stringify(data));
}
function adduser(username) {
var today = "3.3.2020"; // doesnt matter
let obj = {
name: username,
date: today
}
vipJson.push(obj);
saveVIP(vipJson);
}
It doesnt seem to be working. Could anyone help me, to fix my problem, so I can work with .json files ? Thanks a lot!
You need to specify the BufferEncoding options to read the file, something like this.
const fs = require("fs")
const data = fs.readFileSync("./myfile.json", { encoding: "utf8", flag: "r" })
If you are sure about json files, you can read data from file using require.
const fs = require('fs');
const filename = './jsonFile.json';
const data = loadJSON();
function loadJSON() {
return require(filename)
}
function saveJSON(data) {
fs.writeFileSync(filename, JSON.stringify(data));
}
function adduser(username) {
var today = "3.3.2020"; // doesnt matter
let obj = {
name: username,
date: today
}
data.push(obj);
saveJSON(data);
}
Try above code snippet.
I'm trying to write JavaScript code into a js with Nodejs fs module. I managed to write a json file but could wrap my head around on how to write JavaScript to it.
fs.writeFile("config.json", JSON.stringify({name: 'adman'tag: 'batsman',age: 25}), 'utf8',{ flag: "wx" }, function(err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
I need to create a .js file with the following data
const cricketers = [
{
name: 'adman',
tag: 'batsman',
age: 25
},
// other objects
]
module.exports = cricketers ;
Two things:
If all you want to do is to be able to do let someData = require('someFile.json'); Nodejs already supports requiring json files and treats them like Js objects.
Otherwise I don't know of a library that will do exactly this for you, BUT...
You can do this yourself. The fs.writeFile function takes a string, so you just have to generate the string you want to write to the file.
let someData = [{name: 'adman', tag: 'batsman', age: 25}];
let jsonData = JSON.stringify(someData);
let codeStr = `const cricketers = ${jsonData}; module.exports = cricketers;`;
fs.writeFile("someFile.js", codeStr, 'utf8',{ flag: "wx" }, function(err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
Obviously this only works for a very specific use case, but the point is it can be done with simple (or complicated...) string manipulation.
use string templating
const data = `const cricketers = ${JSON.stringify(yourArray)};
module.exports = cricketers;
`
Where yourArray is an array of objects
I have the below json response after running a postMan test of a Rest API:
{
"glossary": {
"title": "example glossary",
"GlossDiv": {
"title": "S",
"GlossList": {
"GlossEntry": {
"ID": "SGML",
"SortAs": "SGML",
"GlossTerm": "Standard Generalized Markup Language",
"Acronym": "SGML",
"Abbrev": "ISO 8879:1986",
"GlossDef": {
"para": "A meta-markup language, used to create markup languages such as DocBook.",
"GlossSeeAlso": ["GML", "XML"]
},
"GlossSee": "markup"
}
}
}
}
}
Now I would like to compare the above json against a predefined json. Say, its the same as above.
How can I compare two jsons via the Postman test?
I had a similar problem to solve except that my JSON also contained an array of objects. I used the following technique that can be modified to deal with the simple array of strings in your question.I created an array of global functions called "assert", which contained helper functions such as "areEqual" and "areArraysOfObjectsEqual" and saved these under the "Tests" tab at a top folder level of my tests.
assert = {
areEqual: (actual, expected, objectName) => {
pm.test(`Actual ${objectName} '` + actual + `' matches Expected ${objectName} '` + expected + `'`, () => {
pm.expect(_.isEqual(actual, expected)).to.be.true;
});
},
areArraysOfObjectsEqual: (actual, expected, objectName) => {
if (!_.isEqual(actual, expected)) {
// Arrays are not equal so report what the differences are
for (var indexItem = 0; indexItem < expected.length; indexItem++) {
assert.compareArrayObject(actual[indexItem], expected[indexItem], objectName);
}
}
else
{
// This fake test will always pass and is just here for displaying output to highlight that the array has been verified as part of the test run
pm.test(`actual '${objectName}' array matches expected '${objectName}' array`);
}
},
compareArrayObject: (actualObject, expectedObject, objectName) => {
for (var key in expectedObject) {
if (expectedObject.hasOwnProperty(key)) {
assert.areEqual(expectedObject[key], actualObject[key], objectName + " - " + key);
}
}
}
};
Your "Pre-request Script" for a test would set your expected object
const expectedResponse =
{
"id": "3726b0d7-b449-4088-8dd0-74ece139f2bf",
"array": [
{
"item": "ABC",
"value": 1
},
{
"item": "XYZ",
"value": 2
}
]
};
pm.globals.set("expectedResponse", expectedResponse);
Your Test would test each item individually or at the array level like so:
const actualResponse = JSON.parse(responseBody);
const expectedResponse = pm.globals.get("expectedResponse");
assert.areEqual(
actualResponse.id,
expectedResponse.id,
"id");
assert.areArraysOfObjectsEqual(
actualResponse.myArray,
expectedResponse.myArray,
"myArrayName");
This technique will give nice "property name actual value matches expected value" output and works with arrays of objects being part of the JSON being compared.
Update:
To test your array of strings "GlossSeeAlso", simply call the supplied global helper method in any of your tests like so:
assert.compareArrayObject(
actualResponse.glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso,
expectedResponse.glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso,
"glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso");
Primitive types in JSON key value pairs can be tested like so:
assert.areEqual(
actualResponse.glossary.title,
expectedResponse.glossary.title,
"glossary.title");
I got it after a while. Add test into your request and use Runner to run all your requests in the collection.
Postman info: Version 7.10.0 for Mac.
Test scripts:
pm.test("Your test name", function () {
var jsonData = pm.response.json();
pm.expect(jsonData).to.eql({
"key1": "value1",
"key2": 100
});
});
You can paste this code into your collection or single request tests tab.
What this code do is to save the request into a global variable with a key for that request. You can change your enviroment and hit the same request and if the response are different the test will fail.
const responseKey = [pm.info.requestName, 'response'].join('/');
let res = '';
try {
res = JSON.stringify(pm.response.json());
} catch(e) {
res = pm.response.text();
}
if (!pm.globals.has(responseKey)) {
pm.globals.set(responseKey, res);
} else {
pm.test(responseKey, function () {
const response = pm.globals.get(responseKey);
pm.globals.unset(responseKey);
try {
const data = pm.response.json();
pm.expect(JSON.stringify(data)).to.eql(response);
} catch(e) {
const data = pm.response.text();
pm.expect(data).to.eql(response);
}
});
}
Hope this help.
You can write javascript code inside Tests tab of Postman. Just write simple code to compare and check result in Tests.
var serverData = JSON.parse(responseBody);
var JSONtoCompare = {}; //set your predefined JSON here.
tests["Body is correct"] = serverData === JSONtoCompare;
Looks like the same question asked at POSTMAN: Comparing object Environment variable with response's object which also lists a solution that works, which is to use JSON.stringify() to turn the objects into strings and then compare the strings.
Came across this issue when migrating from a legacy API to a new one and wanting to assert the new API is exactly the same as the old under different scenarios
For context this clones params of the original get request to the legacy endpoint and validates both match up
LEGACY_API_URL should be defined in the environment and the Request is going to the new API
const { Url } = require('postman-collection');
// Setup the URL for the Legacy API
const legacyRequestUrl = new Url({ host: pm.variables.replaceIn("http://{{LEGACY_API_HOST}}/blah")});
// Add All Parameters From the Source Query
legacyRequestUrl.addQueryParams(pm.request.url.query.all());
// Log out the URL For Debugging Purposes
console.log("URL", legacyRequestUrl.toString());
pm.sendRequest(legacyRequestUrl.toString(), function (err, response) {
pm.test('New API Response Matches Legacy API Response', function () {
// Log Out Responses for Debugging Purposes
console.log("New API Response", pm.response.json())
console.log("Legacy API Response", response.json())
// Assert Both Responses are Equal
pm.expect(_.isEqual(pm.response.json(), response.json())).to.be.true
});
});
Link to an example collection
https://www.getpostman.com/collections/4ff9953237c0ab1bce99
Write JavaScript code under 'Tests' section. Refer below link for more info.
Click Here
I'm trying to send data (as an object) to a child process in Node.js, however, all of my regular expressions get lost in transfer.
var arguments = {
something: {
name: 'test',
age: 28,
active; true
},
otherThing: 'some string',
regex: /test/i,
regex2: new RegExp('test')
};
var child = cp.fork(path.join(__dirname, 'child.js'));
child.on('message', function (data) {
console.log(data);
});
child.send(arguments);
In the child.js file I have this at the top:
process.on('message', function () {
console.log(arguments); // This is where the data has changed
});
When the log is output from the child process the arguments object instead looks like this:
{
something: {
name: 'test',
age: 28,
active: true
},
otherThing: 'some string',
regex: {},
regex2: {}
}
So far unable to find anything elsewhere about why this may be happening, any ideas?
Because they are completely separate JavaScript processes, you can't send objects. When you pass an object, it gets serialized to JSON and parsed by the child. (See the docs.)
JSON does not support serializing regex objects. (Try putting JSON.stringify(/abc/) through your console -- you get back "{}".)
To include regexes in a JSON object, you can use the json-fn module. It supports serializing functions, dates, and regexes. (It was actually thanks to an issue i raised that they added regex support. :))
You could then do something like:
var JSONfn = require('json-fn');
var arguments = {
something: {
name: 'test',
age: 28,
active; true
},
otherThing: 'some string',
regex: /test/i,
regex2: new RegExp('test')
};
var child = cp.fork(path.join(__dirname, 'child.js'));
});
child.send(JSONfn.stringify(arguments));
and:
var JSONfn = require('json-fn');
process.on('message', function (data) {
console.log(JSONfn.parse(data))); // This is where the data has changed
});
You can store the regex as a string like
myRegex.string = "/test/";
myRegex.modif = "i";
Send it to child and then use it like
new RegExp(myRegex.string, myRegex.modif);
I tried json-fn but Date objects stringified are not reverted back to Date. This module JSON4Process stringifies the objects' properties of type Date, RegExp, Function, Set and Map while maintaining the object as a javascript object. You don't need to stringify the whole object if you're using fork, you can directly send it.
const { fork } = require('child_process');
const JSON4Process = require('json4process');
let obj = {
date: new Date(),
regex: new RegExp(/regex/g),
func: () => console.log('func')
}
obj = JSON4Process.stringifyProps(obj);
const child = fork('child.js');
child.send(obj);
And then parse the properties back in the other file:
process.on('message', data => {
let obj = JSON4Process.parseProps(data);
});
In case you need to use spawn or exec you can just use the default JSON.stringify over the modified object with json4process:
let newObj = JSON.stringify(JSON4Process.stringifyProps(obj));
let originalObj = JSON4Process.parseProps(JSON.parse(newObj));