Develop Reference

How to add an 'id' to json file at first line. [javascript][node.js]

I found a working script to add an id to each object;
function addId(id) {
return function iter(o) {
if ('fediverse' in o) {
o.id = id++;
}
Object.keys(o).forEach(function (k) {
Array.isArray(o[k]) && o[k].forEach(iter);
});
};
}
var data = [{"fediverse": "#username#mastodon.online", "name": "alternatenamehere", "alternate_names": "", "gender": "", "category": "", "description": "description here.", "link": "https://mastodon.online/users/username", "image": "https://files.mastodon.online/accounts/avatars/image.jpg", "language": "", "region": "", "user": true, "group": false, "creator": false, "companyOrganization": false, "project": false, "applicationSoftware": false}];
data.forEach(addId(1))
console.dir(data, {depth: null, colors: true, maxArrayLength: null});
This script outputs;
[
{
fediverse: '#username#mastodon.online',
name: 'alternatenamehere',
alternate_names: '',
gender: '',
category: '',
description: 'description here.',
link: 'https://mastodon.online/users/username',
image: 'https://files.mastodon.online/accounts/avatars/image.jpg',
language: '',
region: '',
user: true,
group: false,
creator: false,
companyOrganization: false,
project: false,
applicationSoftware: false,
id: 1
}
]
the problem is the search I use detects ids from the first line only. how can I add the id at the first line instead of the last line?

How do I split nested objects into its separate objects in es6

My json data currently looks like this:
[
{
"timestamp": 345345,
"status": {
"code": 200,
"success": true,
"message": "Success",
"errors": []
},
"size": 1,
"nameList": [
{
"id": 74997
"status": "pending"
}
]
},
{
"timestamp": 45,
"status": {
"code": 200,
"success": true,
"message": "OK",
"errors": []
},
"size": 5,
"content": [
{
"id": 1260087,
"pilot": "Jujuki",
"nameId": 4343,
"RefId": "3453454"
}
]
}
]
How do I split up this object into 2 different object/arrays of data. I want everything with index 0 to go into the first new object, and everything with index 1 to go into the second new object.
What would be best es6 syntax method to use.

you can do Destructuring assignment, like the following:
const data = [{
timestamp: 345345,
status: {
code: 200,
success: true,
message: "Success",
errors: []
},
size: 1,
nameList: [{
id: 74997,
status: "pending"
}]
},
{
timestamp: 45,
status: {
code: 200,
success: true,
message: "OK",
errors: []
},
size: 5,
content: [{
id: 1260087,
pilot: "Jujuki",
nameId: 4343,
RefId: "3453454"
}]
}
];
const [firstObj, secondObj] = data;
console.log("the first Object: ", firstObj);
console.log("the second Object: ", secondObj);

How to transform api response in nestjs?

I am using Nest Serialization to transform api response. However, it returns really crypic result. I expect it to slice some fields out of the data but for some reason, it shows weird result.
product.entity.js
export class ProductEntity {
id: string
description: string
status: boolean
price: string
moq: number
rating: number
last_modified: string
constructor(partial: Partial<ProductEntity>) {
Object.assign(this, partial)
}
}
product.controller.ts
#UseInterceptors(ClassSerializerInterceptor)
#Get()
async findAll(
#Query('page') page: number,
#Query('per_page') per_page: number
): Promise<ProductEntity[]> {
if (!page && !per_page) {
throw new BadRequestException('Proper query params not supplied')
}
const data: ProductEntity[] = []
const products: any = await this.productService.findAll({
page,
per_page
})
products.map(product => {
data.push(new ProductEntity(product))
})
return data
}
product.service.ts
async findAll(query): Promise<Product[]> {
const { page, per_page }: { page: number; per_page: number } = query
const from: number = (page - 1) * per_page
return this.productModel
.find()
.skip(from)
.limit(Number(per_page))
}
which returns,
"$__": {
"strictMode": true,
"selected": {},
"getters": {},
"_id": {
"_bsontype": "ObjectID",
"id": {
"type": "Buffer",
"data": [
93,
15,
149,
140,
78,
148,
77,
10,
10,
105,
51,
130
]
}
},
"wasPopulated": false,
"activePaths": {
"paths": {
"moq_type": "init",
"moq": "init",
"price_type": "init",
"price": "init",
"name": "init",
"category_id": "init",
"company_id": "init",
"short_description": "init",
"long_description": "init",
"active": "init",
"variations": "init",
"keywords": "init",
"photos": "init",
"_id": "init",
"created_at": "init",
"updated_at": "init",
"__v": "init"
},
"states": {
"ignore": {},
"default": {},
"init": {
"_id": true,
"short_description": true,
"long_description": true,
"active": true,
"variations": true,
"keywords": true,
"photos": true,
"name": true,
"price": true,
"price_type": true,
"moq": true,
"moq_type": true,
"category_id": true,
"company_id": true,
"created_at": true,
"updated_at": true,
"__v": true
},
"modify": {},
"require": {}
},
"stateNames": [
"require",
"modify",
"init",
"default",
"ignore"
]
},
"pathsToScopes": {},
"cachedRequired": {},
"session": null,
"$setCalled": [],
"emitter": {
"_events": {},
"_eventsCount": 0,
"_maxListeners": 0
},
"$options": {
"skipId": true,
"isNew": false,
"willInit": true
}
},
Am I doing something wrong in here or is it not correct way to transform API response?

Not sure about NestJS but in regular node.js for mongodb, If you are calling find() that way X().X().X() you need to add at the end of the pipeline the call to exec() method, like :
this.productModel
.find()
.skip(from)
.limit(Number(per_page))
.exec();

I am getting that too. The answer is removed #UseInterceptors(ClassSerializerInterceptor) in your controller.
Maybe that function utilize object inheritance from class extension? I think you don't really need it because that function is used to cast a response (like tranforming camerCase to snake_case, etc). If you really need it, then I don't know because I am still strugle with it too.

API response in the browser different from the one in the code

Here's my code:
async function BuiltWithCall(website) {
var domainCall = `https://api.builtwith.com/v12/api.json?KEY=${keys.builtWith}&LOOKUP=${website}`;
var domainRes = await fetch(domainCall);
console.log(domainRes);
var keywordCall = `https://api.builtwith.com/kw1/api.json?KEY=${keys.builtWith}&LOOKUP=${website}`;
var keywordRes = await fetch(keywordCall);
console.log(keywordRes);
return await {'domRes': domainRes.json(), 'kwRes': keywordRes.json()};
}
It takes the website provided and runs it through the BuiltWith API. But the problem is the response.
Response {
size: 0,
timeout: 0,
[Symbol(Body internals)]:
{ body:
PassThrough {
_readableState: [Object],
readable: true,
domain: null,
_events: [Object],
_eventsCount: 7,
_maxListeners: undefined,
_writableState: [Object],
writable: true,
allowHalfOpen: true,
_transformState: [Object] },
disturbed: false,
error: null },
[Symbol(Response internals)]:
{ url: 'https://api.builtwith.com/v12/api.json?KEY=key&LOOKUP=hotelscombined.com',
status: 200,
statusText: 'OK',
headers: Headers { [Symbol(map)]: [Object] } } }
Response {
size: 0,
timeout: 0,
[Symbol(Body internals)]:
{ body:
PassThrough {
_readableState: [Object],
readable: true,
domain: null,
_events: [Object],
_eventsCount: 3,
_maxListeners: undefined,
_writableState: [Object],
writable: false,
allowHalfOpen: true,
_transformState: [Object] },
disturbed: false,
error: null },
[Symbol(Response internals)]:
{ url: 'https://api.builtwith.com/kw1/api.json?KEY=key&LOOKUP=hotelscombined.com',
status: 200,
statusText: 'OK',
headers: Headers { [Symbol(map)]: [Object] } } }
So this right here makes no sense to me. Because this is the exact same URL, run in a browser:
{
"Keywords": [
{
"Domain": "hotelscombined.com",
"Keywords": [
"compare",
"save",
"cheap",
"hotel",
"deal",
"hotelscombined",
"search",
"... More keywords but you get the idea"
]
}
],
"Errors": []
};
Other call response:
{
"Results": [
{
"Result": {
"IsDB": true,
"Spend": 609,
"Paths": [
{
"FirstIndexed": 1294059600000,
"LastIndexed": 1526338800000,
"Domain": "builtwith.com",
"Url": "",
"SubDomain": "",
"Technologies": [
{
"Categories": [
"Edge Delivery Network"
],
"IsPremium": "yes",
"Name": "Amazon CloudFront",
"Description": "Amazon CloudFront delivers your static and streaming content using a global network of edge locations.",
"Link": "http://aws.amazon.com/cloudfront/",
"Tag": "cdns",
"FirstDetected": 1386284400000,
"LastDetected": 1526338800000
},
]
},
]
},
"Meta": {
"Vertical": "Technology And Computing",
"Social": [
"http://twitter.com/builtwith",
"http://facebook.com/builtwith",
"http://linkedin.com/company/builtwith",
"http://google.com/+builtwithdotcom"
],
"CompanyName": "BuiltWith",
"Telephones": [
"+61-300-558745",
"+1-650-618-3949"
],
"Emails": [
"support#builtwith.com"
],
"City": "Sydney",
"State": "NSW",
"Postcode": "2000",
"Country": "AU",
"Names": [
{
"Name": "N/A",
"Type": 0,
"Email": "n/a#builtwith.com"
},
{
"Name": "N/A",
"Type": 0,
"Email": "n/a#builtwith.com"
}
],
"ARank": 22108,
"QRank": 275921
},
"Attributes": {
"MJRank": 8737,
"MJTLDRank": 4620,
"RefSN": 7402,
"RefIP": 10142,
"TTFB": 129,
"Sitemap": 20,
"GTMTags": 0,
"QubitTags": 0,
"TealiumTags": 0,
"AdobeTags": 0,
"CDimensions": 0,
"CGoals": 0,
"CMetrics": 0,
"SourceBytes": 0
},
"FirstIndexed": 1294059600000,
"LastIndexed": 1526338800000,
"Lookup": "builtwith.com"
}
],
"Errors": []
}
So as you can see the responses are completely different, and I have no idea why. The same fetch method works perfectly for PageSpeed API, but here something is going horribly wrong.
PageSpeed call:
async function PageSpeedCall(website) {
var pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=${keys.pageSpeed}`;
// second call
var results = await fetch(pagespeedCall);
return await results.json();
}
What am I doing wrong?

domainRes is a Response object, not a payload. That is why you see all the stuff in the console output.
To parse the payload as JSON you need to call domainRes.json which also gives you a promise so you have to await for it. Like this.
async function BuiltWithCall(website) {
var domainCall = `https://api.builtwith.com/v12/api.json?KEY=${keys.builtWith}&LOOKUP=${website}`;
var domainRes = await fetch(domainCall);
console.log(domainRes);
var keywordCall = `https://api.builtwith.com/kw1/api.json?KEY=${keys.builtWith}&LOOKUP=${website}`;
var keywordRes = await fetch(keywordCall);
console.log(keywordRes);
return {'domRes': await domainRes.json(), 'kwRes': await keywordRes.json()};
}

How to build the proper mapping/indexing in ElasticSearch with NodeJS

I have been beating my head against this all day, and cannot seem figure out how to get this to work.
I have a source document like this:
{
"created_at": 1454700182,
"message_id": 160,
"user_id": 1,
"establishment_id": 1,
"geo": {
"coordinates": [-4.8767633,
89.7833547
],
"type": "Point"
},
"message": "Venus is in the west",
"active": true,
"score": 0,
"name": {
"first": "First",
"last": "Last"
},
"neighborhood": "Townside"
},
I create a document like this in ElasticSearch:
{
"message_id": 160,
"message": "Venus is in the west",
"first_name": "First",
"last_name": "Last",
"location": {
"lon": -4.8767633,
"lat": 89.7833547
},
"created_at": 1454700182,
"neighborhood": "Townside"
}
I've been trying different ways to create the index.
First:
client.indices.create({
index: 'messages',
type: 'document',
body: {
messages: {
properties: {
message: {
type: 'string',
index: 'not_analyzed'
},
neighborhood: {
type: 'string',
index: 'not_analyzed'
},
first_name: {
type: 'string',
index: 'not_analyzed'
},
last_name: {
type: 'string',
index: 'not_analyzed'
},
created_at: {
type: 'integer',
index: 'not_analyzed'
},
location: {
type: 'geo_point',
lat_lon: true
}
}
}
},
}
);
This allows me to do fuzzy text searches and greater than queries, but doesn't recognize the geo_point. So I tried this:
client.indices.create({
index: 'messages',
type: 'document',
"mappings": {
"messages": {
"properties": {
"message": {
"type": "string",
"index": "not_analyzed"
},
"neighborhood": {
"type": "string",
"index": "not_analyzed"
},
"first_name": {
"type": "string",
"index": "not_analyzed"
},
"last_name": {
"type": "string",
"index": "not_analyzed"
},
"created_at": {
"type": "integer",
"index": "not_analyzed"
},
"location": {
"type": "geo_point",
"lat_lon": true,
"index": "not_analyzed"
}
}
}
}
});
This does recognize the geo_point, but none of the other things work.
Here is the query I've been using for the non geo fields:
query = {
query: {
filtered: {
query: {
multi_match: {
query: message,
fields: ['message', 'neighborhood', 'first_name', 'last_name'],
"fuzziness": "AUTO",
"prefix_length": 2
}
},
filter: {
bool: {
must: {
range: {
"created_at": {
"gte": min_ts
}
}
}
}
}
}
}
};
I've been so turned around on this, just trying to allow text and geo search on the same collection of documents, that I need at least another set of eyes.
Appreciate any help!