var offset = 1;
var limit = 500;
var list = new Promise(function (resolve, reject) {
rets.getAutoLogoutClient(config.clientSettings, (client) => {
var results = client.search.query(SearchType, Class, Query, {
limit: limit,
offset: offset
});
resolve(results);
});
});
var source = Rx.Observable.fromPromise(list);
source.subscribe(results => console.log(results.count));
I am doing a real estate site, using RETS.
What I am trying to do my query are limited from the RETS server, is run this in a loop increasing my Offset until I have all my data. I don't know what the count is until I run the query and find the count value.
I have tried to use expand but I have no clue of how exactly it works. Tried to do these multiple ways, even using the old fashion while loop, which while doesn't work with .then method. So I have turned to RXJS since I been using it in Angular 4.
This is done in express. I need to eventually run corn jobs to fetch for updated properties, but my problem is fetching all the data and increasing the offset each time if the count is higher than my offset. So for example, run a query with an offset of 1 with a limit of 500. Total here is 1690. So next go around my offset would be:
offset += limit
Once I have my data, I need to save it to MongoDB. Which I already been able successfully to do. It's just finding a way to get all the data without having to manually set my offset.
Note the server limit is 2500, yes I can fetch all this in one shot but there are also other data, such as media, which could have well over 2500.
Any suggestions?
This is actually a fairly common use case for RxJS, as there are a lot of paginated data sources, or sources that are otherwise limited in what you can request at one time.
My two cents
To my mind expand is probably the best operator for this given that you are paginating against an unknown data source and you require at least one query in order to determine the final count. If you knew how much data you were going to be querying a more easier option would be to use something like mergeScan, but I digress.
Proposed Solution
This may take a little effort to wrap your head around so I have added annotations wherever possible to break down how this all works. Note I haven't actually tested this, so forgive me any syntax errors.
// Your constant limit for any one query
const limit = 500;
// RxJS helper method that wraps the async call into an Observable
// I am basing this on what I saw of your sample which leads me to believe
// that this should work.
const clientish = Rx.Observable.bindCallback(rets.getAutoLogoutClient);
// A method wrapper around your query call that wraps the resulting promise
// into a defer.
const queryish = (client, params) =>
// Note the use of defer here is deliberate, since the query returns
// a promise that will begin executing immediately, this prevents that behavior
// And forces execution on subscription.
Rx.Observable.defer(() => client.search.query(SearchType, Class, Query, params));
// This does the actual expansion function
// Note this is a higher order function because the client and the parameters
// are available at different times
const expander = (client) => ({limit, count}) =>
// Invoke the query method
queryish(client, {limit, count})
// Remap the results, update offset and count and forward the whole
// package down stream
.map(results => ({
limit,
count: results.count,
offset: offset + limit,
results
}));
// Start the stream by constructing the client
clientish(config.clientSettings)
.switchMap(client =>
// This are the arguments for the initial call
Rx.Observable.of({limit, offset: 0})
// Call the expander function with the client
// The second argument is the max concurrency, you can change that if needed
.expand(expander(client), 1)
// Expand will keep recursing unless you tell it to stop
// This will halt the execution once offset exceeds count, i.e. you have
// all the data
.takeWhile(({count, offset}) => offset < count)
// Further downstream you only care about the results
// So extract them from the message body and only forward them
.pluck('results')
)
.subscribe(results => /*Do stuff with results*/);
const retsConnect = Rx.Observable.create(function(observer) {
rets.getAutoLogoutClient(config.clientSettings, client => {
return searchQuery(client, 500, 1, observer);
});
});
function searchQuery(client, limit, offset, observer) {
let currentOffset = offset === undefined || offset === 0 ? 1 : offset;
return client.search.query(SearchType, Class, Query, {limit: limit, offset: currentOffset})
.then(results => {
offset += limit;
observer.next(results.maxRowsExceeded);
if (results.maxRowsExceeded) {
console.log(offset);
return searchQuery(client, limit, offset, observer);
} else {
console.log('Completed');
observer.complete();
}
});
}
retsConnect.subscribe(val => console.log(val));
So this is getting somewhere with what I have tried here. I am still in process of tweaking this. So what I am looking to do is break the searchQuery down more. Not sure if I should be passing observer.next there, so I am going to is figure out where to map and takeUntil install of returning searchQuery again. I'm not sure takeUntil will take a true or false though. All I need this data to do is be saved into mongodb. So I guess I could leave it like this and have my save method put in there, but I still like to figure this out.
Note: results.maxRowsExceeded returns true when there are still more data. So once the maxRows returns false, it will stop and all the data has been fetched.
Related
I'm trying to make a botinfo command with resource usage stats. I've got memory covered (although percentage for memory usage would be nice too) but I can't seem to get cpu usage. I've tried multiple packages and methods, none worked. My memory thing is:
const usedMemory = os.totalmem() - os.freemem()
How would I make cpu usage and return it in an discord.js embed?
The process.cpuUsage() method returns the user and system CPU time usage of the current process, in an object with properties user and system, whose values are microsecond values (millionth of a second). These values measure time spent in user and system code respectively, and may end up being greater than actual elapsed time if multiple CPU cores are performing work for this process.
The result of a previous call to process.cpuUsage() can be passed as the argument to the function, to get a diff reading.
import { cpuUsage } from 'process';
const startUsage = cpuUsage();
// { user: 38579, system: 6986 }
// spin the CPU for 500 milliseconds
const now = Date.now();
while (Date.now() - now < 500);
console.log(cpuUsage(startUsage));
// { user: 514883, system: 11226 }
For more details check here.
If you want to get in %, you can use existing npm module: node-os-utils. You can use it like this:
const osu = require('node-os-utils')
const cpu = osu.cpu
const count = cpu.count() // 8
cpu.usage()
.then(cpuPercentage => {
console.log(cpuPercentage) // 10.38
})
const osCmd = osu.osCmd
osCmd.whoami()
.then(userName => {
console.log(userName) // admin
})
For more details, check the documentation here.
I found a way to do it. There's a function in node-os-utils (https://www.npmjs.com/package/node-os-utils) called loadavgTime that needs to be used with loadavg-windows (https://www.npmjs.com/package/loadavg-windows) if you're on windows. It takes the os.loadavg from the last minute by default. I then divided it by 2 and then multiplied by 10 cos for some reason it outputs a weird number (between 1.__ and 2.__). It gives it as a percentage, you just need to add the % after it when console.log ing it. Here's the code:
const osu = require('node-os-utils')
require('loadavg-windows')
const cpu = osu.cpu
const usage = (cpu.loadavgTime() / 2) * 10
console.log(usage)
I have a trouble. I receive many objects where I mapping them and I make a external consult using axios and save the return, let's the code:
let savedClients = Object.entries(documents).map(personDocument => {
let [person, document] = personDocument
documentFormated = document
documentNumbers = document.replace(/\D/g, '')
return ConsultDocuments.getResponse(documentNumbers).then(resultScore => { // Calling the axios
const info = { ...resultScore }
return Save.saveClient(info)
})
})
Promise.all(savedClients).then(results => {
console.log(results) // Come only one document, repeted with the total documents passed in map
})
The problem is when it realized the all map first and then make the consults with only the last result many time (the total of documents passed)
This code is legacy and use async/await don't work (serious, if i don't stay here)
I'am tried N ways to make this, and with the libary Q(), it's make the map in correcty order but it's doesn't wait the axios, and all results come with "pending"
Thanks!
I am using twilio javascript sdk for twilio-programmable-chat.
And I want to apply pagination to my channels result but I am not able to figure it out.
Here is my current code.
this.chatClient.getUserChannelDescriptors().then(paginator => {
// All channels are fetched
})
I tried to pass a pageSize similar to how getMessages(10) work but it didn't work.
this.chatClient.getUserChannelDescriptors(10).then(paginator => {
// The result was same, it fetched all the channels instead of just 10
})
I am looking for a example that how pagination can be done on channels.
Thanks.
I've finally found a way how to do it.
It should be done recursively since we get the initial list by calling getUserChannelDescriptors() but then the rest of the records can be fetched by calling nextPage();
async function processChannels(paginator) {
// Now, if hasNextPage is true
// call nextPage() to get the records instead of getUserChannelDescriptors()
if (paginator.hasNextPage) {
const nextPaginator = paginator.nextPage();
processChannels(nextPaginator);
} else {
console.log("END OF RECORDS");
}
}
async function getChannels() {
const paginator = await chatClient.getUserChannelDescriptors();
// Initiate the recursive function
if (paginator.items) {
await processChannels(paginator);
}
}
And this is what you will get in each call.
According to the documentation, the getUserChannelDescriptors method doesn't take any arguments.
But you shouldn't have to do pagination manually, since the method returns a Promise.<Paginator.<ChannelDescriptor>> type..which means you should be able to get access to the Pagination features twilio provides.
Your paginator.items should have only the items in a single page.
EDIT: Basically the point is that your first snippet is correct. Unfortunately, twilio is not open source so I cannot check where they have defined the page_size exactly. But I would encourage you to create say a hundred mock channels, and then check the size of the paginator.items array.
Try this:
this.chatClient.getUserChannelDescriptors().then(paginator => {
console.log(paginator.items, paginator.hasNextPage(), paginator.hasPrevPage());
})
The documentation for the Paginator class is here
I am writing Webdriver automation for a web app. I have a test that looks like this:
it('has five items', async function(done) {
try {
await driver.wait(until.elementLocated(By.className('item-class')),5000);
const items = await driver.findElements(By.className('item-class'));
expect(items.length).toBe(5);
done();
}
catch(err) {
console.log(err)
}
}
This test will pass about 2/3 of the time, but will sometimes fail with:
Expected 0 to be 5.
I would think that there should be no way to get this response, since the first line is supposed to make it wait until some of these items exist. I could understand a result of "Expected 1 to equal 5.", in the case that one item was added to the page, and the rest of the test completed before they were all there, but reaching the expect() call with 0 items on the page does not make sense to me.
The questions, then, are:
1) What am I missing / not understanding, such that this result is in fact possible?
2) Is there a different construct / method I should be using to make it wait until the expected items are on the page?
I checked the source code and elementLocatedBy uses findElements, see here. And findElements can return an empty array of elements after the timeout and hence 0 is expected (learnt something new today).
You can write something custom or use some ready-made method from here that doesn't use findElements
driver.wait(async function() {
const items = await driver.findElements(By.className('item-class'))
return items.length > 0;
}, 5000);
well I think a good way to solve this issue would be
try {
const items = await driver.wait(until.elementsLocated(By.className('item-class')));
return items.length > 0;
}
catch(err) {
console.log(err)
}
this way will always wait for ALL elementS (it's elementSlocated) to be located and will return an array of items (remember that without await it will return an array of promises).
It has no timeout so it will wait until they are all ready (or you can put a limit so if something weird is happening you can see it).
I need to create several deployment scripts like data migration and fixtures for a MongoDB database and I couldn't find enough information about how to drop indexes using Mongoose API. This is pretty straight-forward when using the official MongoDB API:
To delete all indexes on the specified collection:
db.collection.dropIndexes();
However, I would like to use Mongoose for this and I tried to use executeDbCommand adapted from this post, but with no success:
mongoose.connection.db.executeDbCommand({ dropIndexes: collectionName, index: '*' },
function(err, result) { /* ... */ });
Should I use the official MongoDB API for Node.js or I just missed something in this approach?
To do this via the Mongoose model for the collection, you can call dropAllIndexes of the native collection:
MyModel.collection.dropAllIndexes(function (err, results) {
// Handle errors
});
Update
dropAllIndexes is deprecated in the 2.x version of the native driver, so dropIndexes should be used instead:
MyModel.collection.dropIndexes(function (err, results) {
// Handle errors
});
If you want to maintain your indexes in your schema definitions with mongoose (you probably do if you're using mongoose), you can easily drop ones not in use anymore and create indexes that don't exist yet. You can just run a one off await YourModel.syncIndexes() on any models that you need to sync. It will create ones in the background with .ensureIndexes and drop any that no longer exist in your schema definition. You can look at the full docs here:
https://mongoosejs.com/docs/api.html#model_Model.syncIndexes
It looks like you're attempting to drop all of the indexes on a given collection.
According to the MongoDB Docs, this is the correct command.
... I tried to use executeDbCommand adapted from this post, but with no success:
To really help here, we need more details:
What failed? How did you measure "no success"?
Can you confirm 100% that the command ran? Did you output to the logs in the callback? Did you check the err variable?
Where are you creating indexes? Can you confirm that you're not re-creating them after dropping?
Have you tried the command while listing specific index names? Honestly, you should not be using "*". You should be deleting and creating very specific indexes.
This might not be the best place to post this, but I think its worth posting anyway.
I call model.syncIndexes() every time a model is defined/created against the db connection, this ensures the indexes are current and up-to-date with the schema, however as it has been highlighted online (example), this can create issues in distributed architectures, where multiple servers are attempting the same operation at the same time. This is particularly relevant if using something like the cluster library to spawn master/slave instances on multiple cores on the same machine, since they often boot up in close proximity to each other when the whole server is started.
In reference to the above 'codebarbarian' article, the issue is highlighted clearly when they state:
Mongoose does not call syncIndexes() for you, you're responsible for
calling syncIndexes() on your own. There are several reasons for this,
most notably that syncIndexes() doesn't do any sort of distributed
locking. If you have multiple servers that call syncIndexes() when
they start, you might get errors due to trying to drop an index that
no longer exists.
So What I do is create a function which uses redis and redis redlock to gain a lease for some nominal period of time to prevent multiple workers (and indeed multiple workers in multiple servers) from attempting the same sync operation at the same time.
It also bypasses the whole thing unless it is the 'master' that is trying to perform the operation, I don't see any real point in delegating this job to any of the workers.
const cluster = require('cluster');
const {logger} = require("$/src/logger");
const {
redlock,
LockError
} = require("$/src/services/redis");
const mongoose = require('mongoose');
// Check is mongoose model,
// ref: https://stackoverflow.com/a/56815793/1834057
const isMongoModel = (obj) => {
return obj.hasOwnProperty('schema') && obj.schema instanceof mongoose.Schema;
}
const syncIndexesWithRedlock = (model,duration=60000) => new Promise(resolve => {
// Ensure the cluster is master
if(!cluster.isMaster)
return resolve(false)
// Now attempt to gain redlock and sync indexes
try {
// Typecheck
if(!model || !isMongoModel(model))
throw new Error('model argument is required and must be a mongoose model');
if(isNaN(duration) || duration <= 0)
throw new Error('duration argument is required, and must be positive numeric')
// Extract name
let name = model.collection.collectionName;
// Define the redlock resource
let resource = `syncIndexes/${name}`;
// Coerce Duration to Integer
// Not sure if this is strictly required, but wtf.
// Will ensure the duration is at least 1ms, given that duration <= 0 throws error above
let redlockLeaseDuration = Math.ceil(duration);
// Attempt to gain lock and sync indexes
redlock.lock(resource,redlockLeaseDuration)
.then(() => {
// Sync Indexes
model.syncIndexes();
// Success
resolve(true);
})
.catch(err => {
// Report Lock Error
if(err instanceof LockError){
logger.error(`Redlock LockError -- ${err.message}`);
// Report Other Errors
}else{
logger.error(err.message);
}
// Fail, Either LockError error or some other error
return resolve(false);
})
// General Fail for whatever reason
}catch(err){
logger.error(err.message);
return resolve(false);
}
});
I wont go into setting up Redis connection, that is the subject of some other thread, but the point of this above code is to show how you can use syncIndexes() reliably and prevent issues with one thread dropping an index and another trying to drop the same index, or other distributed issues with attempting to modify indexes concurrently.
to drop a particular index you could use
db.users.dropIndex("your_index_name_here")