So i am having a few issues trying to figure out how to fix the snip below. As of right now it is returning the values before the 'request(scanurl,.....' section of the for of loop is running. The loop does run. Also i do not think the 'lines200' var is being updated via the counter. I am a learner so any explanation would be greatly appreciated.
async function processLineByLine(baseData) {
console.log('enter async func')
try {
const fileStream = fs.createReadStream('./file.txt');
let linesTotal = 0;
let lines200 = 0;
const rl = readline.createInterface({
input: fileStream
});
for await (let line of rl) {
console.log('in loop')
const scanurl = (baseData.match(/http/gi)) ? (baseData) : ('https://' + baseData + line);
linesTotal++;
request(scanurl, {json: true}, function (error, response, body) {
let statusCode = response.statusCode;
let htmlBody = body;
//console.log(htmlBody)
//console.log(statusCode)
if (statusCode == "200") {
console.log('in 2nd if')
let $ = cheerio.load(htmlBody);
let titleScrape = $('title').html();
console.log(titleScrape)
if (titleScrape.match(/404 | test/gi)) {
console.log('Matched')
} else {
lines200++;
console.log(lines200)
}
} else {
// Do nothing
}
});
}
return {
total: linesTotal,
count200: lines200,
};
} catch (error) {
console.error(error)
}
}
router.get('/:reqTarget', async (req, res) => {
console.log('starting')
var baseUrl = req.params.reqTarget;
try {
console.log('in the try')
const initTest = await processLineByLine(baseUrl);
const {total, count200} = initTest;
console.log(total, count200)
if (initTest) return res.status(200).send({
message: 'STATUS 200 COUNT: ' + count200 + ' ' + 'TOTAL: ' + total });
} catch (error) {
console.log(error)
}
});
Current Output:
starting
in the try
enter async func
in loop
in loop
in loop
in loop
in loop
in loop
in loop
33 0 //this is the return that is two early
in 2nd if
404 | test
Matched
in 2nd if
404 | test
Matched
When you have a loop containing asynchronous operations, you have one of two options. You can run them all in parallel and somehow track when they are all done. Or, you can run them sequentially one after the other. It appears your loop could be constructed either way, but I'll illustrate the sequential option.
The advent of async/await allows us to "pause" a for loop in the middle with an appropriate await. But, in order to do that, all asynchronous operations have to be promise-based so you can await those promises. To that end, I've switched from the request() library to the request-promise-native library which is a promise wrapper around the request library that uses native, built-in promises. It also has another nice feature in that it automatically checks for a 2xx status code so you don't have to do that yourself.
Here's what that code would look like:
const rp = require('request-promise-native');
async function processLineByLine(baseData) {
console.log('enter async func')
try {
const fileStream = fs.createReadStream('./file.txt');
let linesTotal = 0;
let lines200 = 0;
const rl = readline.createInterface({
input: fileStream
});
for await (let line of rl) {
console.log('in loop')
const scanurl = (baseData.match(/http/gi)) ? (baseData) : ('https://' + baseData + line);
linesTotal++;
try {
let htmlBody = await rp(scanurl, {json: true});
let $ = cheerio.load(htmlBody);
let titleScrape = $('title').html();
console.log(titleScrape);
if (titleScrape.match(/404 | test/gi)) {
console.log('Matched')
} else {
lines200++;
console.log(lines200)
}
} catch(e) {
console.log(`error on request(${scanurl})`, e);
// like your original code, this will only log the error
// and then continue with the rest of the URLs
}
}
return {
total: linesTotal,
count200: lines200,
};
} catch (error) {
console.error(error)
}
}
router.get('/:reqTarget', async (req, res) => {
console.log('starting')
var baseUrl = req.params.reqTarget;
try {
console.log('in the try')
const initTest = await processLineByLine(baseUrl);
const {total, count200} = initTest;
console.log(total, count200)
res.status(200).send({message: 'STATUS 200 COUNT: ' + count200 + ' ' + 'TOTAL: ' + total});
} catch (error) {
console.log(error)
res.sendStatus(500);
}
});
Related
I'm making a scraper that will grab every Uniswap pair and save it to an array using the Graph API.
My problem occurs when I make my 7th request to the API.
Initially, I thought I was being rate limited because I was fetching 1000 tokens at a time, but after adding a 10 second wait between calls and decreasing the fetched tokens from 1000 to 10, it still stops on the 7th loop.
The script works perfectly until this point.
const axios = require('axios');
const fs = require('fs');
async function getTokens(skip) {
try {
const query = `
query tokens($skip: Int!) {
tokens(first: 10, skip: $skip) {
id
name
symbol
}
}
`;
const variables = {
skip: skip
};
const headers = {
"Content-Type": "application/json"
};
const { data } = await axios.post("https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v3", {
query,
variables
}, {
headers
});
return data.data.tokens;
} catch (err) {
console.error(err);
return []
}
}
async function saveTokens(tokens) {
try {
await fs.promises.writeFile("uniTokens.json", JSON.stringify(tokens), { flag: "w" });
} catch (err) {
console.error(err);
}
}
async function main() {
let skip = 0;
let tokens = [];
const retrievedIds = new Set();
while (true) {
const newTokens = await getTokens(skip);
if (newTokens.length === 0) {
console.log("Reached end of tokens, finishing up...");
break;
}
// Only save tokens that haven't been retrieved before
const newIds = new Set(newTokens.map(token => token.id));
newIds.forEach(id => {
if (!retrievedIds.has(id)) {
tokens.push(newTokens.find(token => token.id === id));
retrievedIds.add(id);
}
});
console.log(`Retrieved ${tokens.length} tokens`);
await saveTokens(tokens);
skip += 1000;
// delay the next request by 10 seconds
//await new Promise(resolve => setTimeout(resolve, 10000));
}
}
main();
This is the error that it produces:
TypeError: Cannot read properties of undefined (reading 'tokens')
at getTokens (/root/unipairs/uni:31:26)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async main (/root/unipairs/uni:52:27)
Reached end of tokens, finishing up...
I got a simple async function where I "scrape" site from URLs.
Everything works fine, but now I want to save results into my txt file.
I tried to do simply array where I able to push every result also errors;
Now I got a problem where should I do write to file.
I tried putting it to a separated function then do await function inside my async function but function with write to file i always fired first.
There is full code
const https = require("https");
const fs = require("fs");
const readline = require("readline");
const path = require("path");
let urls = [];
let results = [];
(async function readUrls() {
const fileStream = fs.createReadStream("urls.txt");
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
for await (let line of rl) {
urls.push(line);
}
for await (let url of urls) {
https
.get(url, (res) => {
const {
statusCode
} = res;
const contentType = res.headers["content-type"];
let error;
if (statusCode !== 200) {
error = new Error("Request Failed.\n" + `Status Code: ${statusCode}`);
}
if (error) {
const firstPath = url.split("/")[7];
//there is array
results.push(firstPath);
//--------------
console.error("data : " + firstPath + " - " + " nothing found");
res.resume();
return;
}
res.setEncoding("utf8");
let rawData = "";
res.on("data", (chunk) => {
rawData += chunk;
});
(async () => {
await res.on("end", () => {
try {
const parsedData = JSON.parse(rawData);
const parsedResult = parsedData["data"]["id"] + " - " + parsedData["data"]["price"];
//there is array
results.push(parsedResult);
//--------------
console.log("data : " + parsedData["data"]["id"] + " - " + parsedData["data"]["price"]);
} catch (e) {
console.error(e.message);
}
});
})();
})
.on("error", (e) => {
console.error(`Got error: ${e.message}`);
});
}
})();
There is my simple function to write into file
fs.writeFile('result.txt', results, +(new Date()), function (err) {
if (err) {
console.log("Error occurred", err);
}
console.log("File write successfull");
});
I tried do something
async function secondFunction(){
await firstFunction();
// wait for firstFunction...
};
What I want to achive? I want to scrape every url from my text file and get ID and Price
( this is simple JSON response into browser no html - it works )
At the end I want to save everything into text file.
I made a version of your code that uses node-fetch to call the urls. I prefer this one as it is similar to what one can use on the web
To use it you should install it:
npm install node-fetch
const fetch = require("node-fetch"); // I prefer to use node-fetch for my calls
const fs = require("fs");
const readline = require("readline");
const path = require("path");
let urls = [];
let results = [];
(async function readUrls() {
const fileStream = fs.createReadStream("urls.txt");
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity,
});
for await (let line of rl) {
urls.push(line);
}
// Make the calls one after the other
for (let url of urls) {
try {
// We can call the urls with node-fetch and await the response
const res = await fetch(url);
const { status } = res;
let error;
if (status !== 200)
error = new Error("Request Failed.\n" + `Status Code: ${statusCode}`);
if (error) {
const firstPath = url.split('/')[7];
results.push(firstPath);
console.error("data : " + firstPath + " - " + " nothing found");
// As we are inside a loop here, we use continue instead of return
continue;
}
try {
// Here we try to take the response as json
const parsedData = await res.json();
const parsedResult = parsedData["data"]["id"] + " - " + parsedData["data"]["price"];
//there is array
results.push(parsedResult);
//--------------
console.log(`Data: ${parsedResult}`);
} catch (e) {
// In case we can't get the response as json we log the error
console.error(e.message);
}
} catch (httpError) {
//This is for when the call to fetch fails for some reason
console.error(httpError.message);
}
}
// Here we join the results to a string so that we can save it properly to the file
const resultAsText = results.join("\n");
// Then after all the urls are processed we can write them to a file
fs.writeFile('result.txt', resultAsText, 'utf8', function (err) {
if (err) {
console.log("Error occurred", err);
} else {
console.log("File write successfull");
}
});
})();
I'm currently using https://github.com/pixtron/bybit-api/tree/master
and what im trying to do looks like this:
const {RestClient} = require('#pxtrn/bybit-api');
const API_KEY = 'xxx';
const PRIVATE_KEY = 'yyy';
const client = new RestClient(API_KEY, PRIVATE_KEY);
client.getPosition({symbol: 'BTCUSD'})
.then(msg => {
let btcPositionCheck = msg.result.size;
if (btcPositionCheck == 0) {
console.log("empty")
} else {
let btcPositionSize = btcPositionCheck;
let btcPositionSide = msg.result.side;
}
})
.catch(err => {
console.error(err);
});
client.getPosition({symbol: 'ETHUSD'})
.then(msg => {
let ethPositionCheck = msg.result.size;
if (ethPositionCheck == 0) {
console.log("empty")
} else {
let ethPositionSize = ethPositionCheck;
let ethPositionSide = msg.result.side;
}
})
.catch(err => {
console.error(err);
});
console.log(btcPositionSize + ' ' + ethPositionSize)
returning me the following error:
ReferenceError: btcPositionSize is not defined
simply because im out of scope, I'm aware. however, I'm unsure how to approach rewriting the following to better allow me to construct what I need. (I need to call getPosition for the 3 different assets as well and take the variables from each call to use later in my code.) if that makes any sense. help would be greatly appreciated
edit:
async function getPosition({symbol: BTCUSD}) {
try {
let btcPositionCheck = msg.result.size;
} catch(error) {
return null;
}
}
im trying to rewrite it but im unsure how
Once you get into asynchronous context, you need to stay in asynchronous context to get values. In your case, there are promises to be resolved, you can await them and then print.
Also you need to make sure that your variables are in valid scope
const {RestClient} = require('#pxtrn/bybit-api');
const API_KEY = 'xxx';
const PRIVATE_KEY = 'yyy';
const client = new RestClient(API_KEY, PRIVATE_KEY);
async function doItAll() {
let btcPositionCheck;
let ethPositionCheck;
await client.getPosition({symbol: 'BTCUSD'})
.then(msg => {
btcPositionCheck = msg.result.size;
if (btcPositionCheck == 0) {
console.log("empty")
} else {
let btcPositionSize = btcPositionCheck;
let btcPositionSide = msg.result.side;
}
})
.catch(err => {
console.error(err);
});
await client.getPosition({symbol: 'ETHUSD'})
.then(msg => {
ethPositionCheck = msg.result.size;
if (ethPositionCheck == 0) {
console.log("empty")
} else {
let ethPositionSize = ethPositionCheck;
let ethPositionSide = msg.result.side;
}
})
.catch(err => {
console.error(err);
});
console.log(btcPositionSize + ' ' + ethPositionSize)
}
Note that nicer way is to await promises into value, which means it can look like that (to use await, it needs to be in an async function)
const msg = await client.getPosition({symbol: 'BTCUSD'})
btcPositionCheck = msg.result.size;
if (btcPositionCheck == 0) {
console.log("empty")
} else {
let btcPositionSize = btcPositionCheck;
let btcPositionSide = msg.result.side;
}
I have confusion and I couldn't resolve it from the TypeORM documents.
Here is my function that is used as request handler and it is using transaction taken from getManager()
static makeTransaction = async (req: Request, res: Response) => {
try {
// check if the account ids are same
const { senderAccountId, receiverAccountId, amount } = req.body;
let senderAccount = await findEntityById(getRepository(Account), senderAccountId);
let receiverAccount = await findEntityById(getRepository(Account), receiverAccountId);
senderAccount.balance -= amount;
receiverAccount.balance += amount;
await validateOrReject(senderAccount);
await validateOrReject(receiverAccount);
const tempEntity = MyEntity.create({
amount,
receiverAccount: receiverAccount,
senderAccount: senderAccount,
});
await validateOrReject(tempEntity);
const accounts = await getManager().transaction(async transactionManager => {
await transactionManager.save(senderAccount);
await transactionManager.save(receiverAccount);
return transactionManager.save(tempEntity);
});
return res.status(200).json({
error: null,
data: true,
});
} catch (e) {
return res.status(400).json({ error: "Transaction wasn't successfull" });
}
};
So when I put a catch function for the last saving (only it fails and first 2 of them runs correctly), it gives "Query runner already released. Cannot run queries anymore" error.
When I change the flow like this, it works and I really wonder the reason because in documents there are saving operations for different entity types and this is what I'm trying to do too.
static makeTransaction = async (req: Request, res: Response) => {
try {
// check if the account ids are same
const { senderAccountId, receiverAccountId, amount } = req.body;
let senderAccount = await findEntityById(getRepository(Account), senderAccountId);
let receiverAccount = await findEntityById(getRepository(Account), receiverAccountId);
senderAccount.balance -= amount;
receiverAccount.balance += amount;
await validateOrReject(senderAccount);
await validateOrReject(receiverAccount);
const tempEntity = MyEntity.create({
amount,
receiverAccount,
senderAccount,
});
await validateOrReject(tempEntity);
const transactionRes = await getManager().transaction(async transactionManager => {
return transactionManager.save([tempEntity, senderAccount, receiverAccount]);
});
return res.status(200).json({
error: null,
data: transactionRes[0],
});
} catch (e) {
console.log(e);
return res.status(400).json({ error: "Transaction wasn't successfull" });
}
};
I want to learn the reason for that and any kind of idea or help is appreciated, thank you in advance.
I have an Express route like this in an node server (file is required):
var redis = require('../modules/redis');
module.exports = function (app) {
var redisClient = redis.init();
app.post('/auth/ticket', cors(), function (req, res) {
var hashes = ['hash1','hash2', 'hash3'];
var candidates = []; // An array to collect valid hashes
var key;
// to check each hash against a RedisDB I use a For Loop
for (key in hashes) {
var hash = hashes[key];
console.log("Hash " + hash + " will be proofed now:");
//now I try to collect the valid hashes in the candidates array
if (redisClient.exists(hash) === 1) candidates.push(hash);
}
console.log(JSON.stringify(candidates));
});
};
Now here is the code of my module which shall manage all the redis requests:
exports.init = function () {
Redis = exports.Redis = function () {
var promiseFactory = require("q").Promise,
redis = require('promise-redis')(promiseFactory);
this.client = redis.createClient();
this.client.on('error', function (err) {
console.log('redis error – ' + client.host + ':' + client.port + ' – ' + err);
});
Redis.prototype.exists = function (key) {
this.client.exists(key, function (err, data) {
return data === 1 ? true : false;
});
};
return new Redis();
};
So what I experience is that the module is able to console.log the results properly. If a hash is valid, it returns true and otherwise false. This works as expected.
Problem is, that the for-loop continuous the execution without fetching getting the results. I think this is caused by race-conditions.
As you can see, I have started to workout something there with the use of Q and promise-redis in the top of my code:
var promiseFactory = require("q").Promise,
redis = require('promise-redis')(promiseFactory);
this.client = redis.createClient();
I like to know, how I make my for-loop (in the Express route) waiting for the results of redisClient.exists(hash) or in other words, to get all valid hashes into my candidates array.
Please help
like #brad said, you could use Q.all, it would take an array of promises as input and then return an array of results when all the promises are finished:
there is a mistake in your answer:
Redis.prototype.exists = function (key) {
return this.client.exists(key) // CHANGED, you still need to return a promise.
.then(function (reply) {
console.log("reply " + reply);
return (reply);
})
.catch(console.log);
};
If I understand correctly, what you want is something like
exports.init = function () {
Redis = exports.Redis = function () {
var Q = require("q"),
promiseFactory = Q.Promise,
redis = require('promise-redis')(promiseFactory);
this.client = redis.createClient();
this.client.on('error', function (err) {
console.log('redis error – ' + client.host + ':' + client.port + ' – ' + err);
});
Redis.prototype.exists = function (key) {
return this.client.exists(key).then(function (data) {
return data === 1 ? true : false;
});
};
Redis.prototype.getActive = function (arry) {
var self = this;
return Q.all(arry.map(self.exists.bind(self))
).then(function(res){
return arry.filter(function(val, idx){ return res[idx];});
});
};
return new Redis();
};
# mido22: But did you also recognize that I outsourced all the reds functions to the module file (1st Codeblock) which requires the promise-redid and builds a factory for Q. I changed the code inside the module file to:
Redis.prototype.exists = function (key) {
this.client.exists(key)
.then(function (reply) {
console.log("reply " + reply);
return (reply);
})
.catch(console.log);
};
and this results correctly like the console.log evidently shows.
Your codechange of the for-loop works very well but I think it don't fulfills my needs perfectly. If I could, I would like to have it completely outsourced in to the module file, so that I can use the prototyped method in similar cases from everywhere. Is that possible anyhow?
I see, that it would result in having two promise supported functionalities, if I would create an Instance of Redis Client with promise-redid and Q inside the auth/ticket/ router, too.
like this:
var Q = require('q'),
promiseFactory = Q.Promise,
redis = require("promise-redis")(promiseFactory),
client;
an then the express route (there are a lot of more routes each in a single file) like in your code.
Do you understand what I mean? Of course your solution will be fine for my needs at all, but a module resolving the job completely could have more elegance if possible so far.
Using with redis, bluebird and typescript:
import { RedisClient, createClient, ClientOpts } from "redis";
import { promisifyAll, PromisifyAllOptions } from "bluebird";
export module FMC_Redis {
export class Redis {
opt: ClientOpts;
private rc: RedisClient;
private rcPromise: any;
private static _instance: Redis = null;
public static current(_opt?: ClientOpts): Redis {
if (!Redis._instance) {
Redis._instance = new Redis(_opt);
Redis._instance.redisConnect();
}
return Redis._instance;
}
public get client(): RedisClient {
if (!this.rc.connected) throw new Error("There is no connection to Redis DB!");
return this.rc;
}
/******* BLUEBIRD ********/
public get clientAsync(): any {
// promisifyAll functions of redisClient
// creating new redis client object which contains xxxAsync(..) functions.
return this.rcPromise = promisifyAll(this.client);
}
private constructor(_opt?: ClientOpts) {
if (Redis._instance) return;
this.opt = _opt
? _opt
: {
host: "127.0.0.1",
port: 6379,
db: "0"
};
}
public redisConnect(): void {
this.rc = createClient(this.opt);
this.rc
.on("ready", this.onReady)
.on("end", this.onEnd)
.on("error", this.onError);
}
private onReady(): void { console.log("Redis connection was successfully established." + arguments); }
private onEnd(): void { console.warn("Redis connection was closed."); }
private onError(err: any): void { console.error("There is an error: " + err); }
/****** PROMISE *********/
// promise redis test
public getRegularPromise() {
let rc = this.client;
return new Promise(function (res, rej) {
console.warn("> getKeyPromise() ::");
rc.get("cem", function (err, val) {
console.log("DB Response OK.");
// if DB generated error:
if (err) rej(err);
// DB generated result:
else res(val);
});
});
}
/******* ASYNC - AWAIT *******/
// async - await test function
public delay(ms) {
return new Promise<string>((fnResolve, fnReject) => {
setTimeout(fnResolve("> delay(" + ms + ") > successfull result"), ms);
});
}
public async delayTest() {
console.log("\n****** delayTest ")
let a = this.delay(500).then(a => console.log("\t" + a));
let b = await this.delay(400);
console.log("\tb::: " + b);
}
// async - await function
public async getKey(key: string) {
let reply = await this.clientAsync.getAsync("cem");
return reply.toString();
}
}
}
let a = FMC_Redis.Redis.current();
// setTimeout(function () {
// console.warn(a.client.set("cem", "naber"));
// console.warn(a.client.get("cem"));
// console.warn(a.client.keys("cem"));
// }, 1000);
/***** async await test client *****/
a.delayTest();
/** Standart Redis Client test client */
setTimeout(function () {
a.client.get("cem", function (err, val) {
console.log("\n****** Standart Redis Client")
if (err) console.error("\tError: " + err);
else console.log("\tValue ::" + val);
});
}, 100)
/***** Using regular Promise with Redis Client > test client *****/
setTimeout(function () {
a.getRegularPromise().then(function (v) {
console.log("\n***** Regular Promise with Redis Client")
console.log("\t> Then ::" + v);
}).catch(function (e) {
console.error("\t> Catch ::" + e);
});
}, 100);
/***** Using bluebird promisify with Redis Client > test client *****/
setTimeout(function () {
var header = "\n***** bluebird promisify with Redis Client";
a.clientAsync.getAsync("cem").then(result => console.log(header + result)).catch(console.error);
}, 100);