I have an Express route like this in an node server (file is required):
var redis = require('../modules/redis');
module.exports = function (app) {
var redisClient = redis.init();
app.post('/auth/ticket', cors(), function (req, res) {
var hashes = ['hash1','hash2', 'hash3'];
var candidates = []; // An array to collect valid hashes
var key;
// to check each hash against a RedisDB I use a For Loop
for (key in hashes) {
var hash = hashes[key];
console.log("Hash " + hash + " will be proofed now:");
//now I try to collect the valid hashes in the candidates array
if (redisClient.exists(hash) === 1) candidates.push(hash);
}
console.log(JSON.stringify(candidates));
});
};
Now here is the code of my module which shall manage all the redis requests:
exports.init = function () {
Redis = exports.Redis = function () {
var promiseFactory = require("q").Promise,
redis = require('promise-redis')(promiseFactory);
this.client = redis.createClient();
this.client.on('error', function (err) {
console.log('redis error – ' + client.host + ':' + client.port + ' – ' + err);
});
Redis.prototype.exists = function (key) {
this.client.exists(key, function (err, data) {
return data === 1 ? true : false;
});
};
return new Redis();
};
So what I experience is that the module is able to console.log the results properly. If a hash is valid, it returns true and otherwise false. This works as expected.
Problem is, that the for-loop continuous the execution without fetching getting the results. I think this is caused by race-conditions.
As you can see, I have started to workout something there with the use of Q and promise-redis in the top of my code:
var promiseFactory = require("q").Promise,
redis = require('promise-redis')(promiseFactory);
this.client = redis.createClient();
I like to know, how I make my for-loop (in the Express route) waiting for the results of redisClient.exists(hash) or in other words, to get all valid hashes into my candidates array.
Please help
like #brad said, you could use Q.all, it would take an array of promises as input and then return an array of results when all the promises are finished:
there is a mistake in your answer:
Redis.prototype.exists = function (key) {
return this.client.exists(key) // CHANGED, you still need to return a promise.
.then(function (reply) {
console.log("reply " + reply);
return (reply);
})
.catch(console.log);
};
If I understand correctly, what you want is something like
exports.init = function () {
Redis = exports.Redis = function () {
var Q = require("q"),
promiseFactory = Q.Promise,
redis = require('promise-redis')(promiseFactory);
this.client = redis.createClient();
this.client.on('error', function (err) {
console.log('redis error – ' + client.host + ':' + client.port + ' – ' + err);
});
Redis.prototype.exists = function (key) {
return this.client.exists(key).then(function (data) {
return data === 1 ? true : false;
});
};
Redis.prototype.getActive = function (arry) {
var self = this;
return Q.all(arry.map(self.exists.bind(self))
).then(function(res){
return arry.filter(function(val, idx){ return res[idx];});
});
};
return new Redis();
};
# mido22: But did you also recognize that I outsourced all the reds functions to the module file (1st Codeblock) which requires the promise-redid and builds a factory for Q. I changed the code inside the module file to:
Redis.prototype.exists = function (key) {
this.client.exists(key)
.then(function (reply) {
console.log("reply " + reply);
return (reply);
})
.catch(console.log);
};
and this results correctly like the console.log evidently shows.
Your codechange of the for-loop works very well but I think it don't fulfills my needs perfectly. If I could, I would like to have it completely outsourced in to the module file, so that I can use the prototyped method in similar cases from everywhere. Is that possible anyhow?
I see, that it would result in having two promise supported functionalities, if I would create an Instance of Redis Client with promise-redid and Q inside the auth/ticket/ router, too.
like this:
var Q = require('q'),
promiseFactory = Q.Promise,
redis = require("promise-redis")(promiseFactory),
client;
an then the express route (there are a lot of more routes each in a single file) like in your code.
Do you understand what I mean? Of course your solution will be fine for my needs at all, but a module resolving the job completely could have more elegance if possible so far.
Using with redis, bluebird and typescript:
import { RedisClient, createClient, ClientOpts } from "redis";
import { promisifyAll, PromisifyAllOptions } from "bluebird";
export module FMC_Redis {
export class Redis {
opt: ClientOpts;
private rc: RedisClient;
private rcPromise: any;
private static _instance: Redis = null;
public static current(_opt?: ClientOpts): Redis {
if (!Redis._instance) {
Redis._instance = new Redis(_opt);
Redis._instance.redisConnect();
}
return Redis._instance;
}
public get client(): RedisClient {
if (!this.rc.connected) throw new Error("There is no connection to Redis DB!");
return this.rc;
}
/******* BLUEBIRD ********/
public get clientAsync(): any {
// promisifyAll functions of redisClient
// creating new redis client object which contains xxxAsync(..) functions.
return this.rcPromise = promisifyAll(this.client);
}
private constructor(_opt?: ClientOpts) {
if (Redis._instance) return;
this.opt = _opt
? _opt
: {
host: "127.0.0.1",
port: 6379,
db: "0"
};
}
public redisConnect(): void {
this.rc = createClient(this.opt);
this.rc
.on("ready", this.onReady)
.on("end", this.onEnd)
.on("error", this.onError);
}
private onReady(): void { console.log("Redis connection was successfully established." + arguments); }
private onEnd(): void { console.warn("Redis connection was closed."); }
private onError(err: any): void { console.error("There is an error: " + err); }
/****** PROMISE *********/
// promise redis test
public getRegularPromise() {
let rc = this.client;
return new Promise(function (res, rej) {
console.warn("> getKeyPromise() ::");
rc.get("cem", function (err, val) {
console.log("DB Response OK.");
// if DB generated error:
if (err) rej(err);
// DB generated result:
else res(val);
});
});
}
/******* ASYNC - AWAIT *******/
// async - await test function
public delay(ms) {
return new Promise<string>((fnResolve, fnReject) => {
setTimeout(fnResolve("> delay(" + ms + ") > successfull result"), ms);
});
}
public async delayTest() {
console.log("\n****** delayTest ")
let a = this.delay(500).then(a => console.log("\t" + a));
let b = await this.delay(400);
console.log("\tb::: " + b);
}
// async - await function
public async getKey(key: string) {
let reply = await this.clientAsync.getAsync("cem");
return reply.toString();
}
}
}
let a = FMC_Redis.Redis.current();
// setTimeout(function () {
// console.warn(a.client.set("cem", "naber"));
// console.warn(a.client.get("cem"));
// console.warn(a.client.keys("cem"));
// }, 1000);
/***** async await test client *****/
a.delayTest();
/** Standart Redis Client test client */
setTimeout(function () {
a.client.get("cem", function (err, val) {
console.log("\n****** Standart Redis Client")
if (err) console.error("\tError: " + err);
else console.log("\tValue ::" + val);
});
}, 100)
/***** Using regular Promise with Redis Client > test client *****/
setTimeout(function () {
a.getRegularPromise().then(function (v) {
console.log("\n***** Regular Promise with Redis Client")
console.log("\t> Then ::" + v);
}).catch(function (e) {
console.error("\t> Catch ::" + e);
});
}, 100);
/***** Using bluebird promisify with Redis Client > test client *****/
setTimeout(function () {
var header = "\n***** bluebird promisify with Redis Client";
a.clientAsync.getAsync("cem").then(result => console.log(header + result)).catch(console.error);
}, 100);
Related
So i am having a few issues trying to figure out how to fix the snip below. As of right now it is returning the values before the 'request(scanurl,.....' section of the for of loop is running. The loop does run. Also i do not think the 'lines200' var is being updated via the counter. I am a learner so any explanation would be greatly appreciated.
async function processLineByLine(baseData) {
console.log('enter async func')
try {
const fileStream = fs.createReadStream('./file.txt');
let linesTotal = 0;
let lines200 = 0;
const rl = readline.createInterface({
input: fileStream
});
for await (let line of rl) {
console.log('in loop')
const scanurl = (baseData.match(/http/gi)) ? (baseData) : ('https://' + baseData + line);
linesTotal++;
request(scanurl, {json: true}, function (error, response, body) {
let statusCode = response.statusCode;
let htmlBody = body;
//console.log(htmlBody)
//console.log(statusCode)
if (statusCode == "200") {
console.log('in 2nd if')
let $ = cheerio.load(htmlBody);
let titleScrape = $('title').html();
console.log(titleScrape)
if (titleScrape.match(/404 | test/gi)) {
console.log('Matched')
} else {
lines200++;
console.log(lines200)
}
} else {
// Do nothing
}
});
}
return {
total: linesTotal,
count200: lines200,
};
} catch (error) {
console.error(error)
}
}
router.get('/:reqTarget', async (req, res) => {
console.log('starting')
var baseUrl = req.params.reqTarget;
try {
console.log('in the try')
const initTest = await processLineByLine(baseUrl);
const {total, count200} = initTest;
console.log(total, count200)
if (initTest) return res.status(200).send({
message: 'STATUS 200 COUNT: ' + count200 + ' ' + 'TOTAL: ' + total });
} catch (error) {
console.log(error)
}
});
Current Output:
starting
in the try
enter async func
in loop
in loop
in loop
in loop
in loop
in loop
in loop
33 0 //this is the return that is two early
in 2nd if
404 | test
Matched
in 2nd if
404 | test
Matched
When you have a loop containing asynchronous operations, you have one of two options. You can run them all in parallel and somehow track when they are all done. Or, you can run them sequentially one after the other. It appears your loop could be constructed either way, but I'll illustrate the sequential option.
The advent of async/await allows us to "pause" a for loop in the middle with an appropriate await. But, in order to do that, all asynchronous operations have to be promise-based so you can await those promises. To that end, I've switched from the request() library to the request-promise-native library which is a promise wrapper around the request library that uses native, built-in promises. It also has another nice feature in that it automatically checks for a 2xx status code so you don't have to do that yourself.
Here's what that code would look like:
const rp = require('request-promise-native');
async function processLineByLine(baseData) {
console.log('enter async func')
try {
const fileStream = fs.createReadStream('./file.txt');
let linesTotal = 0;
let lines200 = 0;
const rl = readline.createInterface({
input: fileStream
});
for await (let line of rl) {
console.log('in loop')
const scanurl = (baseData.match(/http/gi)) ? (baseData) : ('https://' + baseData + line);
linesTotal++;
try {
let htmlBody = await rp(scanurl, {json: true});
let $ = cheerio.load(htmlBody);
let titleScrape = $('title').html();
console.log(titleScrape);
if (titleScrape.match(/404 | test/gi)) {
console.log('Matched')
} else {
lines200++;
console.log(lines200)
}
} catch(e) {
console.log(`error on request(${scanurl})`, e);
// like your original code, this will only log the error
// and then continue with the rest of the URLs
}
}
return {
total: linesTotal,
count200: lines200,
};
} catch (error) {
console.error(error)
}
}
router.get('/:reqTarget', async (req, res) => {
console.log('starting')
var baseUrl = req.params.reqTarget;
try {
console.log('in the try')
const initTest = await processLineByLine(baseUrl);
const {total, count200} = initTest;
console.log(total, count200)
res.status(200).send({message: 'STATUS 200 COUNT: ' + count200 + ' ' + 'TOTAL: ' + total});
} catch (error) {
console.log(error)
res.sendStatus(500);
}
});
I am trying to build a simple node.js client using the amqp library, that opens a single connection and then a single channel to a RabbitMQ server. I want to reuse the same connection and channel to send multiple messages. The main problem is, that I don't want to write my entire code inside the callback function of the ceateChannel() function.
How do I reuse the channel outside of the callback function and make sure the callback function has finished before I use the channel?
I've tried both the callback way and the promise way but I can't make either of them work. When using the callback method I run into the described problem.
When using promises, I have the problem that I can't keep a reference of the connection and channel outside of the .then() function because the passed variables get destroyed after setting up the connection and channel.
amqp.connect('amqp://localhost', (err, conn) => {
if (err !== null) return console.warn(err);
console.log('Created connection!');
conn.createChannel((err, ch) => {
if (err !== null) return console.warn(err);
console.log('Created channel!');
//this is where I would need to write the code that uses the variable "ch"
//but I want to move the code outside of this structure, while making sure
//this callback completes before I try using "ch"
});
});
amqp.connect('amqp://localhost').then((conn) => {
return conn.createChannel();
}).then((ch) => {
this.channel = ch;
return ch.assertQueue('', {}).then((ok) => {
return this.queueName = ok.queue;
});
}).catch(console.warn);
why you don't use async\await ?
const conn = await amqp.connect('amqp://localhost');
const ch = await conn.createChannel();
// after that you can use ch anywhere, don't forget to handle exceptions
Also if you use amqplib, don't forget to handle close and internal error events, for example like this:
conn.on('error', function (err) {
console.log('AMQP:Error:', err);
});
conn.on('close', () => {
console.log("AMQP:Closed");
});
Try with a class, like this:
RabbitConnection.js
const amqp = require('amqplib');
const RabbitSettings = {
protocol: 'amqp',
hostname: 'localhost',
port: 5672,
username: 'guest',
password: 'guest',
authMechanism: 'AMQPLAIN',
vhost: '/',
queue: 'test'
}
class RabbitConnection {
constructor() {
RabbitConnection.createConnection();
this.connection = null;
this.channel = null;
}
static getInstance() {
if (!RabbitConnection.instance) {
RabbitConnection.instance = new RabbitConnection();
}
return RabbitConnection.instance;
}
//create connection to rabbitmq
static async createConnection() {
try {
this.connection = await amqp.connect(`${RabbitSettings.protocol}://${RabbitSettings.username}:${RabbitSettings.password}#${RabbitSettings.hostname}:${RabbitSettings.port}${RabbitSettings.vhost}`);
this.channel = await this.connection.createChannel();
this.channel.assertQueue(RabbitSettings.queue);
console.log('Connection to RabbitMQ established');
} catch (error) {
console.log(error);
}
}
//send message to rabbitmq queue
static async sendMessage(message, queueName) {
try {
let msg = await this.channel.sendToQueue(queueName, Buffer.from(message));
console.log('Message sent to RabbitMQ');
return msg;
} catch (error) {
console.log(error);
}
}
}
module.exports = { RabbitConnection };
ServerExpress.js
const express = require('express');
const { RabbitConnection } = require('./RabbitConnection');
const serverUp = () => {
const app = express();
app.get('/', (req, res) => {
RabbitConnection.sendMessage('Hello World', 'test');
res.send('Hello World!');
});
app.listen(3000, () => {
console.log('Server is running on port 3000');
});
};
module.exports = { serverUp };
index.js
const { RabbitConnection } = require("./RabbitConnection");
const { serverUp } = require("./ServerExpress");
serverUp();
RabbitConnection.getInstance();
Thank you for taking the time to read this,
I have a little problem with node-soap, so basically I'm trying to verify the identity of the client before sending back a response, after following the documentation I found the server.authenticate function.
server.authenticate = async function (security: any) {
const binarySecurityTokenAsBase64String = security.BinarySecurityToken.$value;
const pemKeyFromRequestAsString = "-----BEGIN CERTIFICATE-----" + "\n" + binarySecurityTokenAsBase64String.replace(/(.{64})/g, "$1\n") + "\n" + "-----END CERTIFICATE-----";
const success = await validateCertificate(pemKeyFromRequestAsString);
if (success) {
return true;
} else {
winston.warn("Failed to validate Certificate - Either Certificate Verification with CA Chain Failed or the system encountered an error");
return false;
}
};
That's where I do my verification business and return true or false based on the result:
const success = await validateCertificate(pemKeyFromRequestAsString);
My problem is, no matter what is the result, I still get the response back, on the logs, everything is fine and confirm that the verification failed, maybe this because of Async/Sync stuff.. I'm really new to Javascript/Typescript World, any help would be greatly appreciated.
Here is my a preview of my code:
try {
const myService = {
Calculate_Service: {
Calculate_Port: {
multiply: function(args, callback) {
const a = 1;
try {
winston.debug("Reached the multiply Function");
const n = args.a * args.b;
callback({
multiplicationResult : n
});
} catch (e) {
winston.error(e);
throw {
Fault: {
Code: {
Value: "soap:Sender",
Subcode: { value: "rpc:BadArguments" }
},
Reason: { Text: JSON.stringify(e) },
statusCode: 500
}
};
}
},
}
}
}
const xml = fs.readFileSync(AppConfiguration.responseServerWsdlPath, "utf8");
app.use(bodyParser.raw({
type: function () {
return true;
}, limit: "5mb"
}));
app.listen(port, async function () {
winston.info("Express server listening on port " + port);
const server = ArcNodeSoap.listen(app, "/calculatorService", myService, xml);
server.authenticate = async function (security: any) {
const binarySecurityTokenAsBase64String = security.BinarySecurityToken.$value;
const pemKeyFromRequestAsString = "-----BEGIN CERTIFICATE-----" + "\n" + binarySecurityTokenAsBase64String.replace(/(.{64})/g, "$1\n") + "\n" + "-----END CERTIFICATE-----";
const success = await validateCertificate(pemKeyFromRequestAsString);
if (success) {
return true;
} else {
winston.warn("Failed to validate Certificate - Either Certificate Verification with CA Chain Failed or the system encountered an error");
return false;
}
};
server.log = function (type, data) {
winston.debug("type: " + type);
winston.debug(JSON.stringify(data));
};
server.on("headers", function (headers, methodName) {
//More debug stuff;
winston.debug("****** HEADERS **********");
winston.debug(JSON.stringify(headers));
winston.debug("methodName: " + methodName);
winston.debug("*************************")
});
});
} catch (err) {
winston.error(err);
}
I appreciate the time guys, thank you!
I finally fixed my problem, if anyone has the same problem with the async/sync code:
I fixed it by using the Async method from the documentation
server.authenticate = function (security: any, callback): any {
//Get the Binary Security Token coming from the request as a Base 64 String
const binarySecurityTokenAsBase64String = security.BinarySecurityToken.$value;
//Creating a new certificate with header, footer and line breaks from the binarySecurityTokenAsBase64String
const pemKeyFromRequestAsString = "-----BEGIN CERTIFICATE-----" + "\n" + binarySecurityTokenAsBase64String.replace(/(.{64})/g, "$1\n") + "\n" + "-----END CERTIFICATE-----";
//Validate the certificate
//This is an async wrapper where I added all my verification steps;
validateCertificateWrapper(pemKeyFromRequestAsString).then((authorized: boolean) => {
//If the certificate is valid
if (authorized) {
winston.info("Verification successfully Passed");
return callback(true);
} else { //If the certificate is invalid
winston.error("Failed to validate Certificate");
return callback(false);
}
}, () => {
winston.error("Failed to validate Certificate");
return callback(false);
} );
};
I am new to Node.js and Javascript and am developing an Alexa application using Lambda function and DynamoDB.
I have a table in DynamoDB named: Chat
with PrimaryKey: 'Said' and a column 'say'. Whenever the Alexa skills is launched I just want to fetch a record based on what is said by the user and return. So its basically a single Query on the primary key which works fine.
However, I dont get any response from the lambda function in speech output variable as the API doesn't wait for the response builder to complete the async call to DynamoDB and returns a null response. Is there any way to enforce the async call to be resolved before sending the response?
const WelcomeMessage = {
canHandle(handlerInput) {
const request = handlerInput.requestEnvelope.request;
return request.type === 'LaunchRequest' ||
(request.type === 'IntentRequest');
},
handle(handlerInput) {
var ans;
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({
region: 'us-east-1'
});
// Create the DynamoDB service object
var dynamodb = new AWS.DynamoDB();
var params = {
TableName: 'chat',
Key: {
'said': {
S: 'Hi Sir' + ''
}
},
ProjectionExpression: 'say'
};
dynamodb.getItem(params, function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
if (data) {
return handlerInput.responseBuilder
.speak(data.Item.say.S + '')
.getResponse();
} else {
ans = 'You dint train me for that!';
return handlerInput.responseBuilder
.speak(ans)
.getResponse();
}
}
});
}
};
Wrong Output:
I found a workaround. I return a promise and resolve it before I return it ensuring the callback to be completed before a response is sent.
const WelcomeMessage = {
canHandle(handlerInput) {
const request = handlerInput.requestEnvelope.request;
return request.type === 'LaunchRequest'
|| (request.type === 'IntentRequest');
},
handle(handlerInput) {
return new Promise((resolve) => {
var ans;
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
// Create the DynamoDB service object
//ddb = new AWS.DynamoDB({apiVersion: '2012-10-08'});
var dynamodb = new AWS.DynamoDB();
var params = {
TableName : 'chat',
Key: {
'said':{S: handlerInput.requestEnvelope.request.intent.slots.input.value+''}
}
};
dynamodb.getItem(params, function(err, data) {
if (err){
console.log(err, err.stack);
}
else{
if(data.Item){
return resolve(handlerInput.responseBuilder
.speak(data.Item.say.S+'')
.withShouldEndSession(false)
.getResponse());
}
else{
ans='You dint train me for that!';
return resolve(handlerInput.responseBuilder
.speak(ans)
.withShouldEndSession(false)
.getResponse());
}
}
});
});
}
};
I am playing with NodeJS and for this purpose created an email extractor. Somehow when i create multiple http requests the node.exe memory useage in windows task manager keeps increasing. I understand that the node needs more memory to process the requests but what i noticed that this memory usage does not come down even after all requests have been successfully processed.
When i start nodejs it consumes about 35000K memory but after about 80-100 request this goes upto 50000K and stays.
Here is my simple email extractor module:
var request = require('request'),
cheerio = require('cheerio'),
async = require('async'),
urlHelper = require('url');
function Extractor(config) {
this.baseUrl = config.url;
this.parsedUrl = urlHelper.parse(config.url);
this.urls = [];
this.emails = [];
}
Extractor.prototype.getEmails = function getEmails(html) {
var foundEmails = html.match(/([a-zA-Z0-9._-]+#[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+)/gi) || [];
if(foundEmails.length) this.emails = this.emails.concat(foundEmails);
}
Extractor.prototype.extract = function extract(html) {
var $ = cheerio.load(html),
that = this;
if($('body')){
this.getEmails($('body').html());
}
if(!this.emails.length){
$("a[href^='http://" + this.parsedUrl.host + "'], a[href^='https://" + this.parsedUrl.host + "'], a[href^='/'], a[href^='./'], a[href^='../']").each(function(k, v) {
that.urls.push(urlHelper.resolve(that.baseUrl, $(v).attr('href')));
});
}
};
/**
* Process the base URL
*/
Extractor.prototype.processBase = function processBase(next) {
request(this.baseUrl, function(err, response, body) {
return next(err, body);
});
}
/**
* Process the internal pages
*/
Extractor.prototype.processInternal = function processInternal(cb) {
var that = this;
async.whilst(
// while this condition returns true
function () { return that.emails.length === 0 && that.urls.length > 0; },
// do this
function (callback) {
request(that.urls.shift(), function (err, response, body) {
var $ = cheerio.load(body);
if($(body)){
that.getEmails($('body').html());
}
callback(); // async internal, needs to be called after we are done with our thing
});
},
// call this if any errors occur. An error also stops the series
// this is also called on successful completion of the series
function (err) {
cb(that);
}
);
}
Extractor.prototype.process = function process(next) {
var that = this;
this.processBase(function(err, html) {
if(err) {
console.log(err);
} else {
that.extract(html);
if(!that.emails.length) {
that.processInternal(function(res) {
return next(null, that);
});
}
}
});
}
module.exports = Extractor;
and here is how i call it:
var express = require('express');
var router = express.Router();
var Extractor = require('../services/Extractor');
router.get('/', function(req, res) {
res.json({msg: 'okay'});
var extractor = new Extractor({url: 'http://lior-197784.use1-2.nitrousbox.com:4000/crawl'});
extractor.process(function(err, res) {});
});
module.exports = router;