Develop Reference

How to pass a Postgres Pool or Client through a socket

So I've deployed my server to Heroku with the following:
const pool = new Pool({
connectionString: process.env.DATABASE_URL,
ssl: {
rejectUnauthorized: false
}
});
const http = require('http').Server(app);
const io = require('socket.io')(http);
I have a function that sends a request to my Raspberry Pi (which is acting as a socket client in this case) asking it to run a scraping function and update a table in my Heroku Postgres database accordingly. The only problem is on the Pi, the client.query() function is not recognized...not sure why. Here's my function that sends the request to the Pi:
async function F1() {
const client = await pool.connect();
try {
await client.query('BEGIN')
//do some database updating here in some table updates
let date = ("0" + terminationTime.getDate()).slice(-2);
let month = ("0" + (terminationTime.getMonth() + 1)).slice(-2);
let year = terminationTime.getFullYear();
let hours = terminationTime.getHours();
let minutes = terminationTime.getMinutes();
let seconds = terminationTime.getSeconds();
let timestamp = year + "-" + month + "-" + date + " " + hours + ":" + minutes + ":" + seconds
let data = {
pool: pool, //here I tried passing in the pool, also tried passing in client
query: query,
timestamp: timestamp
}
io.to(RPiMiniServerID).emit("requestToPi", data)
} catch(err) {
console.log(err);
}
}
And here's my code on the Pi/Client side:
const io = require("socket.io-client");
const socket = io.connect("MY HEROKU APP URL HERE")
const { Pool, Client } = require("pg");
socket.on("requestToPi", (data) => sync function() {
let client = await data.pool.connect()
let query = data.query
let timestamp = data.timestamp
//a bunch more code and THEN
await client.query('INSERT INTO table (columnnames) VALUES($1)', [value]); //this is the line with the error
}());
I've tried passing in the client directly, but that also doesn't work. Not really sure what to do here--any help is much appreciated!

Let me get this straight, you're trying to send the instance of the pg library object that was started on your heroku server to your Pi and continue operation from that 'pg' library object state?
The first rule of HTTP is that its stateless. I know its Socket io but it still relies on http and the data is recreated on the other end and its state is lost over the connection. So in a way, you're starting a new instance of the pg instance on the Pi and the one on the heroku server is not the same as the one on the Pi.
await client.query('INSERT INTO table (columnnames) VALUES($1)', [value]); //this is the line with the error
That is because the data has completely lost its state. That means, you're not connected to database, you have not started the commit process yet, ect. You have a new instance of the pg library on your hand and everything you did on heroku server is completely different.
One solution to this problem is to use some sort of real time model data synchronization like backbone.io or other similar solutions out there.

node.js How to communicate between modules

I'm fairly new to node.js and recently started to make some modules. However I've come to a point where communication between modules is required. Since this is not a problem I've encountered in the past I'm stuck with finding a clean solution.
This is the boilerplate I currently got (Left out some checks to make the code a bit smaller). The basic idea atm is joining any irc channel given by an http post.
bot.js
//Include services
var Webservice = require('./Webservice');
var Ircservice = require('./Ircservice');
//Create service instances
var webservice = new Webservice();
var ircservice = new Ircservice();
//Initialize services
webservice.init(1337);
ircservice.init('alt-irc.snoonet.org', 80, 'User');
//Handle events
ircservice.on('irc-registered', function(msg){
console.log(ircservice.connected);
ircservice.joinChannel('#testchannel')
});
ircservice.on('irc-join', function(channel){
console.log('Successfuly joined: ' + channel);
});
webservice.on('web-join', function(streamer){
ircservice.joinChannel('#' + streamer);
});
Webservice.js
var express = require('express');
var app = express();
var bodyParser = require('body-parser');
var util = require('util');
var EventEmitter = require('events').EventEmitter;
app.use(bodyParser.urlencoded({
extended: true
}));
var Webservice = function(){
EventEmitter.call(this);
};
Webservice.prototype.init = function(port){
app.listen(port, function () {
console.log('Webserver listening on ' + port);
});
this.initRoutes();
};
Webservice.prototype.initRoutes = function(){
var self = this;
//join a irc-channel
app.post('/join', function (req, res) {
var streamer = req.body.name;
self.emit('web-join', streamer);
res.send('Received')
});
};
util.inherits(Webservice, EventEmitter);
module.exports = Webservice;
Ircservice.js
var irc = require('irc');
var util = require('util');
var EventEmitter = require('events').EventEmitter;
var Ircservice = function(){
EventEmitter.call(this);
}
Ircservice.prototype.init = function(server, port, nick){
this.client = new irc.Client(server, nick, {
port: parseInt(port)
});
this.initListerners();
};
Ircservice.prototype.initListerners = function(){
var self = this;
this.client.addListener('message', function (from, to, message) {
console.log(from + ' => ' + to + ': ' + message);
});
this.client.addListener('join', function(channel, nick, message){
self.emit('irc-join', channel);
});
};
Ircservice.prototype.joinChannel = function(channel){
this.client.join(channel, null);
};
util.inherits(Ircservice, EventEmitter);
module.exports = Ircservice;
This example works perfectly, but as you can see the communication between my webservice and ircservice is handled by the bot.js. While this is perfectly fine for this example, I cannot use this method whenever I want.
Let say in the future I want to keep a list in my ircservice of all channels he has joined and display this through a webpage. I could keep a local array on my ircservice and on the join event add that channel to the array. But how do I continue on the webservice end. I can write an endpoint '/getchannels' but my webservice itself is not aware of the ircserver to get the channels (ircservice.getChannels or something similar) and firing an event in my web request doesn't feel like the way to go.
One solution that came up in my mind was passing the instances of the services to each other like webservice.setIrcservice(ircservice) and the other way around in the bot.js. But this feels like dirty code and a hard depency.
So how can I communicate between modules when I need data instantaneously and events are no option?

AWS S3 with SAML

I am trying to upload files to S3 from an app written in JavaScript. For this reason - mobile app, I am limited to libraries that I could use. Got the thing to work by using FormData until it was decided to use SAML and delegate authentication. Now, the temporary credentials are being obtained OK. However, AWS::S3 does not want to recognize them. It throws an error: The AWS Access Key Id you provided does not exist in our records.
My code is below:
console.log("AWS temp credentials: " + JSON.stringify(delegated_jwt.Credentials));
var aws_creds = delegated_jwt.Credentials;
var secret = aws_creds.SecretAccessKey;
var policyBase64 = base64.encode(JSON.stringify(POLICY_JSON));
console.log ("policy base64: " + policyBase64 );
var signature = CryptoJS.enc.Base64.stringify(CryptoJS.HmacSHA1(policyBase64, secret));
console.log("signature: " + signature);
var key = "user_uploads" + "/" + delegated_jwt.Subject + '/' + (new Date).getTime() + ".jpg";
console.log("AWS::S3 key: " + key);
var params = new FormData();
params.append('key', key);
params.append('acl', 'private');
params.append('Content-Type', "image/jpeg");
params.append('AWSAccessKeyId', aws_creds.AccessKeyId);
params.append('policy', policyBase64);
params.append('signature', signature);
params.append('file', captured.uri);
var xhr = new XMLHttpRequest();
xhr.open('POST', 'https://mybucket.s3.amazonaws.com/', true);
xhr.onload = () => {
...
When I used permanent access and secret keys, it worked fine. If this there is something wrong with my AWS settings, how do I debug this? What else should I check?

retry sql connection using JavaScript

I know using javascript is not the best way to connect to a SQL server but this is for an in-house application. I connect using the following:
dbNSConnection = new ActiveXObject("ADODB.Connection") ;
var sNSConnectionString="Driver={SQL Server};TrustedConnection=Yes;Server=" + sNSServer + ";Database=" + sNSDatabase + ";UID=" + sNSUID + ";PWD=" + sNSPWD;
dbNSConnection.Open(sNSConnectionString);
How can I make sure connection has gone thru and how do I retry if not connected?

Isn't there any other way you could get this done on the codebehind, and pass the results to the javascript? This may be an example of what I'm talking about originally.
If not, can't you simply do the following?
try
{
dbNSConnection = new ActiveXObject("ADODB.Connection") ;
var sNSConnectionString="Driver={SQL Server};TrustedConnection=Yes;Server=" + sNSServer + ";Database=" + sNSDatabase + ";UID=" + sNSUID + ";PWD=" + sNSPWD;
dbNSConnection.Open(sNSConnectionString);
//Run rest of query here
}
catch(err)
{
var message = err.message;
//Finish building errors message here
}

node js azure SDK getBlobToStream uses lots of memory

I am writing a backup script that simply downloads all the blobs in all the blob containers of a specific Azure account.
The script uses async.js to make sure only so much threads can run at the same time so it doesn't overload the server. When I run this script it works fine, but when it hits large files it runs out of memory. I'm guessing the download runs faster than the disk can write, and it eventually fills up the in-memory buffer so badly that I run out of memory entirely, but debugging the exact cause has been impossible so far.
The specific function which appears to use a lot of memory is called as follows:
blobService.getBlobToStream(
containerName,
blob.name,
fs.createWriteStream(fullPath),
function(error) {
if(error){ //Something went wrong, write it to the console but finish the queue item and continue.
console.log("Failed writing " + blob.name + " (" + error + ")");
callback();
}
else if(!error) { //Write the last modified date and finish the queue item silently
fs.writeFile(fullPath + ".date", blobLastModified, function(err)
{ if(err) console.log("Couldn't write .date file: " + err); });
callback();
}
});
Even a single 700MB download will easily fill up 1GB of memory on my side.
Is there any way around this? Am I missing a parameter which magically prevents the Azure SDK from buffering everything and the kitchen sink?
Full code:
#!/usr/bin/env node
//Requires
var azure = require('azure');
var fs = require('fs');
var mkdirp = require('mkdirp');
var path = require('path');
var async = require('async');
var maxconcurrency = 1; //Max amount of simultaneous running threads of getBlobsAndSaveThem() running through async.js.
var blobService = azure.createBlobService();
backupPrefix='/backups/azurebackup/' //Always end with a '/'!!
//Main flow of the script is near the bottom of the file.
var containerProcessingQueue = async.queue(
function getBlobsAndSaveThem(containerName) {
console.log(containerName); //DEBUG
blobService.listBlobs(containerName,
function(error, blobs) {
if(!error){
var blobProcessingQueue =
async.queue(function(index,callback) {
var blob = blobs[index];
console.log(blob); //DEBUG
var fullPath = backupPrefix + containerName + '/' + blob.name;
var blobLastModified = new Date(blob.properties['last-modified']);
//Only create if the directoy doesn't exist, since mkdirp fails if the directory exists.
if(!fs.existsSync(path.dirname(fullPath))){ //And do it sync, because otherwise it'll check 99999 times if the directory exists simultaneously, doesn't find it, then fails to create it 99998 times.
mkdirp.sync(path.dirname(fullPath), function(err) { console.log('Failed to create directory ' + path.dirname(fullPath) + " ("+ err + ")"); });
}
if(fs.existsSync(fullPath + ".date")){
if(blobLastModified == fs.readFileSync(fullPath + ".date").toString()) {
callback();
return; //If the file is unmodified, return. No this won't exit the program, because it's called within a function definition (async.queue(function ...))
}
}
blobService.getBlobToStream(
containerName,
blob.name,
fs.createWriteStream(fullPath),
function(error) {
if(error){ //Something went wrong, write it to the console but finish the queue item and continue.
console.log("Failed writing " + blob.name + " (" + error + ")");
callback();
}
else if(!error) { //Write the last modified date and finish the queue item silently
fs.writeFile(fullPath + ".date", blobLastModified, function(err)
{ if(err) console.log("Couldn't write .date file: " + err); });
callback();
}
});
},maxconcurrency);
for(var blobindex in blobs){
blobProcessingQueue.push(blobindex);
} //Push new items to the queue for processing
}
else {
console.log("An error occurred listing the blobs: " + error);
}
});
},1);
blobService.listContainers(function(err, result){
for(var i=0;i<result.length;i++) {
containerProcessingQueue.push(result[i].name);
}
});

For all those now curious the variables for the start and end have changed. They are now just rangeStart and rangeEnd.
Here is the azure node documentation for more help
http://dl.windowsazure.com/nodestoragedocs/BlobService.html

One thing that you could possibly do is read only a chunk of data into stream instead of whole blob data, append that to the file and read next chunk. Blob Storage service supports that. If you look at the source code for getBlobToStream (https://github.com/WindowsAzure/azure-sdk-for-node/blob/master/lib/services/blob/blobservice.js), you can specify from/to bytes in the options - rangeStartHeader and rangeEndHeader. See if that helps.
I have hacked some code which does just that (as you can see from my code, my knowledge about node.js is quite primitive :)). [Please use this code just to get an idea about how you can do chunked download as I think it still has some glitches]
var azure = require('azure');
var fs = require('fs');
var blobService = azure.createBlobService("account", "accountkey");
var containerName = "container name";
var blobName = "blob name";
var blobSize;
var chunkSize = 1024 * 512;//chunk size -- we'll read 512 KB at a time.
var startPos = 0;
var fullPath = "D:\\node\\";
var blobProperties = blobService.getBlobProperties(containerName, blobName, null, function (error, blob) {
if (error) {
throw error;
}
else {
blobSize = blob.contentLength;
fullPath = fullPath + blobName;
console.log(fullPath);
doDownload();
}
}
);
function doDownload() {
var stream = fs.createWriteStream(fullPath, {flags: 'a'});
var endPos = startPos + chunkSize;
if (endPos > blobSize) {
endPos = blobSize;
}
console.log("Downloading " + (endPos - startPos) + " bytes starting from " + startPos + " marker.");
blobService.getBlobToStream("test", blobName, stream,
{ "rangeStartHeader": startPos, "rangeEndHeader": endPos-1 }, function(error) {
if (error) {
throw error;
}
else if (!error) {
startPos = endPos;
if (startPos <= blobSize - 1) {
doDownload();
}
}
});
}