Develop Reference

Watson Conversation - Oracle DB Integration

Good morning/afternoon all, I am trying to get Watson to return a response manually set from our Oracle Databases.
I am using async to sequentially access the database and return the response because the first problem I had was the DB query would not happen until after Watson had already returned a response. Async.waterfall fixes this.
My current problem: In the console I see everything logged correctly. The Database is queried, response.output.text is set correctly and then returned to Watson but no response ever appears in my chat.. If I set the response.output.text = "asdfa" before the async, asdfa is returned as expected.
I am at a loss trying to figure this out so I appreciate any and all help. Please let me know if any more information is required.
// Send the input to the conversation service
conversation.message(payload, function (err, data) {
if (err) {
return res.status(err.code || 500).json(err);
}
return res.json(updateMessage(payload, data));
});
});
function updateMessage(input, response) {
var responseText = null;
if (!response.output) {
response.output = {};
} else {
// checkNames check
if (response.output.nodes_visited[0] === 'slot_11_1519333387192' && response.entities[0].entity === 'confirm') {
/* This code actually returns asdfa as a response from Watson.. */
// response.output.text = "asdfa";
// return response;
async.waterfall([
// this function queries the database
// TODO: module out the Oracle connection parts once POC is completed
function queryDB(callback) {
console.log('Starting queryDB');
var query = "SELECT column_name FROM table#prod WHERE column_id = '" + response.context.VSUID + "'";
oracledb.getConnection('hr',
function (err, connection) {
var conn = oracleGetConnection(err, connection);
conn.execute(query, {}, {
outFormat: oracledb.OBJECT
},
function (err, result) {
console.log('result from Oracle: ', result);
// pass a null error and the result of the query
callback(null, result.rows[0]);
});
});
},
// this function formats the result of the query
// TODO: this should not be it's own function. This can happen at the same time the db gets the row
function formatName (arg1, callback) {
console.log('this should happen after query..');
console.log('arg1: ', arg1);
var r = JSON.stringify(arg1);
r = r.substring((r.indexOf(':') + 1) + 1, r.length - 2);
console.log('Name is: ', r);
// pass a null error and the formatted name
callback(null, r);
}
],
// Final function to be ran after the two above have completed
function finalFunction (err, result) {
if (err) {
console.log('uh oh async err: ', err);
} else {
console.log('This is final Function');
// set output text
response.output.text = 'Is your name ' + result + '?';
// response.context.dbResponse = 'Is your name ' + result + '?';
// response.output.text = "asdfasdfasd";
// console.log('This is the value of response\n\n', response);
// var resp = returnResponse(input, response);
response.context.dbResponse = response.output.text[0];
return returnResponse(input, response);
// return response;
}
});
// response.output.text = "asdfa";
console.log('This is response.output.text ', response.output.text);
return response;
} else {
//If no special if case to query the db just run Watson Conversation stock
return returnResponse(input, response);
}
}
}
Here is a sample console log.
This logs the Input from the user:
name 111111111
This logs the Response from Watson:
Is 111111111correct?
This logs the intent recognized, if any:
nameCheck
This logs the entity recognized, if any:
VSUID
This logs the text that is being returned to the user: [ 'Is 111111111correct?'
]
Starting queryDB
Connected to database
result from Oracle: { outBinds: undefined,
rowsAffected: undefined,
metaData: [ { name: 'TABLE_FIRST_NAME' } ],
rows: [ [ 'Tyler' ], [ 'Tyler' ] ],
resultSet: undefined }
this should happen after query..
arg1: [ 'Tyler' ]
Name is: "Tyler
This is final Function
This logs the Input from the user:
yes
This logs the Response from Watson:
Is your name "Tyler?
This logs the entity recognized, if any:
confirm
This logs the text that is being returned to the user: Is your name "Tyler?

Writing code like the following will open you up to SQL injection vulnerabilities (and likely performance problems too):
var query = "SELECT column_name FROM table#prod WHERE column_id = '" + response.context.VSUID + "'";
Please read the section of the documentation on bind variables.
On to your question...
You are treating updateMessage as though it is a synchronous function, but it is performing asynchronous work. A function that is performing asynchronous work will need an asynchronous API, such as Node.js style callbacks, Promises, or AsyncFunctions (async/await).
If you see line 73 of the code you supplied, you're "returning" the response object, but that is outside of the async.waterfall call. Even the return on line 67 will not work because of the asynchronous nature of Node.js.
Here's my latest attempt at describing how all of this works:
https://www.youtube.com/watch?v=iAdeljxq_hs
You can access the slides and sample code here:
https://www.dropbox.com/s/quu7oxiug0gh6ua/Understanding%20Async%20Processing%20and%20Patterns%20in%20Node.js.zip?dl=0
In the sample code's code > header-detail directory, you'll see 5 different files that start with header-detail-with- followed but the name of a different API choice you could make. You will have to make a similar choice with your updateMessage API.
To run a test, use the ddl.sql file to create the target tables, then edit db-config.js as needed for your environment, and finally run node test.js 1 from a terminal in that directory. You can change the number at the end to run a different test file.

alexa Steam custom skill api integration

currently trying to develop one of my first alexa skills. trying to build a skill that updates the user on who out of their Steam friends, are online.
Curerntly it's pretty messy and not in the format i hope it to be in the end, but for testing purposes, I'm not yet using intents, im just testing using the launchrequest.
So far I've managed to get a users (mine) friends list from steam and put all the friends ids in a url that should be able to grab the details for these
users.
The issue I'm having is performing the second API call to grab the players details using the steamIds. i keep getting an 'undefined' return and am stumped on what i'm doing wrong.
I'm very much new to JS so there's bound to be mistakes in here, but i can work on tidying up later once i've got it working.
This works fine
/**
* Called when the user invokes the skill without specifying what they want.
*/
function onLaunch(launchRequest, session, callback) {
console.log("onLaunch requestId=" + launchRequest.requestId
+ ", sessionId=" + session.sessionId);
var cardTitle = "Hello, World!"
testGet(function (response) {
var speechOutput = "Here is the result of your query: " + response;
var shouldEndSession = true;
callback(session.attributes,
buildSpeechletResponse(cardTitle, speechOutput, "", true));
});
//var speechOutput = "You can tell Hello, World! to say Hello, World!"
//callback(session.attributes,
// buildSpeechletResponse(cardTitle, speechOutput, "", true));
}
This is the function that is grabbing the details from my friendlist
function testGet(response) {
var http = require('http')
var url = " http://api.steampowered.com/ISteamUser/GetFriendList/v0001/?key=XXXXXXXXXX&steamid=76561198068311091&relationship=friend"
http.get(url, function (res) {
// data is streamed in chunks from the server
// so we have to handle the "data" event
var buffer = "",
data,
friendsList,
i,
address,
textResponse,
route;
res.on("data", function (chunk) {
buffer += chunk;
});
res.on("end", function (err) {
// finished transferring data
// dump the raw data
console.log(buffer);
console.log("\n");
data = JSON.parse(buffer);
friendsList = data.friendslist.friends;
textResponse = isOnline(friendsList);
response("Friends online: " + textResponse);
}).on('error', function (e) {
console.log("Error message: " + e.message);
});
})
}
and this is the final function which i'm having difficulties with.
function isOnline(friendsList){
var http = require('http'),
i,
comma,
friendsIDs = "";
// for loop to get all friends ids in string
for (i = 0; i < friendsList.length; i++) {
// if i equals 0 then it is the start of the loop so no
//comma needed, otherwise add a comma to seperate the ids.
if(i === 0) {comma = ""}
else{comma = ","}
//place the ids in a comma seperate string
friendsIDs += comma + friendsList[i].steamid;
}
var playerurl = "http://api.steampowered.com/ISteamUser/GetPlayerSummaries/v0002/?key=xxxxx&steamids=" + friendsIDs;
// works fine up to this point
// run the api call to get player details
http.get(playerurl, function (response) {
// data is streamed in chunks from the server
// so we have to handle the "data" event
var buffer = "",
playerdata,
returnText,
textResponse,
friendsInformation,
route;
response.on("playerdata", function (chunk) {
buffer += chunk;
});
response.on("end", function (err) {
// finished transferring data
// dump the raw data
console.log(buffer);
console.log("\n");
playerdata = JSON.parse(buffer);
friendsInformation = playerdata.response.players;
for (i = 0; i < friendsInformation.length; i++) {
if(friendsInformation[i].personastate == 1) {
returnText += friendsInformation[i].personaname + " chicken";
}
}
return returnText;
}).on('error', function (e) {
console.log("Error message: " + e.message);
});
});
}
Been going round in circles for hours and feel so close to doing this but have no idea where I'm going wrong?!
thanks

I have managed to solve my problem by using javascript promises. I'm completely new to promises so took some trial and error but have managed to get it to work. here is the simplest video i could find to explain the concept, it definately helped me understand how to rearrange my code.
If you wish to do two API calls in an Alexa skill, using the data from the first api call to construct and inform the second call, you will need to use promises to do them sequentially.
it took the code from the isOnline() and testGet() functions and moved them into a promise, which allowed me to complete 1 api call (the original call to get friends list info) before executing the second call (the second api to get player details based on the results of the friendslist api call)
I can now check with alexa to see which of my steam friends are online! Hopefully i'll be able to build in some more functionality (eg what they are playing, if they are offline or just away/busy/snoozing)
thanks for contributing

This code doesn't seem to fire in order?

My problem is that the code does not seem to be running in order, as seen below.
This code is for my discord.js bot that I am creating.
var Discord = require("discord.js");
var bot = new Discord.Client();
var yt = require("C:/Users/username/Documents/Coding/Discord/youtubetest.js");
var youtubetest = new yt();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var prefix = "!";
var vidid;
var commands = {
play: {
name: "!play ",
fnc: "Gets a Youtube video matching given tags.",
process: function(msg, query) {
youtubetest.respond(query, msg);
var vidid = youtubetest.vidid;
console.log(typeof(vidid) + " + " + vidid);
console.log("3");
}
}
};
bot.on('ready', () => {
console.log('I am ready!');
});
bot.on("message", msg => {
if(!msg.content.startsWith(prefix) || msg.author.bot || (msg.author.id === bot.user.id)) return;
var cmdraw = msg.content.split(" ")[0].substring(1).toLowerCase();
var query = msg.content.split("!")[1];
var cmd = commands[cmdraw];
if (cmd) {
var res = cmd.process(msg, query, bot);
if (res) {
msg.channel.sendMessage(res);
}
} else {
let msgs = [];
msgs.push(msg.content + " is not a valid command.");
msgs.push(" ");
msgs.push("Available commands:");
msgs.push(" ");
msg.channel.sendMessage(msgs);
msg.channel.sendMessage(commands.help.process(msg));
}
});
bot.on('error', e => { console.error(e); });
bot.login("mytoken");
The youtubetest.js file:
var youtube_node = require('youtube-node');
var ConfigFile = require("C:/Users/username/Documents/Coding/Discord/json_config.json");
var mybot = require("C:/Users/username/Documents/Coding/Discord/mybot.js");
function myyt () {
this.youtube = new youtube_node();
this.youtube.setKey(ConfigFile.youtube_api_key);
this.vidid = "";
}
myyt.prototype.respond = function(query, msg) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
msg.channel.sendMessage("There was an error finding requested video.");
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
}
});
console.log("2");
};
module.exports = myyt;
As the code shows, i have an object for the commands that the bot will be able to process, and I have a function to run said commands when a message is received.
Throughout the code you can see that I have put three console.logs with 1, 2 and 3 showing in which order I expect the parts of the code to run. When the code is run and a query is found the output is this:
I am ready!
string +
2
3
1
This shows that the code is running in the wrong order that I expect it to.
All help is very highly appreciated :)
*Update! Thank you all very much to understand why it isn't working. I found a solution where in the main file at vidid = youtubetest.respond(query, msg) when it does that the variable is not assigned until the function is done so it goes onto the rest of my code without the variable. To fix I simply put an if statement checking if the variable if undefined and waiting until it is defined.*

Like is mentioned before, a lot of stuff in javascript runs in async, hence the callback handlers. The reason it runs in async, is to avoid the rest of your code being "blocked" by remote calls. To avoid ending up in callback hell, most of us Javascript developers are moving more and more over to Promises. So your code could then look more like this:
myyt.prototype.respond = function(query, msg) {
return new Promise(function(resolve, reject) {
this.youtube.search(query, 1, function(error, result) {
if (error) {
reject("There was an error finding requested video."); // passed down to the ".catch" statement below
} else {
vidid = 'http://www.youtube.com/watch?v=' + result.items[0].id.videoId;
myyt.vidid = vidid;
console.log("1");
resolve(2); // Resolve marks the promises as successfully completed, and passes along to the ".then" method
}
});
}).then(function(two) {
// video is now the same as myyt.vidid as above.
console.log(two);
}).catch(function(err) {
// err contains the error object from above
msg.channel.sendMessage(err);
})
};
This would naturally require a change in anything that uses this process, but creating your own prototypes seems.. odd.
This promise returns the vidid, so you'd then set vidid = youtubetest.response(query, msg);, and whenever that function gets called, you do:
vidid.then(function(id) {
// id is now the vidid.
});
Javascript runs async by design, and trying to hack your way around that leads you to dark places fast. As far as I can tell, you're also targetting nodeJS, which means that once you start running something synchronously, you'll kill off performance for other users, as everyone has to wait for that sync call to finish.
Some suggested reading:
http://callbackhell.com/
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://stackoverflow.com/a/11233849/3646975
I'd also suggest looking up ES6 syntax, as it shortens your code and makes life a hellofalot easier (native promises were only introduced in ES6, which NodeJS 4 and above supports (more or less))

In javascript, please remember that any callback function you pass to some other function is called asynchronously. I.e. the calls to callback function may not happen "in order". "In order" in this case means the order they appear on the source file.
The callback function is simply called on certain event:
When there is data to be processed
on error
in your case for example when the youtube search results are ready,
'ready' event is received or 'message' is received.
etc.

JavaScript How to make a node wait before returning in node.js/Node-RED?

experienced programmer, inexperienced javascript programmer.
I am working with Node-RED and in one of my nodes, I am calling an external program with 'child_process' which sends data back to my node via stdout.
I am using console.info(stdout) to log the child process data, so I know that it is working appropriately. This is the code within the function node:
/* required modules */
var cp = context.global.child_process;
/* variables */
var packet = {};
var cmd = '~/packet/packet.py ';
/* construct the packet object */
packet.networkTime = (msg.payload[3] << 24) + (msg.payload[2] << 16) + (msg.payload[1] << 8) + msg.payload[0];
packet.sampleCount = msg.payload[4];
packet.uncompressedWidth = msg.payload[5];
packet.compressedWidth = msg.payload[6];
packet.sample0 = (msg.payload[8] << 8) + msg.payload[7];
var compressedData = msg.payload.slice(9);
/* change buffer type to array */
packet.compressedData = [];
compressedData.forEach(
function(element){
packet.compressedData.push(element);
}
);
/* stringify the object */
var packet_str = "'" + JSON.stringify(packet) + "'";
/* this section will execute the command */
cp.exec(
cmd + packet_str,
function(error, stdout, stderr){
console.info(stdout);
return JSON.parse(stdout);
}
);
//return data_msg;
I tried setting data_msg to {} and using JSON.parse(stdout) to place into that object, but the 'return data_msg' always executes before the value is returned. I simply want to wait for my external process to return data before returning this node.
Thank you for your assistance,

Node.js has asynchronous execution. Put some things in functions and handle sequential execution in callbacks or use promises. Below is my use of promises that I use to ensure a scipt is finished executing before using it's stdout data within it's calling function.
Upon stderr the promise is rejected and an error message is propagated. Upon stdout data is stored in result. When the process finishes the data is returned as the promise is resolved.
var spawn = require('child_process').spawn;
anExecutable = function(cmd, packet_str){
return new Promise(function(resolve, reject){
var aProcess = spawn(cmd, packet_str);
aProcess.stdout.on('data', (data) =>{
var result = JSON.parse(data);
});
aProcess.on('error', (error) => {
reject(error);
});
aProcess.on('exit', (code) => {
resolve(result);
});
});
}
Call like this:
anExecutable(cmd, packet_str).then(function(data_msg){
return data_msg;
}).catch(function(error){
return error;
});
Let me know if you have any Q's :D

Maybe you're looking for execSync?

Import sql file in node.js and execute against PostgreSQL

I'm looking for an efficient way to take a raw sql file and have it executed synchronously against a postgres database, akin to if you ran it through psql.
I have an sql file which creates all databases, imports data, etc. I need to execute this using node.js but cannot find any module which does this automatically. For the node.js application itself, we use node-postgres ('pg'), knex.js and bookshelf.js. I assume though that pg is best for this.
One alternative I can think of is to read the full file, split it by semicolons, replace newlines with spaces, trim any duplicate space, then feed it into pg one by one in a manner that they're executed sequentially, not asynchronously. I'm a little surprised if this is truly the most efficient way and also if no libraries exist yet to solve this. I'm a little hesitant to jump into it seeing as SQL syntax can itself be a little challenging and I might accidentally mash it up.
Some clarifications in advance:
psql cannot be used as it's not installed on the target machine
I've chosen to develop and source control sql statements in sql native form, because it's a lot easier for a DBA to use and manipulate it

You can just separate consequent queries with a semicolon when passed to client.query
That works:
var pg = require('pg');
pg.connect('postgres://test:test#localhost/test', function(err, client, done){
client.query('CREATE TABLE test (test VARCHAR(255)); INSERT INTO test VALUES(\'test\') ');
done();
});
And consequently, that works too:
var pg = require('pg');
var fs = require('fs');
var sql = fs.readFileSync('init_database.sql').toString();
pg.connect('postgres://test:test#localhost/test', function(err, client, done){
if(err){
console.log('error: ', err);
process.exit(1);
}
client.query(sql, function(err, result){
done();
if(err){
console.log('error: ', err);
process.exit(1);
}
process.exit(0);
});
});

I've written the following function which works for my case. It would have been much more simpler if it weren't for:
Using batch to manage concurrency
Having the tricky PostgreSQL COPY case to consider
Code snippet:
function processSQLFile(fileName) {
// Extract SQL queries from files. Assumes no ';' in the fileNames
var queries = fs.readFileSync(fileName).toString()
.replace(/(\r\n|\n|\r)/gm," ") // remove newlines
.replace(/\s+/g, ' ') // excess white space
.split(";") // split into all statements
.map(Function.prototype.call, String.prototype.trim)
.filter(function(el) {return el.length != 0}); // remove any empty ones
// Execute each SQL query sequentially
queries.forEach(function(query) {
batch.push(function(done) {
if (query.indexOf("COPY") === 0) { // COPY - needs special treatment
var regexp = /COPY\ (.*)\ FROM\ (.*)\ DELIMITERS/gmi;
var matches = regexp.exec(query);
var table = matches[1];
var fileName = matches[2];
var copyString = "COPY " + table + " FROM STDIN DELIMITERS ',' CSV HEADER";
var stream = client.copyFrom(copyString);
stream.on('close', function () {
done();
});
var csvFile = __dirname + '/' + fileName;
var str = fs.readFileSync(csvFile);
stream.write(str);
stream.end();
} else { // Other queries don't need special treatment
client.query(query, function(result) {
done();
});
}
});
});
}
Beware that this would fail if you used semicolons anywhere except to terminate SQL statements.

The #databases/pg client supports running SQL files out of the box:
const createPool = require('#databases/pg');
const {sql} = require('#databases/pg');
const db = createPool();
db.query(sql.file('my-file.sql')).catch(ex => {
console.error(ex);
process.exitCode = 1;
}).then(() => db.dispose());
It also supports having multiple statements in a single call to db.query:
const createPool = require('#databases/pg');
const {sql} = require('#databases/pg');
const db = createPool();
db.query(sql`
INSERT INTO users (name) VALUES (${'Forbes'});
SELECT * FROM users;
`)).then(
results => console.log(results)
).catch(ex => {
console.error(ex);
process.exitCode = 1;
}).then(() => db.dispose());
In this example, each statement is run in sequence, and the result of the last statement is returned.

The following, which just reads a file into a string and runs it using query seems to work for me:
const { Pool } = require("pg");
const pool = new Pool({ host, port, user, password, database });
dbClient = await pool.connect();
var sql = fs.readFileSync("/path/to/file.sql", "utf8");
await dbClient.query(sql);
In case it also helps, here is further code to run all "*.sql" files in a directory in alphabetical order:
const pathWithSqlFiles = "/path/to/sqldir";
const filenames = fs
.readdirSync(pathWithSqlFiles, { withFileTypes: true })
.filter((item) => !item.isDirectory() && item.name.toLowerCase().endsWith(".sql"))
.map((item) => item.name);
for (const filename of filenames) {
var sql = fs.readFileSync(`${pathWithSqlFiles}/${filename}`, "utf8");
await dbClient.query(sql);
}
(Don't forget to close the client connection at some point after this using await dbClient.end()).

There are many ways to import a database through SQL file the simplest and fasted way is to just run this command in you cmd where your file is saved:
psql -h localhost -U postgres -d myDataBase -a -f myFile.sql
Or you can read and parse the file through node.js and run it. But it would take time.
function processSQLFile(fileName) {
// Extract SQL queries from files. Assumes no ';' in the fileNames
var queries = fs.readFileSync(fileName).toString()
.replace(/(\r\n|\n|\r)/gm," ") // remove newlines
.replace(/\s+/g, ' ') // excess white space
.split(";") // split into all statements
.map(Function.prototype.call, String.prototype.trim)
.filter(function(el) {return el.length != 0}); // remove any empty ones
// Execute each SQL query sequentially
queries.forEach(function(query) {
batch.push(function(done) {
if (query.indexOf("COPY") === 0) { // COPY - needs special treatment
var regexp = /COPY\ (.*)\ FROM\ (.*)\ DELIMITERS/gmi;
var matches = regexp.exec(query);
var table = matches[1];
var fileName = matches[2];
var copyString = "COPY " + table + " FROM STDIN DELIMITERS ',' CSV HEADER";
var stream = client.copyFrom(copyString);
stream.on('close', function () {
done();
});
var csvFile = __dirname + '/' + fileName;
var str = fs.readFileSync(csvFile);
stream.write(str);
stream.end();
} else { // Other queries don't need special treatment
client.query(query, function(result) {
done();
});
}
});
});
}