I'm trying to call spark job server API from node js. The API which is the python egg file does provide the count of nulls from the file. So once I call the API from the node, it is reaching the SJS server and the job starts which triggers res.on('data') event and soon after that, it triggers res.on('end') as well before the job finishes it's execution and returns the result. So due to this, I am not able to get the data once the job completes.
Below is the code snippet, Please let me know what is the mistake done here.
var postData = {
'input': {
'strings': {
'file': 'file path to be passed'
}
}
};
var options = {
hostname: 'localhost',
port: 8090,
path: '/jobs?appName=my_ml_job&classPath=my_py_package.NullCheck.nullcheck&context=py-context',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(JSON.stringify(postData))
}
};
var post_req = Http.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
res.on('end', () => {
return{
'STATUS': 'FINISHED'
}
});
});
post_req.on('error', e => {
rj(e);
});
// post the data
post_req.write(postData);
post_req.end();
Related
Background: I have a function that needs a small debugging which is related to API requests(POST,GET) and promises. For the past few days, I have been trying to figure how asynchronous functions work and it turns out it's impossible (or so I heard) to directly return a value from such functions to synchronous functions.
My initial idea was to make a POST request function that returns an access token which I could plug to the authorization header of my GET function (which works perfectly so long as a token is provided). From my discovery, however, I can't do this on two separate methods because the POST request returns undefined outside the function. So my idea now is to include the GET request within the POST so that the access token can be passed directly without any method call. I tried to do this using Promises which I'm not very familiar with but manage to make some sense. My expectation was that I can call the httpGetData() function which would first get token, then pass it to a .then() portion which is supposed to return actual data or [object Promise] instead of undefined.
Question: The issue is that when I call the httpGetData() function it returns undefined but my console.log(result[0]) shows clearly that the GET request has fetched the data I want. I was wondering if there is a tweak that would allow my code to return this data (console.log is useless for me other than to see what's happening on background of api request). I saw some SO questions like these answered by returning the values in the .then() but my attempts all return undefined.
I've attached screenshots of the outputs. Keep in mind this is an Alexa skill Lambda code so the return appears after .speak() and the console.log shows below.
function httpGetData() {
var promise1 = new Promise(((resolve, reject) => {
var options = {
method: "POST", // POST request intended to get access token
host: 'hostinfo',
path: "pathinfo",
headers: {
"content-type": "stuff here",
"cache-control": "no-cache",
}
};
var request = http.request(options, function (response) {
var returnData = [];
response.on("data", function (chunk) {
returnData += chunk;
});
response.on("end", function () {
var data = JSON.parse(returnData);
var accessToken = data.access_token; // the access token value of the POST api request like "039509544932n3249fc21232nc32423nci"
resolve(accessToken);
});
response.on('error', (error) => {
reject(error);
});
});
request.write("------stuff: form-data; name=\"client_secret\"\r\n\r\stuff here\r\n------stuff: form-data; name=\"client_id\"\r\n\r\stuff here\r\n------stuff: form-data; name=\"grant_type\"\r\n\r\nclient_credentials\r\n------stuff");
request.end();
}));
promise1.then(function(value) {
return new Promise(((resolve, reject) => {
var options = {
method: "GET", //this is a GET request so it's intended to get usable data using access token
host: 'hostinfo',
path: "pathinfo",
headers: {
"content-type": "stuff here",
"Authorization": "Bearer " + value, // value is the access token that was resolved earlier ("039509544932n3249fc21232nc32423nci") so I'm trying to concatonate this to the authorization header
"cache-control": "no-cache",
}
};
var request = http.request(options, function (response) {
var returnData = [];
response.on("data", function (chunk) {
returnData += chunk;
});
response.on("end", function () {
console.log(value); //this is the line that prints access token in screenshot
resolve(JSON.parse(returnData));
});
response.on('error', (error) => {
reject(error);
});
});
request.write("------stuff: form-data; name=\"client_secret\"\r\n\r\stuff here\r\n------stuff: form-data; name=\"client_id\"\r\n\r\stuff here\r\n------stuff: form-data; name=\"grant_type\"\r\n\r\nclient_credentials\r\n------stuff");
request.end();
}))
}).then((result) => {
console.log('hello');
return result; //I'm not sure here but wanted to return the result
}).then((result) => {
console.log(result[0]); // just a check which shows that result[0] actually executes and can be printed as shown in screenshot
return result[0]; //other desperate attempt
});
}// httpGetData() function ends here
return image
Console log image
I've finally solved the problem thanks to #Bergi. Although this answer can be shortened, I want to post it so that it can serve as an option. What Bergi suggested was that instead of having both the POST and GET requests done in one function, we can separate them and use another function to do execute them simultaneously. In the code below, httpToken() performs the POST request and returns an access token. The httpResource(accessToken) function performs a GET request where it will take an accessToken parameter. The other function, finalData() will help us make the POST and GET requests simultaneously hence returning an [object Promise] instead of undefined, which we can access. Our last step will be to have an async/await function (retrieveInfo()), which will use finalData() to get the response from our requests. Enjoy!
function httpToken() {
return new Promise(((resolve, reject) => {
var options = {
method: "POST",
host: 'hostinfo',
path: "pathinfo",
headers: {
"content-type": "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW",
"cache-control": "no-cache",
}
};
var request = http.request(options, function (response) {
var returnData = [];
response.on("data", function (chunk) {
returnData += chunk;
});
response.on("end", function () {
var data = JSON.parse(returnData);
var accessToken = data.access_token;
resolve(accessToken); //resolving access token (can be changed to resolve(data) to include timestamp etc)
});
response.on('error', (error) => {
reject(error);
});
});
request.write("------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"client_secret\"\r\n\r\n"client secret here"\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"client_id\"\r\n\r\n"client id here"\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"grant_type\"\r\n\r\n"grant type here"\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW--");
request.end();
}));
}
function httpResource(accessToken){
return new Promise(((resolve, reject) => {
var options = {
method: "GET",
host: 'hostinfo',
path: "pathinfo",
headers: {
"content-type": "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW",
"Authorization": "Bearer " + accessToken,
"cache-control": "no-cache",
}
};
var request = http.request(options, function (response) {
var returnData = [];
response.on("data", function (chunk) {
returnData += chunk;
});
response.on("end", function () {
resolve(JSON.parse(returnData));
});
response.on('error', (error) => {
reject(error);
});
});
request.write("------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"client_secret\"\r\n\r\n"client secret here"\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"client_id\"\r\n\r\n"client id here"\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"grant_type\"\r\n\r\n"grant type here"\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW--");
request.end();
}));
}
function finalData()
{
return httpToken().then(token => httpResource(token));
}
async function retrieveInfo()
{
response = await finalData(); //response will have the data returned from our GET request using finalData()
//do stuff with response...
}
So I'm having trouble getting one javascript function to finish before the next one starting. I've spent quite a lot of time trying to use callback methods described on other stackoverflow posts. I could get simple examples that used timeouts to work but couldn't get it to work with my API request. I stumbled upon async.js and thought that perhaps using async.series would be a good idea to get my two functions to perform one after another. So I tried this approach, however I still seem to be having the problem where the first function takes a bit longer to execute (which is fine) but the execution process moves past this function instead of waiting for it to end. I feel I have a misconception of some sort since I have tried several methods but to no avail.
What is strange is, is that that when running server.js, it goes into the first function but then it leaves the async.series() function even before the request is finished. When I print inside of tokenReq(), I can see that the request was successful as a token code is returned successfully however this happens to late as execution has moved on. The output is shown below.
server.js:
var access_code;
async.series([
function() {
access_code = queries.data.tokenReq(code);
console.log("Finished inside function 1");
},
function() {
console.log("\n Starting function 2 \n");
if (access_code === "error") {
res.json("An error has occured");
} else {
var response = queries.data.messagesReq(access_code);
res.json(response);
}
}
],
function(err, access_code) {
});
console.log("Outside");
queries.js:
tokenReq: function(code) {
var tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
var form = {
code: code,
client_id: "__ID__",
redirect_uri: "__Site__/",
grant_type: "authorization_code",
client_secret: "__Secret__",
};
var formData = querystring.stringify(form);
var contentLength = formData.length;
request({
headers: {
'Content-Length': contentLength,
'Content-Type': 'application/x-www-form-urlencoded'
},
uri: tokenUrl,
body: formData,
method: 'POST'
}, function (error, response, body) {
if (error != "null") {
var access_token = JSON.parse(body).access_token;
console.log("\n INSIDE FUNCTION REQUEST, Token: " + access_token + " \n");
return access_token;
} else {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
// console.log('body:', body); // Print the HTML for the Google homepage.
return "error";
}
});
},
Output:
Finished inside function 1
Outside
INSIDE FUNCTION REQUEST, Token: 8Swhd.......
You missed a major point here. Since node.js is asynchronous there should not be a way to know when a function completes its execution. That is why we specify callbacks so that the invoking function knows whom to call when it finishes its execution. Once you have functions with callbacks, you can enforce series/parallel/waterfall behavior with async module.
tokenReq: function(code, cb) {
var tokenUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
var form = {
code: code,
client_id: "__ID__",
redirect_uri: "__Site__/",
grant_type: "authorization_code",
client_secret: "__Secret__",
};
var formData = querystring.stringify(form);
var contentLength = formData.length;
request({
headers: {
'Content-Length': contentLength,
'Content-Type': 'application/x-www-form-urlencoded'
},
uri: tokenUrl,
body: formData,
method: 'POST'
}, function (error, response, body) {
if (error != "null") {
var access_token = JSON.parse(body).access_token;
console.log("\n INSIDE FUNCTION REQUEST, Token: " + access_token + " \n");
return cb(null, access_token);
} else {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
// console.log('body:', body); // Print the HTML for the Google homepage.
return cb(new Error("whatever"));
}
});
},
Now, you can use the callback inside server.js
var access_code;
async.series([
function(cb) {
return queries.data.tokenReq(code, cb);
},
function(access_code, cb) {
console.log("\n Starting function 2 \n");
if (access_code === "error") {
res.json("An error has occured");
} else {
var response = queries.data.messagesReq(access_code);
res.json(response);
}
// do whatever you want after this
return cb();
}
],
function(err, access_code) {
if (err) {
console.log(err);
}
// wrap your logic around a function and call the correspoding callback here
});
I am writing a new json file when I run my results.html page and then using express.static to make the public folder files accessible in the browser.
I have my app working but am having to click the button multiple times in order to have the updated file be written and accessed.
How should I send the JSON info to the browser without writing a file? Or not experience this lag?
Send the data back in response
function setData(req, res) {
var data = '';
req.on('data', function(streamData) {
data += streamData;
});
req.on('end', function() {
fs.writeFile('data/data.json', data, 'utf8', function(err) {
if (!err) {
res.writeHead(201, {
'Content-Type': 'text/json'
});
res.end(data)
} else {
res.writeHead(400, {
'Content-Type': 'text/json'
});
res.end("File write error" + err)
}
})
})
}
access the data from front end
$.ajax({
url: '/set/data',
method: 'POST',
data: data
})
.done(function(responseData) {
// update your ui with responseData
})
.fail(function(error) {
console.log(error)
})
I'm running this code using the request module for node.js
var hsKey = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
var hsForm = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
var hsHost = "https://docs.google.com/"
var url = hsHost + "forms/d/" + hsForm + "/formResponse"
var form = {
"entry.129401737": pointsAvg,
"entry.2000749128": hiddenNeurons,
"submit": "Submit",
"formkey": hsKey
};
request.post({
url: url,
form: form
}, function (err, res, body) {
console.log("Sent data");
});
I have tried running the above code just using standard Node.JS libraries, to no avail. The callback function is never fired and the request doesn't go through. I don't know why.
I believe I've found the answer to my own problem. The issue seems to be that I'm not allocating any time in the Node.js event loop to allow the request to be executed.
Have a look at this question:
your code should look something like
var hsKey = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
var hsForm = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
var hsHost = "https://docs.google.com/"
var url = hsHost + "forms/d/" + hsForm + "/formResponse"
var form = {
"entry.129401737": pointsAvg,
"entry.2000749128": hiddenNeurons,
"submit": "Submit",
"formkey": hsKey
};
request.post({
url: url,
form: form
}, function (response) {
response.setEncoding('utf8');
response.on('data', function(chunk){
//do something with chunk
});
});
The data event should get fired on receiving a response.
So if you read the docs for the request module at npm
request
.get('http://google.com/img.png')
.on('response', function(response) {
console.log(response.statusCode) // 200
console.log(response.headers['content-type']) // 'image/png'
});
There is a response event that should get fired.
I ran into this as well. I ended up creating a separate js file containing only the request, without the describe and it methods, and running it with 'mocha mynewbarebonesreq.js'. suddenly I could see that there was an exception being thrown and swallowed by mocha (with the standard reporter, spec).
I finally installed and enabled mocha_reporter which shows the exceptions
now it looks like this:
describe('CMSLogin', function () {
it('should log in as user ' + JSON.stringify(USER_PASS), function (done) {
request({
url: "http://cms.lund.multiq.com:3000/api/CMSUsers/login",
method: "POST",
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
json: false,
body: JSON.stringify(USER_PASS)
}, (err, res, body) => {
var parsedBody = JSON.parse(body);
this.token = parsedBody.id;
console.log(this.token)
assert.equal(USER_PASS.userId, parsedBody.userId);
assert.doesNotThrow(() => Date.parse(parsedBody.created));
if (err) { done.fail(err); }
done();
});
});
}
I'm calling the Firebase REST API from a Node.js process. The problem I'm seeing is that POSTS fail when the post body contains non-ASCII characters. This is despite the request returning a "200" status, and the name of a node (which doesn't actually get created).
I'm currently trying something like this:
function push(path, object, callback) {
console.log("Pushing to "+path+" on: "+firebase.host);
var fullPath=firebase.basePath+path;
console.log("fullPath="+fullPath);
var body = JSON.stringify(object);
var options = {
host: firebase.host,
port: 80,
method: "POST",
path: fullPath, //gamma.firebase.com/...
agent: false,
headers: {
'content-type': 'application/json',
'Content-Length': body.length,
}
};
var req = http.request(options, function(response) {
var result = "";
console.dir(response.headers);
response.on('data', function(chunk) {
result+=chunk;
});
response.on('end', function() {
console.error("POST response result: "+result);
try {
callback(JSON.parse(result));
} catch(e) {
callback({ error: e });
}
});
response.on('error', function(e) {
console.error("POST response error: "+error);
callback({error: e});
});
});
req.on('error', function(error) {
console.error("POST request error: "+error);
});
req.write(body);
req.end();
}
The contents of "object" can be as simple as:
{"text": "test\u00a0text"}
The result I get back is status 200, and an reasonable-looking child name, which doesn't actually get created.
I've tried setting content-type to a bunch of different things (adding ; charset="UTF-8", for example), and it doesn't seem to affect the results at all.
There is an error in the way we are handling certain types of input which is yielding the erroneous 200 status. We will roll out a fix shortly. To work around the problem in the meantime you can omit sending the Content-Length header. This will allow you to post ASCII and non-ASCII data.