From my API(nodejs), I'm accessing a third-party API (using http) to download files.
The service returns a Base64 string, chopped into smaller pieces, to be able to handle larger files.
Is it possible to do multiple http-requests (loop ?) to the third-party service, send each piece in response, to the browser until there is no longer any response from the third-party service?
The reason i want to do this, is because I don't want to consume to much memory on the node server.
I will put the pieces back together in the browser.
Any suggestions on how to do this?
See my current code below.
var request = require('request');
router.post('/getfiledata', function(req, res) {
var fileid = req.body.fileid;
var token = req.headers.authorization;
getFileData(req, res, dbconfig, fileid, token, function(err, chunkOfFile) {
if (err) {
res.status(500).send({
status: 500,
message: err
});
return;
}
res.send(chunkOfFile);
});
});
function getFileData(req, res, dbconfig, fileid, token, next) {
var url ="http://*ip*/service/rest/getfiledata";
var reqbody = {
fileId: fileid
};
var options = {
url: url,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': token
},
json: true,
body: reqbody
};
/*SOME LOOP HERE TO GET EACH CHUNK AND SEND TO BROWSER*/
request(options, function(err, resp, body) {
if (err) {
console.log(err);
next(err, undefined);
return;
} else {
next(undefined, body)
};
});
};
I think you need Socket.io to push chunks to the browser.
Server :
socket.send("chunk", chunkOfFile)
Client :
let fullString = ""
socket.on("chunk", chunkOfFile => fullString += chunkOfFile )
Something like that
The request library you are using allows for streaming of data from one source to another. Check out the documentation on github.
Here is an example from that page:
request
.get(source)
.on('response', function(response) {
console.log(response.statusCode) // 200
console.log(response.headers['content-type']) // 'image/png'
})
.pipe(request.put(destination))
You may choose to use the http module from Nodejs, as it implements the EventEmitter class too.
I ended up doing a recursive loop from the client. Sending http-requests to my API(node) until the response no longer returns any base64 data chunks.
Thank you guys!
Related
I'm working on a game, i'm trying to get the client to post/put data about the player to the server, but after 6 requests the server seems to crash, then after a minute or so it will accept another 6 requests and repeat. This is the post code:
app.post('/entityData', function(req, res) {
//test = req.body;
//console.log(test);
console.log(req.body);
//entityList[req.params.uid] = req.body;
});
I've got a fair amount of other code but these are all for get requests, which seem to work fine with the client. Here is the code my client is sending:
async function sendPlayerData() {
let playerData = {
num: player.getPos().x
};
console.log(playerData);
try {
let response = await fetch(serverIP + "/entityData", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(playerData)
});
if(!response.ok) {
throw new Error("Error sending player data to server. ");
}
} catch (error) {
console.log(error);
}
}
I get no errors on the client or server, so have absolutely no idea what i'm doing wrong. The only modules i've got on the server is a custom utility module and fs.
The issue was that i was not sending back a response, after i fixed that it worked.
I'm trying attach an image using the bot emulator tool and sending this image off to the microsofts customvision api, the issue I'm having is that I get
{ Code: 'BadRequestImageFormat', Message: '' }
back from custom the custom vision api call.
I'm using the the request module from npm to handle the calls
// Receive messages from the user and respond by echoing each message back (prefixed with 'You said:')
var bot = new builder.UniversalBot(connector, function (session) {
session.send("Hello"); //session.message.text
// If there is an attachment
if (session.message.attachments.length > 0){
console.log(session.message.attachments[0])
request.post({
url: 'xxx',
encoding: null,
json: true,
headers: {
'Content-Type': 'application/octet-stream',
'Prediction-Key': 'xxx'
},
body: session.message.attachments[0]
}, function(error, response, body){
console.log(body);
});
}
});
I believe that I may be sending the wrong format through to custom vision however I have been unable to figure it out as of yet.
I replicated your issue and it looks like the problem is your 'Content-Type'. You're attempting to pass JSON in your request, but setting the content-type as octet-stream. See my modified code below:
var bot = new builder.UniversalBot(connector, function (session) {
session.send("Hello"); //session.message.text
// If there is an attachment
if (session.message.attachments.length > 0){
console.log(session.message.attachments[0])
request.post({
url: 'https://northeurope.api.cognitive.microsoft.com/vision/v1.0/analyze?visualFeatures',
encoding: null,
json: true,
headers: {
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': 'Your API Key...'
},
body: session.message.attachments[0]
},
function (err, response, body) {
if (err) return console.log(err)
console.log(body);
});
}
});
When I run this, I get the error InvalidImageUrl which is to be expected as it's looking for a content on localhost. You could get round this by exposing your localhost using Ngrok.
I am trying to POST an image from my Node JS app to another REST API. I have the image in Mongo DB (as binary array data) that is read by Node JS and then is supposed to be POSTed to another API.
The problem I face is how do I send request data along with the image? I have this raw data (that is in JSON format) that should be POSTed along with image:
{"data":{"client":"abc","address": "123"},"meta":{"owner": "yourself","host": "hostishere"}}
I am required to do this using the 'request' module. I can use 'multer' if that helps better. But, I am stuck on how do I send the above request data along with the image stream. Below is my current code. Could you please help me finish it?
var options = {
host: 'hostname.com',
port: 80,
path: '/api/content',
method: 'POST',
headers:{
'Content-Type' : 'multipart/form-data'
}
};
var request = http.request(options, function(response) {
var str = '';
var respTime ='';
response.on('data', function (chunk) {
str = str.concat(chunk);
});
response.on('end', () => {
console.log('No more data in response.');
});
setTimeout(function() {
res.send(JSON.stringify(
{
'imageURL': IMG_URL,
'imageId': IMG_ID,
'body': JSON.parse(str)
}
));
}, 1000);
});
request.on('error', (e) => {
console.error('**** problem with request: ', e);
});
request.write(image.IMG_STR); //image.IMG_STR is the binary array representation of the image.
request.end();
UPDATE: 06/06/2017
So, I happened to talk to the REST team that provides the end point and found out that the data should be sent in the following specific format. Below is a snapshot of the request that succeeded. Could someone help me with the Node code that I should use? I have tried form-data package but have been getting the same error:
if you have control over "the other API" too, you could include the image as base64 representation of the binary data in the post-body (and decode it on the API side)
answer to the update 06/06/2017:
according to the screenshot the API requires multipart/formdata.
such requests with the "request"-module are documented in https://github.com/request/request#multipartform-data-multipart-form-uploads
quick example (not tested):
var formData = {
Data: {data: {client: "abc" ...},
file: fs.createReadStream('testImage_2.jpg'),
};
request.post({url:'<YourUrl>', formData: formData}, function optionalCallback(err, httpResponse, body) {
if (err) {
return console.error('upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
});
If you add the body to your request with the JSON data, you should be able to send it:
var options = {
host: 'hostname.com',
port: 80,
path: '/api/content',
method: 'POST',
headers:{
'Content-Type' : 'multipart/form-data'
},
body: {
"data": {"client":"abc","address": "123"},
"meta":{"owner": "yourself","host": "hostishere"}
}
};
What I don't understand is why you have a setTimeout with res.send when there is no res variable defined anywhere.
I have a external web service, which returns image.
and I have node express route which call that external web service.
I am struggling to pass the returned object of external web service (which is image) as express route's return object
here is this example, I am trying to get image from external URL and passing it as it is.. it doesn't work, can someone help me know to do so?
exports.getImage = function (req, res) {
var http = require('http');
var options = {
host: 'http://www.gettyimages.co.uk',
path: '/CMS/StaticContent/1391099215267_hero2.jpg',
method: 'GET',
headers: {
"content-type": "image/jpeg"
}
};
var request = http.request(options, function(response) {
var imagedata = '';
response.setEncoding('binary');
response.on('data', function(chunk){
imagedata += chunk
});
response.on('end', function() {
console.log('imagedata: ', imagedata);
res.writeHead(200, {'Content-Type': 'image/jpeg' });
res.send(imagedata);
});
}).on("error", function(e) {
console.log("Got error: " + e.message, e);
});
request.end();
};
The issue with the existing code is that res.send() defaults to a non-binary encoding when you pass it a string, so your data will end up getting mangled because of that.
Secondly, you're better off just streaming the data instead, that way you aren't buffering the entire image in memory every time. Example:
var request = http.get(options, function(response) {
res.writeHead(response.statusCode, {
'Content-Type': response.headers['content-type']
});
response.pipe(res);
}).on("error", function(e) {
console.log("Got error: " + e.message, e);
});
Lastly, the host value is wrong, it should just be hostname only (no scheme): www.gettyimages.co.uk. Also, setting "content-type": "image/jpeg" in the request headers doesn't make sense (and can be removed) since you're not sending a jpeg image in your request.
After struggling a few days trying to get something to work and getting no where, I was wondering if someone has gotten iOS Receipt Validation working on Node.js. I have tried the node module iap_verifier found here but I could not get it to work properly for me. the only response I received back form Apples servers is 21002, data was malformed.
One thing that has worked for me was a client side validation request to apples servers that I got directly from the tutorials provided by Apple here, with the code shown below.
// The transaction looks ok, so start the verify process.
// Encode the receiptData for the itms receipt verification POST request.
NSString *jsonObjectString = [self encodeBase64:(uint8_t *)transaction.transactionReceipt.bytes
length:transaction.transactionReceipt.length];
// Create the POST request payload.
NSString *payload = [NSString stringWithFormat:#"{\"receipt-data\" : \"%#\", \"password\" : \"%#\"}",
jsonObjectString, ITC_CONTENT_PROVIDER_SHARED_SECRET];
NSData *payloadData = [payload dataUsingEncoding:NSUTF8StringEncoding];
// Use ITMS_SANDBOX_VERIFY_RECEIPT_URL while testing against the sandbox.
NSString *serverURL = ITMS_SANDBOX_VERIFY_RECEIPT_URL;
// Create the POST request to the server.
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:serverURL]];
[request setHTTPMethod:#"POST"];
[request setHTTPBody:payloadData];
NSURLConnection *conn = [[NSURLConnection alloc] initWithRequest:request delegate:self];
[conn start];
I have a bunch of different code I have been using to send a wide array of things to my node server. and all of my different attempts have failed. I have even tried just funneling the "payloadData" I constructed in the client side validation example above to my server and sending that to Apples servers with the following code:
function verifyReceipt(receiptData, responder)
{
var options = {
host: 'sandbox.itunes.apple.com',
port: 443,
path: '/verifyReceipt',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(receiptData)
}
};
var req = https.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log("body: " + chunk);
});
});
req.write(receiptData);
req.end();
}
Where the function is passed the payloadData. The response received from Apple is always 21002. I'm still basically a node novice,so I can't figure out what exactly is going wrong. I think there might be some data corruption happening when I am sending the data from ObjC to my Node server, so perhaps I am not transmitting right.
If anyone can point me in the right direction, or provide some example of how they got receipt validation to work in node for them, it would be a great help. It would be great if anyone has had any experience with the iap_verifier module, and exactly what data it requires. I'll provide any code example I need to, as I have been fighting this process for a few days now.
Thanks!
For anyone using the npm library "request", here's how to avoid that bothersome 21002 error.
formFields = {
'receipt-data': receiptData_64
'password': yourAppleSecret
}
verifyURL = 'https://buy.itunes.apple.com/verifyReceipt' // or 'https://sandbox.itunes.apple.com/verifyReceipt'
req = request.post({url: verifyURL, json: formFields}, function(err, res, body) {
console.log('Response:', body);
})
This is my working solution for auto-renewable subscriptions, using the npm request-promise library.
Without JSON stringify-ing the body form, I was receiving 21002 error (The data in the receipt-data property was malformed or missing)
const rp = require('request-promise');
var verifyURL = 'https://sandbox.itunes.apple.com/verifyReceipt';
// use 'https://buy.itunes.apple.com/verifyReceipt' for production
var options = {
uri: verifyURL,
method: 'POST',
headers: {
'User-Agent': 'Request-Promise',
'Content-Type': 'application/x-www-form-urlencoded',
},
json: true
};
options.form = JSON.stringify({
'receipt-data': receiptData,
'password': password
});
rp(options).then(function (resData) {
devLog.log(resData); // 0
}).catch(function (err) {
devLog.log(err);
});
Do you have composed correctly receiptData? Accordlying with Apple specification it should have the format
{"receipt-data": "your base64 receipt"}
Modifying your code wrapping the base64 receipt string with receipt-data object the validation should works
function (receiptData_base64, production, cb)
{
var url = production ? 'buy.itunes.apple.com' : 'sandbox.itunes.apple.com'
var receiptEnvelope = {
"receipt-data": receiptData_base64
};
var receiptEnvelopeStr = JSON.stringify(receiptEnvelope);
var options = {
host: url,
port: 443,
path: '/verifyReceipt',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(receiptEnvelopeStr)
}
};
var req = https.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log("body: " + chunk);
cb(true, chunk);
});
res.on('error', function (error) {
console.log("error: " + error);
cb(false, error);
});
});
req.write(receiptEnvelopeStr);
req.end();
}