I'm new to Stack Overflow and this is my first question, so please bear with me. I never had to ask a question until I had a problem and couldn't find a solution. Anyways, my problem is my Node JS HTTPS server that I use to host my website has a problem where when the client requests a resource such as a video or an image, it's a 50% chance that the resource will properly load for the client. The times it doesn't load I get an SSL error in the console (for example, net::ERR_SSL_PROTOCOL_ERROR 200 (OK)). Sometimes it will load, sometimes it won't. I don't know if I have a misconfiguration with my server or not, but I need to get this fixed. I need the resource to load for the client 100% of the time. Any help would be great.
Here's how my server is configured.
const reqHandler = (req, res) => {
let body = '';
req.on('data', chunk => body += chunk.toString());
req.on('end', () => {
var path = req.url.split('?')[0];
var query = req.url.split('?')[1] || '';
res.statusCode = 200; // Assume a good request unless otherwise indicated
if (path.split('/').slice(-1)[0] === '') { // If client requests folder
if (existsSync(`./${root}${path}index.js`)) { // node-http has priority
let type = readFileSync(`./${root}${path}index.js`, 'utf-8').split('\n')[0];
type = type.replace('//', '').trim();
if (type === 'node-http') {
try {
delete require.cache[require.resolve(`./${root}${path}index.js`)];
require(`./${root}${path}index.js`).callback(req, res, body, query);
} catch (e) {
res.write(`Node HTTP exception: ${e.message}`);
res.end();
}
return;
}
} else { // Otherwise, fallback to index.html
path = path + 'index.html';
}
} else { // If the client didn't request a folder
if (path.split('.').length === 1) { // No extension? No problem!
// See if it's a folder
if (existsSync(`./${root}${path}/`)) { // Redirect, if exists
res.statusCode = 301;
res.setHeader('Location', path + '/');
res.end();
return;
} else { // If not folder, assume try to find it
var extensions = getType('get-extensions');
for (var i of extensions) {
if (existsSync(`./${root}${path}.${i}`)) {
path = `${path}.${i}`;
break;
}
}
}
}
var extension = path.split('.').slice(-1)[0];
// node-http
if (extension === 'js') {
if (!existsSync(`./${root}${path}`)) {
res.statusCode = 404;
res.end();
return;
}
let type = readFileSync(`./${root}${path}`, 'utf-8').split('\n')[0];
type = type.replace('//', '').trim();
if (type === 'node-http') {
try {
delete require.cache[require.resolve(`./${root}${path}`)];
require(`./${root}${path}`).callback(req, res, body, query);
} catch (e) {
res.write(`Node HTTP exception: ${e.message}`);
res.end();
}
return;
}
}
if (extension === 'ws') {
// Websocket connection
return;
}
// videos
if (extension === 'mp4') {
var vidPath = `./${root}${path}`;
readFile(vidPath, (err, data) => {
if (err) {
if (err.code === 'ENOENT') {
res.statusCode = 404;
res.end('404 Not Found');
}
res.end();
return
} else {
var stats = statSync(vidPath);
let size = stats.size;
let chunkSize = 10 ** 6; // 1 megabyte (1,000,000 bytes)
let start = req.headers.range ? Number(req.headers.range.replace(/\D/g, "")) : 0;
let end = Math.min(start + chunkSize, size - 1);
let contentLength = end - start + 1;
res.setHeader('Content-Range', `bytes ${start}-${end}/${size}`);
res.statusCode = 206;
res.writeHead(206, {
'Content-Range': `bytes ${start}-${end}/${size}`,
'Content-Type': 'video/mp4',
'Content-Length': contentLength,
'Accept-Ranges': 'bytes',
'Date': new Date().toUTCString()
});
let stream = createReadStream(vidPath, {start, end});
stream.pipe(res);
}
});
return;
}
}
readFile(`./${root}${path}`, (err, data) => {
if (err) {
if (err.code === 'ENOENT') {
res.statusCode = 404;
res.end('404 Not Found');
}
} else {
let type = getType(path.split('.').slice(-1)[0]);
if (type !== null) {
res.setHeader('Content-Type', type);
}
res.end(data);
}
});
});
}
// Redirect HTTP traffic to HTTPS
http.createServer((req, res) => {
if (req.headers.host === 'domain.com') {
let redirectUrl = `https://${req.headers.host}${req.url}`;
res.statusCode = 301;
res.setHeader('Location', redirectUrl);
res.end();
return;
}
reqHandler(req, res);
}).listen(80);
// HTTPS server
const server = https.createServer({
cert: readFileSync('/path/to/cert.pem').toString(),
key: readFileSync('/path/to/key.pem').toString(),
ca: readFileSync('/path/to/chain.pem').toString()
}, reqHandler);
server.listen(443);
Related
So, I am trying to write code for a bot on discord to search google for a desired image. I was able to get the bot to respond to simple commands with a version of code (A1), only with more options, however I have been unable to get the image search code (A2) to work nor have I been able to combine the two codes together. I would greatly appreciate some advice as to where I'm going wrong. :-) thank you.
(A1):
const Discord = require('discord.js');
const client = new Discord.Client();
client.on('ready', () => {
console.log(`Logged in as ${client.user.tag}!`);
});
client.on('message', msg => {
if (msg.content === 'ping') {
msg.reply('pong');
}
});
client.login('insert discord code here');
(A2)
var cheerio = require("cheerio");
var request = require("request");
var discord = require("discord.js");
var client = new discord.Client();
client.login("insert discord code here");
client.on("ready", function() {
console.log("logged in");
});
client.on("message", function(message) {
var parts = message.content.split("search.", "search query");
/* Simple command manager */
if (parts[0] === "search.") { //I want to check this, but I am not sure how as I want to start the message with the command
image(message, parts);
}
});
function image(message, parts) {
var search = parts.slice(1).join(["search."]);
var options = {
url: "http://results.dogpile.com/serp?qc=images&q=" + search,
method: "GET",
headers: {
"Accept": "text/html",
"User-Agent": "Chrome"
}
};
request(options, function(error, response, responseBody) {
if (error) {
return;
}
$ = cheerio.load(responseBody);
var links = $(".image a.link");
var urls = new Array(links.length).fill(0).map((v, i) => links.eq(i).attr("href"));
console.log(urls);
if (!urls.length) {
return;
}
message.channel.send( urls[0] );
});
}
I have corrected the errors:
var cheerio = require("cheerio"); /* Used to extract html content, based on
jQuery || install with npm install cheerio */
var request = require("request"); /* Used to make requests to URLs and fetch
response || install with npm install request */
var discord = require("discord.js");
var client = new discord.Client();
client.on("ready", function() {
console.log("logged in");
});
client.on("message", function(message) {
var parts = message.content.split(" ");
/* Simple command manager */
if (parts[0] === "search.") {
image(message, parts);
}
});
function image(message, parts) {
/* extract search query from message */
var search = parts.slice(1).join(" ");
var options = {
url: "http://results.dogpile.com/serp?qc=images&q=" + search,
method: "GET",
headers: {
"Accept": "text/html",
"User-Agent": "Chrome"
}
};
request(options, function(error, response, responseBody) {
if (error) {
// handle error
return;
}
/* Extract image URLs from responseBody using cheerio */
$ = cheerio.load(responseBody);
var links = $(".image a.link");
var urls = new Array(links.length).fill(0).map((v, i) =>
links.eq(i).attr("href"));
console.log(urls);
if (!urls.length) {
return;
}
// Send result
message.channel.send( urls[0] );
});
client.on('message', msg => {
if (msg.content === 'ping') {
msg.reply('pong');
} else if (msg.content === '') {
msg.reply('');
} else if (msg.content === '') {
msg.reply('');
} else if (msg.content === '') {
msg.reply('');
} else if (msg.content === '') {
msg.reply('');
}
});
}
I have an API that downloads multiple files from AWS S3, creates a zip which is saved to disk, and sends that zip back to the client. The API works, but I have no idea how to handle the response / download the zip to disk on the client side.
This is my API:
reports.get('/downloadMultipleReports/:fileKeys', async (req, res) => {
var s3 = new AWS.S3();
var archiver = require('archiver');
const { promisify } = require('util');
var str_array = req.params.fileKeys.split(',');
console.log('str_array: ',str_array);
for (var i = 0; i < str_array.length; i++) {
var filename = str_array[i].trim();
var filename = str_array[i];
var localFileName = './temp/' + filename.substring(filename.indexOf("/") + 1);
console.log('FILE KEY >>>>>> : ', filename);
const params = { Bucket: config.reportBucket, Key: filename };
const data = await (s3.getObject(params)).promise();
const writeFile = promisify(fs.writeFile);
await writeFile(localFileName, data.Body);
}
// create a file to stream archive data to.
var output = fs.createWriteStream('reportFiles.zip');
var archive = archiver('zip', {
zlib: { level: 9 } // Sets the compression level.
});
// listen for all archive data to be written
// 'close' event is fired only when a file descriptor is involved
output.on('close', function() {
console.log(archive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
// This event is fired when the data source is drained no matter what was the data source.
// It is not part of this library but rather from the NodeJS Stream API.
// #see: https://nodejs.org/api/stream.html#stream_event_end
output.on('end', function() {
console.log('Data has been drained');
});
// good practice to catch warnings (ie stat failures and other non-blocking errors)
archive.on('warning', function(err) {
if (err.code === 'ENOENT') {
// log warning
} else {
// throw error
throw err;
}
});
// good practice to catch this error explicitly
archive.on('error', function(err) {
throw err;
});
// pipe archive data to the file
archive.pipe(output);
// append files from a sub-directory, putting its contents at the root of archive
archive.directory('./temp', false);
// finalize the archive (ie we are done appending files but streams have to finish yet)
// 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand
archive.finalize();
output.on('finish', () => {
console.log('Ding! - Zip is done!');
const zipFilePath = "./reportFiles.zip" // or any file format
// res.setHeader('Content-Type', 'application/zip');
fs.exists(zipFilePath, function(exists){
if (exists) {
res.writeHead(200, {
"Content-Type": "application/octet-stream",
"Content-Disposition": "attachment; filename=" + "./reportFiles.zip"
});
fs.createReadStream(zipFilePath).pipe(res);
} else {
response.writeHead(400, {"Content-Type": "text/plain"});
response.end("ERROR File does not exist");
}
});
});
return;
});
And this is how I am calling the API / expecting to download the response:
downloadMultipleReports(){
var fileKeysString = this.state.awsFileKeys.toString();
var newFileKeys = fileKeysString.replace(/ /g, '%20').replace(/\//g, '%2F');
fetch(config.api.urlFor('downloadMultipleReports', { fileKeys: newFileKeys }))
.then((response) => response.body())
this.closeModal();
}
How can I handle the response / download the zip to disk?
This is what ended up working for me:
Server side:
const zipFilePath = "./reportFiles.zip";
fs.exists(zipFilePath, function(exists){
if (exists) {
res.writeHead(200, {
"Content-Type": "application/zip",
"Content-Disposition": "attachment; filename=" + "./reportFiles.zip"
});
fs.createReadStream(zipFilePath).pipe(res);
} else {
response.writeHead(400, {"Content-Type": "text/plain"});
response.end("ERROR File does not exist");
}
});
Client side:
downloadMultipleReports(){
var fileKeysString = this.state.awsFileKeys.toString();
var newFileKeys = fileKeysString.replace(/ /g, '%20').replace(/\//g, '%2F');
fetch(config.api.urlFor('downloadMultipleReports', { fileKeys: newFileKeys }))
.then((res) => {return res.blob()})
.then(blob =>{
download(blob, 'reportFiles.zip', 'application/zip');
this.setState({isOpen: false});
})
}
I have an issue with Ajax in a form with node.js, I'm developing a simple node.js currency converter application and populating the data on the frontend (HTML) using Ajax. However it's not working, any help is seriously appreciated. Thanks.
1. Frontend
Change this
xmlhttp.open("GET","http://localhost:9099/", true);
to
xmlhttp.open("POST","http://localhost:9099/", true);
as your backend server accepts POST for getting the answer.
2. Backend
Remove response.end and response.writeHead from bottom, and move it to where you are calculating store.
Your final code:
http.createServer(function(request, response) {
switch (request.method) {
case 'POST':
if (request.url === "/") {
var requestBody = '';
request.on('data', function(data) {
requestBody += data;
if (requestBody.length > 1e7) {
response.writeHead(413, {
'Content-Type': 'text/plain'
});
response.end('Request Entity is too large');
}
});
request.on('end', function(data) {
console.log(requestBody);
var formData = qs.parse(requestBody);
var requestBofy = '';
// I renamed the callback parameter to response2
https.get('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml?8f65cd5d1af1727c40d2a89a31c6a0f1', function(response2) {
if (response2.statusCode >= 200 && response2.statusCode < 400) {
response2.on('data', function(data_) {
requestBofy += data_.toString();
});
response2.on('end', function() {
console.log(requestBofy);
parser.parseString(requestBofy, function(err, result) {
console.log('FINISHED', err, result);
var xml = requestBofy;
var parseString = require('xml2js').parseString;
parseString(xml, function(err, result) {
var jFile = JSON.stringify(result);
var parsedResponse = JSON.parse(jFile);
var rateHUF = parsedResponse['gesmes:Envelope']['Cube'][0]['Cube'][0]['Cube'][6]['$'].rate;
var rateINR = parsedResponse['gesmes:Envelope']['Cube'][0]['Cube'][0]['Cube'][22]['$'].rate;
var store = 'No value';
if (formData.vSelectedValue == 'HUF' && formData.vSelectedValue2 == 'INR') {
store = Math.round(formData.vFirstNo * (rateINR / rateHUF));
} else {
store = Math.round(formData.vFirstNo * (rateHUF / rateINR));
}
// Your response should end here
response.writeHead(200, {
"Content-Type": "text/html"
});
response.end('Your Answer: ' + store);
});
});
});
}
});
});
} else {
response.writeHead(404, {
'Content-Type': 'text/plain'
});
response.end('404 - Page not found');
}
break;
case 'GET':
if (request.url === "/") {
getFileContent(response, 'public/home.html', 'text/html');
} else {
response.writeHead(404, {
'Content-Type': 'text/plain'
});
response.end('404 - Page not found');
}
break;
default:
response.writeHead(404, {
'Content-Type': 'text/plain'
});
response.end('404 - Page not found');
}
}).listen(9099);
I'm attempting to post an image onto the twitter api, v1.1
I've tried just about all the example out there, and nothing seems to be able to post it.
include Posting images to twitter in Node.js using Oauth
I'm using the oauth library mentioned there, and I also had jsOauth, which I thought I'd give a shot according to https://gist.github.com/lukaszkorecki/1038408
Nothing has worked, and at this point I'm starting to lose hope on whether I can even do this.
function postStatusWithMedia(status, file) {
var err = new Object();
if(fs.existsSync(file) === false) {
err.message = "File not found :(";
parseTwitterError(err);
} else {
var oauth = OAuth(options = {
"consumerKey": consumer_key,
"consumerSecret": consumer_secret,
"accessTokenKey": access_token,
"accessTokenSecret": access_token_secret
});
callbacks = {
onSuccess : function() {
console.log('upload worked!')
},
onFailure : function() {
console.log('upload failed!');
console.dir(arguments);
}
},
uploadData = {
'status' : status,
'media' : Base64.encode(fs.readFileSync(file))
};
oauth.post('https://api.twitter.com/1.1/statuses/update_with_media.json',uploadData, callbacks.onSuccess, callbacks.onFailure);
return false;
}
}
If it can't be done, can you please explain why?
Otherwise, anything that could lead me to the right direction would be great.
var fs = require('fs');
var request = require('request');
var FormData = require('form-data');
var utf8 = require('utf8');
// Encode in UTF-8
status = utf8.encode(status);
var form = new FormData();
form.append('status', status)
form.append('media[]', fs.createReadStream(file));
// Twitter OAuth
form.getLength(function(err, length){
if (err) {
return requestCallback(err);
}
var oauth = {
consumer_key: consumer_key,
consumer_secret: consumer_secret,
token: access_token,
token_secret: access_token_secret
};
var r = request.post({url:"https://api.twitter.com/1.1/statuses/update_with_media.json", oauth:oauth, host: "api.twitter.com", protocol: "https:"}, requestCallback);
r._form = form;
r.setHeader('content-length', length);
});
function requestCallback(err, res, body) {
if(err) {
throw err;
} else {
console.log("Tweet and Image uploaded successfully!");
}
}
I ended up using request and node-form-data to manually construct a multipart/form-data request and send it with the status request, utf8 was for encoding the status into UTF-8, not doing so caused issues with '<3' and other characters.
I have not tested these code.Its from my colleague.sure the code is working.
Perhaps this will help.
//twitter_update_with_media.js
(function() {
var fs, path, request, twitter_update_with_media;
fs = require('fs');
path = require('path');
request = require('request');
twitter_update_with_media = (function() {
function twitter_update_with_media(auth_settings) {
this.auth_settings = auth_settings;
this.api_url = 'https://api.twitter.com/1.1/statuses/update_with_media.json';
}
twitter_update_with_media.prototype.post = function(status, imageUrl, callback) {
var form, r;
r = request.post(this.api_url, {
oauth: this.auth_settings
}, callback);
form = r.form();
form.append('status', status);
return form.append('media[]', request(imageUrl));
};
return twitter_update_with_media;
})();
module.exports = twitter_update_with_media;
}).call(this);
next file
//upload_to_twitter.js
var tuwm = new twitter_update_with_media({
consumer_key: TWITTER_OAUTH_KEY,
consumer_secret: TWITTER_OAUTH_SECRET,
token: access[0],
token_secret: access[1]
});
media_picture.picture = imageURL;
if (media_picture.picture) {
console.log('with media upload');
request.head(media_picture.picture,
function (error, response, body) {
if (!error && response.statusCode == 200) {
var image_size = response.headers['content-length'];
if (image_size > 2000000) { // 2mb max upload limit
console.log('greater than 2mb');
sendMessageWithoutImage(err, req, res, next, twit, wallpost, access);
} else {
console.log('less than 2mb');
console.log('twitter text', content);
tuwm.post(content, media_picture.picture, function(err, response) {
if (err) {
console.log('error', err);
return next(err);
}
error_parse = JSON.parse(response.body);
console.log('with media response', response.body);
if (error_parse.errors) {
console.log('have errors', error_parse);
res.json({
status: 500,
info: error_parse.errors[0].code + ' ' + error_parse.errors[0].message
});
} else {
res.json({
status: 200,
info: "OK",
id: response.id
});
}
});
}
}
});
for example:
knox.js:
knox.putFile("local.jpeg", "upload.jpeg", {
"Content-Type": "image/jpeg"
}, function(err, result) {
if (err != null) {
return console.log(err);
} else {
return console.log("Uploaded to amazon S3");
I have two images in the same directory as knox.js, local.jpeg and local2.jpeg, i am able to upload local.jpeg to s3, but not local2.jpeg, both files have the same permissions. am i missing anything here? thanks
My implementation without store in locale. With express, knox, mime, fs.
var knox = require('knox').createClient({
key: S3_KEY,
secret: S3_SECRET,
bucket: S3_BUCKET
});
exports.upload = function uploadToAmazon(req, res, next) {
var file = req.files.file;
var stream = fs.createReadStream(file.path)
var mimetype = mime.lookup(file.path);
var req;
if (mimetype.localeCompare('image/jpeg')
|| mimetype.localeCompare('image/pjpeg')
|| mimetype.localeCompare('image/png')
|| mimetype.localeCompare('image/gif')) {
req = knox.putStream(stream, file.name,
{
'Content-Type': mimetype,
'Cache-Control': 'max-age=604800',
'x-amz-acl': 'public-read',
'Content-Length': file.size
},
function(err, result) {
console.log(result);
}
);
} else {
next(new HttpError(HTTPStatus.BAD_REQUEST))
}
req.on('response', function(res){
if (res.statusCode == HTTPStatus.OK) {
res.json('url: ' + req.url)
} else {
next(new HttpError(res.statusCode))
}
});
That's because your code does not uploade local2.jpeg!
You code will only pushes the file named local.jpeg. You should, for every file, invoke the knox.put() method. I also advise you to have some helper function that will do some string formatting to rename to uploaded file on s3 (or just keep it as it is :) )
var files = ["local.jpeg", "local1.jpeg"];
for (file in files){
var upload_name = "upload_"+ file; // or whatever you want it to be called
knox.putFile(file, upload_name, {
"Content-Type": "image/jpeg"
}, function (err, result) {
if (err != null) {
return console.log(err);
} else {
return console.log("Uploaded to amazon S3");
}
});
}