A lot of HTTP requests and save as json: connect ETIMEDOUT - javascript

I have a problem with my code. I need to capture a lot of data by API. First, i had a problem with memory and now I see: connect ETIMEDOUT error
My code:
var fetch = require('node-fetch');
const async = require('async');
const request = require('request');
var jsonfile = require('jsonfile');
var file = '/temp/data.json'
let urls = [];
for (var i = 1; i <= 608469; i++) {
let url = "https://api.demo.com/v2.1/people?user_key=auth_key&page="+i+"&sort_order=created_at%20DESC";
urls.push(url);
}
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function(err, res, body) {
callback(err, body);
}
);
}
async.map(urls, httpGet, function (err, res){
if (err) return console.log(err);
console.log(typeof(res));
jsonfile.writeFileSync(file, res);
});

Just replace async's map method with eachLimit. Key here is limit. It still will run http request for each link, but in no more than limit paraller threads. Here is a doc: https://caolan.github.io/async/docs.html#eachLimit

Related

Downloading a lot of images with NodeJS

I got a large list of URLs (8000+) I want to download the images from.
I created a script that will download the files, as long as I'm setting the limit to about a 100.
If I try to download more I'm getting errors like
(node:6740) UnhandledPromiseRejectionWarning: Error: read ECONNRESET
(node:3808) UnhandledPromiseRejectionWarning: Error: connect ETIMEDOUT
(node:7052) UnhandledPromiseRejectionWarning: Error: Client network socket disconnected before secure TLS connection was established
This is the code that reads my CSV with URL's:
const fotoDownload = require('./async-foto');
const csv = require('csv-parser');
const fs = require('fs');
const results = [];
fs.createReadStream('\\\\hk-nas02\\import\\Partij\\Files\\partij.csv')
.pipe(csv( { separator: ';'}))
.on('data', (data) => results.push(data))
.on('end', () => {
console.log(results.length)
let NoPartijen = results.length;
for(i = 0;i < 50; i++){
//console.log(results[i]);
itemno = i
path = '..\\files\\images\\'+results[i]['partij.VPARTIJNR']+'_'+results[i]['partij.PARTIJNR']+'_H.jpg';
console.log(path)
fotoDownload.fotoDownload(results[i]['partij.EXFOTOURL'], path, itemno)
}
console.log('Test');
});
and calls the following code to download:
const util = require('util')
const fs = require('fs')
const axios = require("axios").default;
module.exports = {
fotoDownload: async (url, path, itemno) => {
try {
const response = await axios({
method: "GET",
url: url,
responseType: "stream",})
await response.data.pipe(fs.createWriteStream(path));
console.log('Start foto download' + itemno);
return;
} catch (err) {
throw new Error(err)
}
}
}
I'm assuming I need the chop the data into chunks or something, but I'm a bit lost here. Can someone put me in the right direction?
You need to set a timeout for downloading the images & add connection keep-alive
something like this
axios.defaults.timeout = 30000; //or whateve your desired timeout
axios.defaults.httpsAgent = new https.Agent({ keepAlive: true });
module.exports = {
fotoDownload: async (url, path, itemno) => {
try {
const response = await axios({
method: "GET",
url: url,
responseType: "stream",})
await response.data.pipe(fs.createWriteStream(path));
console.log('Start foto download' + itemno);
return;
} catch (err) {
throw new Error(err)
}
}
}
Note: make sure u have https installed
run npm i http or npm i https

Node Js as server http to consume external web service

I want to install Node Js to consume a external web service in SOAP.
This is the architecture:
|Webshere application| ---HTTP---> |Node Js| ----HTTPS--> ||External Web Service||
This is the script I am doing:
var http = require('http');
var httpntlm = require('httpntlm');
var options = {
url: "https://serverexterno/WebServiceSOAP?wsdl"
}
const server = http.createServer( (req, res) => {
let body = '';
let headers = req.headers
let as400Response ='';
req.setEncoding('ascii');
req.on('data', (chunk) => {
body += chunk;
});
httpntlm.post(options,
function (err, resp) {
console.log (resp);
if(err) {
return console.log(err);
}
console.log( resp.headers);
console.log("resp.body" + resp.body);
body = resp.body;
res.statusCode = 200;
return res.end(resp.body);
});
console.log ('Finish ....');
} catch (er) {
res.statusCode = 400;
return res.end(`error: ${er.message}`);
}
});
});
server.listen(8084, function(){
console.log("Server listening on: http://localhost:%s", 8087);
});
However, I don't receive as expected. I receive the message in html with the message "Hello! This is an Axis2 Web Service!". It seems like the webservice is not executing.
Can someone help me why get this error?

Headers, Node.js , Can\'t set headers after they are sent

I'm new in node.js and I'm implementing a Static file server.
what i'm trying to do is just get the file name from the url and display it. running the code id get this error:
_http_outgoing.js:489
throw new Error('Can\'t set headers after they are sent.');
Error: Can't set headers after they are sent.
this is my code
#!/usr/bin/env node
/*
* Basic node.js HTTP server
*/
const http = require('http');
const url = require('url');
const fs = require('fs');
const routes = Object.create(null);
function file(rew, res, serched) {
let fileSerched = serched[serched.length - 1]
fileSerched = __dirname + "/NodeStaticFiles/" + fileSerched;
fs.readFile(fileSerched, function(err, data) {
if (err) {
res.statusCode = 500;
res.end(`Error getting the file: ${err}.`);
} else {
res.setHeader('Content-type', 'text/plain');
res.writeHead(200)
res.end(data);
}
})
}
routes['file'] = file;
function onRequest(req, res) {
const pathname = url.parse(req.url).pathname
const uri = pathname.split('/', 3)[1]
let splittedPathname = pathname.split('/')
splittedPathname.shift()
splittedPathname.shift()
if (typeof routes[uri] === 'function') {
routes[uri](req, res, splittedPathname);
} else {
res.statusCode = 404;
res.end(`File not found!`);
}
res.end()
}
http.createServer(onRequest).listen(3000);
console.log('Server started at localhost:3000')
You need to make sure your code won't call something like res.end() more than once. For example:
function onRequest(req, res) {
const pathname = url.parse(req.url).pathname
const uri = pathname.split('/', 3)[1]
let splittedPathname = pathname.split('/')
splittedPathname.shift()
splittedPathname.shift()
if (typeof routes[uri] === 'function') {
routes[uri](req, res, splittedPathname);
} else {
// THIS WILL CALL res.end() and continue on
res.statusCode = 404;
res.end(`File not found!`);
}
// THIS WILL CALL res.end() AGAIN!!
res.end()
}
try adding a return after you call res.end() in an if/else

API with Node & Express gives back old cached response until newer is retrieved

I cannot find how to prevent this behavior. I have a very simple Express API that works well and gives the intended response. It´s connected to a database storing press articles.
The problem comes when I request article with id 1. It shows up. Then I try to request article with id number 2, and instead I am getting the cached response of the previous article. This goes on for as long as it takes for the database to actually deliver the article 2.
This is what I have:
var sql = require('mssql');
var express = require('express');
var path = require('path');
var response;
var dbConfig = {
//connection details
};
function getArt(an) {
var conn = new sql.Connection(dbConfig);
conn.connect().then(function() {
var req = new sql.Request(conn);
req.query(/* query to the DB */).then(function(recordset) {
response = recordset;
conn.close();
})
.catch(function(err) {
response = err;
conn.close();
});
})
.catch(function(err) {
response = err;
conn.close();
})
return response;
}
var app = express();
app.get('/sourcedeletion/api/v2.0/article/:an', function(req, res) {
res.send(getArt(req.params.an.toString()));
});
app.listen(3000);
What part am I missing, that could help me to get the real response on every request, rather than the previous cached one until the new one arrives from the server?
You're using a global variable response that is not enclosed in the scope of the request. This is prone to error! getArt returns the value of the global variable response without waiting for the promise to execute, so it returns an old value. You should do:
const sql = require('mssql');
const express = require('express');
const path = require('path');
const dbConfig = {
//connection details
};
const getArt = (an) => {
const conn = new sql.Connection(dbConfig);
return conn.connect()
.then(() => {
const req = new sql.Request(conn);
return req.query(/* query to the DB */)
})
.catch(err => err)
.then(response => {
conn.close();
return response;
});
});
};
const app = express();
app.get('/sourcedeletion/api/v2.0/article/:an', (req, res) => {
getArt(req.params.an.toString())
.then(response => {
res.send(response);
});
});
app.listen(3000);
con.connection()
.. is an async call, so before the new response is set, the function returns what ever was there before.
If you want to make sure that the function waits for the connection to finish before returning a value try to change to the following:
return conn.connect().then(function() {
var req = new sql.Request(conn);
return req.query(/* query to the DB */).then(function(recordset) {
response = recordset;
return response;
})
.catch(function(err) {
response = err;
return response;
});
})

Node.js beginner- request without query strings

I'm new to node.js. A project I'm working on requires that a large array (625 items) be stored in a MySQL database as a string and I'm using Node MySQL to accomplish that.
Right now I have it so that the row is updated when a request is made to "/(column name)?(value)". Unfortunately, it didn't take long to learn that passing 625 items as a string to node isn't the most efficient way of doing things.
Do you have any suggestions for alternate methods of passing a large array as a string besides querystrings?
var http = require('http'),
mysql = require("mysql"),
url = require("url"),
express = require("express");
var connection = mysql.createConnection({
user: "root",
database: "ballot"
});
var app = express();
app.use(express.bodyParser());
app.options('/', function (request, response)
{
response.header("Access-Control-Allow-Origin", "*");
response.end();
});
app.get('/', function (request, response)
{
connection.query("SELECT * FROM pathfind;", function (error, rows, fields) {
for(i=0;i<rows.length;i++)
{
response.send('<div id="layers">'+rows[i].layers+'</div> \
<div id="speed">'+rows[i].speed+'</div> \
<div id="direction">'+rows[i].direction+'</div>');
}
response.end();
});
});
app.post('/', function (request, response)
{
console.log(request.param('data', null));
var urlData = url.parse(request.url, true);
if(urlData.pathname = '/layers')
{
col = "layers";
}
else if(urlData.pathname = '/speed')
{
col = "speed";
}
else if(urlData.pathname = '/direction')
{
col = "direction";
}
req.addListener("data", function(data) {
connection.query("UPDATE pathfind SET "+col+"="+data+" WHERE num=0", function (error, rows, fields) {
if (error)
{
app.close();
}
});
});
});
app.listen(8080);**
EDIT: So now I know I need to use posts. I've rewritten the above code using express.js. Most examples that I look at online use posts with HTML forms, which is fine, but my project needs to post data via an AJAX request. Can someone show me how that data is parsed in node.js?
You can extract post data like this.
if (req.method == 'POST') {
var body = '';
req.on('data', function (chunk) {
body += chunk;
});
req.on('end', function () {
var postData = body.toString();
console.log(postData);
});
}

Categories

Resources