My project contains few methods that uses node-horseman, but after every server restart, the process will die after I executed something, whatever method it is. For example, I have one method that retrieves a list of Alerts I have in Google Alerts:
* getRSSNames() {
return new Promise((resolve, reject) => {
horseman
.userAgent(this.userAgent)
.open('https://www.google.com/alerts')
.evaluate(function() {
var item = []
jQuery('#manage-alerts-div li').each(function(key, val) {
item.push({
dataId: jQuery(val).attr('data-id'),
name: jQuery(val).find('.query_div').text(),
url: jQuery(val).find('a').attr('href')
})
})
return item
}).then(function(data) {
resolve(data)
})
.close()
})
}
Success, if I've just initiated the local server. Then, rendering the list in a select, choose one and try to delete it:
* deleteRSSFeed() {
let feedID = this.feedID.replace(/%3A/g, ':')
return new Promise((resolve, reject) => {
horseman
.userAgent(this.userAgent)
.open('https://google.com/alerts')
.evaluate(function(selector) {
return jQuery('[data-id="' + selector + '"]').find('.delete_button').click()
}, feedID)
.then(function(data) {
resolve(data)
})
.close()
})
}
It will cause the error:
Unhandled rejection HeadlessError: Phantom Process died
at poll_func (/home/gabriel/Sites/co-report/api/node_modules/node-phantom-simple/node-phantom-simple.js:584:10)
at /home/gabriel/Sites/co-report/api/node_modules/node-phantom-simple/node-phantom-simple.js:52:7
at ClientRequest.<anonymous> (/home/gabriel/Sites/co-report/api/node_modules/node-phantom-simple/node-phantom-simple.js:501:11)
at emitOne (events.js:96:13)
at ClientRequest.emit (events.js:188:7)
at Socket.socketErrorListener (_http_client.js:308:9)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at emitErrorNT (net.js:1272:8)
at _combinedTickCallback (internal/process/next_tick.js:74:11)
at process._tickCallback (internal/process/next_tick.js:98:9)
I also have a function that logs in in Google. If I login and I refresh my page with the server running, since I logged in and when refreshing it will retrieve the feed list, it will cause the same error. Please, help me.
Ok... My solution was to instantiate horseman inside each method. That's it.
Related
im trying to scrap a website using pupeteer , this website has captcha and i use a 3rd party library/api to bypass the captcha
here is the simplified version of my code
const puppeteer = require('puppeteer');
const dbc = require('./dpc/deathbycaptcha');
async function init()
{
let page = launchPuppeteer();
try {
await page.goto(`https://www.example.com`, {timeout: 60000});
await captcha_element.screenshot({ path: setting.captcha , omitBackground: true, });
let captchaResult = await solve_captcha().catch((error)=> {throw new Error( error) });
log('ALL DONE');
}
catch (e) {
log('=====================ERROR============================');
log(e);
}
}
here is solve_captcha function
function solve_captcha(){
return new Promise(( resolve , reject )=>{
try
{
const client = new dbc.HttpClient('myUsername', 'myPassword');
client.decode({captcha: setting.captcha}, (captcha) => {
if (captcha) {
resolve(captcha['text']);
}
else
{
reject('error')
}
});
}
catch (e) {
reject(e.toString());
}
})
}
so sometimes i cant connect to captcha solving api due to down server or network problems , it would throw this error
C:\pathToProject\dpc\deathbycaptcha.js:218
throw new Error(err.message);
^
Error: read ECONNRESET
at ClientRequest.<anonymous> (C:\pathToProject\dpc\deathbycaptcha.js:218:15)
←[90m at ClientRequest.emit (events.js:310:20)←[39m
←[90m at Socket.socketErrorListener (_http_client.js:426:9)←[39m
←[90m at Socket.emit (events.js:310:20)←[39m
←[90m at emitErrorNT (internal/streams/destroy.js:92:8)←[39m
←[90m at emitErrorAndCloseNT (internal/streams/destroy.js:60:3)←[39m
←[90m at processTicksAndRejections (internal/process/task_queues.js:84:21)←[39m
this error happens in the deathbycaptcha.js which i've required in my code with
const dbc = require('./dpc/deathbycaptcha');
here is the simplified version of the code in the in the deathbycaptcha.js which is cuzing the error
class HttpClient extends Client {
_call(arguments here) {
var options = {someoptions};
var form = new FormData();
const request = form.submit(options, (err, response) => {
if (err) {
console.log(err.toString())
throw new Error(err.message);
}
// more code here
}
}
so here is the problem , i have this code running inside 2 try/catch blocks
first in the caller function (init()) and second one in the solve_captcha function but none of them are able to catch this error and it will break all the code and the program will exit and i have to run it again
why cant i catch and handle this error in my code ?
I am working on a Discord bot that needs to update someone's nickname, and then reset it back to normal after 2 hours.
I have the code all set up and working when the timeout is set to a few seconds, but when I push the timeout to 2 hours I get a failed request to the discord server. I'm thinking this is because that channel doesn't stay connected that long -- not sure.
Here is the error:
UnhandledPromiseRejectionWarning: FetchError: request to https://discord.com/api/v7/guilds/297251675616444416/members/711322838556213319 failed, reason: getaddrinfo ENOTFOUND discord.com
at RequestHandler.execute (C:\Users\japhy\projects\enhanced\node_modules\discord.js\src\rest\RequestHandler.js:93:15)
at runMicrotasks (<anonymous>)
at processTicksAndRejections (internal/process/task_queues.js:93:5)
at async RequestHandler.push (C:\Users\japhy\projects\enhanced\node_modules\discord.js\src\rest\RequestHandler.js:39:14)
at async GuildMember.edit (C:\Users\japhy\projects\enhanced\node_modules\discord.js\src\structures\GuildMember.js:312:5)
Here is the code itself:
function nicknameTimeout(memberID, memberGuildID) {
guildVar = client.guilds.cache.get(memberGuildID);
member = guildVar.members.cache.get(memberID);
setTimeout(() => {
guildVar = client.guilds.cache.get(memberGuildID);
member = guildVar.members.cache.get(memberID);
nickname = member.nickname;
if (typeof nickname === "object") {
nickname = member.user.username;
}
if (nickname.endsWith("UPDATED***")) {
nickname = nickname.slice(0, -11);
member.setNickname(nickname, "");
}
}, 7200000);
}
Any ideas on how to forcibly reconnect to the client after the timeout, or any other workaround would be much appreciated.
im trying to scrape some website , i use a service to bypass captcha using their library
here simplified version of my code
const dbc = require('./deathbycaptcha.js');
open_tab();
async function open_tab()
{
try {
const captcha = await solvecaptcha('xxxx' ,'somedomain' ) .catch(function (error) {throw new Error('caperror'); });
console.log('captcha is solved !');
}
catch(e)
{
console.log('--------ERRRO--------------------------');
console.log(e.message);
}
}
function solvecaptcha(datakey , url )
{
return new Promise(function(resoulve, reject){
const client = new dbc.HttpClient(config.captcha.username, config.captcha.password);
try
{
client.decode({extra: {type: 4, token_params: some_params}}, (captcha) => {
if (captcha) {
resoulve(captcha['text']);
else
reject('cant solve captcha!');
});
}
catch (e) {
reject('cant solve captcha!');
}
});
}
so it starts in the open_Tab function , i call solvecaptcha there and it returnes the solved captcha and its all good ... but sometimes there is a connection error in deathbycaptcha.js which will break the whole code
like if i close my internet connection and try this i get this error and the program crashes
C:\wamp64\www\domain_check\robot\DBC\deathbycaptcha.js:208
throw new Error(err.message);
^
Error: getaddrinfo ENOTFOUND api.dbcapi.me api.dbcapi.me:80
at ClientRequest.form.submit (C:\wamp64\www\domain_check\robot\DBC\deathbycaptcha.js:208:15)
at ClientRequest.emit (events.js:189:13)
at Socket.socketErrorListener (_http_client.js:392:9)
at Socket.emit (events.js:189:13)
at emitErrorNT (internal/streams/destroy.js:82:8)
at emitErrorAndCloseNT (internal/streams/destroy.js:50:3)
at process._tickCallback (internal/process/next_tick.js:63:19)
i like to be able to control this exception so it wouldn't break the whole code , but no matter how many try/catch i use to control this ... ti wouldn't work and still breaks my code
and its this line that cuzes the problem
client.decode({extra: {type: 4, token_params: some_params}}, (captcha) => {
if (captcha) {
resoulve(captcha['text']);
else
reject('cant solve captcha!');
});
even though i've wrapped it inside try/catch its no use !
Im trying to get the idsFromMongo I saved in my neo4j database and search for those ids in the mongodb to return the objects I want. It works 1 time but then my server crashes and it returns the error Can't set headers after they are sent.
This is my code:.
routes.get('/advertisements/recommended/:brand', function(req, res) {
res.contentType('application/json');
var brandFromUrl = req.params.brand;
var advertisementIds = [];
Advertisement.find({'car.brand': brandFromUrl})
.then(function (ads) {
// res.status(200).json(ads);
ads.forEach(function (record) {
console.log('ids: ' + record._id)
session
.run("MERGE(a:Advertisement {idFromMongo: {idParam}, brand: {brandParam}}) WITH a MATCH(b: Advertisement {brand: {brandParam}}) MERGE(a)-[:SHARED_BRAND]->(b)", {idParam: record._id.toString(), brandParam: brandFromUrl})
.then(function(result) {
session
.run("MATCH (n:Advertisement{brand: {brandParam}}) RETURN (n)", {brandParam: brandFromUrl})
.then(function(result) {
result.records.forEach(function(record){
advertisementIds.push(record._fields[0].properties.idFromMongo);
});
Advertisement.find({
'_id': { $in: advertisementIds}
}, function(err, docs){
res.status(200).json(docs);
})
})
})
});
})
.catch((error) => {
res.status(400).json(error);
});
});
This is my error:
Error: Can't set headers after they are sent.
at validateHeader (_http_outgoing.js:494:11)
at ServerResponse.setHeader (_http_outgoing.js:501:3)
at ServerResponse.header (c:\dev\individueel-project\individueel-database\node_modules\express\lib\response.js:767:10)
at ServerResponse.send (c:\dev\individueel-project\individueel-database\node_modules\express\lib\response.js:170:12)
at ServerResponse.json (c:\dev\individueel-project\individueel-database\node_modules\express\lib\response.js:267:15)
at c:\dev\individueel-project\individueel-database\api\advertisement.routes.v1.js:70:35
at model.Query.<anonymous> (c:\dev\individueel-project\individueel-database\node_modules\mongoose\lib\model.js:4046:16)
at c:\dev\individueel-project\individueel-database\node_modules\kareem\index.js:273:21
at c:\dev\individueel-project\individueel-database\node_modules\kareem\index.js:131:16
at _combinedTickCallback (internal/process/next_tick.js:131:7)
at process._tickCallback (internal/process/next_tick.js:180:9)
The problem here is you are executing an array and into the array you are sending the response. So, eventually it will send the responde several times, but only first time will work, after that it will raise an error.
To solve it, you should answer the request after the all promises into array has finished (Promise.all) or if you don't need to wait until the whole array has finished, check if you have already responded and don't do it again.
I'm loading a .ndjson file via https. I want to close it after reading 100 lines of the file.
const amount = 100;
https.get(url, (res) => {
var { statusCode } = res;
if (statusCode !== 200) {
throw new Error(`Request Failed.\n Status Code: ${statusCode}`);
}
res.setEncoding('utf8');
let rows = [];
res
.pipe(ndjson.parse())
.on('data', function (obj) {
rows.push(obj);
if (rows.length === amount) {
this.end();
}
})
.on('end', () => {
resolve(rows);
});
}).on('error', (e) => {
throw new Error(e.message);
});
But every way I have tried the close the stream, the same error message appears:
Error: Could not parse row {"username"...
at DestroyableTransform.parseRow [as mapper] (C:\Users\thoma\Documents\GitHub\test\node_modules\ndjson\index.js:19:28)
at DestroyableTransform.flush [as _flush] (C:\Users\thoma\Documents\GitHub\test\node_modules\split2\index.js:44:21)
at DestroyableTransform.<anonymous> (C:\Users\thoma\Documents\GitHub\test\node_modules\readable-stream\lib\_stream_transform.js:138:49)
at Object.onceWrapper (events.js:314:30)
at emitNone (events.js:105:13)
at DestroyableTransform.emit (events.js:207:7)
at prefinish (C:\Users\thoma\Documents\GitHub\test\node_modules\readable-stream\lib\_stream_writable.js:596:14)
at finishMaybe (C:\Users\thoma\Documents\GitHub\test\node_modules\readable-stream\lib\_stream_writable.js:604:5)
at afterWrite (C:\Users\thoma\Documents\GitHub\test\node_modules\readable-stream\lib\_stream_writable.js:470:3)
at _combinedTickCallback (internal/process/next_tick.js:144:20)
And the stream works fine when not forcefully closed, so it isn't related to the ndjson file. Is it possible to close the stream in the middle of the request?
There are several solutions :
Send a header to close the connection : this.set("Connection", "close")
Ending stream thanks to this.end()
I do not know which one is the best.
So, in your example, you have an issue due to this (I recommand you to take a look to this link). Try the following code :
res
.pipe(ndjson.parse())
.on('data', obj => {
rows.push(obj);
if (rows.length === amount) {
this.end();
}
})
.on('end', () => {
resolve(rows);
});