im trying to scrape some website , i use a service to bypass captcha using their library
here simplified version of my code
const dbc = require('./deathbycaptcha.js');
open_tab();
async function open_tab()
{
try {
const captcha = await solvecaptcha('xxxx' ,'somedomain' ) .catch(function (error) {throw new Error('caperror'); });
console.log('captcha is solved !');
}
catch(e)
{
console.log('--------ERRRO--------------------------');
console.log(e.message);
}
}
function solvecaptcha(datakey , url )
{
return new Promise(function(resoulve, reject){
const client = new dbc.HttpClient(config.captcha.username, config.captcha.password);
try
{
client.decode({extra: {type: 4, token_params: some_params}}, (captcha) => {
if (captcha) {
resoulve(captcha['text']);
else
reject('cant solve captcha!');
});
}
catch (e) {
reject('cant solve captcha!');
}
});
}
so it starts in the open_Tab function , i call solvecaptcha there and it returnes the solved captcha and its all good ... but sometimes there is a connection error in deathbycaptcha.js which will break the whole code
like if i close my internet connection and try this i get this error and the program crashes
C:\wamp64\www\domain_check\robot\DBC\deathbycaptcha.js:208
throw new Error(err.message);
^
Error: getaddrinfo ENOTFOUND api.dbcapi.me api.dbcapi.me:80
at ClientRequest.form.submit (C:\wamp64\www\domain_check\robot\DBC\deathbycaptcha.js:208:15)
at ClientRequest.emit (events.js:189:13)
at Socket.socketErrorListener (_http_client.js:392:9)
at Socket.emit (events.js:189:13)
at emitErrorNT (internal/streams/destroy.js:82:8)
at emitErrorAndCloseNT (internal/streams/destroy.js:50:3)
at process._tickCallback (internal/process/next_tick.js:63:19)
i like to be able to control this exception so it wouldn't break the whole code , but no matter how many try/catch i use to control this ... ti wouldn't work and still breaks my code
and its this line that cuzes the problem
client.decode({extra: {type: 4, token_params: some_params}}, (captcha) => {
if (captcha) {
resoulve(captcha['text']);
else
reject('cant solve captcha!');
});
even though i've wrapped it inside try/catch its no use !
Related
im trying to scrap a website using pupeteer , this website has captcha and i use a 3rd party library/api to bypass the captcha
here is the simplified version of my code
const puppeteer = require('puppeteer');
const dbc = require('./dpc/deathbycaptcha');
async function init()
{
let page = launchPuppeteer();
try {
await page.goto(`https://www.example.com`, {timeout: 60000});
await captcha_element.screenshot({ path: setting.captcha , omitBackground: true, });
let captchaResult = await solve_captcha().catch((error)=> {throw new Error( error) });
log('ALL DONE');
}
catch (e) {
log('=====================ERROR============================');
log(e);
}
}
here is solve_captcha function
function solve_captcha(){
return new Promise(( resolve , reject )=>{
try
{
const client = new dbc.HttpClient('myUsername', 'myPassword');
client.decode({captcha: setting.captcha}, (captcha) => {
if (captcha) {
resolve(captcha['text']);
}
else
{
reject('error')
}
});
}
catch (e) {
reject(e.toString());
}
})
}
so sometimes i cant connect to captcha solving api due to down server or network problems , it would throw this error
C:\pathToProject\dpc\deathbycaptcha.js:218
throw new Error(err.message);
^
Error: read ECONNRESET
at ClientRequest.<anonymous> (C:\pathToProject\dpc\deathbycaptcha.js:218:15)
←[90m at ClientRequest.emit (events.js:310:20)←[39m
←[90m at Socket.socketErrorListener (_http_client.js:426:9)←[39m
←[90m at Socket.emit (events.js:310:20)←[39m
←[90m at emitErrorNT (internal/streams/destroy.js:92:8)←[39m
←[90m at emitErrorAndCloseNT (internal/streams/destroy.js:60:3)←[39m
←[90m at processTicksAndRejections (internal/process/task_queues.js:84:21)←[39m
this error happens in the deathbycaptcha.js which i've required in my code with
const dbc = require('./dpc/deathbycaptcha');
here is the simplified version of the code in the in the deathbycaptcha.js which is cuzing the error
class HttpClient extends Client {
_call(arguments here) {
var options = {someoptions};
var form = new FormData();
const request = form.submit(options, (err, response) => {
if (err) {
console.log(err.toString())
throw new Error(err.message);
}
// more code here
}
}
so here is the problem , i have this code running inside 2 try/catch blocks
first in the caller function (init()) and second one in the solve_captcha function but none of them are able to catch this error and it will break all the code and the program will exit and i have to run it again
why cant i catch and handle this error in my code ?
I have written a small error handling function, which is invoked after an AXIOS request, like so:
try {
...
} catch (error) {
handleAxiosError(error);
}
The function is as follows:
function handleAxiosError(error) {
if (error.response !== undefined) {
logger.error(`Received a HTTP error. Status code: ${error.response.status}, Data: ${error.response.data}`);
} else if (error.request !== undefined) {
logger.error(error.request);
} else {
logger.error(error.message);
}
throw new Error(error);
}
Although an error is thrown:
(node:94324) UnhandledPromiseRejectionWarning: Error: Error: connect ECONNREFUSED 127.0.0.1:6557
at handleAxiosError (C:\pathtoapp\utils\utils.js:66:11)
Pino only saves the following to the log. I can't find the problem. Is this an async issue?
{"level":50,"time":1567435455281,"pid":94324,"hostname":"host","name":"app","res":{},"v":1}
Thanks!
When using async logging (the default for the Pino logger), the process might have exited before all of the logging has been processed.
See https://github.com/pinojs/pino/blob/HEAD/docs/asynchronous.md
You can also change the logging to be synchronous, and you won't have this problem:
const dest = pino.destination({ sync: true })
When I am trying to load the following package in Meteor https://github.com/vsivsi/meteor-job-collection
It gets downloaded 100% and extracted, but at the time of loading it throws the following error:
{ [
Error: ENOTEMPTY: directory not empty, rmdir 'C:\Users\LALITS~1\AppData\Local\Temp\mt-16riklk\npm\job\node_modules']
errno: -4051,
code: 'ENOTEMPTY',
syscall: 'rmdir',
path: 'C:\\Users\\LALITS~1\\AppData\\Local\\Temp\\mt-16riklk\\npm\\job\\node_modules' }
I am using windows 8.1 64 bit.
I have tried to delete the folder manually, but again it created a new one and throws the same error. Can anyone tell me what is the problem? Am I missing something?
Thanks in advance.
Your issue looks like this known Meteor bug:
https://github.com/meteor/meteor/issues/8663. This bug occurs under Windows when updating to the next Meteor version.
Maybe you can try the proposed solution, which is to edit the following file:
C:\Users\[yourName]\AppData\Local\.meteor\packages\meteor-tool\[yourMeteorVersion]\mt-os.windows.x86_32\tools\fs\files.js
...and replace functions files.rm_recursive_async and files.rm_recursive with this code:
files.rm_recursive_async = function (path) {
return new Promise(function (resolve, reject) {
rimraf(files.convertToOSPath(path), function (err) {
err && console.log(err);
resolve();
//return err ? reject(err) : resolve();
});
});
}; // Like rm -r.
files.rm_recursive = Profile("files.rm_recursive", function (path) {
try {
rimraf.sync(files.convertToOSPath(path));
} catch (e) {
if (e.code === "ENOTEMPTY" && canYield()) {
files.rm_recursive_async(path).await();
return;
}
console.log(e);
//throw e;
}
}); // Makes all files in a tree read-only.
My project contains few methods that uses node-horseman, but after every server restart, the process will die after I executed something, whatever method it is. For example, I have one method that retrieves a list of Alerts I have in Google Alerts:
* getRSSNames() {
return new Promise((resolve, reject) => {
horseman
.userAgent(this.userAgent)
.open('https://www.google.com/alerts')
.evaluate(function() {
var item = []
jQuery('#manage-alerts-div li').each(function(key, val) {
item.push({
dataId: jQuery(val).attr('data-id'),
name: jQuery(val).find('.query_div').text(),
url: jQuery(val).find('a').attr('href')
})
})
return item
}).then(function(data) {
resolve(data)
})
.close()
})
}
Success, if I've just initiated the local server. Then, rendering the list in a select, choose one and try to delete it:
* deleteRSSFeed() {
let feedID = this.feedID.replace(/%3A/g, ':')
return new Promise((resolve, reject) => {
horseman
.userAgent(this.userAgent)
.open('https://google.com/alerts')
.evaluate(function(selector) {
return jQuery('[data-id="' + selector + '"]').find('.delete_button').click()
}, feedID)
.then(function(data) {
resolve(data)
})
.close()
})
}
It will cause the error:
Unhandled rejection HeadlessError: Phantom Process died
at poll_func (/home/gabriel/Sites/co-report/api/node_modules/node-phantom-simple/node-phantom-simple.js:584:10)
at /home/gabriel/Sites/co-report/api/node_modules/node-phantom-simple/node-phantom-simple.js:52:7
at ClientRequest.<anonymous> (/home/gabriel/Sites/co-report/api/node_modules/node-phantom-simple/node-phantom-simple.js:501:11)
at emitOne (events.js:96:13)
at ClientRequest.emit (events.js:188:7)
at Socket.socketErrorListener (_http_client.js:308:9)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at emitErrorNT (net.js:1272:8)
at _combinedTickCallback (internal/process/next_tick.js:74:11)
at process._tickCallback (internal/process/next_tick.js:98:9)
I also have a function that logs in in Google. If I login and I refresh my page with the server running, since I logged in and when refreshing it will retrieve the feed list, it will cause the same error. Please, help me.
Ok... My solution was to instantiate horseman inside each method. That's it.
I am trying to perform HTTP.get on a set of URLs asynchronously using futures.
Meteor version: 0.8.3
Platform: Windows server 2012
The code is somewhat like this:
var futures = _.map(test, function(url) {
var future = new Future();
var onComplete = future.resolver();
try{
// Make async http call
var httpGet = HTTP.get(url,{followRedirects: true},function(error, result) {
if(error)
{
apiLogger.error("%s is error",error);
onComplete(error, null);
}
else
{
if(result!=null){
//operations done here
onComplete(error, JSON.stringify(object1));
}
else {
apiLogger.error('%s - User encountered an error. URL not parsed: %s',user,url);
onComplete(error, null);
}
}
});
}
catch(e)
{
apiLogger.error('%s - URsarsed: %s - %s',user,url,result.statusCode);
onComplete(error, null);
}
return future;
});
The issue I am facing is improper error handling.
I am getting the following error on some URLs:
I20140904-17:57:38.609(-4)? Exception while invoking method 'parallelAsyncJob' E
rror: failed [404] <html><head><title>Apache Tomcat/7.0.12 - Error report</title
><style><!--H1 {font-family:Tahoma,Arial,sans-serif;color:white;background-color
:#525D76;font-size:22px;} H2 {font-family:Tahoma,Arial,sans-serif;color:white;ba
ckground-color:#525D76;font-size:16px;} H3 {font-family:Tahoma,Arial,sans-serif;
color:white;background-color:#525D76;font-size:14px;} BODY {font-family:Tahoma,A
rial,sans-serif;color:black;background-color:white;} B {font-family:Tahoma,Arial
,sans-serif;color:white;background-colo...
I20140904-17:57:38.617(-4)? at packages/underscore/underscore.js:255
I20140904-17:57:38.619(-4)? at Array.map (native)
I20140904-17:57:38.621(-4)? at Function._.map._.collect (packages/underscore
/underscore.js:123)
I20140904-17:57:38.623(-4)? at Function._.invoke (packages/underscore/unders
core.js:254)
I20140904-17:57:38.626(-4)? at Meteor.methods.parallelAsyncJob (app/server/m
ethods.js:1:9355)
I20140904-17:57:38.628(-4)? at maybeAuditArgumentChecks (packages/livedata/l
ivedata_server.js:1488)
I20140904-17:57:38.631(-4)? at packages/livedata/livedata_server.js:650
I20140904-17:57:38.632(-4)? at _.extend.withValue (packages/meteor/dynamics_
nodejs.js:37)
I20140904-17:57:38.635(-4)? at packages/livedata/livedata_server.js:649
I20140904-17:57:38.644(-4)? at _.extend.withValue (packages/meteor/dynamics_
nodejs.js:37)
I20140904-17:57:38.646(-4)? - - - - -
I20140904-17:57:38.648(-4)? at makeErrorByStatus (packages/http/httpcall_com
mon.js:12)
I20140904-17:57:38.650(-4)? at Request._callback (packages/http/httpcall_ser
ver.js:99)
I20140904-17:57:38.652(-4)? at Request.self.callback (C:\Users\Administrator
\AppData\Local\.meteor\tools\edf8981bb6\lib\node_modules\request\request.js:122:
22)
I20140904-17:57:38.655(-4)? at Request.EventEmitter.emit (events.js:98:17)
I20140904-17:57:38.657(-4)? at Request.<anonymous> (C:\Users\Administrator\A
ppData\Local\.meteor\tools\edf8981bb6\lib\node_modules\request\request.js:888:14
)
I20140904-17:57:38.660(-4)? at Request.EventEmitter.emit (events.js:117:20)
I20140904-17:57:38.662(-4)? at IncomingMessage.<anonymous> (C:\Users\Adminis
trator\AppData\Local\.meteor\tools\edf8981bb6\lib\node_modules\request\request.j
s:839:12)
I20140904-17:57:38.665(-4)? at IncomingMessage.EventEmitter.emit (events.js:
117:20)
I20140904-17:57:38.668(-4)? at _stream_readable.js:920:16
I20140904-17:57:38.669(-4)? at process._tickCallback (node.js:415:13)
Am I doing something wrong? Or is it some GET issue?
Update:
I am using futures because the final operation can only be performed after getting all the URLs.
Interesting thing, I am able to open the URL via browser, and even POSTMAN and getting 200 status. But meteor get is receiving 404.
Per your error message, you’re getting a 404 error code; some of your URLs are invalid. And you say yourself that it only happens on certain URLs.
Why is your code so complicated, with all these futures? HTTP.get() itself offers an asyncCallback, and you’re already using inline callbacks in your code, so why not just strip out all the futures stuff?
_.map(test, function(url) {
try {
// Make async http call
HTTP.get(url, {followRedirects: true}, function(error, result) {
if (error) {
if (result.statusCode == 404)
apiLogger.error('Error 404, URL not found: %s', url);
else
apiLogger.error('Error %s from %s for user %s',
result.statusCode, url, user);
return false;
} else {
if (result != null) {
// operations done here
} else {
apiLogger.error('Empty or invalid result returned from %s for user %s',
url, user);
}
return false;
}
});
} catch (error) {
return false;
}
});