Develop Reference

Selenium won't work unless I actually look at the Web page (perhaps anti-crawler mechanism by JavaScript?)

The following code works fine ONLY when I look at the Web page (aka the Chrome tab being manipulated by Selenium).
Is there a way to make it work even when I'm browsing another tab/window?
(I wonder how the website knows I'm actually looking at the web page or not...)
#This is a job website in Japanese
login_url = "https://mypage.levtech.jp/"
driver = selenium.webdriver.Chrome("./chromedriver")
#Account and password are required to log in.
#I logged in and got to the following page, which displays a list of companies that I have applied for:
#https://mypage.levtech.jp/recruits/screening
#Dictionary to store company names and their job postings
jobs = {}
for i, company in enumerate(company_names):
time.sleep(1)
element = driver.find_elements_by_class_name("ScreeningRecruits_ListItem")[i]
while element.text == "":
#While loops and time.sleep() are there because the webpage seems to take a while to load
time.sleep(0.1)
element = driver.find_elements_by_class_name("ScreeningRecruits_ListItem")[i]
td = element.find_element_by_tag_name("td")
while td.text == "":
time.sleep(0.1)
td = element.find_element_by_tag_name("td")
if td.text == company:
td.click()
time.sleep(1)
jobs[company] = get_job_desc(driver) #The get_job_desc function checks HTML tags and extract info from certain elements
time.sleep(1)
driver.back()
time.sleep(1)
print(jobs)
By the way, I have tried adding a user agent and scroll down the page using the following code, in the hope that the Web page would believe that I'm "looking at it." Well, I failed :(
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

So, I think the answer to your question is due to window_handles. Whenever we open a new tab, Selenium changes the window's focus on us ( obviously ). Because the focus is on another page, we need to use the driver.switch_to.window(handle_here) method. This way, we can switch to our proper tab. In order to do this, I found a website that has a similar functionality ( also in Japanese / Kanji? ) that might help you out.
MAIN PROGRAM - For Reference
from selenium import webdriver
from selenium.webdriver.chrome.webdriver import WebDriver as ChromeDriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait as DriverWait
from selenium.webdriver.support import expected_conditions as DriverConditions
from selenium.common.exceptions import WebDriverException
import time
def get_chrome_driver():
"""This sets up our Chrome Driver and returns it as an object"""
path_to_chrome = "F:\Selenium_Drivers\Windows_Chrome85_Driver\chromedriver.exe"
chrome_options = webdriver.ChromeOptions()
# Browser is displayed in a custom window size
chrome_options.add_argument("window-size=1500,1000")
return webdriver.Chrome(executable_path = path_to_chrome,
options = chrome_options)
def wait_displayed(driver : ChromeDriver, xpath: str, int = 5):
try:
DriverWait(driver, int).until(
DriverConditions.presence_of_element_located(locator = (By.XPATH, xpath))
)
except:
raise WebDriverException(f'Timeout: Failed to find {xpath}')
# Gets our chrome driver and opens our site
chrome_driver = get_chrome_driver()
chrome_driver.get("https://freelance.levtech.jp/project/search/?keyword=&srchbtn=top_search")
wait_displayed(chrome_driver, "//div[#class='l-contentWrap']//ul[#class='asideCta']")
wait_displayed(chrome_driver, "//div[#class='l-main']//ul[#class='prjList']")
wait_displayed(chrome_driver, "//div[#class='l-main']//ul[#class='prjList']//li[contains(#class, 'prjList__item')][1]")
# Click on the first item title link
titleLinkXpath = "(//div[#class='l-main']//ul[#class='prjList']//li[contains(#class, 'prjList__item')][1]//a[contains(#href, '/project/detail/')])[1]"
chrome_driver.find_element(By.XPATH, titleLinkXpath).click()
time.sleep(2)
# Get the currently displayed window handles
tabs_open = chrome_driver.window_handles
if tabs_open.__len__() != 2:
raise Exception("Failed to click on our Link's Header")
else:
print(f'You have: {tabs_open.__len__()} tabs open')
# Switch to the 2nd tab and then close it
chrome_driver.switch_to.window(tabs_open[1])
chrome_driver.close()
# Check how many tabs we have open
tabs_open = chrome_driver.window_handles
if tabs_open.__len__() != 1:
raise Exception("Failed to close our 2nd tab")
else:
print(f'You have: {tabs_open.__len__()} tabs open')
# Switch back to our main tab
chrome_driver.switch_to.window(tabs_open[0])
chrome_driver.quit()
chrome_driver.service.stop()
For scrolling, you could use this method
def scroll_to_element(driver : ChromeDriver, xpath : str, int = 5):
try:
webElement = DriverWait(driver, int).until(
DriverConditions.presence_of_element_located(locator = (By.XPATH, xpath))
)
driver.execute_script("arguments[0].scrollIntoView();", webElement)
except:
raise WebDriverException(f'Timeout: Failed to find element using xpath {xpath}\nResult: Could not scroll')

Protractor : How to detect if an unexpected alert is open

In one of our tests, sometimes we get the browser alert message when the user is trying to log out.
I do not want the tests failed due to these alerts. We are getting the below error when there is an alert:
An error was thrown in an afterAll
AfterAll Failed: unexpected alert open: {Alert text : You have unsaved changes! If you leave, your changes will be lost.}
I have tried to resolve it by
1. added unexpectedAlertBehaviour: 'accept' to the conf.js file, which failed, and 2, modifying the code using an if/else block as follows:
exports.logOutfromESY =function(){
var G = GV;
var EC = protractor.ExpectedConditions;
expect(G.User_Menu_Dropdown.isPresent()).toBeTruthy();
G.User_Menu_Dropdown.click();
browser.wait(EC.presenceOf(G.logOut_Button), 2000, 'The Logout Buttons taking too long to appear in the DOM');
G.logOut_Button.click();
browser.sleep(500);
if(alert.isPresent()){
browser.switchTo().alert().then(function (alert) {
alert.accept();
});
}
browser.wait(EC.presenceOf(G.Email_Input_box), 3000, 'The Login Page redirection taking long time');
browser.sleep(500);
};
It did not work either. Note that the "alert" variable was my last ditch effort to make protractor to identify the alert.
Is there any way to achieve this?

Seems that this solution may be helpful:
https://stackoverflow.com/a/29873887/6331748
EDIT:
You can also consider to use --disable-notifications flag.
You need to add it in your config file like that:
export const config = {
capabilities: {
chromeOptions: {
args: ['--disable-notifications']
}
}
}

Push username & password to alert using Selenium python 3+

Alert on website
Hi Everyone
i am trying to accept alert using below code but i am getting timeout exception everytime. tried increasing wait time still same error
wait = WebDriverWait(driver, 60)
driver.get('abc.com')
alert = wait.until(EC.alert_is_present()) ----------Error at this line
time.sleep(4)
#WebDriverWait(driver, 3).until(EC.alert_is_present())
alert = driver.switch_to.alert
alert.send_keys('username' + Keys.TAB + 'password')
alert.accept()
Error :-
Traceback (most recent call last):
File "C:\Users\Gaurav Chhabra\Desktop\FAIL_RATE.py", line 30, in <module>
alert = wait.until(EC.alert_is_present())
File "C:\Users\Gaurav Chhabra\AppData\Local\Programs\Python\Python35\lib\site-packages\selenium\webdriver\support\wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
What could be the reason as have used the same code on another website and its working, is this is because of some website features which are unable etc ?
i have also attached image of alert
or if somebody can help me with some new way to pass username & password in this alert ?
thanks

Selenium-Java-geckodriver : Test Execution halts with JavaScript error as "document.getElementById(...) is null"

I had been encountering this issue for a while now. Browsed through a lot of threads on SO & other forums but still clueless.
While automating a simple flow on a Web Application with Selenium 3.4.0, geckodriver v0.16.1 & Mozilla Firefox 53.0 within Eclipse Neon v2 IDE, I am facing an error intermittently on the console as:
JavaScript error: https://www.url.com/my, line 1715: TypeError: document.getElementById(...) is null
Though using chromedriver v2.29/Google Chrome 58.0 or using Python I don't face any such issue.
Once this error appears, the Test Execution halts and finally shows TimeoutException as follows:
Exception in thread "main" org.openqa.selenium.TimeoutException: Timeout loading page after 300000ms
URL of the website is: https://www.shareinvestor.com/my
The HTML DOM is:
<div id="sic_sitemap">
<div id="sic_container">
<div id="sic_header">
<h1 id="sic_siteTitle">
<div id="sic_headerMembershipLink">
<a id="sic_mobileEdition" href="/mobile">
<div id="sic_loginContainer" class="sic_logIn" style="">
<div class="sic_login-wrap">
<div class="sic_logIn-subscribe">
<div class="sic_logIn-bg">
<a href="/user/login.html">
</div>
</div>
</div>
</div>
<div id="sic_subHeader">
<div id="sic_mainNav" class="sic_withRightCorner">
<div id="sic_sideBar" class="sic_expanded { selected_market_suffix: 'MY'}">
<div class="sic_superBanner sic_superBannerTop">
<div id="sic_content" class="sic_collapsed">
<div id="sic_footer" class="si_fixed">
</div>
As of now, I have tried out the following options but of no avail:
Java Click
JavascriptExecutor Click
Actions Click
Here is my code:
import org.openqa.selenium.By;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.interactions.Actions;
import org.openqa.selenium.remote.DesiredCapabilities;
public class 78644072022 {
public static void main(String[] args) {
System.setProperty("webdriver.gecko.driver", "C:\\Utility\\BrowserDrivers\\geckodriver.exe");
DesiredCapabilities dc = DesiredCapabilities.firefox();
dc.setCapability("marionette", true);
WebDriver driver = new FirefoxDriver(dc);
driver.manage().window().maximize();
driver.get("https://www.shareinvestor.com/my");
WebElement login_button = driver.findElement(By.xpath("//div[#id='sic_loginContainer']/div/div[#class='sic_logIn-bg']/a"));
//Java Click
login_button.click();
//JavascriptExecutor Click
/*JavascriptExecutor jse = (JavascriptExecutor)driver;
jse.executeScript("arguments[0].click();", login_button);*/
//Actions Click
/*Actions act = new Actions(driver);
act.moveToElement(login_button).click().build().perform();*/
driver.findElement(By.xpath("//input[#id='sic_login_header_username']")).sendKeys("debanjan");
driver.findElement(By.xpath("//input[#id='sic_login_header_password']")).sendKeys("5786");
driver.findElement(By.xpath("//input[#id='sic_login_submit']")).click();
}
}
I am looking for a Java Solution with geckodriver to overcome the error JavaScript error:TypeError: document.getElementById(...) is null
In one of the SO threads I saw a solution like:
You need to do a null check in updateHTML like this:
function updateHTML(elmId, value) {
var elem = document.getElementById(elmId);
if(typeof elem !== 'undefined' && elem !== null) {
document.getElementById(elmId).innerHTML = value;
}
}
Can we implement this?
Any suggestions & pointers will be helpful.

I am looking for a Java Solution with geckodriver to overcome the error JavaScript error:TypeError: document.getElementById(...) is null
To answer your question, I don't believe there's anything you can do to "fix" this via Java/Selenium. This is a JavaScript error, which originates from the website that you are trying to access. You might want to consider contacting their support team, maybe one of their developers can look at this issue.
Instead of clicking on the login button, maybe consider navigating directly to the login page?
driver.get("https://www.shareinvestor.com/user/login.html");

Firstly, Those javascript errors are not triggered due to any of the selenium code. Ofcourse, the timeout has been triggered by the selenium(will discuss on this on a later point).
You will get that javascript error irrespective of any kind of browser you launch the URL with. But in case of the gecko, you are notified in the eclipse console with but not in case of Chrome. If you need to see the java script error in chrome, just launch the url in chrome and go to devtools/console(F12). You can also see the same in firefox console too.
Chrome Img:
Secondly, We are getting timeout exception because the site is really taking too much time to load. I have been waited for 7 minutes and the page is still loading even now. Selenium won't executes its script unless the page has been completely launched. As a result we are getting timeout exception(not sure about the default page launch time permitted). I thought of bypassing directly to the login page ("https://www.shareinvestor.com/user/login.html") and that's also not taking any finite time to load completely.
Intepolation: Those java script errors are not an issue for automation But those page loads are really. This site doesn't seems like a good candidate for automation with this issue.
Update1: otherwise we can also stop the page loading via another thread like sending "esc" key sequence using Action class.
Update2: I tried the same code today and it works fine. Below is the code snippet that i have tried(There is no change at all)
public static void main(String[] args) throws InterruptedException
{
System.setProperty("webdriver.gecko.driver", "F:\\Softwares\\Selenium\\Webdriver\\geckodriver.exe");
WebDriver driver = new FirefoxDriver();
driver.manage().window().maximize();
driver.manage().timeouts().implicitlyWait(5, TimeUnit.SECONDS);
driver.get("https://www.shareinvestor.com/my");
WebElement login_button = driver.findElement(By.xpath("//div[#id='sic_loginContainer']/div/div[#class='sic_logIn-bg']/a"));
//Java Click
login_button.click();
System.out.println("In Login PAge");
driver.findElement(By.xpath("//input[#id='sic_login_header_username']")).sendKeys("debanjan");
driver.findElement(By.xpath("//input[#id='sic_login_header_password']")).sendKeys("5786");
System.out.println("Entered password");
driver.findElement(By.xpath("//input[#id='sic_login_submit']")).click();
}
Selenium version - 3.4.0
Gecko driver - v0.16.1(mine is 32 bit)
Mozilla - 51.0.1 (Update=>It's working on 53.02 also)
Hope this helps you. Thanks.

I think I've managed to find what is causing this uproar in your script.
I inspected your HTML and it seems javascript method function showRemaining() is causing this problem; because showRemaining() contains this statement
document.getElementById('countdown').innerHTML = '';
where it tries to set innerHTML attribute for element having id as countdown to ''.
But countdown doesn't exist anywhere on your web page hence the error.
TypeError: document.getElementById(...) is null
and somehow selenium isn't able to look past this error. So I think getting it fixed from developers should help you.
UPDATE :
Basically you need to wait all elements to load using implicit wait, once all elements are loaded, then your Javascript error gets resolved and ultimately interaction with the webpage is not hindered by that javascript error:
driver.get("https://www.shareinvestor.com/my");
driver.manage().timeouts().implicitlyWait(20, TimeUnit.SECONDS);
driver.manage().window().maximize();
/*String script = "function updateHTML(countdown, value) {" +
"var elem = document.getElementById('countdown');"+
"if(typeof elem !== 'undefined' && elem !== null) {"+
" document.getElementById('countdown').innerHTML = value;"+
"}"+
"}";
((JavascriptExecutor) driver).executeScript(script);*/
WebElement login_button = driver.findElement(By.xpath("//div[#id='sic_loginContainer']/div/div[#class='sic_logIn-bg']/a"));
login_button.click();
driver.findElement(By.xpath("//input[#id='sic_login_header_username']")).sendKeys("debanjan");
driver.findElement(By.xpath("//input[#id='sic_login_header_password']")).sendKeys("5786");
driver.findElement(By.xpath("//input[#id='sic_login_submit']")).click();

It looks like the XPath for the login_button is incorrect:
By.xpath("//div[#id='sic_loginContainer']/div/div[#class='sic_logIn-bg']/a");
Should be:
By.xpath("//div[#id='sic_loginContainer']/div/div/div[#class='sic_logIn-bg']/a");
This may explain the TimeoutException, since Selenium is unable to locate the non-existent element. EDIT: My mistake, you should see a NoSuchElementException if the element cannot be located.
As for the JavaScript error, unless you were trying to use Selenium to access the web element being manipulated by the JavaScript code (id='countdown' element), Selenium should just ignore the error. Otherwise, there's the potential for other Selenium errors, such as StaleElementReferenceException, but that does not seem to be the case.

This is due to your application HTML page using the async javascript and it uses some variable reference that is not available at the time of execution. we had same issue and asked developer to follow some best practices for javascript in the HTML like putting the script tag at end of the page. I have checked the HTML page source of the site and it contains many script tags in between the code. This will block the DOM rendering. Better, ask the developer to follow some best practices to include script tag in HTML. you refer the link Where should I put <script> tags in HTML markup? for the best practices.

Firefox hang on jQuery based site after loading halfway when automating Selenium using Python

I'm trying to scrape a site that's jQuery based and I'm having trouble with getting the page to load completely before extracting the elements with Selenium. The page has multiple modules, each of which is a different query. I tried using the wait commands I found in the documentation, but it would usually hang the browser after one of the multiple queries load.
For reference, my OS is Windows 7, Firefox 30.0, Python 2.7 and Selenium 2.42.1
The commands and results I tried are as follows:
Explicit Wait: Browser hangs after loading the first query (Firefox Not Responding)
try:
element = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH, path)))
finally:
browser.quit()
Expected Conditions: Browser hangs after loading the first query (Firefox Not Responding)
wait = WebDriverWait(browser, 10)
element = wait.until(EC.element_to_be_clickable((By.XPATH,path)))
Implicit Wait: Firefox hangs after loading the first query (Firefox Not Responding)
browser.implicitly_wait(10) # seconds
myDynamicElement = browser.find_element_by_xpath(path)
Custom Function: Page loads, but selenium starts scraping before the second query is loaded resulting in an error
def wait_for_condition(browser,c):
for x in range(1,10):
print "Waiting for jquery: " + c
x = browser.execute_script("return " + c)
if(x):
return
time.sleep(1)
def main():
wait_for_condition(browser,"jQuery.active == 0")
#First element to be clicked on to scrape:
path="//a[starts-with(#class, 'export db')]"
browser.find_element_by_xpath(path).click()
The error is:
selenium.common.exceptions.NoSuchElementException: Message: u'Unable to locate element: {"method":"xpath","selector":"//a[starts-with(#class, \'export db\')]"}' ;
Catching this exception and running wait_for_condition again in the except block causes the browser to stop loading the rest of the queries and hang:
wait_for_condition(browser,"jQuery.active == 0")
try:
path="//a[starts-with(#class, 'export db')]"
browser.find_element_by_xpath(path).click()
except NoSuchElementException:
wait_for_condition(browser,"jQuery.active == 0")
path="//a[starts-with(#class, 'export db')]"
browser.find_element_by_xpath(path).click()
Please let me know if you have any suggestions to solving the problems.
Thanks in advance,
Teresa