I have some code that opens a website, submits search criteria, and loops through each resulting table row for the link to the details behind it. The results may be broken into pages so it keeps track of that too. It works largely fine with the exception that occasionally, and usually on the first row of a newly loaded page, it will just sit there and not follow the link. I can see the link indicate that it was hovered over, but nothing happens, until I click on it with the mouse, then the process continues. I just don't know what I am missing, as it is placed within a loop that should retry the click as long as the right page is there, which is identified by the presence of known elements. The while loop that starts at line 29 is the functional part in question, and line 31 is where the link is clicked. Code below:
def pagescrape(driver,page_no): global page_count, resume_record_number results_table_details_link_xpath="//table[@class='results']/tbody/tr/td[1]/a[contains(@href,'inquiryDetail')]" captcha_page_element_xpath="//*[@class='captcha__human__title' and text()='You have been blocked.']" try: wait=WebDriverWait(driver,20) wait.until(EC.element_to_be_clickable((By.XPATH,results_table_details_link_xpath))) try: result_banner=driver.find_element(By.CLASS_NAME,"pagebanner").text #Search Results page result_banner_list=result_banner.split() search_total=int(result_banner_list[0]) if search_total<=25: min_rec=1 max_rec=search_total else: min_rec = int(result_banner_list[4]) max_rec = int(result_banner_list[6].replace('.', '')) record_count=max_rec-min_rec+1 page_header_tag=str(str(page_no)+'/'+str(page_count)).ljust(10,' ') print(f"Page:{page_header_tag} Search Total:{str(search_total).ljust(6,' ')} Search Letter:{letter_current} Profile:{profile_current}") wait=WebDriverWait(driver,6) for i in range(resume_record_number,record_count+1): row_tag=f"{i}/{record_count}".ljust(10,' ') case_status=driver.find_element("xpath", f"//table[@class='results']/tbody/tr[{i}]/td[7]").text if case_status=='Open' or case_status=='ACTIVE CASE': Case_Number=driver.find_element("xpath", f"/html/body/div/table[4]/tbody/tr[{i}]/td[1]/a").text case_type=driver.find_element("xpath", f"/html/body/div/table[4]/tbody/tr[{i}]/td[6]").text time.sleep(1) while True: try: details_link=driver.find_element("xpath", f"/html/body/div/table[4]/tbody/tr[{i}]/td[1]/a") driver.execute_script("return arguments[0].scrollIntoView();", details_link) details_link.click() wait.until(EC.any_of (EC.text_to_be_present_in_element((By.CLASS_NAME,'InfoStatement'),'This is an electronic case record'), EC.element_to_be_clickable((By.XPATH,"//div[@class='Subheader']/a[text()='Go Back']")) )) html=driver.find_element(By.TAG_NAME, "html") #Details page for j in range(randrange(5)): html.send_keys(Keys.PAGE_DOWN) driver.find_element("xpath",f"/html/body").click(); for j in range(randrange(5)): html.send_keys(Keys.PAGE_UP) driver.find_element("xpath",f"/html/body").click(); time.sleep(1) details=getrecord(driver,Case_Number,page_no,case_type) values_list=list(details.values()) print(f"{row_tag} {values_list[8]}") with open(proc_path, 'a',encoding="utf-8",newline="") as f: writer = csv.writer(f) writer.writerow(values_list) driver.back() time.sleep(1) break except TimeoutException as timeout_results_page: try: wait.until( EC.any_of( EC.element_to_be_clickable((By.XPATH,results_table_details_link_xpath)), EC.text_to_be_present_in_element((By.CLASS_NAME,'captcha__human__title'),'You have been blocked.'), EC.text_to_be_present_in_element_value((By.NAME,'button'),'Go Back'), EC.element_to_be_clickable((By.ID,"btnDisclaimerAgree")), ) ) if len(driver.find_elements(By.XPATH,results_table_details_link_xpath))>0: #Search results page time.time(1) print(f'Retrying record: {i}') continue elif len(driver.find_elements(By.XPATH,captcha_page_element_xpath))>0: #captcha page continue elif len(driver.find_elements(By.NAME,"button"))>0: #Error page. Title: An Error has occurred print(f"{row_tag} -- Error page loaded") backbutton=driver.find_element(By.NAME,"button") backbutton.click() time.sleep(2) break else: #Back at home/disclaimer page resume_record_number=i return i except Exception as err_long_timeout: if 'Search Results' in driver.title: continue else: print(f"Page Title: {driver.title} - {err_long_timeout} -- in-search page wait") continue except Exception as err_other: print(err_other) continue while True: try: wait.until(EC.element_to_be_clickable((By.XPATH,results_table_details_link_xpath))) break except Exception as err_results: print(f"Page Title: {driver.title} - {err_results} -- Results page expected") else: print(f"{row_tag} -- Closed Case") continue resume_record_number=1 return -1 except Exception as err_results_page: return -3 except Exception as e: print(f"{e} -- Catch-all") return -3Any insight would be greatly appreciated 