Update CDP Mode

Author: mdmintz

seleniumbase/core/browser_launcher.py

@@ -938,6 +938,14 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
  driver.solve_captcha = CDPM.solve_captcha
  driver.find_element_by_text = CDPM.find_element_by_text
  driver._is_using_cdp = True
+ if (
+ hasattr(sb_config, "_cdp_proxy")
+ and sb_config._cdp_proxy
+ and "@" in sb_config._cdp_proxy
+ ):
+ time.sleep(0.077)
+ loop.run_until_complete(page.wait(0.25))
+ time.sleep(0.022)
 
 
 def uc_activate_cdp_mode(driver, url=None, **kwargs):
@@ -2746,7 +2754,7 @@ def _set_chrome_options(
  d_f_string = ",".join(included_disabled_features)
  chrome_options.add_argument("--disable-features=%s" % d_f_string)
  chrome_options.add_argument("--enable-unsafe-extension-debugging")
- if proxy_auth:
+ if proxy_string:
  chrome_options.add_argument("--test-type")
  if proxy_auth or sb_config._ext_dirs:
  if not is_using_uc(undetectable, browser_name):
@@ -2770,7 +2778,7 @@ def _set_chrome_options(
  chrome_options.add_argument("--disable-popup-blocking")
  # Skip remaining options that trigger anti-bot services
  return chrome_options
- if not proxy_auth:
+ if not proxy_string:
  chrome_options.add_argument("--test-type")
  chrome_options.add_argument("--log-level=3")
  chrome_options.add_argument("--no-first-run")
@@ -5812,6 +5820,7 @@ def get_local_driver(
  driver, *args, **kwargs
  )
  )
+ driver.activate_cdp_mode = driver.uc_activate_cdp_mode
  driver.uc_open_with_cdp_mode = (
  lambda *args, **kwargs: uc_open_with_cdp_mode(
  driver, *args, **kwargs

seleniumbase/core/sb_cdp.py

@@ -1823,7 +1823,11 @@ def _on_a_cf_turnstile_page(self, source=None):
  time.sleep(0.2)
  source = self.get_page_source()
  if (
- 'data-callback="onCaptchaSuccess"' in source
+ (
+ 'data-callback="onCaptchaSuccess"' in source
+ and not 'title="reCAPTCHA"' in source
+ and not 'id="recaptcha-token" in source'
+ )
  or "/challenge-platform/scripts/" in source
  or 'id="challenge-widget-' in source
  or "challenges.cloudf" in source
@@ -1836,13 +1840,19 @@ def _on_a_g_recaptcha_page(self, source=None):
  if not source or len(source) < 400:
  time.sleep(0.2)
  source = self.get_page_source()
+ self.loop.run_until_complete(self.page.wait(0.1))
  if (
- 'id="recaptcha-token"' in source
- or 'title="reCAPTCHA"' in source
+ (
+ 'id="recaptcha-token"' in source
+ or 'title="reCAPTCHA"' in source
+ )
+ and self.is_element_visible('iframe[title="reCAPTCHA"]')
  ):
+ self.loop.run_until_complete(self.page.wait(0.1))
  return True
  elif "/recaptcha/api.js" in source:
  time.sleep(1.6) # Still loading
+ self.loop.run_until_complete(self.page.wait(0.1))
  return True
  return False
 
@@ -1852,8 +1862,10 @@ def __gui_click_recaptcha(self, use_cdp=False):
  selector = 'iframe[title="reCAPTCHA"]'
  else:
  return
+ time.sleep(0.25)
+ self.loop.run_until_complete(self.page.wait())
+ time.sleep(0.25)
  with suppress(Exception):
- time.sleep(0.08)
  element_rect = self.get_gui_element_rect(selector, timeout=1)
  e_x = element_rect["x"]
  e_y = element_rect["y"]
@@ -1884,7 +1896,9 @@ def gui_click_captcha(self):
 
  def __click_captcha(self, use_cdp=False):
  """Uses PyAutoGUI unless use_cdp == True"""
- self.sleep(0.056)
+ self.sleep(0.075)
+ self.loop.run_until_complete(self.page.wait())
+ self.sleep(0.025)
  source = self.get_page_source()
  if self._on_a_cf_turnstile_page(source):
  pass
@@ -1894,61 +1908,32 @@ def __click_captcha(self, use_cdp=False):
  else:
  return
  selector = None
- if (
- self.is_element_present('[name*="cf-turnstile-"]')
- and self.is_element_present("#challenge-form div > div")
- ):
+ if self.is_element_present('[class="cf-turnstile"]'):
+ selector = '[class="cf-turnstile"]'
+ elif self.is_element_present("#challenge-form div > div"):
  selector = "#challenge-form div > div"
- elif (
- self.is_element_present('[name*="cf-turnstile-"]')
- and self.is_element_present(
- '[style="display: grid;"] div div'
- )
- ):
+ elif self.is_element_present('[style="display: grid;"] div div'):
  selector = '[style="display: grid;"] div div'
- elif (
- self.is_element_present('[name*="cf-turnstile-"]')
- and self.is_element_present("[class*=spacer] + div div")
- ):
+ elif self.is_element_present("[class*=spacer] + div div"):
  selector = '[class*=spacer] + div div'
- elif (
- self.is_element_present('[name*="cf-turnstile-"]')
- and self.is_element_present(".spacer div:not([class])")
- ):
+ elif self.is_element_present(".spacer div:not([class])"):
  selector = ".spacer div:not([class])"
- elif (
- self.is_element_present('script[src*="challenges.c"]')
- and self.is_element_present(
- '[data-testid*="challenge-"] div'
- )
- ):
+ elif self.is_element_present('[data-testid*="challenge-"] div'):
  selector = '[data-testid*="challenge-"] div'
- elif self.is_element_present(
- "div#turnstile-widget div:not([class])"
- ):
+ elif self.is_element_present("div#turnstile-widget div:not([class])"):
  selector = "div#turnstile-widget div:not([class])"
  elif self.is_element_present("ngx-turnstile div:not([class])"):
  selector = "ngx-turnstile div:not([class])"
  elif self.is_element_present(
  'form div:not([class]):has(input[name*="cf-turn"])'
  ):
  selector = 'form div:not([class]):has(input[name*="cf-turn"])'
- elif (
- self.is_element_present('[src*="/turnstile/"]')
- and self.is_element_present("form div:not(:has(*))")
- ):
+ elif self.is_element_present("form div:not(:has(*))"):
  selector = "form div:not(:has(*))"
- elif (
- self.is_element_present('[src*="/turnstile/"]')
- and self.is_element_present(
- "body > div#check > div:not([class])"
- )
- ):
+ elif self.is_element_present("body > div#check > div:not([class])"):
  selector = "body > div#check > div:not([class])"
  elif self.is_element_present(".cf-turnstile-wrapper"):
  selector = ".cf-turnstile-wrapper"
- elif self.is_element_present('[class="cf-turnstile"]'):
- selector = '[class="cf-turnstile"]'
  elif self.is_element_present(
  '[id*="turnstile"] div:not([class])'
  ):
@@ -2048,7 +2033,7 @@ def __click_captcha(self, use_cdp=False):
  self.loop.run_until_complete(self.page.evaluate(script))
  self.loop.run_until_complete(self.page.wait())
  with suppress(Exception):
- time.sleep(0.08)
+ time.sleep(0.05)
  element_rect = self.get_gui_element_rect(selector, timeout=1)
  e_x = element_rect["x"]
  e_y = element_rect["y"]
@@ -2059,15 +2044,17 @@ def __click_captcha(self, use_cdp=False):
  x = e_x + x_offset
  y = e_y + y_offset
  sb_config._saved_cf_x_y = (x, y)
- time.sleep(0.08)
+ time.sleep(0.05)
+ if hasattr(sb_config, "_cdp_proxy") and sb_config._cdp_proxy:
+ time.sleep(0.22) # CAPTCHA may load slower with proxy
  if use_cdp:
  self.sleep(0.03)
  gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
  with gui_lock: # Prevent issues with multiple processes
  self.bring_active_window_to_front()
- time.sleep(0.056)
+ time.sleep(0.05)
  self.click_with_offset(selector, x_offset, y_offset)
- time.sleep(0.056)
+ time.sleep(0.05)
  else:
  self.gui_click_x_y(x, y)
 

seleniumbase/fixtures/base_case.py

@@ -227,6 +227,28 @@ def open(self, url):
  if self.__is_cdp_swap_needed():
  self.cdp.open(url)
  return
+ elif (
+ hasattr(self.driver, "_is_using_uc")
+ and self.driver._is_using_uc
+ and "@" in self.proxy_string
+ and (
+ not hasattr(self.driver, "_is_using_cdp")
+ or not self.driver._is_using_cdp
+ )
+ ):
+ # Auth in UC Mode requires CDP Mode
+ logging.info("UC Mode requires CDP Mode for auth. Activating now.")
+ self.activate_cdp_mode(url)
+ return
+ elif (
+ hasattr(self.driver, "_is_using_uc")
+ and self.driver._is_using_uc
+ and hasattr(self.driver, "_is_using_cdp")
+ and self.driver._is_using_cdp
+ ):
+ self.disconnect()
+ self.cdp.open(url)
+ return
  self._check_browser()
  if self.__needs_minimum_wait():
  time.sleep(0.04)
@@ -5005,7 +5027,7 @@ def activate_cdp_mode(self, url=None, **kwargs):
  self.driver.connect()
  current_url = self.get_current_url()
  if not current_url.startswith(("about", "data", "chrome")):
- self.open("about:blank")
+ self.driver.get("about:blank")
  self.driver.uc_open_with_cdp_mode(url, **kwargs)
  else:
  self.get_new_driver(undetectable=True)
@@ -5015,6 +5037,13 @@ def activate_cdp_mode(self, url=None, **kwargs):
  self.solve_captcha = self.cdp.solve_captcha
  if hasattr(self.cdp, "find_element_by_text"):
  self.find_element_by_text = self.cdp.find_element_by_text
+ if (
+ hasattr(sb_config, "_cdp_proxy")
+ and sb_config._cdp_proxy
+ and "@" in sb_config._cdp_proxy
+ ):
+ with suppress(Exception):
+ self.cdp.loop.run_until_complete(self.cdp.page.wait(0.25))
  self.undetectable = True
 
  def activate_recorder(self):
@@ -14527,7 +14556,8 @@ def __activate_virtual_display(self):
  sb_config._virtual_display = self._xvfb_display
  if "DISPLAY" not in os.environ.keys():
  print(
- "\nX11 display failed! Will use regular xvfb!"
+ "\n X11 display failed! Is Xvfb installed? "
+ "\n Try this: `sudo apt install -y xvfb`"
  )
  self.__activate_standard_virtual_display()
  else:
@@ -14540,7 +14570,10 @@ def __activate_virtual_display(self):
  print("\n" + str(e.msg))
  else:
  print(e)
- print("\nX11 display failed! Will use regular xvfb!")
+ print(
+ "\n X11 display failed! Is Xvfb installed? "
+ "\n Try this: `sudo apt install -y xvfb`"
+ )
  self.__activate_standard_virtual_display()
  return
  pyautogui_is_installed = False