diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 43f1f6d4e44..514a19332ef 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -298,13 +298,13 @@ with SB(uc=True, test=True, ad_block=True) as sb: items = sb.find_elements('[data-item-id]') for item in items: if required_text.lower() in item.text.lower(): - description = item.querySelector( + description = item.query_selector( '[data-automation-id="product-title"]' ) if description and description.text not in unique_item_text: unique_item_text.append(description.text) print("* " + description.text) - price = item.querySelector( + price = item.query_selector( '[data-automation-id="product-price"]' ) if price: diff --git a/examples/cdp_mode/playwright/raw_nordstrom_sync.py b/examples/cdp_mode/playwright/raw_nordstrom_sync.py index 2ac2d9d8dc2..16ae7fced6e 100644 --- a/examples/cdp_mode/playwright/raw_nordstrom_sync.py +++ b/examples/cdp_mode/playwright/raw_nordstrom_sync.py @@ -19,11 +19,11 @@ unique_item_text = [] items = sb.find_elements("article") for item in items: - description = item.querySelector("article h3") + description = item.query_selector("article h3") if description and description.text not in unique_item_text: unique_item_text.append(description.text) price_text = "" - price = item.querySelector('div div span[aria-hidden="true"]') + price = item.query_selector('div div span[aria-hidden="true"]') if price: price_text = price.text print("* %s (%s)" % (description.text, price_text)) diff --git a/examples/cdp_mode/raw_cdp_immoscout.py b/examples/cdp_mode/raw_cdp_immoscout.py new file mode 100644 index 00000000000..8be1af245a2 --- /dev/null +++ b/examples/cdp_mode/raw_cdp_immoscout.py @@ -0,0 +1,25 @@ +"""An example that bypasses the DataDome Slider CAPTCHA. +(PyAutoGUI is installed at runtime if not yet installed.)""" +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome(use_chromium=True, locale="en") +sb.goto("https://www.immoscout24.ch/en/real-estate/rent") +sb.sleep(1.6) +sb.solve_captcha() +sb.sleep(2.5) +sb.click_if_visible("#onetrust-accept-btn-handler", timeout=3) +sb.sleep(0.5) +sb.click('[data-cy="Locations_searchFieldOpener"]') +sb.sleep(0.6) +sb.press_keys('[data-cy="Locations_searchField"]', "Bern") +sb.sleep(0.6) +sb.click_if_visible('b:contains("Bern")', timeout=2) +sb.sleep(2.1) +sb.click('button[data-cy="SearchBar_button"]') +sb.sleep(3.5) +print("*** " + sb.get_text("h1")) +items = sb.find_elements('[data-test="result-list-item"]') +for item in items: + item.flash(color="44CC88") + print(item.query_selector('[class*="mainTitle"]').text) + print(item.query_selector("address").text) diff --git a/examples/cdp_mode/raw_cdp_nordstrom.py b/examples/cdp_mode/raw_cdp_nordstrom.py index 2a3d5d0ba87..eea45c06aec 100644 --- a/examples/cdp_mode/raw_cdp_nordstrom.py +++ b/examples/cdp_mode/raw_cdp_nordstrom.py @@ -12,11 +12,11 @@ unique_item_text = [] items = sb.find_elements("article") for item in items: - description = item.querySelector("article h3") + description = item.query_selector("article h3") if description and description.text not in unique_item_text: unique_item_text.append(description.text) price_text = "" - price = item.querySelector('div div span[aria-hidden="true"]') + price = item.query_selector('div div span[aria-hidden="true"]') if price: price_text = price.text print("* %s (%s)" % (description.text, price_text)) diff --git a/examples/cdp_mode/raw_cdp_target.py b/examples/cdp_mode/raw_cdp_target.py index 84412ba1f6b..7380b639a04 100644 --- a/examples/cdp_mode/raw_cdp_target.py +++ b/examples/cdp_mode/raw_cdp_target.py @@ -17,11 +17,11 @@ items = sb.find_elements('[data-test="product-details"]') for item in items: if required_text.lower() in item.text.lower(): - description = item.querySelector('a[data-test*="Card/title"]') + description = item.query_selector('a[data-test*="Card/title"]') if description and description.text not in unique_item_text: unique_item_text.append(description.text) print("* " + description.text) - price = item.querySelector('[data-test="current-price"]') + price = item.query_selector('[data-test="current-price"]') if price: print(" (" + price.text + ")") item.scroll_into_view() diff --git a/examples/cdp_mode/raw_cdp_walmart.py b/examples/cdp_mode/raw_cdp_walmart.py index 4e4500af62c..47d2ff1ce9c 100644 --- a/examples/cdp_mode/raw_cdp_walmart.py +++ b/examples/cdp_mode/raw_cdp_walmart.py @@ -22,13 +22,13 @@ items = sb.find_elements('[data-item-id]') for item in items: if required_text.lower() in item.text.lower(): - description = item.querySelector( + description = item.query_selector( '[data-automation-id="product-title"]' ) if description and description.text not in unique_item_text: unique_item_text.append(description.text) print("* " + description.text) - price = item.querySelector( + price = item.query_selector( '[data-automation-id="product-price"]' ) if price: diff --git a/examples/cdp_mode/raw_chatgpt.py b/examples/cdp_mode/raw_chatgpt.py index c31776422d3..c36669294ea 100644 --- a/examples/cdp_mode/raw_chatgpt.py +++ b/examples/cdp_mode/raw_chatgpt.py @@ -5,8 +5,9 @@ sb.activate_cdp_mode() sb.open("https://chatgpt.com/") sb.sleep(1) - sb.click_if_visible('button[aria-label="Close dialog"]') - sb.click_if_visible('button[data-testid="close-button"]') + close_1 = 'button[aria-label="Close dialog"]' + close_2 = 'button[data-testid="close-button"]' + sb.click_if_visible("%s, %s" % (close_1, close_2)) query = "Compare Playwright to SeleniumBase in under 178 words" sb.press_keys("#prompt-textarea", query) sb.click('button[data-testid="send-button"]') diff --git a/examples/cdp_mode/raw_idealista.py b/examples/cdp_mode/raw_idealista.py index 772bb83b5ca..51d1c040eef 100644 --- a/examples/cdp_mode/raw_idealista.py +++ b/examples/cdp_mode/raw_idealista.py @@ -13,5 +13,5 @@ items = sb.find_elements("div.item-info-container") for item in items: item.flash(color="44CC88") - print(item.querySelector("a.item-link").text) - print(item.querySelector("span.item-price").text) + print(item.query_selector("a.item-link").text) + print(item.query_selector("span.item-price").text) diff --git a/examples/cdp_mode/raw_immoscout.py b/examples/cdp_mode/raw_immoscout.py deleted file mode 100644 index 769e5654840..00000000000 --- a/examples/cdp_mode/raw_immoscout.py +++ /dev/null @@ -1,26 +0,0 @@ -"""An example that bypasses the DataDome Slider CAPTCHA. -(PyAutoGUI is installed at runtime if not yet installed.)""" -from seleniumbase import SB - -with SB(uc=True, test=True, locale="en") as sb: - sb.activate_cdp_mode() - sb.goto("https://www.immoscout24.ch/en/real-estate/rent") - sb.sleep(1.5) - sb.solve_captcha() - sb.sleep(2.5) - sb.click_if_visible("#onetrust-accept-btn-handler") - sb.sleep(0.5) - sb.click('[data-cy="Locations_searchFieldOpener"]') - sb.sleep(0.5) - sb.press_keys('[data-cy="Locations_searchField"]', "Bern") - sb.sleep(0.6) - sb.click('b:contains("Bern")') - sb.sleep(2.1) - sb.click('button[data-cy="SearchBar_button"]') - sb.sleep(3.5) - print("*** " + sb.get_text("h1")) - items = sb.find_elements('[data-test="result-list-item"]') - for item in items: - item.flash(color="44CC88") - print(item.querySelector('[class*="mainTitle"]').text) - print(item.querySelector("address").text) diff --git a/examples/cdp_mode/raw_nordstrom.py b/examples/cdp_mode/raw_nordstrom.py index e75e608473f..5d03acbc23f 100644 --- a/examples/cdp_mode/raw_nordstrom.py +++ b/examples/cdp_mode/raw_nordstrom.py @@ -13,11 +13,11 @@ unique_item_text = [] items = sb.find_elements("article") for item in items: - description = item.querySelector("article h3") + description = item.query_selector("article h3") if description and description.text not in unique_item_text: unique_item_text.append(description.text) price_text = "" - price = item.querySelector('div div span[aria-hidden="true"]') + price = item.query_selector('div div span[aria-hidden="true"]') if price: price_text = price.text print("* %s (%s)" % (description.text, price_text)) diff --git a/examples/cdp_mode/raw_softpedia.py b/examples/cdp_mode/raw_softpedia.py index bc513a4735e..4d37c5c40a1 100644 --- a/examples/cdp_mode/raw_softpedia.py +++ b/examples/cdp_mode/raw_softpedia.py @@ -16,10 +16,10 @@ sb.wait_for_element(item_container) items = sb.find_elements(item_container) for item in items: - result = item.querySelector("h4 a") + result = item.query_selector("h4 a") links.append(result.get_attribute("href")) print("* " + result.text) - print(item.querySelector("p").get_attribute("title")) + print(item.query_selector("p").get_attribute("title")) for link in links: sb.goto(link) sb.remove_elements("div.ad") diff --git a/examples/cdp_mode/raw_target.py b/examples/cdp_mode/raw_target.py index 5e891846c87..946ee2c783c 100644 --- a/examples/cdp_mode/raw_target.py +++ b/examples/cdp_mode/raw_target.py @@ -18,11 +18,11 @@ items = sb.find_elements('[data-test="product-details"]') for item in items: if required_text.lower() in item.text.lower(): - description = item.querySelector('a[data-test*="Card/title"]') + description = item.query_selector('a[data-test*="Card/title"]') if description and description.text not in unique_item_text: unique_item_text.append(description.text) print("* " + description.text) - price = item.querySelector('[data-test="current-price"]') + price = item.query_selector('[data-test="current-price"]') if price: print(" (" + price.text + ")") item.scroll_into_view() diff --git a/examples/cdp_mode/raw_walmart.py b/examples/cdp_mode/raw_walmart.py index 6882061f611..1b47ea4fa2c 100644 --- a/examples/cdp_mode/raw_walmart.py +++ b/examples/cdp_mode/raw_walmart.py @@ -23,13 +23,13 @@ items = sb.find_elements('[data-item-id]') for item in items: if required_text.lower() in item.text.lower(): - description = item.querySelector( + description = item.query_selector( '[data-automation-id="product-title"]' ) if description and description.text not in unique_item_text: unique_item_text.append(description.text) print("* " + description.text) - price = item.querySelector( + price = item.query_selector( '[data-automation-id="product-price"]' ) if price: diff --git a/examples/cdp_mode/raw_xpath.py b/examples/cdp_mode/raw_xpath.py index 82559e328d9..70d90a8532b 100644 --- a/examples/cdp_mode/raw_xpath.py +++ b/examples/cdp_mode/raw_xpath.py @@ -7,6 +7,7 @@ sb.cdp.goto("https://seleniumbase.io/demo_page") sb.cdp.highlight('//input[@id="myTextInput"]') sb.cdp.type('//*[@id="myTextInput"]', "XPath Test!") + sb.cdp.type('//input[@name="text" or @name="preText2"]', "Xpath Here!") sb.cdp.sleep(0.5) sb.cdp.highlight('//button[contains(text(),"(Green)")]') sb.cdp.click('//button[starts-with(text(),"Click Me")]') diff --git a/examples/presenter/uc_presentation_4.py b/examples/presenter/uc_presentation_4.py index a503faae661..2efd7fb5024 100644 --- a/examples/presenter/uc_presentation_4.py +++ b/examples/presenter/uc_presentation_4.py @@ -531,7 +531,7 @@ def test_presentation_4(self): items = sb.find_elements('[data-item-id]') for item in items: if required_text.lower() in item.text.lower(): - description = item.querySelector( + description = item.query_selector( '[data-automation-id="product-title"]' ) if ( @@ -540,7 +540,7 @@ def test_presentation_4(self): ): unique_item_text.append(description.text) print("* " + description.text) - price = item.querySelector( + price = item.query_selector( '[data-automation-id="product-price"]' ) if price: @@ -879,11 +879,11 @@ def test_presentation_4(self): unique_item_text = [] items = sb.find_elements("article") for item in items: - description = item.querySelector("article h3") + description = item.query_selector("article h3") if description and description.text not in unique_item_text: unique_item_text.append(description.text) price_text = "" - price = item.querySelector( + price = item.query_selector( 'div div span[aria-hidden="true"]' ) if price: diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index eff8d496fa8..28aece1764b 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.51.0" +__version__ = "4.51.1" diff --git a/seleniumbase/config/proxy_list.py b/seleniumbase/config/proxy_list.py index fd5c289a5d8..be0948e4a40 100644 --- a/seleniumbase/config/proxy_list.py +++ b/seleniumbase/config/proxy_list.py @@ -16,16 +16,14 @@ If you don't already have a proxy server to connect to, you can try finding one from one of following sites: * https://www.sslproxies.org/ -* https://bit.ly/36GtZa1 * https://www.us-proxy.org/ * https://hidemy.name/en/proxy-list/ -* http://free-proxy.cz/en/proxylist/country/all/https/ping/all """ PROXY_LIST = { - "example1": "98.8.195.160:443", # (Example) - set your own proxy here - "example2": "200.174.198.86:8888", # (Example) - "example3": "socks5://184.178.172.5:15303", # (Example) + "example1": "65.109.191.98:3080", # (Example) - set your own proxy here + "example2": "157.254.194.57:1080", # (Example) + "example3": "socks4://142.54.228.193:4145", # (Example) "proxy1": None, "proxy2": None, "proxy3": None, diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index e3d7475207b..f664002df95 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -2219,7 +2219,8 @@ def _on_a_datadome_slider_page(self, *args, **kwargs): self.loop.run_until_complete(self.page.wait(0.05)) if ( self.is_element_visible( - 'body > iframe[src*="/geo.captcha-delivery.com/captcha/"]' + 'body > iframe[src*="/geo.captcha-delivery.com/captcha/"], ' + 'body > iframe[src*="/geo.captcha-delivery.com/interstitial/"]' ) ): return True @@ -2305,14 +2306,18 @@ def __gui_click_recaptcha(self, use_cdp=False): return False def __gui_slide_datadome_captcha(self): - iframe = 'body > iframe[src*="/geo.captcha-delivery.com/captcha/"]' + iframe = ( + 'body > iframe[src*="/geo.captcha-delivery.com/captcha/"], ' + 'body > iframe[src*="/geo.captcha-delivery.com/interstitial/"]' + ) if not self.is_element_visible(iframe): return False src = self.get_attribute(iframe, "src") tab = self.get_active_tab() + time.sleep(0.05) self.open_new_tab(url=src) - time.sleep(0.41) - self.loop.run_until_complete(self.page.wait(0.1)) + time.sleep(0.42) + self.loop.run_until_complete(self.page.wait(0.12)) time.sleep(0.25) x1, y1 = self.get_gui_element_center("div.slider") x2, y2 = self.get_gui_element_center("div.sliderTarget") diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 669f5ef6f6b..a9fd291a918 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -301,7 +301,15 @@ def open(self, url, **kwargs): shared_utils.check_if_time_limit_exceeded() self._check_browser() time.sleep(0.8) - self.driver.get(url) + try: + self.driver.get(url) + except Exception as e2: + if "ERR_CONNECTION_RESET" in e2.msg and self.proxy_string: + message = "ERR_CONNECTION_RESET: " + message += "Invalid proxy and/or Internet unreachable!" + raise ProxyConnectionException(message) + else: + raise elif ( "ERR_INTERNET_DISCONNECTED" in e.msg or "neterror?e=dnsNotFound" in e.msg @@ -355,6 +363,10 @@ def open(self, url, **kwargs): logging.debug("Invalid session id. Will open new browser.") self.driver = self.get_new_driver() self.driver.get(url) + elif "ERR_TUNNEL_CONNECTION_FAILED" in e.msg and self.proxy_string: + message = "ERR_CONNECTION_RESET: " + message += "Bad proxy and/or Internet unreachable!" + raise ProxyConnectionException(message) else: raise try: diff --git a/seleniumbase/fixtures/xpath_to_css.py b/seleniumbase/fixtures/xpath_to_css.py index 852ebb83ae4..618adfdb157 100644 --- a/seleniumbase/fixtures/xpath_to_css.py +++ b/seleniumbase/fixtures/xpath_to_css.py @@ -188,6 +188,25 @@ def convert_xpath_to_css(xpath): s_val2 = data.group(5) return '%s[%s="%s"][%s="%s"]' % (s_tag, s_atr1, s_val1, s_atr2, s_val2) + # Find instance of: //tag[@attribute1='value1' or @attribute2='value2'] + # This tracks single/double quotes and maps them to comma-separated CSS. + data = re.match( + r"^\s*//([\w\-*]*)\s*\[\s*@?([\w\-:]+)\s*=\s*[\"']([^\"']*)[\"']\s+" + r"or\s+@?([\w\-:]+)\s*=\s*[\"']([^\"']*)[\"']\s*\]", + xpath, + ) + if data: + s_tag = data.group(1) + if s_tag == "*": + s_tag = "" + s_atr1 = data.group(2) + s_val1 = data.group(3) + s_atr2 = data.group(4) + s_val2 = data.group(5) + return '%s[%s="%s"], %s[%s="%s"]' % ( + s_tag, s_atr1, s_val1, s_tag, s_atr2, s_val2 + ) + # **** End of handling special xpath edge cases instantly **** if xpath[0] != '"' and xpath[-1] != '"' and xpath.count('"') % 2 == 0: diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index a2d88eaa7ee..4e581d686a9 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -493,11 +493,13 @@ async def get( "*.pubmatic.com*", "*.id5-sync.com*", "*.moatads.com*", + "*.viously.com*", "*.dotomi.com*", "*.adsrvr.org*", "*.atmtd.com*", "*.liadm.com*", "*.loopme.me*", + "*.flashb.id*", "*.adnxs.com*", "*.openx.net*", "*.tapad.com*", diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index 4d222ebc22f..1f4639c8f04 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -606,6 +606,8 @@ async def start( proxy_pass, proxy_scheme, ) + elif proxy: + proxy = proxy_helper.validate_proxy_string(proxy_string) if "binary_location" in kwargs and not browser_executable_path: browser_executable_path = kwargs["binary_location"] if not user_data_dir and "--user-data-dir" in arg_join: