omkarcloud / botasaurus-proxy-authentication Goto Github PK

class Crawler:
    def __init__(self, proxy_url, remote_url):
        self.proxy = proxy_url

        self.chrome_options = Options()
        add_proxy_options(self.chrome_options, self.proxy)

        if remote_url:
            self.driver = webdriver.Remote(
                command_executor=remote_url, options=self.chrome_options
            )
        else:
            self.driver = webdriver.Chrome(options=self.chrome_options)

    def crawl(self, url):
        self.driver.get(url)
        print("Current URL:", self.driver.current_url)
        return self.driver.page_source

    def close(self):
        self.driver.quit()