[SOLVED] Selenium + 2Captcha Proxy Not Working for Website

I’m trying to use a paid 2Captcha proxy with Selenium to access Finland insurance site. The proxy works with requests.get(), but when I try to access the website using Selenium, it fails.

I’ve verified the proxy manually, and it works. I’ve tried adding --proxy-server to ChromeOptions. No clear error messages appear, the page just doesn’t load.

What am I missing?

Here is what I get when the page opens (sorry it is in Croatian):

Here’s a minimal reproducible example of the code:

import requests
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

# Proxy credentials
USERNAME = "your_username"
PASSWORD = "your_password"
PROXY_DNS = "your_proxy_ip:port"


def test_proxy():
    """Test if the proxy is working."""
    proxy_url = f"http://{USERNAME}:{PASSWORD}@{PROXY_DNS}"
    proxies = {"http": proxy_url, "https": proxy_url}

    try:
        response = requests.get("http://ip-api.com/json", proxies=proxies, timeout=10)
        print("Proxy Response:", response.json())
    except requests.RequestException as e:
        print("Proxy failed:", e)


def setup_driver():
    """Setup Selenium WebDriver with proxy."""
    chrome_options = webdriver.ChromeOptions()
    proxy = f"http://{USERNAME}:{PASSWORD}@{PROXY_DNS}"
    chrome_options.add_argument(f"--proxy-server={proxy}")

    driver = webdriver.Chrome(options=chrome_options)
    return driver


def test_website():
    """Test if Selenium can access the website."""
    driver = setup_driver()
    driver.get("https://www.if.fi/henkiloasiakkaat/vakuutukset/autovakuutus")

    try:
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.TAG_NAME, "body"))
        )
        time.sleep(10)
        print("Page loaded successfully.")
    except:
        print("Failed to load the page.")

    driver.quit()


if __name__ == "__main__":
    test_proxy()  # Check if proxy works via requests
    test_website()  # Check if proxy works with Selenium

I saw your post on UpWork and I don't know anything about python but I have access to Claude 3.7, maybe this will help you:

I see the issue with your Selenium proxy configuration. When using authenticated proxies with Selenium, there are a few specific approaches required that are different from how requests handles proxies.

Here's what's missing in your current setup:

For authenticated proxies in Selenium, you can't pass credentials directly in the proxy URL when using --proxy-server. Instead, you need to use a separate approach to handle the authentication.
The error ERR_NO_SUPPORTED_PROXIES typically indicates that Chrome couldn't establish a connection with the provided proxy configuration.

Here's a corrected version of your code:

import requests
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.common.proxy import Proxy, ProxyType

# Proxy credentials
USERNAME = "your_username"
PASSWORD = "your_password"
PROXY_DNS = "your_proxy_ip:port"


def test_proxy():
    """Test if the proxy is working."""
    proxy_url = f"http://{USERNAME}:{PASSWORD}@{PROXY_DNS}"
    proxies = {"http": proxy_url, "https": proxy_url}

    try:
        response = requests.get("http://ip-api.com/json", proxies=proxies, timeout=10)
        print("Proxy Response:", response.json())
    except requests.RequestException as e:
        print("Proxy failed:", e)


def setup_driver():
    """Setup Selenium WebDriver with proxy."""
    chrome_options = webdriver.ChromeOptions()
    
    # Method 1: Using Chrome extension for authentication
    # Create proxy extension
    proxy_host = PROXY_DNS.split(':')[0]
    proxy_port = PROXY_DNS.split(':')[1]
    
    manifest_json = """
    {
        "version": "1.0.0",
        "manifest_version": 2,
        "name": "Chrome Proxy",
        "permissions": [
            "proxy",
            "tabs",
            "unlimitedStorage",
            "storage",
            "webRequest",
            "webRequestBlocking"
        ],
        "background": {
            "scripts": ["background.js"]
        },
        "minimum_chrome_version":"22.0.0"
    }
    """
    
    background_js = """
    var config = {
        mode: "fixed_servers",
        rules: {
            singleProxy: {
                scheme: "http",
                host: "%s",
                port: parseInt(%s)
            },
            bypassList: []
        }
    };

    chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});

    function callbackFn(details) {
        return {
            authCredentials: {
                username: "%s",
                password: "%s"
            }
        };
    }

    chrome.webRequest.onAuthRequired.addListener(
        callbackFn,
        {urls: ["<all_urls>"]},
        ['blocking']
    );
    """ % (proxy_host, proxy_port, USERNAME, PASSWORD)
    
    import os
    import zipfile
    import tempfile
    
    # Create a temporary directory
    plugin_dir = tempfile.mkdtemp()
    
    # Write the extension files
    with open(os.path.join(plugin_dir, 'manifest.json'), 'w') as f:
        f.write(manifest_json)
    with open(os.path.join(plugin_dir, 'background.js'), 'w') as f:
        f.write(background_js)
    
    # Create a zip file
    plugin_path = os.path.join(plugin_dir, 'proxy_auth_plugin.zip')
    with zipfile.ZipFile(plugin_path, 'w') as zp:
        zp.write(os.path.join(plugin_dir, 'manifest.json'), 'manifest.json')
        zp.write(os.path.join(plugin_dir, 'background.js'), 'background.js')
    
    # Add the extension to Chrome
    chrome_options.add_extension(plugin_path)
    
    # Additional recommended options
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    
    # Create and return the driver
    driver = webdriver.Chrome(options=chrome_options)
    return driver


def test_website():
    """Test if Selenium can access the website."""
    driver = setup_driver()
    
    try:
        # First verify our IP through the proxy
        driver.get("https://api.ipify.org")
        print(f"Current IP via proxy: {driver.find_element(By.TAG_NAME, 'pre').text}")
        
        # Then try the actual website
        driver.get("https://www.if.fi/henkiloasiakkaat/vakuutukset/autovakuutus")
        
        WebDriverWait(driver, 20).until(
            EC.presence_of_element_located((By.TAG_NAME, "body"))
        )
        print("Page loaded successfully.")
        print(f"Current URL: {driver.current_url}")
        
    except Exception as e:
        print(f"Failed to load the page. Error: {e}")
    finally:
        driver.quit()


if __name__ == "__main__":
    test_proxy()  # Check if proxy works via requests
    test_website()  # Check if proxy works with Selenium

Key changes and explanations:

Chrome Extension Method: For authenticated proxies, creating a Chrome extension to handle authentication is one of the most reliable approaches. This is what I've implemented in the setup_driver() function.
IP Verification: I added a check to verify the IP address you're using with the proxy before trying to access the target website.
Error Handling: I've improved error handling to display more specific error messages.
Timeout: Increased the wait time to 20 seconds to accommodate potentially slower proxy connections.

Alternative approaches if this doesn't work:

You could try using a PAC (Proxy Auto-Configuration) file:

chrome_options.add_argument(f"--proxy-pac-url=data:text/javascript,{{'FindProxyForURL': function(url, host) {{ return 'PROXY {PROXY_DNS}'; }}}}")

Or use Selenium's built-in proxy configuration (though this doesn't handle authentication as well):

proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = PROXY_DNS
proxy.ssl_proxy = PROXY_DNS
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities, options=chrome_options)