Scrapinghub Support Center

Since it’s not so trivial to set up proxy authentication in Selenium, a popular option is to employ Polipo as a proxy. Update Polipo configuration file /etc/polipo/config to include Crawlera credentials (if the file is not present, copy and rename config.sample found in Polipo source folder):

parentProxy = "proxy.crawlera.com:8010"
parentAuthCredentials = "<CRAWLERA_APIKEY>:"

For password safety reasons this content is displayed as (hidden) in the Polipo web interface manager. The next step is to specify Polipo proxy details in the Selenium automation script, e.g. for Python and Firefox:

from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains

profile = webdriver.FirefoxProfile()
profile.set_preference('network.proxy.type', 1)  # manual
profile.set_preference('network.proxy.http', '127.0.0.1')
profile.set_preference('network.proxy.http_port', 8123)
profile.set_preference('network.proxy.ftp', '127.0.0.1')
profile.set_preference('network.proxy.ftp_port', 8123)
profile.set_preference('network.proxy.ssl', '127.0.0.1')
profile.set_preference('network.proxy.ssl_port', 8123)
profile.set_preference('network.proxy.no_proxies_on', 'localhost, 192.168.0.1')

driver = webdriver.Firefox(firefox_profile=profile)

driver.get("https://www.whatismyip.com/")
elem = driver.find_element_by_css_selector("a.btn.btn-success.btn-md.btn-block")
actions = ActionChains(driver)
actions.click(on_element=elem)
actions.perform()
print("Clicked on a button!")
driver.close()

For Python and Chrome:

from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

polipo_proxy = "127.0.0.1:8123"
proxy = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': polipo_proxy,
    'ftpProxy' : polipo_proxy,
    'sslProxy' : polipo_proxy,
    'noProxy'  : ''
})

chrome_options = webdriver.ChromeOptions()

# disable images to speed up the page loading
prefs = {"profile.managed_default_content_settings.images": 2}
chrome_options.add_experimental_option("prefs", prefs)

capabilities = dict(DesiredCapabilities.CHROME)
proxy.add_to_capabilities(capabilities)

driver = webdriver.Chrome(desired_capabilities=capabilities, chrome_options=chrome_options)
driver.set_page_load_timeout(600)

driver.get("https://www.whatismyip.com/")
elem = driver.find_element_by_css_selector("a.btn.btn-success.btn-md.btn-block")
actions = ActionChains(driver)
actions.click(on_element=elem)
actions.perform()
print("Clicked on a button!")
driver.close()

For Java and Chrome:

import org.openqa.selenium.By;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

public class CrawleraTest {

    public static void main(String[] args) {

        Proxy proxy = new Proxy(); 
        proxy.setHttpProxy("127.0.0.1:8123"); 
        proxy.setSslProxy("127.0.0.1:8123"); 

        DesiredCapabilities capabilities = DesiredCapabilities.chrome(); 
        capabilities.setCapability("proxy", proxy); 
        
        ChromeOptions options = new ChromeOptions(); 
        options.addArguments("start-maximized"); 

        capabilities.setCapability(ChromeOptions.CAPABILITY, options); 

        WebDriver driver = new ChromeDriver(capabilities);
        
        driver.get("https://www.whatismyip.com/");

        WebDriverWait wait = new WebDriverWait(driver, 60);    
        wait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//a[@role='button'][contains(text(),'Questions & Answers')]")));

        driver.quit();
    }
}