Since it’s not so trivial to set up proxy authentication in Selenium, a popular option is to employ Polipo as a proxy. Update Polipo configuration file /etc/polipo/config to include Crawlera credentials (if the file is not present, copy and rename config.sample found in Polipo source folder):


parentProxy = "proxy.crawlera.com:8010"
parentAuthCredentials = "<CRAWLERA_APIKEY>:"


For password safety reasons this content is displayed as (hidden) in the Polipo web interface manager. The next step is to specify Polipo proxy details in the Selenium automation script, e.g. for Python and Firefox:


from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains

profile = webdriver.FirefoxProfile()
profile.set_preference('network.proxy.type', 1)  # manual
profile.set_preference('network.proxy.http', '127.0.0.1')
profile.set_preference('network.proxy.http_port', 8123)
profile.set_preference('network.proxy.ftp', '127.0.0.1')
profile.set_preference('network.proxy.ftp_port', 8123)
profile.set_preference('network.proxy.ssl', '127.0.0.1')
profile.set_preference('network.proxy.ssl_port', 8123)
profile.set_preference('network.proxy.no_proxies_on', 'localhost, 192.168.0.1')

driver = webdriver.Firefox(firefox_profile=profile)

driver.get("https://www.whatismyip.com/")
elem = driver.find_element_by_css_selector("a.btn.btn-success.btn-md.btn-block")
actions = ActionChains(driver)
actions.click(on_element=elem)
actions.perform()
print("Clicked on a button!")
driver.close()


Another example using Chrome:


from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

polipo_proxy = "127.0.0.1:8123"
proxy = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': polipo_proxy,
    'ftpProxy' : polipo_proxy,
    'sslProxy' : polipo_proxy,
    'noProxy'  : ''
})

capabilities = dict(DesiredCapabilities.CHROME)
proxy.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)

driver.get("https://www.whatismyip.com/")
elem = driver.find_element_by_css_selector("a.btn.btn-success.btn-md.btn-block")
actions = ActionChains(driver)
actions.click(on_element=elem)
actions.perform()
print("Clicked on a button!")
driver.close()