This is an example of using Crawlera with R.


# helper functions
safe_GET <- purrr::safely(httr::GET)
safely_htmlParse <- purrr::safely(htmlParse)
safely_xpathSApply <- purrr::safely(xpathSApply)

GoogleHits <- function(gs_input)
{
  handle_it <- httr::handle('')
  gs_url <- paste("https://www.google.com/search?q=",
                  gsub(" ","+",gs_input), sep = "") # modified line
  ua <- random.ua() # get a random user agent
  CAINFO = paste(system.file(package="RCurl"), "/CurlSSL/crawlera-ca.crt", sep = "")

  gs.script <- safe_GET(gs_url, user_agent(ua),
                        use_proxy(url = "proxy.crawlera.com", port = 8010, username = "YOUR_USER_KEY", password = ""),
                        config(cainfo = CAINFO),
                        handle = handle_it
                        ,verbose()
  )

  # if we can parse the response and extract the information we're interested in
  # ...then it means we got valid content from the site
  # ...if not then we retry and pause as necessary reduce how hard we're hitting Crawlera
  # ...this also helps avoid 503 errors
  gs_doc <- safely_htmlParse(gs.script$result)
  gs_res <- safely_xpathSApply(gs_doc$result, '//*/div[@id="resultStats"]', xmlValue)

  # if we didn't get a valid result back, retry
  retry_count <- 0
  while (!is.null(gs_doc$error) | !is.null(gs_res$error)) {
    write(paste0(" >>> retry(", retry_count, ") <<<"), stderr())

    gs.script <- safe_GET(gs_url, user_agent(ua),
                          use_proxy(url = "proxy.crawlera.com", port = 8010, username = "YOUR_USER_KEY", password = ""),
                          config(cainfo = CAINFO),
                          handle = handle_it
                          ,verbose()
    )

    gs_doc <- safely_htmlParse(gs.script$result)
    gs_res <- safely_xpathSApply(gs_doc$result, '//*/div[@id="resultStats"]', xmlValue)

    retry_count = retry_count + 1
    if (retry_count == 3) {
      write(paste0(" !!!! NO VALID RESPONSE IN 3 RETRIES !!!! (", gs_url, ") "), stderr())
      Sys.sleep(10)
    } else if (retry_count == 10) {
      write(paste0(" !!!! NO VALID RESPONSE IN 10 RETRIES !!!! (", gs_url, ") "), stderr())
      Sys.sleep(300)
      retry_count <- 0
    }
  }

  gs.res.without.secs <- gsub("\\s*\\([^\\)]+\\)","",gs_res$result)
  gs.res <- as.numeric(gsub("[^0-9]", "", gs.res.without.secs))

  return(gs.res)
}

google_result <- GoogleHits(search_string)