Warning

Zyte API is replacing Smart Proxy Manager. See Migrating from Smart Proxy Manager to Zyte API.

Using Smart Proxy Manager with C#

Warning

For the code below to work you must first install the Zyte CA certificate.

Note

All the code in this documentation has been tested with .NET 5.0.401 and C# 9.0.

Using WebRequest

using System;
using System.IO;
using System.Net;

namespace ProxyRequest
{
    class MainClass
    {
        public static void Main(string[] args)
        {
            var myProxy = new WebProxy("http://proxy.zyte.com:8011", true);
            myProxy.Credentials = new NetworkCredential("<CRAWLERA_APIKEY>", "");

            var request = (HttpWebRequest)WebRequest.Create("https://httpbin.scrapinghub.com/headers");
            request.Proxy = myProxy;
            request.PreAuthenticate = true;
            request.AllowAutoRedirect = false;
            request.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => true;

            var response = request.GetResponse();
            Console.WriteLine("Response Status: " + ((HttpWebResponse)response).StatusDescription);
            Console.WriteLine("\nResponse Headers:\n" + ((HttpWebResponse)response).Headers);
            var dataStream = response.GetResponseStream();
            var reader = new StreamReader(dataStream);
            string responseFromServer = reader.ReadToEnd();
            Console.WriteLine("Response Body:\n" + responseFromServer);
            reader.Close();
            response.Close();
        }
    }
}

Asynchronous Request with HttpClient

Make sure to reference System.Net.Http while running the below code in case of any error.

using System;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;

namespace AsyncProxyRequest
{
    class MainClass
    {
        static void Main(string[] args)
        {
            _ = AsyncMain();
            Console.ReadKey();
        }

        public static async Task AsyncMain()
        {
            string requestUri = "https://httpbin.scrapinghub.com/headers";
            string crawleraApiKey = "<CRAWLERA_APIKEY>";
            string crawleraAuth = Convert.ToBase64String(
                Encoding.Default.GetBytes(crawleraApiKey + ":"));
            string crawleraUri = string.Format(
                "{0}:{1}", "proxy.zyte.com", "8010");

            WebProxy proxy = new WebProxy(crawleraUri, true);
            CookieContainer cookieContainer = new CookieContainer();
            HttpClientHandler httpClientHandler = new HttpClientHandler()
            {
                Proxy = proxy,
                PreAuthenticate = false,
                UseDefaultCredentials = false,
                UseProxy = true,
                UseCookies = true,
                CookieContainer = cookieContainer
            };
            HttpClient client = new HttpClient(httpClientHandler);

            int _TimeoutSec = 300;
            client.Timeout = new TimeSpan(0, 0, _TimeoutSec);
            client.DefaultRequestHeaders.Add("Proxy-Authorization", "Basic " + crawleraAuth);

            try
            {
                HttpResponseMessage response = await client.GetAsync(requestUri);
                HttpContent content = response.Content;
                Console.WriteLine("Response Status Code: " + (int)response.StatusCode);

                string scrapedPage = await content.ReadAsStringAsync();
                Console.WriteLine("Response Content: " + scrapedPage);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
        }
    }
}

Note

You may need to update the following setting in the applications App.config so proxy autherization works:

<system.net>
  <settings>
    <httpWebRequest useUnsafeHeaderParsing="true" />
  </settings>
</system.net>

Warning

Some HTTP client libraries, including Apache HttpComponents Client and .NET, don’t send authentication headers by default. This can result in doubled requests, so preemptive authentication should be enabled where this is the case.

If you use WebClient and receive 407s from Smart Proxy Manager, try setting AllowAutoRedirect to false.