C# HttpWebRequest 超时

Question

使用 Fiddler，我模仿了实际浏览器 (Chrome) 发送的所有标头。 然后我尝试通过

调用该函数

GetResponse("https://www.example.com/");

但得到错误：

System.dll 中出现“System.Net.WebException”类型的未处理异常

附加信息：操作已超时

代码：

public static string GetResponse(string sURL, CookieContainer cookies = null, string sParameters = "", string sUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36")
    {
        HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(sURL);
        httpRequest.UserAgent = sUserAgent;

        if (cookies == null) cookies = new CookieContainer();
        httpRequest.CookieContainer = cookies;

        System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Tls12;

        httpRequest.AllowAutoRedirect = true;
        httpRequest.KeepAlive = true;
        httpRequest.ProtocolVersion = HttpVersion.Version11;

        httpRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
        httpRequest.Headers.Add("Accept-Encoding", "gzip, deflate, br");
        httpRequest.Headers.Add("Accept-Language", "en-US,en;q=0.8");
        httpRequest.Headers.Add("upgrade-insecure-requests", "1");
        httpRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
        httpRequest.KeepAlive = true;

        if (sParameters == "")
        {
            httpRequest.Method = "GET";
        }
        else
        {
            httpRequest.Method = "POST";
            httpRequest.ContentType = "application/x-www-form-urlencoded";
            httpRequest.ContentLength = sParameters.Length;

            using (Stream stream = httpRequest.GetRequestStream())
            {
                stream.Write(Encoding.UTF8.GetBytes(sParameters), 0, sParameters.Length);
            }

        }
                    
        HttpWebResponse httpWebResponse = (HttpWebResponse)httpRequest.GetResponse();
        
        string sResponse;
        using (Stream stream = httpWebResponse.GetResponseStream())
        {
            StreamReader reader = new StreamReader(stream, System.Text.Encoding.GetEncoding(936));
            sResponse = reader.ReadToEnd();
        }

        return sResponse;
    }

在 Fiddler 中检查后，与常规浏览器操作相比，缺少的部分是 cookie，但是代码中的 httpwebrequest 对象已附加了一个 cookie 容器。所以我不知道为什么cookie丢失了。

感谢您的帮助。

Answer 1

它可能看起来很多东西，但你不需要关心这一切现在做什么。

只需在Form 上创建一个Button，然后将其作为Click 事件处理程序进行异步处理，如此处所示（此处的主要方法使用所有异步Http/IO .Net 方法）。
剩下的代码只需要粘贴（在同一个表单中，以使其快速）。

用于来回传递数据的StreamObject，在连接完成后会包含一些关于其ResourceURI属性中指定的网站的信息（如下）。

StreamObject.Payload 是完整的 Html 页面，使用其内部 CodePage 或服务器检测到的 CodePage 进行解码。
如果您想查看它，只需将其传递给 WebBrowser。

注意：
我可能遗漏了一些内容，将其修改为发布在此处。这将立即 很明显。
以防万一，告诉我它是什么，我会更新它 代码。
另外：
禁用 Fiddler！

Visual Studio Version: VS Pro 15.7.5
.Net FrameWork: 4.7.1


private async void TestConnection_Click(object sender, EventArgs e)
{
    StreamObject sObject = new StreamObject()
    {
        ResourceURI = new Uri(@"https://www.bestbuy.com/?intl=nosplash"),
        ProcessStream = true
    };

    sObject = await HTTP_GetStream(sObject);
    Console.WriteLine(sObject.Payload.Length);
}


using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Net.Security;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Security;
using System.Security.Authentication;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Security.Permissions;
using System.Security.Principal;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;


public class StreamObject
{
    public StreamObject()
    {
        this.Cookies = new CookieContainer();
    }

    public Stream ContentStream { get; set; }
    public bool ProcessStream { get; set; }
    public Uri ResourceURI { get; set; }
    public Uri ResponseURI { get; set; }
    public string Referer { get; set; }
    public string Payload { get; set; }
    public string ServerType { get; set; }
    public string ServerName { get; set; }
    public IPAddress[] ServerIP { get; set; }
    public string ContentName { get; set; }
    public string ContentType { get; set; }
    public string ContentCharSet { get; set; }
    public string ContentLanguage { get; set; }
    public long ContentLenght { get; set; }
    public HttpStatusCode StatusCode { get; set; }
    public string StatusDescription { get; set; }
    public WebExceptionStatus WebException { get; set; }
    public string WebExceptionDescription { get; set; }
    public CookieContainer Cookies { get; set; }
}

const uint COR_E_INVALIDOPERATION = 0x80131509;


public async Task<StreamObject> HTTP_GetStream(StreamObject RequestObject)
{
    if (string.IsNullOrEmpty(RequestObject.ResourceURI.ToString().Trim()))
        return null;

    MemoryStream memstream = new MemoryStream();
    HttpWebRequest httpRequest;
    CookieContainer CookieJar = new CookieContainer();
    HttpStatusCode StatusCode = HttpStatusCode.OK;

    ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 |
                                           SecurityProtocolType.Tls |
                                           SecurityProtocolType.Tls11 |
                                           SecurityProtocolType.Tls12;

    ServicePointManager.Expect100Continue = false;
    ServicePointManager.DefaultConnectionLimit = 10;
    ServicePointManager.ServerCertificateValidationCallback += TlsValidationCallback;

    httpRequest = WebRequest.CreateHttp(RequestObject.ResourceURI);

    try
    {
        HTTP_RequestHeadersInit(ref httpRequest, CookieJar, RequestObject);
        httpRequest.Method = "GET";
        using (HttpWebResponse httpResponse = (HttpWebResponse)await httpRequest.GetResponseAsync())
        {
            Stream ResponseStream = httpResponse.GetResponseStream();
            //SslProtocols Protocol = ExtractSslProtocol(ResponseStream);

            if (StatusCode == HttpStatusCode.OK)
            {
                await ResponseStream.CopyToAsync(memstream);
                RequestObject.ContentStream = memstream;
                RequestObject.ResponseURI = httpResponse.ResponseUri;
                RequestObject.ContentLenght = memstream.Length;
                RequestObject.ContentCharSet = httpResponse.CharacterSet ?? string.Empty;
                RequestObject.ContentLanguage = httpResponse.Headers["Content-Language"] ?? string.Empty;
                RequestObject.ContentType = httpResponse.ContentType.ToLower();
                if (RequestObject.ContentType.IndexOf(@"/") > -1)
                {
                    do {
                        RequestObject.ContentType = RequestObject.ContentType.Substring(RequestObject.ContentType.IndexOf(@"/") + 1);
                        if (RequestObject.ContentType.IndexOf(@"/") < 0)
                            break;
                    } while (true);
                    if (RequestObject.ContentType.IndexOf(";")> -1)
                        RequestObject.ContentType = RequestObject.ContentType.Substring(0, RequestObject.ContentType.IndexOf(@";"));
                    RequestObject.ContentType = "." + RequestObject.ContentType;
                }
                RequestObject.ContentName = httpResponse.Headers["Content-Disposition"] ?? string.Empty;
                if (RequestObject.ContentName.Length == 0)
                    RequestObject.ContentName = RequestObject.ResourceURI.Segments.Last();
                RequestObject.ServerType = httpResponse.Server;
                RequestObject.ServerName = RequestObject.ResponseURI.DnsSafeHost;
                RequestObject.ServerIP = await Dns.GetHostAddressesAsync(RequestObject.ServerName);
                RequestObject.StatusCode = StatusCode;
                RequestObject.StatusDescription = httpResponse.StatusDescription;
                if (RequestObject.ProcessStream)
                    RequestObject.Payload = ProcessResponse(RequestObject.ContentStream,
                                                            Encoding.GetEncoding(RequestObject.ContentCharSet), 
                                                            httpResponse.ContentEncoding);
            }
        }
    }
    catch (WebException exW)
    {
        if (exW.Response != null)
        {
            RequestObject.StatusCode = ((HttpWebResponse)exW.Response).StatusCode;
            RequestObject.StatusDescription = ((HttpWebResponse)exW.Response).StatusDescription;
        }
        RequestObject.WebException = exW.Status;
        RequestObject.WebExceptionDescription = exW.Message;

    }
    catch (Exception exS)
    {
        if ((uint)exS.HResult == COR_E_INVALIDOPERATION)
        {
            //RequestObject.WebException = PingHostAddress("8.8.8.8", 500) > 0
            //                            ? WebExceptionStatus.NameResolutionFailure
            //                            : WebExceptionStatus.ConnectFailure;
            RequestObject.WebException = WebExceptionStatus.ConnectFailure;
            RequestObject.WebExceptionDescription = RequestObject.WebException.ToString();
        }
        else
        {
            RequestObject.WebException = WebExceptionStatus.RequestCanceled;
            RequestObject.WebExceptionDescription = RequestObject.WebException.ToString();
        }
    }
    finally
    {
        ServicePointManager.ServerCertificateValidationCallback -= TlsValidationCallback;
    }

    RequestObject.Cookies = httpRequest.CookieContainer;
    RequestObject.StatusCode = StatusCode;
    return RequestObject;

}   //HTTP_GetStream


private bool TlsValidationCallback(object sender, X509Certificate CACert, X509Chain CAChain, SslPolicyErrors sslPolicyErrors)
{
    //if (sslPolicyErrors == SslPolicyErrors.None)
    //    return true;

    X509Certificate2 _Certificate = new X509Certificate2(CACert);
    //X509Certificate2 _CACert = new X509Certificate2(@"[localstorage]/ca.cert");
    //CAChain.ChainPolicy.ExtraStore.Add(_CACert);


    //X509Certificate2 cert =  GetCertificateFromStore(thumbprint);
    X509Certificate2 cert = (X509Certificate2)CACert;

    //CspKeyContainerInfo cpsKey = (CspKeyContainerInfo)((RSACryptoServiceProvider)cert.PublicKey.Key).CspKeyContainerInfo;
    //if (cert.HasPrivateKey) { RSA rsaKey = (RSA)cert.GetRSAPrivateKey(); }
    //if (cpsKey.Accessible) { Console.WriteLine("Exportable: {0}", cpsKey.Exportable); }

    // next line generates exception "Key does not exist"
    //bool isexportable = provider.CspKeyContainerInfo.Exportable;

    CAChain.Build(_Certificate);
    foreach (X509ChainStatus CACStatus in CAChain.ChainStatus)
    {
        if ((CACStatus.Status != X509ChainStatusFlags.NoError) &
            (CACStatus.Status != X509ChainStatusFlags.UntrustedRoot))
            return false;
    }
    return true;
}


private void HTTP_RequestHeadersInit(ref HttpWebRequest httpreq, CookieContainer cookiecontainer, StreamObject postdata)
{
    httpreq.Date = DateTime.Now;
    httpreq.Timeout = 30000;
    httpreq.ReadWriteTimeout = 30000;
    httpreq.CookieContainer = cookiecontainer;
    httpreq.KeepAlive = true;
    httpreq.AllowAutoRedirect = true;
    httpreq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
    httpreq.ServicePoint.MaxIdleTime = 30000;
    httpreq.Referer = postdata.Referer;
    httpreq.UserAgent = "Mozilla / 5.0(Windows NT 6.1; WOW64; Trident / 7.0; rv: 11.0) like Gecko";
    //httpreq.UserAgent = "Mozilla/5.0 (Windows NT 10; Win64; x64; rv:56.0) Gecko/20100101 Firefox/61.0";
    httpreq.Accept = "ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
    httpreq.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US;q=0.8,en-GB;q=0.5,en;q=0.3");
    httpreq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate;q=0.8");
    httpreq.Headers.Add(HttpRequestHeader.CacheControl, "no-cache");
    httpreq.Headers.Add("DNT", "1");

    if (postdata != null && postdata.UseProxy)
    {
        if (postdata.ProxyParameters != null)
        {
            if ((postdata.ProxyParameters.Host.Length > 0))
            {
                httpreq.Proxy = new WebProxy(postdata.ProxyParameters.Host, postdata.ProxyParameters.Port);
            }
            else
            {
                httpreq.Proxy = new WebProxy(postdata.ProxyParameters.Uri, postdata.ProxyParameters.BypassLocal);
            }
                httpreq.Proxy.Credentials = new NetworkCredential(postdata.ProxyParameters.Credentials.UserID,
                                                                  postdata.ProxyParameters.Credentials.Password);
        }
        else
        {
            httpreq.Proxy = WebRequest.GetSystemWebProxy();
        }
    }
}


private string ProcessResponse(Stream stream, Encoding encoding, string ContentEncoding)
{
    string html = string.Empty;
    stream.Position = 0;
    try
    {
        using (MemoryStream memStream = new MemoryStream())
        {
            if (ContentEncoding.Contains("gzip"))
            {
                using (GZipStream gzipStream = new GZipStream(stream, CompressionMode.Decompress))
                {
                    gzipStream.CopyTo(memStream);
                };
            }
            else if (ContentEncoding.Contains("deflate"))
            {
                using (DeflateStream deflStream = new DeflateStream(stream, CompressionMode.Decompress))
                {
                    deflStream.CopyTo(memStream);
                };
            }
            else
            {
                stream.CopyTo(memStream);
            }

            memStream.Position = 0;
            using (StreamReader reader = new StreamReader(memStream, encoding))
            {
                html = reader.ReadToEnd();
                html = DecodeMetaCharSetEncoding(memStream, html, encoding);
            };
        };
    }
    catch (Exception)
    {
        return string.Empty;
    }
    return html;
}


private string DecodeMetaCharSetEncoding(Stream memStream, string _html, Encoding _encode)
{
    Match _match = new Regex("<meta\\s+.*?charset\\s*=\\s*\"?(?<charset>[A-Za-z0-9_-]+)\"?",
                             RegexOptions.Singleline |
                             RegexOptions.IgnoreCase).Match(_html);
    if (_match.Success)
    {
        string charset = _match.Groups["charset"].Value.ToLower() ?? "utf-8";
        if ((charset == "unicode") | (charset == "utf-7") | (charset == "utf-16"))
           charset = "utf-8";

        try
        {
            Encoding metaEncoding = Encoding.GetEncoding(charset);
            if (_encode.WebName != metaEncoding.WebName)
            {
                memStream.Position = 0L;
                using (StreamReader recodeReader = new StreamReader(memStream, metaEncoding))
                { _html = recodeReader.ReadToEnd().Trim(); }
            }
        }
        catch (ArgumentException)
        {
            _html = string.Empty;
        }
        catch (Exception)
        {
            _html = string.Empty;
        }
    }
    return _html;
}