C# - HttpWebRequest 获取数据下一页问题答案

【问题标题】：C# - HttpWebRequest Get Data Next Page(s) IssueC# - HttpWebRequest 获取数据下一页问题
【发布时间】：2016-02-29 01:04:27
【问题描述】：

我尝试使用 httpwebrequest 从网站中提取数据。我能够从第一个“页面”中提取数据，但是当我尝试将数据获取到下一页或上一页时出现错误。

Next Page Action - Click on the number page

要查看 Post 参数，我使用 Fiddler 来比较我的代码和网站中的 Post Request Header。

Post Request - Header 2ºPage

我的实际 C# 代码是这样的：

string URI = "https://www.citius.mj.pt/Portal/consultas/ConsultaVendaAnuncios.aspx";                

CookieContainer cc = new CookieContainer();
HttpWebRequest h = (HttpWebRequest)WebRequest.Create(URI);
h.CookieContainer = cc;
HttpWebResponse hr = (HttpWebResponse)h.GetResponse();
using (var s = hr.GetResponseStream())
{
    using (var r = new StreamReader(s, Encoding.UTF8))
    {
        string html = r.ReadToEnd();                        

        Match mVS = Regex.Match(html, "id=\"__VIEWSTATE\" value=\"(.*?)\"", RegexOptions.Singleline);                        
        Match mEV = Regex.Match(html, "id=\"__EVENTVALIDATION\" value=\"(.*?)\"", RegexOptions.Singleline);

        ///////////////////////////////////////SEARCH///////////////////////////////////////////////////

        Dictionary<string, string> param = new Dictionary<string, string>();
        param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$ScriptManager1|ctl00$ContentPlaceHolder1$btnSearch");                        
        param.Add("__EVENTTARGET", "");
        param.Add("__EVENTARGUMENT", "");
        param.Add("__LASTFOCUS", "");
        param.Add("__VIEWSTATE", mVS.Groups[1].Value);                        
        param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
        param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
        param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
        param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
        param.Add("__ASYNCPOST", "true");
        param.Add("ctl00$ContentPlaceHolder1$btnSearch", "Pesquisar");

        string post = "";
        foreach (var i in param)
        {
            post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
        }

        var data = Encoding.UTF8.GetBytes(post);                        

        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URI);
        request.CookieContainer = cc;
        request.Referer = URI;
        request.Method = "POST";                        
        request.Headers.Add("Cache-Control", "no-cache");
        request.Accept = "*/*";                                                
        request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
        request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
        request.ContentLength = data.Length;
        request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
        request.Headers.Add("Origin", "https://www.citius.mj.pt");
        request.Headers.Add("X-Requested-With", "XMLHttpRequest");
        request.Headers.Add("X-MicrosoftAjax: Delta=true");

        using (var stream = request.GetRequestStream())
        {
            stream.Write(data, 0, data.Length);
        }

        HttpWebResponse response = (HttpWebResponse)request.GetResponse();                        

        var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();                        

        int x = 1;

        ////////////////////////////////////////////NEXT PAGE//////////////////////////////////////////

        param.Clear();

        param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$upResultados|ctl00$ContentPlaceHolder1$Pager1$lnkNext");
        param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
        param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
        param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
        param.Add("__EVENTTARGET", "ctl00$ContentPlaceHolder1$Pager1$lnkNext");
        param.Add("__EVENTARGUMENT", "");
        param.Add("__LASTFOCUS", "");
        param.Add("__VIEWSTATE", mVS.Groups[1].Value);
        param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
        param.Add("__ASYNCPOST", "true");

        post = "";
        foreach (var i in param)
        {
            post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
        }

        data = Encoding.UTF8.GetBytes(post);

        request = (HttpWebRequest)WebRequest.Create(URI);
        request.CookieContainer = cc;
        request.Referer = URI;
        request.Method = "POST";
        request.Headers.Add("Cache-Control", "no-cache");
        request.Accept = "*/*";
        request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
        request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
        request.ContentLength = data.Length;
        request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
        request.Headers.Add("Origin", "https://www.citius.mj.pt");
        request.Headers.Add("X-Requested-With", "XMLHttpRequest");
        request.Headers.Add("X-MicrosoftAjax: Delta=true");

        using (var stream = request.GetRequestStream())
        {
            stream.Write(data, 0, data.Length);
        }

        response = (HttpWebResponse)request.GetResponse();

        responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();

        int y = 1;

    }
}

当我尝试获取下一页数据时得到的结果： Result from fiddler

比较来自我的代码和网站的调用之间的标头，我看到它的唯一区别是“_EVENTEVALUATION”。

我做错了什么？

【问题讨论】：

这两个值是由 javascript 计算出来的。这是给你的建议：响应字符串是由| 字符连接的字符串数组。您必须找到它（不要使用拆分功能）。有些元素是字符串，有些是数字，其余的是整数，描述字符串的长度（可能包含|）跟随该整数。然后您使用 chrome（或其他工具）调试页面以查找页面如何计算两个这些值。
@Sakura 我更新了代码。我尝试使用 $InkNext（单击数字）而不是使用 btnNextPage。我认为这是一种简单的方法，而且我的参数更少。不幸的是，我没有得到我需要的结果。任何想法为什么这不起作用？我知道该网站制作了 __doPostBack('ctl00$ContentPlaceHolder1$Pager1$lnkNext','')

标签： c# post httpwebrequest

【解决方案1】：

按照我的建议，您应该拆分响应字符串。

string URI = "https://www.citius.mj.pt/Portal/consultas/ConsultaVendaAnuncios.aspx";

CookieContainer cc = new CookieContainer();
HttpWebRequest h = (HttpWebRequest)WebRequest.Create(URI);
h.CookieContainer = cc;
HttpWebResponse hr = (HttpWebResponse)h.GetResponse();
using (var s = hr.GetResponseStream())
{
    using (var r = new StreamReader(s, Encoding.UTF8))
    {
        string html = r.ReadToEnd();

        Match mVS = Regex.Match(html, "id=\"__VIEWSTATE\" value=\"(.*?)\"", RegexOptions.Singleline);
        Match mEV = Regex.Match(html, "id=\"__EVENTVALIDATION\" value=\"(.*?)\"", RegexOptions.Singleline);

        ///////////////////////////////////////SEARCH///////////////////////////////////////////////////

        Dictionary<string, string> param = new Dictionary<string, string>();
        param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$ScriptManager1|ctl00$ContentPlaceHolder1$btnSearch");
        param.Add("__EVENTTARGET", "");
        param.Add("__EVENTARGUMENT", "");
        param.Add("__LASTFOCUS", "");
        param.Add("__VIEWSTATE", mVS.Groups[1].Value);
        param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
        param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
        param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
        param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
        param.Add("__ASYNCPOST", "true");
        param.Add("ctl00$ContentPlaceHolder1$btnSearch", "Pesquisar");

        string post = "";
        foreach (var i in param)
        {
            post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
        }

        var data = Encoding.UTF8.GetBytes(post);

        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URI);
        request.CookieContainer = cc;
        request.Referer = URI;
        request.Method = "POST";
        request.Headers.Add("Cache-Control", "no-cache");
        request.Accept = "*/*";
        request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
        request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
        request.ContentLength = data.Length;
        request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
        request.Headers.Add("Origin", "https://www.citius.mj.pt");
        request.Headers.Add("X-Requested-With", "XMLHttpRequest");
        request.Headers.Add("X-MicrosoftAjax: Delta=true");

        using (var stream = request.GetRequestStream())
        {
            stream.Write(data, 0, data.Length);
        }

        HttpWebResponse response = (HttpWebResponse)request.GetResponse();

        var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();

        File.WriteAllText("D:\\tmp1.txt", responseString);

        ////////////////////////////////////////////NEXT PAGE//////////////////////////////////////////

        param.Clear();

        string[] split = responseString.Split('|');
        for (int i = 0; i < split.Length; i++)
        {
            if (split[i] == "__VIEWSTATE") param.Add("__VIEWSTATE", split[i + 1]);
            if (split[i] == "__EVENTVALIDATION") param.Add("__EVENTVALIDATION", split[i + 1]);
        }
        param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$upResultados|ctl00$ContentPlaceHolder1$Pager1$lnkNext");
        param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
        param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
        param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
        param.Add("__EVENTTARGET", "ctl00$ContentPlaceHolder1$Pager1$lnkNext");
        param.Add("__EVENTARGUMENT", "");
        param.Add("__LASTFOCUS", "");
        //param.Add("__VIEWSTATE", mVS.Groups[1].Value);
        //param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
        param.Add("__ASYNCPOST", "true");

        post = "";
        foreach (var i in param)
        {
            post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
        }

        data = Encoding.UTF8.GetBytes(post);

        request = (HttpWebRequest)WebRequest.Create(URI);
        request.CookieContainer = cc;
        request.Referer = URI;
        request.Method = "POST";
        request.Headers.Add("Cache-Control", "no-cache");
        request.Accept = "*/*";
        request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
        request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
        request.ContentLength = data.Length;
        request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
        request.Headers.Add("Origin", "https://www.citius.mj.pt");
        request.Headers.Add("X-Requested-With", "XMLHttpRequest");
        request.Headers.Add("X-MicrosoftAjax: Delta=true");

        using (var stream = request.GetRequestStream())
        {
            stream.Write(data, 0, data.Length);
        }

        response = (HttpWebResponse)request.GetResponse();

        responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();

        File.WriteAllText("D:\\tmp2.txt", responseString);
        int y = 1;

    }
}

【讨论】：

现在我明白你的意思了。在此解决方案之前，我认为我只需要使用与第一个请求相同的 _VIEWSTATE 和相同的 _EVENTVALIDATION。 Tnks 为您的患者和建议。你帮了我很多。