知道我使用了 selenium、splash、模仿 xhr 请求但我失败了，我怎么能刮掉这个网站

Question

我需要抓取这个带有“加载更多”按钮的website。

1. 我使用了 selenium，但执行时间太长，我未能抓取所有需要的项目。
2. 我使用了 splash 但没有得到任何结果
3. 我尝试模拟 XHR 请求，但也失败了

什么工具适合抓取这样的网站？

Answer 1

首先，让我们弄清楚新部分项目的下载过程是如何工作的。在浏览器中，例如。 G。 Chrome，按 F12 打开 DevTools，导航到 https://www.tayara.tn/sc/immobilier/appartements，向下滚动，然后加载一些新项目，转到 Network 选项卡，将过滤器设置为 XHR，如下所示：

您可能会注意到，每次单击“Montrer plus”按钮时，都会记录大小约为 5 KB 的新请求。响应中有所有必要的数据：

要制作这样的 XHR，你需要从之前的响应中检索 data.listings.pageInfo.endCursor 值，并将其作为 variables.page.offset 属性放入请求负载中，当然，你还需要保留整个负载结构，并添加相关的标题：

关于variables.page.offset 属性。实际上，它由三个 Base64 编码部分组成，解码后很明显 e. G。 cDEwbg==.MjAxOS0wMS0yNlQyMDoyMTo1OFo=.NjAwMA== 是一些前缀 p10n + 开始日期 2019-01-26T20:21:58Z + 检索到的项目总数 6000。因此，您可以通过更改最后一个值来请求项目的任何其他部分。此外，您可以在 variables.page.count 属性中指定每个请求的项目数量（似乎限制为 100）。

这里是 VBA 示例，展示了如何进行这种抓取。 将JSON.bas 模块导入VBA 项目进行JSON 处理。

Option Explicit

Sub Test()

    Dim sCat As String
    Dim oResSht As Worksheet
    Dim oResCell As Range
    Dim lNextOutput As Long
    Dim sOffset As String
    Dim oRes As Object
    Dim sPayload As String
    Dim sJSONString As String
    Dim vJSON
    Dim sState As String
    Dim aItems
    Dim oItem

    ' Set category for parsing
    sCat = "2"
    ' Set output sheet
    Set oResSht = ThisWorkbook.Sheets(1)
    With oResSht
        .Cells.Delete
        .Cells.WrapText = False
        Set oResCell = .Cells(1, 1)
    End With
    lNextOutput = 1000
    sOffset = ""
    Set oRes = CreateObject("Scripting.Dictionary")
    Do
        ' Retrieve JSON content
        sPayload = _
            "{""query"":""query ListingsPage($page: Page, $filter: SearchFilter, $sortBy: SortOrder) {\n  listings: searchAds(page: $page, filter: $filter, sortBy: $sortBy) " & _
            "{\n    items {\n      uuid\n      title\n      price\n      currency\n      thumbnail\n      createdAt\n      state\n      category " & _
            "{\n        id\n        name\n        engName\n        __typename\n      }\n      user {\n        uuid\n        displayName\n        avatar(width: 96, height: 96) " & _
            "{\n          url\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    trackingInfo " & _
            "{\n      transactionId\n      listName\n      recommenderId\n      experimentId\n      variantId\n      __typename\n    }\n    totalCount\n    pageInfo " & _
            "{\n      startCursor\n      hasPreviousPage\n      endCursor\n      hasNextPage\n      __typename\n    }\n    __typename\n  }\n}\n""," & _
            """variables"":{""page"":{""count"":100,""offset"":""" & sOffset & """},""filter"":{""queryString"":null,""category"":""" & sCat & """,""regionId"":null,""attributeFilters"":[]},""sortBy"":""CREATED_DESC""},""operationName"":""ListingsPage""}"
        With CreateObject("MSXML2.XMLHTTP")
            .Open "POST", "https://www.tayara.tn/graphql", True
            .setRequestHeader "content-type", "application/json"
            '.setRequestHeader "user-agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
            .setRequestHeader "content-length", Len(sPayload)
            .send (sPayload)
            Do Until .readyState = 4: DoEvents: Loop
            sJSONString = .responseText
        End With
        ' Parse JSON sample
        JSON.Parse sJSONString, vJSON, sState
        Select Case True
            Case sState <> "Object"
                Debug.Print Now & " Invalid JSON response"
            Case IsNull(vJSON("data"))
                Debug.Print Now & " Response contains no data"
            Case Else
                ' Retrieve items
                aItems = vJSON("data")("listings")("items")
                ' Add retrieved items to resulting dataset
                For Each oItem In aItems
                    Set oRes(oRes.Count) = oItem
                Next
                ' Check if the page is last
                If vJSON("data")("listings")("pageInfo")("hasNextPage") = False Then Exit Do
                ' Retrieve offset property for next page request
                sOffset = vJSON("data")("listings")("pageInfo")("endCursor")
                Debug.Print Now & " " & sOffset
                ' Output once per 1000 parsed items
                If oRes.Count >= lNextOutput Then
                    Output oRes, oResCell
                    lNextOutput = oRes.Count + 1000
                End If
        End Select
        DoEvents
    Loop
    ' Finally output results
    Output oRes, oResCell
    MsgBox "Completed" & vbCrLf & "Actually parsed: " & oRes.Count & vbCrLf & """totalCount"" from API response: " & vJSON("data")("listings")("totalCount")

End Sub

Sub Output(vData, oTarget As Range)

    Dim aData()
    Dim aHeader()

    ' Convert raw JSON to 2d array and output to target range
    JSON.ToArray vData, aData, aHeader
    With oTarget
        OutputArray oTarget.Cells(1, 1), aHeader
        Output2DArray oTarget.Cells(1, 1).Offset(1, 0), aData
        .Parent.Columns.AutoFit
    End With

End Sub

Sub OutputArray(oDstRng As Range, aCells As Variant)

    With oDstRng
        .Parent.Select
        With .Resize(1, UBound(aCells) - LBound(aCells) + 1)
            .NumberFormat = "@"
            .Value = aCells
        End With
    End With

End Sub

Sub Output2DArray(oDstRng As Range, aCells As Variant)

    With oDstRng
        .Parent.Select
        With .Resize( _
                UBound(aCells, 1) - LBound(aCells, 1) + 1, _
                UBound(aCells, 2) - LBound(aCells, 2) + 1)
            .NumberFormat = "@"
            .Value = aCells
        End With
    End With

End Sub

我的输出如下：

顺便说一句，类似的方法适用于in other answers。