如何使用 Puppeteer 抓取无限滚动网站 [重复]答案

【问题标题】：How to scrape infinite scroll websites using Puppeteer [duplicate]如何使用 Puppeteer 抓取无限滚动网站 [重复]
【发布时间】：2021-01-21 10:27:27
【问题描述】：

我正在尝试抓取一个无限滚动的网站。

我正在控制滚动，但仍然在到达网页末尾后退出。

这是我的代码：

const puppeteer = require("puppeteer");

module.exports.scraper = async (url, callBack) => {
    const browser = await puppeteer.launch({ headless: false });
    const page = await browser.newPage();

    await page.setUserAgent(
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
    );

    await page.setViewport({ width: 1200, height: 768 });

    function wait(ms) {
        return new Promise((resolve) => setTimeout(() => resolve(), ms));
    }

    await page.goto(`${url}/products/?department=men&l2_category=polos-t-shirts`, {
        waitUntil: "networkidle0",
    });

    // Get the height of the rendered page
    const bodyHandle = await page.$("body");
    const { height } = await bodyHandle.boundingBox();
    await bodyHandle.dispose();

    // Scroll one viewport at a time, pausing to let content load
    const viewportHeight = page.viewport().height;
    let viewportIncr = 0;
    while (viewportIncr + viewportHeight < height) {
        await page.evaluate((_viewportHeight) => {
            window.scrollBy(0, _viewportHeight);
        }, viewportHeight);
        await wait(1600);
        viewportIncr = viewportIncr + viewportHeight;
    }

    let data = await page.evaluate(() => {
        window.scrollTo(0, 0);
        let products = [];
        let productElements = document.querySelectorAll(".product-wrap");

        productElements.forEach((productElement) => {
            let productJson = {};
            try {
                productJson.imageUrl = productElement.querySelector(".renderedImg").src;
                productJson.brandName = productElement.querySelector(
                    ".brand-name",
                ).innerText;
            } catch (e) {
                console.log(e);
            }
            products.push(productJson);
        });
        return products;
    });
    await wait(100);
    callBack(data, true);
    await browser.close();
};

这种情况怎么刮？

【问题讨论】：

“到达网页末尾后退出”是什么意思？它应该做什么呢？我还不清楚您要解决什么问题。
我需要我的脚本在滚动到达页面底部后等待它加载内容。我正在抓取具有无限滚动功能的网站。

标签： node.js web-scraping puppeteer

【解决方案1】：

这是处理无限滚动的一种策略。它在循环中重复滚动/比较，直到滚动无效。即当我们告诉它滚动，但我们仍然在与上次迭代相同的 scrollTop 值时，认为它完成了。在极端情况下，浏览器最终会耗尽堆内存并崩溃，但这是我们对于普通站点的起点：

const puppeteer = require('puppeteer');
const url = 'https://example.com';

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  page.on('console', async msg => {
    const args = msg.args();
    const vals = [];
    for (let i = 0; i < args.length; i++) {
      vals.push(await args[i].jsonValue());
    }
    console.log(vals.join('\t'));
  });
  await page.goto(url);
  await page.evaluate(()=> {
    
    const wait = (duration) => { 
      console.log('waiting', duration);
      return new Promise(resolve => setTimeout(resolve, duration)); 
    };

    (async () => {
      
      window.atBottom = false;
      const scroller = document.documentElement;  // usually what you want to scroll, but not always
      let lastPosition = -1;
      while(!window.atBottom) {
        scroller.scrollTop += 1000;
        // scrolling down all at once has pitfalls on some sites: scroller.scrollTop = scroller.scrollHeight;
        await wait(300);
        const currentPosition = scroller.scrollTop;
        if (currentPosition > lastPosition) {
          console.log('currentPosition', currentPosition);
          lastPosition = currentPosition;
        }
        else {
          window.atBottom = true;
        }
      }
      console.log('Done!');

    })();

  });

  await page.waitForFunction('window.atBottom == true', {
    timeout: 900000,
    polling: 1000 // poll for finish every second
  });

  await page.screenshot({path: 'example.png', fullPage: true});

  await browser.close();
})();

【讨论】：