单个 URL 中的 Web Scrape 分页（cheerio 和 axios）答案

【问题标题】：Web Scrape pagination in a single URL (cheerio and axios)单个 URL 中的 Web Scrape 分页（cheerio 和 axios）
【发布时间】：2022-05-11 22:10:45
【问题描述】：

这里是新手。我正在进行网络抓取项目。我想要一些关于网页抓取分页技术的指南。我正在抓取这个网站https://www.imoney.my/unit-trust-investments。如您所见，我想根据 X 年检索不同的“总回报”百分比。现在我正在使用cheerio和axios。

const http = require("http");
const axios = require("axios");
const cheerio = require("cheerio");

http
    .createServer(async function (_, res) {
        try {
            const response = await axios.get(
                "https://www.imoney.my/unit-trust-investments"
            );

            const $ = cheerio.load(response.data);

            const funds = [];
            $("[class='list-item']").each((_i, row) => {
                const $row = $(row);

                const fund = $row.find("[class*='product-title']").find("a").text();
                const price = $row.find("[class*='is-narrow product-profit']").find("b").text();
                const risk = $row.find("[class*='product-title']").find("[class*='font-xsm extra-info']").text().replace('/10','');;
                const totalreturn = $row.find("[class*='product-return']").find("[class='font-lg']").find("b").text().replace('%','');

                funds.push({ fund, price, risk, totalreturn});
            });
            
            res.statusCode = 200;
            res.write(JSON.stringify(funds, null, 4));
        } catch (err) {
            res.statusCode = 400;
            res.write("Unable to process request.");
        }
        res.end();
    })
    .listen(8080);

请注意，选择不同年份时，URL 不会改变，只会改变总回报的值

【问题讨论】：

标签： javascript node.js web-scraping axios cheerio

【解决方案1】：

这是因为页面使用 javascript 来生成内容。在这种情况下，您需要像 Puppeteer 这样的东西。这就是你需要的：

const puppeteer = require("puppeteer");

const availableFunds = "10000";
const years = 2; // 3 for 0.5 years; 2 for 1 year; 1 for 2 years, 0 for 3 years.

async function start() {
  const browser = await puppeteer.launch({
    headless: false,
  });

  const page = await browser.newPage();
  await page.goto("https://www.imoney.my/unit-trust-investments");
  await page.waitForSelector(".product-item");

  await page.focus("#amount");
  for (let i = 0; i < 5; i++) {
    await page.keyboard.press("Backspace");
  }
  await page.type("#amount", availableFunds);

  await page.click("#tenure");
  for (let i = 0; i < years; i++) {
    await page.keyboard.press("ArrowUp");
  }
  await page.keyboard.press("Enter");
  const funds = await page.evaluate(() => {
    const funds = [];
    Array.from(document.querySelectorAll(".product-item")).forEach((el) => {
      const fund = el.querySelector(".title")?.textContent.trim();
      const price = el.querySelector(".investmentReturnValue")?.textContent.trim();
      const risk = el.querySelector(".col-title .info-desc dd")?.textContent.trim();
      const totalreturn = el.querySelector(".col-rate.text-left .info-desc .ir-value")?.textContent.trim();
      if (fund && price && risk && totalreturn) funds.push({ fund, price, risk, totalreturn });
    });
    return funds;
  });

  console.log(funds);

  browser.close();
}

start();

输出：

[
  {
    fund: 'Aberdeen Standard Islamic World Equity Fund - Class A',
    price: 'RM 12,651.20',
    risk: 'Medium\n                                7/10',
    totalreturn: '26.51'
  },
  {
    fund: 'Affin Hwang Select Balanced Fund',
    price: 'RM 10,355.52',
    risk: 'Medium\n                                5/10',
    totalreturn: '3.56'
  },
... and others

【讨论】：