puppeteer 在抓取谷歌地图评论时获取元素句柄

Answer 1

您可以使用以下代码（也称为check it in the online IDE）从 Google 地图评论页面获取所有信息：

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

const placeUrl = 'https://www.google.com/maps/place/Eiffel+Tower/@48.8583701,2.2901039,16z/data=!4m5!3m4!1s0x47e66e2964e34e2d:0x8ddca9ee380ef7e0!8m2!3d48.8583701!4d2.2944813'

async function scrollPage(page, scrollContainer) {
  let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
  while (true) {
    await page.evaluate(`document.querySelector("${scrollContainer}").scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
    await page.waitForTimeout(2000);
    let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
    if (newHeight === lastHeight) {
      break;
    }
    lastHeight = newHeight;
  }
}

async function getReviewsFromPage(page) {
  const reviews = await page.evaluate(() => {
    return Array.from(document.querySelectorAll(".jftiEf")).map((el) => {
      return {
        user: {
          name: el.querySelector(".d4r55")?.textContent.trim(),
          link: el.querySelector(".WNxzHc a")?.getAttribute("href"),
          thumbnail: el.querySelector(".NBa7we")?.getAttribute("src"),
          localGuide: el.querySelector(".RfnDt span:first-child").style.display === "none" ? undefined : true,
          reviews: parseInt(el.querySelector(".RfnDt span:last-child")?.textContent.replace("·", "")),
        },
        rating: parseFloat(el.querySelector(".kvMYJc")?.getAttribute("aria-label")),
        date: el.querySelector(".rsqaWe")?.textContent.trim(),
        snippet: el.querySelector(".MyEned")?.textContent.trim(),
        likes: parseFloat(el.querySelector(".GBkF3d:nth-child(2)")?.getAttribute("aria-label")),
        images: Array.from(el.querySelectorAll(".KtCyie button")).length
          ? Array.from(el.querySelectorAll(".KtCyie button")).map((el) => {
              return {
                thumbnail: getComputedStyle(el).backgroundImage.slice(5, -2),
              };
            })
          : undefined,
        date: el.querySelector(".rsqaWe")?.textContent.trim(),
      };
    });
  });
  return reviews;
}

async function fillPlaceInfo(page) {
  const placeInfo = await page.evaluate(() => {
    return {
      title: document.querySelector(".DUwDvf").textContent.trim(),
      address: document.querySelector("button[data-item-id='address']")?.textContent.trim(), // data-item-id attribute may be different if the language is not English
      rating: document.querySelector("div.F7nice").textContent.trim(),
      reviews: document.querySelector("span.F7nice").textContent.trim().split(" ")[0],
    };
  });
  return placeInfo;
}

async function getLocalPlaceReviews() {
  const browser = await puppeteer.launch({
    headless: false,
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });

  const page = await browser.newPage();

  await page.setDefaultNavigationTimeout(60000);
  await page.goto(placeUrl);
  await page.waitForSelector(".DUwDvf");

  const placeInfo = await fillPlaceInfo(page);

  await page.click(".mgr77e .DkEaL");
  await page.waitForTimeout(2000);
  await page.waitForSelector(".jftiEf");

  // await scrollPage(page, '.DxyBCb');

  const reviews = await getReviewsFromPage(page);

  await browser.close();

  return { placeInfo, reviews };
}

getLocalPlaceReviews().then(console.log);

如果您想获得所有评论（不仅来自第一页），您需要取消注释行

await scrollPage(page, '.DxyBCb');

输出：

{
   "placeInfo":{
      "title":"Eiffel Tower",
      "address":"Champ de Mars, 5 Av. Anatole France, 75007 Paris, France",
      "rating":"4.7",
      "reviews":"296,235"
   },
   "reviews":[
      {
         "user":{
            "name":"Chris Petrik",
            "link":"https://www.google.com/maps/contrib/105283592710083789031/reviews?hl=en-US",
            "thumbnail":"https://lh3.googleusercontent.com/a-/AFdZucoZXWYCGWuPQpSzxrEQ6_QAs1FfuUYCGsH8e4tj8KM=w36-h36-p-c0x00000000-rp-mo-ba5-br100",
            "localGuide":true,
            "reviews":48
         },
         "rating":5,
         "date":"5 months ago",
         "snippet":"I mean, come on...it's the Eiffel Tower! This is a magical place. The view of Paris from the top is absolutely breathtaking. The day we went it was not too crowded, so it made things much easier. This was part of our Honeymoon and I am.so …",
         "likes":63,
         "images":[
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipPgClEw3JwTLJOuf-DqC2xtZRodoavkpYVFBYqu=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipOIbD6g6nNgBH6ojj_OyqIFnQEDs8mHJOejDEs0=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipM7jHcKnsTdnOc4ZDZ4O22BA7WKSldfLTbTcLq3=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipPIeo83uGbdZLFFxfU0s6EQcUtHASohuUq5015U=w300-h225-p-k-no"
            }
         ]
      },
      {
         "user":{
            "name":"Rudro Rana",
            "link":"https://www.google.com/maps/contrib/113301929922891906080/reviews?hl=en-US",
            "thumbnail":"https://lh3.googleusercontent.com/a-/AFdZucoz9H1zqYCq4Z8nj1tBkzwFOCQaQepigI6vvaOrFQ=w36-h36-p-c0x00000000-rp-mo-ba4-br100",
            "localGuide":true,
            "reviews":86
         },
         "rating":5,
         "date":"6 months ago",
         "snippet":"The most spectacular and wonderful place in Paris. The Eiffel tower is historical place, but looks modern at the same time. The view from the top of the Eiffel is magnificent and breathtaking. The view of skyscraper and different historical …",
         "likes":24,
         "images":[
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipOs9TSNoyYmW1GL4SH9PlkAihvWsUbMTn-8O2Sj=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipMqIy_ik9TMotJPBn7uuIat-6ulc42OG95wN0WB=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipNZfqb-BSQ2gETyJd3ye4l2s7Cv7ttWkH9aOc00=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipPrTQSSiUCMMDgVgvLPE9zSpCPj4E2YNp0IPIS3=w300-h225-p-k-no"
            }
         ]
      },
      {
         "user":{
            "name":"Laura",
            "link":"https://www.google.com/maps/contrib/103073735774223183506/reviews?hl=en-US",
            "thumbnail":"https://lh3.googleusercontent.com/a/AItbvmk2xfd1DFgsFDiKkUJF8qTe_Alam2fJJoSor1sW=w36-h36-p-c0x00000000-rp-mo-ba4-br100",
            "localGuide":true,
            "reviews":44
         },
         "rating":4,
         "date":"3 months ago",
         "snippet":"We went with a tour company which allowed us to get through the queues faster- that was worth it. We also bought summit tickets. It was cool going up to the summit (a little nauseating) but in terms of enjoying the views the second floor …",
         "likes":12,
         "images":[
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipMBRGdJb3zL1rME20osajG-bosdIV8U82VTYS1n=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipN2t_lVvQzucqwV_kPE8YfrNANJVc73e3oT-c-p=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipOynJwoev37FdtckuYCrNeDHczSngytMITHsid7=w300-h225-p-k-no"
            },
            {
               "thumbnail":"https://lh5.googleusercontent.com/p/AF1QipPPxtgyiV9N85IpbeekCX2dweCAMB2_FI0_CfOx=w300-h225-p-k-no"
            }
         ]
      }
   ]
}

您可以从我的博文Web Scraping Google Maps Reviews with Nodejs 中了解有关抓取 Google 地图评论的更多信息。