使用 javascript 获取页面上的所有元素

Question

我写了一个网页爬虫，它成功爬取了一个网页，并使用puppeteer抓取了一个职位发布的数据。我现在正在尝试获取页面上所有具有.opportunity 类的元素，然后将其传递给一个函数，该函数将获取该特定职位发布的数据。但是getElementsByClassName 的列表返回的是一个空对象？

const puppeteer = require('puppeteer');
const fs = require('fs');

async function crawlOpo(opo) {

    const opportunity = {
        title: '',
        desc: '',
        category: '',
        reqName: '',
        hours: '',
        postingDate: '',
        locationName: '',
        address: ''
    };

    const title = await page.evaluate(() => {
        try {
            return opo.querySelector('.row .col-lg-20 h3 a').innerText
        } catch(err) {
            return err
        }
    });
    const desc = await page.evaluate(() => {
        try {
            return opo.querySelector('.hidden-xs.paragraph').innerText
        } catch(err) {
            return err
        }
    });

    const category = await page.evaluate(() => {
        try {
            return opo.querySelector('.row.paragraph .col-sm-18 .row .col-md-8 .label-with-icon span').innerText
        } catch(err) {
            return err
        }
    });

    const reqName = await page.evaluate(() => {
        try {
            return opo.querySelector('.row.paragraph .col-sm-18 .row .col-md-8:nth-of-type(2) .label-with-icon span').innerText
        } catch(err) {
            return err
        }
    });

    const hours = await page.evaluate(() => {
        try {
            return opo.querySelector('.row.paragraph .col-sm-18 .row .col-md-8:nth-of-type(3) .label-with-icon span').innerText
        } catch(err) {
            return err
        }
    });

    const postingDate = await page.evaluate(() => {
        try {
            return opo.querySelector('.row .col-lg-4 h3 small').innerText
        } catch(err) {
            return err
        }
    });

    const locationName = await page.evaluate(() => {
        try {
            return opo.querySelector('.row.paragraph:nth-of-type(2) .col-lg-20 div div candidate-physical-location address span:nth-of-type(2) span').innerText
        } catch(err) {
            return err
        }
    });

    opportunity.title = title;
    opportunity.desc = desc;
    opportunity.category = category;
    opportunity.reqName = reqName;
    opportunity.hours = hours;
    opportunity.postingDate = postingDate;
    opportunity.locationName = locationName;

    opportunities.push(opportunity)

    console.log(opportunities); 

    browser.close();
}

(async () => {
  try {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();

    const opportunities = [];

    let url = "https://recruiting2.ultipro.com/PUB1004PSCU/JobBoard/d433f5c3-37c8-4bcf-a3af-248a707c7d31/?q=&o=postedDateDesc"
    await page.goto(url, {timeout: 0, waitUntil: 'networkidle0'});

    const oportunitiesDOM = await page.evaluate(() => {
        return document.getElementsByClassName('opportunity');
    });

    oportunitiesDOM.forEach(opo => {
        await crawlOpo(opo)
    });
  } catch (err) {
    console.error(err)
  }
})()

这里的逻辑是它运行 async 箭头函数，该函数将启动浏览器 -> 加载页面 -> 评估页面 -> 获取类 .opporuntity 的所有元素 -> 遍历列表并传递每个机会进入crawlOpo 函数，然后获取该机会所需的特定数据，然后将该对象分配给一个数组。

Answer 1

在document.getElementsByClassName('.opportunity') 的参数中，您有CSS 选择器'.opportunity'。

getElementsByClassName 方法将类名作为参数，而不是 CSS 选择器。

很可能应该更正为document.getElementsByClassName('opportunity')。