网络抓取谷歌标签管理器数据层中的嵌入表？答案

【问题标题】：Web scrape an embedded table in google tag manager's datalayer?网络抓取谷歌标签管理器数据层中的嵌入表？
【发布时间】：2020-03-02 03:21:46
【问题描述】：

我正在尝试从OECD Data 抓取表格。我正在学习在 Jupyter 中使用 Python 和 BeautifulSoup 来实现这一目标。

我找到的最接近的类似帖子是How to web scrape tables embedded in websites using Python。在我的例子中，被抓取的对象有一个嵌套的 JSON 对象，它与我从页面中检查的内容不匹配。

抓取的对象

<!DOCTYPE html>
<html lang="en"><head>
    <script>
        dataLayer = [{
            'siteName':'data',
            'siteEnvironment':'live',
            'pageLanguage':'en',
            'pageTopic':'',
            'pageSubTopic':'',
            'pageCategory':'oecdChart',
            'indicatorCode':'FBORNPOP'
        }];
    </script>
    <title>Foreign-born population, Total, % of population, 2013 – 2018</title>
    <meta content="IE=edge" http-equiv="X-UA-Compatible"/>
    <meta content="noindex" name="robots"/>
    <link href="//data.oecd.org/media/dataportal/stylesheets/main.min.css" rel="stylesheet" type="text/css"/>
    <link href="//data.oecd.org/media/oecdorg/styleassets/css/responsive/bootstrap.min.css" rel="stylesheet" type="text/css"/>
    <meta content="summary_large_image" name="twitter:card"/>
    <meta content="@OECD" name="twitter:site"/>
    <meta content="@OECD" name="twitter:creator"/>
    <meta content="Interactive charts by the OECD" name="twitter:title"/>
    <meta content="Foreign-born population, Total, % of population, 2013 – 2018" name="twitter:description"/>
    <meta content="https://data.oecd.org/custom/share/txqy.png" name="twitter:image:src"/>
    <meta content="website" property="og:type"/>
    <meta content="OECD Data" property="og:site_name"/>
    <meta content="https://data.oecd.org/chart/5Rmq" property="og:url"/>
    <meta content="Interactive charts by the OECD" property="og:title"/>
    <meta content="Foreign-born population, Total, % of population, 2013 – 2018" property="og:description"/>
    <meta content="https://data.oecd.org/custom/share/txqy.png" property="og:image"/>
    <meta content="en_GB" property="og:locale"/>
    <meta content="fr_FR" property="og:locale:alternate"/>
    <meta content="embedded" itemprop="context"/>
</head>
<body>
    <!-- New Google Tag Manager -->
    <noscript><iframe height="0" src="https://www.googletagmanager.com/ns.html?id=GTM-P5JSM4P" style="display:none;visibility:hidden" width="0"></iframe></noscript>
    <script>
    (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
            new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
            j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
            'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
    })(window, document, 'script', 'dataLayer', 'GTM-P5JSM4P');
    </script>
    <!-- End New Google Tag Manager -->

<!-- Google Tag Manager -->
<noscript>
    <iframe height="0" src="//www.googletagmanager.com/ns.html?id=GTM-WZ3XJR" style="display: none; visibility: hidden" width="0"></iframe>
</noscript>
<script>
    (function(w, d, s, l, i) {
        w[l] = w[l] || [];
        w[l].push({
            'gtm.start':
                new Date().getTime(),
            event: 'gtm.js'
        });
        var f = d.getElementsByTagName(s)[0],
            j = d.createElement(s),
            dl = l != 'dataLayer' ? '&l=' + l : '';
        j.async = true;
        j.src =
            '//www.googletagmanager.com/gtm.js?id=' + i + dl;
        f.parentNode.insertBefore(j, f);
    })(window, document, 'script', 'dataLayer', 'GTM-WZ3XJR');
</script>
<!-- End Google Tag Manager -->
<div class="ddp-root">
    <div class="embedded-chart">
        <div class="ddp-chart" data-baseline-comparison="" data-compact-header="false" data-data='{"header":{"id":"6bda3584-cfa5-46cb-afd3-d2512d0d52f8","test":false,"prepared":"2020-03-01T09:16:26.3887278Z","sender":{"id":"OECD","name":"Organisation for Economic Co-operation and Development"},"links":[{"href":"https://stats.oecd.org:443/sdmx-json/data/DP_LIVE/.FBORNPOP.TOT.PC_POP.A/OECD?json-lang=en&amp;dimensionAtObservation=allDimensions&amp;startPeriod=2013&amp;endPeriod=2018","rel":"request"}]},"dataSets":[{"action":"Information","observations":{"0:0:0:0:0:0":[27.54,0,null],"0:0:0:0:0:1":[27.814,0,null],"0:0:0:0:0:2":[28.077,0,null],"0:0:0:0:0:3":[28.65,0,null],"0:0:0:0:0:4":[29.2,0,null],"0:0:0:0:0:5":[29.641,0,null],"1:0:0:0:0:0":[15.676,0,null],"1:0:0:0:0:1":[15.816,0,null],"1:0:0:0:0:2":[15.807,0,null],"1:0:0:0:0:3":[16.281,0,null],"1:0:0:0:0:4":[16.566,0,null],"1:0:0:0:0:5":[16.809,0,null],"2:0:0:0:0:0":[19.952,0,null],"2:0:0:0:0:1":[20.108,0,null],"2:0:0:0:0:2":[20.275,0,null],"2:0:0:0:0:3":[20.779,0,null],"3:0:0:0:0:0":[8.115,0,null],"3:0:0:0:0:1":[8.43,0,null],"3:0:0:0:0:2":[8.838,0,null],"3:0:0:0:0:3":[9.463,0,null],"3:0:0:0:0:4":[9.952,0,null],"3:0:0:0:0:5":[10.282,0,null],"4:0:0:0:0:0":[5.235,0,null],"4:0:0:0:0:1":[5.553,0,null],"4:0:0:0:0:2":[5.85,0,null],"4:0:0:0:0:3":[6.127,0,null],"4:0:0:0:0:4":[6.473,0,null],"4:0:0:0:0:5":[6.726,0,null],"5:0:0:0:0:0":[12.539,0,null],"5:0:0:0:0:1":[12.976,0,null],"5:0:0:0:0:2":[13.45,0,null],"5:0:0:0:0:3":[13.982,0,null],"5:0:0:0:0:4":[15.513,0,null],"5:0:0:0:0:5":[16.006,0,null],"6:0:0:0:0:0":[4.274,0,null],"6:0:0:0:0:1":[4.527,0,null],"6:0:0:0:0:2":[4.831,0,null],"6:0:0:0:0:3":[5.171,0,null],"6:0:0:0:0:4":[5.289,0,null],"6:0:0:0:0:5":[5.534,0,null],"7:0:0:0:0:0":[41.495,0,null],"7:0:0:0:0:1":[42.707,0,null],"7:0:0:0:0:2":[43.887,0,null],"7:0:0:0:0:3":[45.258,0,null],"7:0:0:0:0:4":[46.404,0,null],"7:0:0:0:0:5":[47.563,0,null],"8:0:0:0:0:0":[0.787,0,null],"8:0:0:0:0:1":[0.79,0,null],"8:0:0:0:0:2":[0.74,0,null],"8:0:0:0:0:3":[0.79,0,null],"9:0:0:0:0:0":[11.468,0,null],"9:0:0:0:0:1":[11.581,0,null],"9:0:0:0:0:2":[11.795,0,null],"9:0:0:0:0:3":[12.106,0,null],"9:0:0:0:0:4":[12.545,0,null],"9:0:0:0:0:5":[12.97,0,null],"10:0:0:0:0:0":[13.059,0,null],"10:0:0:0:0:1":[13.685,0,null],"10:0:0:0:0:2":[14.236,0,null],"10:0:0:0:0:3":[14.701,0,null],"10:0:0:0:0:4":[15.075,0,null],"10:0:0:0:0:5":[15.362,0,null],"11:0:0:0:0:0":[8.628,0,null],"11:0:0:0:0:1":[8.455,0,null],"11:0:0:0:0:2":[8.555,0,null],"11:0:0:0:0:3":[8.613,0,null],"12:0:0:0:0:0":[15.308,0,null],"12:0:0:0:0:1":[15.804,0,null],"12:0:0:0:0:2":[16.397,0,null],"12:0:0:0:0:3":[17.039,0,null],"12:0:0:0:0:4":[18.006,0,null],"12:0:0:0:0:5":[18.803,0,null],"13:0:0:0:0:0":[27.325,0,null],"13:0:0:0:0:1":[27.883,0,null],"13:0:0:0:0:2":[28.376,0,null],"13:0:0:0:0:3":[28.761,0,null],"13:0:0:0:0:4":[29.259,0,null],"13:0:0:0:0:5":[29.483,0,null],"14:0:0:0:0:0":[12.29,0,null],"14:0:0:0:0:1":[12.535,0,null],"14:0:0:0:0:2":[13.107,0,null],"14:0:0:0:0:3":[13.662,0,null],"14:0:0:0:0:4":[14.157,0,null],"14:0:0:0:0:5":[13.794,0,null],"15:0:0:0:0:0":[12.846,0,null],"15:0:0:0:0:1":[12.942,0,null],"15:0:0:0:0:2":[13.174,0,null],"15:0:0:0:0:3":[13.436,0,null],"15:0:0:0:0:4":[13.481,0,null],"15:0:0:0:0:5":[13.626,0,null],"16:0:0:0:0:0":[2.364,0,null],"16:0:0:0:0:1":[2.486,0,null],"16:0:0:0:0:2":[2.593,0,null],"17:0:0:0:0:0":[6.603,0,null],"17:0:0:0:0:1":[0.0,0,null],"17:0:0:0:0:2":[6.641,0,null],"17:0:0:0:0:4":[5.811,0,null],"18:0:0:0:0:0":[16.039,0,null],"18:0:0:0:0:1":[16.132,0,null],"18:0:0:0:0:2":[16.376,0,null],"18:0:0:0:0:3":[17.148,0,null],"19:0:0:0:0:0":[22.435,0,null],"19:0:0:0:0:1":[23.36,0,null],"19:0:0:0:0:2":[24.478,0,null],"19:0:0:0:0:3":[25.077,0,null],"20:0:0:0:0:0":[15.031,0,null],"20:0:0:0:0:1":[14.934,0,null],"20:0:0:0:0:2":[14.831,0,null],"20:0:0:0:0:3":[14.767,0,null],"20:0:0:0:0:4":[14.701,0,null],"20:0:0:0:0:5":[15.014,0,null],"21:0:0:0:0:0":[11.89,0,null],"21:0:0:0:0:1":[12.032,0,null],"21:0:0:0:0:2":[12.186,0,null],"21:0:0:0:0:3":[12.276,0,null],"21:0:0:0:0:4":[12.476,0,null],"21:0:0:0:0:5":[12.486,0,null],"22:0:0:0:0:0":[9.529,0,null],"22:0:0:0:0:1":[9.596,0,null],"22:0:0:0:0:2":[9.708,0,null],"22:0:0:0:0:3":[9.94,0,null],"22:0:0:0:0:4":[10.199,0,null],"22:0:0:0:0:5":[10.415,0,null],"23:0:0:0:0:0":[14.515,0,null],"23:0:0:0:0:1":[16.023,0,null],"23:0:0:0:0:2":[16.504,0,null],"23:0:0:0:0:3":[16.378,0,null],"23:0:0:0:0:4":[16.844,0,null],"23:0:0:0:0:5":[17.593,0,null],"24:0:0:0:0:0":[16.081,0,null],"24:0:0:0:0:1":[16.61,0,null],"24:0:0:0:0:2":[17.375,0,null],"24:0:0:0:0:3":[18.305,0,null],"24:0:0:0:0:4":[18.96,0,null],"24:0:0:0:0:5":[19.392,0,null],"25:0:0:0:0:0":[7.063,0,null],"25:0:0:0:0:1":[7.161,0,null],"25:0:0:0:0:2":[7.3,0,null],"25:0:0:0:0:3":[7.523,0,null],"26:0:0:0:0:0":[13.292,0,null],"26:0:0:0:0:1":[12.88,0,null],"26:0:0:0:0:2":[12.773,0,null],"26:0:0:0:0:3":[12.769,0,null],"26:0:0:0:0:4":[12.997,0,null],"26:0:0:0:0:5":[13.365,0,null],"27:0:0:0:0:0":[0.0,0,null],"27:0:0:0:0:1":[1.883,0,null],"27:0:0:0:0:2":[2.024,0,null],"27:0:0:0:0:3":[2.235,0,null],"27:0:0:0:0:4":[2.383,0,null],"27:0:0:0:0:5":[2.781,0,null],"28:0:0:0:0:0":[0.0,0,null],"28:0:0:0:0:1":[0.0,0,null],"28:0:0:0:0:2":[0.0,0,null],"29:0:0:0:0:0":[10.891,0,null],"29:0:0:0:0:1":[11.374,0,null],"29:0:0:0:0:2":[11.906,0,null],"29:0:0:0:0:3":[12.639,0,null],"29:0:0:0:0:4":[13.884,0,null],"29:0:0:0:0:5":[16.162,0,null],"30:0:0:0:0:0":[3.186,0,null],"30:0:0:0:0:1":[3.225,0,null],"30:0:0:0:0:2":[3.273,0,null],"30:0:0:0:0:3":[3.336,0,null],"30:0:0:0:0:4":[3.418,0,null],"30:0:0:0:0:5":[3.492,0,null],"31:0:0:0:0:0":[23.472,0,null],"31:0:0:0:0:1":[22.936,0,null],"31:0:0:0:0:2":[22.532,0,null],"31:0:0:0:0:3":[22.187,0,null],"31:0:0:0:0:4":[21.78,0,null],"32:0:0:0:0:0":[13.879,0,null],"32:0:0:0:0:1":[13.629,0,null],"32:0:0:0:0:2":[13.47,0,null],"32:0:0:0:0:3":[13.138,0,null],"32:0:0:0:0:4":[12.898,0,null],"32:0:0:0:0:5":[12.749,0,null],"33:0:0:0:0:1":[4.807,0,null],"33:0:0:0:0:2":[9.251,0,null],"33:0:0:0:0:3":[4.463,0,null],"33:0:0:0:0:4":[4.404,0,null],"33:0:0:0:0:5":[4.554,0,null]}}],"structure":{"links":[{"href":"https://stats.oecd.org/sdmx-json/dataflow/DP_LIVE/OECD","rel":"dataflow"}],"name":"OECD Data Live dataset","description":"OECD Data Live dataset","dimensions":{"observation":[{"keyPosition":0,"id":"LOCATION","name":"Country","values":[{"id":"AUS","name":"Australia"},{"id":"BEL","name":"Belgium"},{"id":"CAN","name":"Canada"},{"id":"DNK","name":"Denmark"},{"id":"FIN","name":"Finland"},{"id":"DEU","name":"Germany"},{"id":"HUN","name":"Hungary"},{"id":"LUX","name":"Luxembourg"},{"id":"MEX","name":"Mexico"},{"id":"NLD","name":"Netherlands"},{"id":"NOR","name":"Norway"},{"id":"PRT","name":"Portugal"},{"id":"SWE","name":"Sweden"},{"id":"CHE","name":"Switzerland"},{"id":"GBR","name":"United Kingdom"},{"id":"USA","name":"United States"},{"id":"CHL","name":"Chile"},{"id":"GRC","name":"Greece"},{"id":"IRL","name":"Ireland"},{"id":"NZL","name":"New Zealand"},{"id":"EST","name":"Estonia"},{"id":"FRA","name":"France"},{"id":"ITA","name":"Italy"},{"id":"SVN","name":"Slovenia"},{"id":"AUT","name":"Austria"},{"id":"CZE","name":"Czech Republic"},{"id":"ESP","name":"Spain"},{"id":"TUR","name":"Turkey"},{"id":"POL","name":"Poland"},{"id":"ISL","name":"Iceland"},{"id":"SVK","name":"Slovak Republic"},{"id":"ISR","name":"Israel"},{"id":"LVA","name":"Latvia"},{"id":"LTU","name":"Lithuania"}],"role":"REF_AREA"},{"keyPosition":1,"id":"INDICATOR","name":"Indicator","values":[{"id":"FBORNPOP","name":"Foreign-born population"}]},{"keyPosition":2,"id":"SUBJECT","name":"Subject","values":[{"id":"TOT","name":"Total"}]},{"keyPosition":3,"id":"MEASURE","name":"Measure","values":[{"id":"PC_POP","name":"% of population"}]},{"keyPosition":4,"id":"FREQUENCY","name":"Frequency","values":[{"id":"A","name":"Annual"}],"role":"FREQ"},{"id":"TIME_PERIOD","name":"Time","values":[{"id":"2013","name":"2013"},{"id":"2014","name":"2014"},{"id":"2015","name":"2015"},{"id":"2016","name":"2016"},{"id":"2017","name":"2017"},{"id":"2018","name":"2018"}],"role":"TIME_PERIOD"}]},"attributes":{"dataSet":[],"series":[],"observation":[{"id":"TIME_FORMAT","name":"Time Format","values":[{"id":"P1Y","name":"Annual"}]},{"id":"OBS_STATUS","name":"Observation Status","values":[]}]},"annotations":[{"title":"Copyright OECD - All rights reserved","uri":"","text":""},{"title":"Terms and Conditions","uri":"http://www.oecd.org/termsandconditions/","text":""},{"title":"Privacy Policy","uri":"http://www.oecd.org/privacy/","text":""},{"title":"MyOECD","uri":"https://www.oecd.org/login","text":""},{"title":"Contact Us","uri":"http://www.oecd.org/contact/","text":""}]}}' data-editable="false" data-embedded="true" data-end-time="2018" data-external-footer-id="embedded-chart-footer" data-fixed-height="true" data-frequency="A" data-highlighted-locations="" data-indicator="FBORNPOP" data-language="en" data-locations="" data-measure="PC_POP" data-referrer="//data.oecd.org/migration/foreign-born-population.htm" data-show-baseline="False" data-show-non-baseline="True" data-source="OECD International Migration Statistics: International migration database" data-start-time="2013" data-stored-configuration="impose" data-subject="TOT" data-table-note="Information on data for Israel: http://oe.cd/israel-disclaimer" data-type="table" data-use-latest-data="False">
    </div>
    <footer class="embedded-chart-footer" id="embedded-chart-footer">
        <div class="copyright-tooltip info-tooltip" tabindex="0">
            <i class="icon-ddp-copyright"></i>
            <p class="info-tooltip-content">
                © OECD
                <span class="separator">·</span>
                <a href="//www.oecd.org/termsandconditions/" target="_top">Terms &amp; Conditions</a>
            </p>
        </div>
        <p class="explore">Compare countries on <a class="chart-open-referrer" href="//data.oecd.org/migration/foreign-born-population.htm" target="_top">data.oecd.org</a></p>
        <p class="logo"><a class="chart-open-referrer" href="//data.oecd.org/migration/foreign-born-population.htm" target="_top" title="OECD Data">OECD Data</a></p>
    </footer>
</div>
</div>
<script src="//data.oecd.org/media/dataportal/scripts/ddp-chart-config.js"></script>
<script src="//data.oecd.org/media/dataportal/scripts/jquery.min.js"></script>
<script src="//data.oecd.org/media/dataportal/scripts/application.js"></script>


</body></html>

我是在抓取缓存的网页吗？

已编辑

下面是我使用的代码，上面是响应。

from bs4 import BeautifulSoup
from requests_html import HTMLSession

session = HTMLSession()
oecd_url = "https://data.oecd.org/chart/5Rmq"
response = session.get(oecd_url)
soup = BeautifulSoup(response.html.html,"html5lib")
soup

【问题讨论】：

该网站中有 2 个 divs 类 ddp-root。也许你弄错了？
请分享您迄今为止尝试过的代码示例以及所需的输出。

标签： python web-scraping

【解决方案1】：

它有一个 googletaskmanager，用于分析和电子商务服务，它将所有项目作为 api 部署到您的页面。这就是为什么你找不到你选择的元素

<noscript>
<iframe height="0" src="//www.googletagmanager.com/ns.html?id=GTM-WZ3XJR" 
 style="display: none; visibility: hidden" width="0"></iframe>
</noscript>

【讨论】：