无法使用谷歌应用脚​​本从下一页解析名称

Question

我一直在尝试从webpage 获取表的第一个提供程序名称。由于该表的数据分布在下一页中，我在脚本中创建了一个循环，以从所有下一页中获取第一个提供者名称。作为I'm always getting the first provider name of the table located in the first page, not from the next pages，我只是想不通我到底做错了什么。

如果您想知道如何从该页面手动填充结果：

1. 去这个网站website
2. 从Country 旁边的下拉列表中选择US
3. 点击Search按钮

我在 python 中使用了相同的逻辑并找到了成功。但是，与这里唯一不同的是我在 python 脚本中创建了会话，并在每个循环中重用了相同的会话。

这是我目前的方法的样子：

function fetchInformation() {
  const url = 'http://carf.org/advancedProviderSearch.aspx';
  
  var options = {
    'method': 'GET',
    'muteHttpExceptions': true,
    'headers': {
      'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'

    },
  };
  
  var response = UrlFetchApp.fetch(url, options);
  var $ = Cheerio.load(response.getContentText());
  var payload = {};
  payload['__VIEWSTATE'] = $('[id="__VIEWSTATE"]').first().attr('value'); 
  payload['EktronClientManager'] = $('[id="EktronClientManager"]').first().attr('value');
  payload['__VIEWSTATEGENERATOR'] = $('[id="__VIEWSTATEGENERATOR"]').first().attr('value'); 
  payload['__EVENTVALIDATION'] = $('[id="__EVENTVALIDATION"]').first().attr('value'); 
  payload['__EVENTTARGET'] = '';
  payload['ctl00$usrWebsiteSearch$txtWebsiteSearch'] = 'Search CARF.org';
  payload['ctl00$body$ddlCity'] = '0';
  payload['ctl00$body$ddlZipcode'] = '0';
  payload['ctl00$body$ddlCountry'] = 'US';
  payload['ctl00$body$ddlProgram'] = '';
  payload['ctl00$body$ddlProgramFocus'] = ''; 
  payload['ctl00$body$ddlPopulation'] = '';
  payload['ctl00$body$btnAdvSearch'] = 'Search';

  var i = 0;
  
  while (i<5){
      var options = {
        'method': 'POST',
        'payload': payload,
        'muteHttpExceptions': true,
        'headers': {
          'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'

        },
      };

      var response = UrlFetchApp.fetch(url, options);
      var $ = Cheerio.load(response.getContentText());
      var elem = $('table#ctl00_body_gvProviderAdvSearch tr td > a').first().text();
      
      console.log(elem);
      
      var payload = {};
      payload['__VIEWSTATE'] = $('[id="__VIEWSTATE"]').first().attr('value'); 
      payload['EktronClientManager'] = $('[id="EktronClientManager"]').first().attr('value');
      payload['__VIEWSTATEGENERATOR'] = $('[id="__VIEWSTATEGENERATOR"]').first().attr('value'); 
      payload['__EVENTVALIDATION'] = $('[id="__EVENTVALIDATION"]').first().attr('value'); 
      payload['ctl00$usrWebsiteSearch$txtWebsiteSearch'] = 'Search CARF.org';
      payload['ctl00$body$ddlCity'] = '0';
      payload['ctl00$body$ddlZipcode'] = '0';
      payload['ctl00$body$ddlCountry'] = 'US';
      payload['ctl00$body$ddlProgram'] = '';
      payload['ctl00$body$ddlProgramFocus'] = ''; 
      payload['ctl00$body$ddlPopulation'] = '';
      payload['__EVENTTARGET'] = 'ctl00$body$gvProviderAdvSearch$ctl13$ctl01';
      i++;
  }

}

如何从分布在下一页的每个页面的表格中获取第一个提供者名称？

Answer 1

修改点：

• 当我看到您的脚本时，我认为在您的情况下，可能需要包含 cookie。
• 不幸的是，在当前阶段，User-Agent 无法使用 UrlFetchApp 更改。

当这些点反映在你的脚本中时，它变成如下。

修改脚本：

function fetchInformation() {
  const url = 'http://carf.org/advancedProviderSearch.aspx';

  // Modified
  var options = {
    'method': 'GET',
    'muteHttpExceptions': true,
  };

  var response = UrlFetchApp.fetch(url, options);
  var $ = Cheerio.load(response.getContentText());
  var payload = {};
  payload['__VIEWSTATE'] = $('[id="__VIEWSTATE"]').first().attr('value');
  payload['EktronClientManager'] = $('[id="EktronClientManager"]').first().attr('value');
  payload['__VIEWSTATEGENERATOR'] = $('[id="__VIEWSTATEGENERATOR"]').first().attr('value');
  payload['__EVENTVALIDATION'] = $('[id="__EVENTVALIDATION"]').first().attr('value');
  payload['__EVENTTARGET'] = '';
  payload['ctl00$usrWebsiteSearch$txtWebsiteSearch'] = 'Search CARF.org';
  payload['ctl00$body$ddlCity'] = '0';
  payload['ctl00$body$ddlZipcode'] = '0';
  payload['ctl00$body$ddlCountry'] = 'US';
  payload['ctl00$body$ddlProgram'] = '';
  payload['ctl00$body$ddlProgramFocus'] = '';
  payload['ctl00$body$ddlPopulation'] = '';
  payload['ctl00$body$btnAdvSearch'] = 'Search';

  var cookie = response.getAllHeaders()["Set-Cookie"][1]; // Added

  var i = 0;

  while (i < 5) {
    var options = {
      'method': 'POST',
      'payload': payload,
      'muteHttpExceptions': true,
      'headers': { cookie }, // Modified
    };

    var response = UrlFetchApp.fetch(url, options);
    var $ = Cheerio.load(response.getContentText());
    var elem = $('table#ctl00_body_gvProviderAdvSearch tr td > a').first().text();

    console.log(elem);

    var payload = {};
    payload['__VIEWSTATE'] = $('[id="__VIEWSTATE"]').first().attr('value');
    payload['EktronClientManager'] = $('[id="EktronClientManager"]').first().attr('value');
    payload['__VIEWSTATEGENERATOR'] = $('[id="__VIEWSTATEGENERATOR"]').first().attr('value');
    payload['__EVENTVALIDATION'] = $('[id="__EVENTVALIDATION"]').first().attr('value');
    payload['ctl00$usrWebsiteSearch$txtWebsiteSearch'] = 'Search CARF.org';
    payload['ctl00$body$ddlCity'] = '0';
    payload['ctl00$body$ddlZipcode'] = '0';
    payload['ctl00$body$ddlCountry'] = 'US';
    payload['ctl00$body$ddlProgram'] = '';
    payload['ctl00$body$ddlProgramFocus'] = '';
    payload['ctl00$body$ddlPopulation'] = '';
    payload['__EVENTTARGET'] = 'ctl00$body$gvProviderAdvSearch$ctl13$ctl01';
    i++;
  }

}

结果：

上面修改的脚本运行后，在日志中可以看到如下结果。

Greenleaf Job Training Services Inc
06Maren
10th District Substance Abuse Program dba New Beginnings, C.A.S.A.
12 Queen Street
12th Street House

参考：

• fetch(url, params)