Google 数据洞察社区连接器 getData() 未按预期工作

Question

function getData(request){
  try{  
  var options = {
  'method' : 'post',
  'contentType': 'application/json',
  'payload' : JSON.stringify(request)
  };
  response=UrlFetchApp.fetch(getDataUrl, options);

  resData = JSON.parse(response.getContentText())

  return resData

  }catch (e) { 
    e = (typeof e === 'string') ? new Error(e) : e;
    Logger.log("Catch", e);
    throw e;
  }
}

以上是我的 getData() 函数。

我的 isAdminUser() 返回 true。

当我尝试可视化我的数据时，我收到以下错误

数据集配置错误

Data Studio 无法连接到您的数据集。

从社区连接器请求数据时出错。如果此问题仍然存在，请将此问题报告给此社区连接器的提供商。

错误 ID：3d11b88b https://i.stack.imgur.com/x3Hki.png

每次刷新数据时错误代码都会发生变化，我找不到任何字典来将 错误 id 映射到 错误

我尝试通过记录 request 参数、response.getContentText() 和 resData 变量进行调试，以确保我的数据格式正确.

以下是 Stackdriver 日志

中打印的日志

请求

{configParams={/个人配置数据/}，fields=[{name=LASTNAME}]}

response.getContentText()

{"schema":[{"name":"LASTNAME","dataType":"STRING"}],"rows":[{"values":["test"]},{"values": ["test"]},{"values":["Dummy"]},{"values":["One"]},{"values":["Nagargoje"]},{"values":[" "]},{"values":[""]},{"values":[""]},{"values":[""]},{"values":[""]}]," filtersApplied":false}

resData

{rows=[{values=[test]}, {values=[test]}, {values=[Dummy]}, {values=[One]}, {values=[Nagargoje]}, {values=[]}, {values=[]}, {values=[]}, {values=[]}, {values=[]}], filtersApplied=false, 架构=[{name=LASTNAME, dataType=STRING}]}

我不确定我的 getData() 函数出了什么问题。

我返回的对象似乎与此处给出的结构相匹配 https://developers.google.com/datastudio/connector/reference#getdata

Answer 1

所以我的 getData() 函数没有问题，问题存在于清单文件中。 我正在搜索有关通过 URL 传递参数的信息，偶然发现了一个名为 dataStudio.useQueryConfig 并将其添加到我的清单文件并将其值设置为 true。 谷歌数据工作室希望我返回getData() 的查询配置。 但我真正想要的是this。

无论如何，感谢Matthias 建议我看看Open-Source implementations，我能够调试它

我实现了JSON connect，它工作得很好，所以我记录了它在 getData() 中返回的内容并在我的代码中使用了该格式/结构，但我的连接器仍然无法工作。

我的下一个假设是我的 getSchema() 返回值可能有问题。所以我也记录了这一点，然后复制粘贴了来自JSON connect 的 getData() 和 getSchema() 返回变量的硬编码值。

即使这样也没有用，所以我最后的赌注是清单文件一定有问题，也许我在其中添加的虚拟链接一定是问题所在。然后，在进行实地比较之后，我终于能够让我的社区连接器工作了。

如果错误消息有点帮助并且看起来不那么通用，这将更容易调试。

Answer 2

首先：您可以随时查看其他人为自定义 Google Data Studio 连接器所做的Open-Source implementations。他们是一个很好的信息来源。有关更多信息，请查看Open Source Community Connectors 上的文档。

第二：我的实现是用于时间跟踪系统，因此具有机密的 GDPR 相关数据。这就是为什么我不能只给你回复消息。但我组装了这段代码。它包含身份验证、HTTP GET 数据获取和数据转换。解释在代码下方。如果您需要进一步的帮助，请再次查看开源连接器。

var cc = DataStudioApp.createCommunityConnector();

const URL_DATA = 'https://www.myverysecretdomain.com/api';
const URL_PING = 'https://www.myverysecretdomain.com/ping';
const AUTH_USER = 'auth.user'
const AUTH_KEY = 'auth.key';
const JSON_TAG = 'user';

String.prototype.format = function() {
  // https://coderwall.com/p/flonoa/simple-string-format-in-javascript
  a = this;
  for (k in arguments) {
    a = a.replace("{" + k + "}", arguments[k])
  }
  return a
}

function httpGet(user, token, url, params) {
  try {
    // this depends on the URL you are connecting to
    var headers = {
      'ApiUser': user,
      'ApiToken': token,
      'User-Agent': 'my super freaky Google Data Studio connector'
    };

    var options = {
      headers: headers
    };

    if (params && Object.keys(params).length > 0) {
      var params_ = [];
      for (const [key, value] of Object.entries(params)) {
        var value_ = value;
        if (Array.isArray(value))
          value_ = value.join(',');

        params_.push('{0}={1}'.format(key, encodeURIComponent(value_)))
      }

      var query = params_.join('&');
      url = '{0}?{1}'.format(url, query);
    }

    var response = UrlFetchApp.fetch(url, options);

    return {
      code: response.getResponseCode(),
      json: JSON.parse(response.getContentText())
    }  
  } catch (e) {
    throwConnectorError(e);
  }
}

function getCredentials() {
  var userProperties = PropertiesService.getUserProperties();
  return {
    username: userProperties.getProperty(AUTH_USER),
    token: userProperties.getProperty(AUTH_KEY)
  }
}

function validateCredentials(user, token) {
  if (!user || !token) 
    return false;

  var response = httpGet(user, token, URL_PING);

  if (response.code == 200)
    console.log('API key for the user %s successfully validated', user);
  else
    console.error('API key for the user %s is invalid. Code: %s', user, response.code);

  return response;
}  

function getAuthType() {
  var cc = DataStudioApp.createCommunityConnector();
  return cc.newAuthTypeResponse()
    .setAuthType(cc.AuthType.USER_TOKEN)
    .setHelpUrl('https://www.myverysecretdomain.com/index.html#authentication')
    .build();
}

function resetAuth() {
  var userProperties = PropertiesService.getUserProperties();
  userProperties.deleteProperty(AUTH_USER);
  userProperties.deleteProperty(AUTH_KEY);

  console.info('Credentials have been reset.');
}

function isAuthValid() {
  var credentials = getCredentials()
  if (credentials == null) {
    console.info('No credentials found.');
    return false;
  }

  var response = validateCredentials(credentials.username, credentials.token);
  return (response != null && response.code == 200);
}

function setCredentials(request) {
  var credentials = request.userToken;
  var response = validateCredentials(credentials.username, credentials.token);

  if (response == null || response.code != 200) return { errorCode: 'INVALID_CREDENTIALS' };

  var userProperties = PropertiesService.getUserProperties();
  userProperties.setProperty(AUTH_USER, credentials.username);
  userProperties.setProperty(AUTH_KEY, credentials.token);

  console.info('Credentials have been stored');

  return {
    errorCode: 'NONE'
  };
}

function throwConnectorError(text) {
  DataStudioApp.createCommunityConnector()
    .newUserError()
    .setDebugText(text)
    .setText(text)
    .throwException();
}

function getConfig(request) {
  // ToDo: handle request.languageCode for different languages being displayed
  console.log(request)

  var params = request.configParams;
  var config = cc.getConfig();

  // ToDo: add your config if necessary

  config.setDateRangeRequired(true);
  return config.build();
}

function getDimensions() {
  var types = cc.FieldType;

  return [
    {
      id:'id',
      name:'ID',
      type:types.NUMBER
    },
    {
      id:'name',
      name:'Name',
      isDefault:true,
      type:types.TEXT
    },
    {
      id:'email',
      name:'Email',
      type:types.TEXT
    }
  ];
}

function getMetrics() {
  return [];
}

function getFields(request) {
  Logger.log(request)

  var fields = cc.getFields();

  var dimensions = this.getDimensions();
  var metrics = this.getMetrics();
  dimensions.forEach(dimension => fields.newDimension().setId(dimension.id).setName(dimension.name).setType(dimension.type));  
  metrics.forEach(metric => fields.newMetric().setId(metric.id).setName(metric.name).setType(metric.type).setAggregation(metric.aggregations));

  var defaultDimension = dimensions.find(field => field.hasOwnProperty('isDefault') && field.isDefault == true);
  var defaultMetric = metrics.find(field => field.hasOwnProperty('isDefault') && field.isDefault == true);

  if (defaultDimension)
    fields.setDefaultDimension(defaultDimension.id);
  if (defaultMetric)
    fields.setDefaultMetric(defaultMetric.id);

  return fields;
}

function getSchema(request) {
  var fields = getFields(request).build();
  return { schema: fields };
}

function convertValue(value, id) {  
  // ToDo: add special conversion if necessary
  switch(id) {      
    default:
      // value will be converted automatically
      return value[id];
  }
}

function entriesToDicts(schema, data, converter, tag) {

  return data.map(function(element) {

    var entry = element[tag];
    var row = {};    
    schema.forEach(function(field) {

      // field has same name in connector and original data source
      var id = field.id;
      var value = converter(entry, id);

      // use UI field ID
      row[field.id] = value;
    });

    return row;
  });
}

function dictsToRows(requestedFields, rows) {
  return rows.reduce((result, row) => ([...result, {'values': requestedFields.reduce((values, field) => ([...values, row[field]]), [])}]), []);
}

function getParams (request) { 
  var schema = this.getSchema();
  var params;

  if (request) {
    params = {};

    // ToDo: handle pagination={startRow=1.0, rowCount=100.0}
  } else {
    // preview only
    params = {
      limit: 20
    }
  }

  return params;
}

function getData(request) {
  Logger.log(request)

  var credentials = getCredentials()
  var schema = getSchema();
  var params = getParams(request);

  var requestedFields;  // fields structured as I want them (see above)
  var requestedSchema;  // fields structured as Google expects them
  if (request) {
    // make sure the ordering of the requested fields is kept correct in the resulting data
    requestedFields = request.fields.filter(field => !field.forFilterOnly).map(field => field.name);
    requestedSchema = getFields(request).forIds(requestedFields);
  } else {
    // use all fields from schema
    requestedFields = schema.map(field => field.id);
    requestedSchema = api.getFields(request);
  }

  var filterPresent = request && request.dimensionsFilters;
  //var filter = ...
  if (filterPresent) {
    // ToDo: apply request filters on API level (before the API call) to minimize data retrieval from API (number of rows) and increase speed
    // see https://developers.google.com/datastudio/connector/filters

    // filter = ...   // initialize filter
    // filter.preFilter(params);  // low-level API filtering if possible
  }

  // get HTTP response; e.g. check for HTTT RETURN CODE on response.code if necessary
  var response = httpGet(credentials.username, credentials.token, URL_DATA, params);  

  // get JSON data from HTTP response
  var data = response.json;

  // convert the full dataset including all fields (the full schema). non-requested fields will be filtered later on  
  var rows = entriesToDicts(schema, data, convertValue, JSON_TAG);

  // match rows against filter (high-level filtering)
  //if (filter)
  //  rows = rows.filter(row => filter.match(row) == true);

  // remove non-requested fields
  var result = dictsToRows(requestedFields, rows);

  console.log('{0} rows received'.format(result.length));
  //console.log(result);

  return {
    schema: requestedSchema.build(),
    rows: result,
    filtersApplied: filter ? true : false
  };
}

过滤所有名称以 J 开头的用户的示例请求。

{
    configParams={}, 
    dateRange={
        endDate=2020-05-14, 
        startDate=2020-04-17
    }, 
    fields=[
        {name=name}
    ], 
    scriptParams={
        lastRefresh=1589543208040
    }, 
    dimensionsFilters=[
        [
            {
                values=[^J.*], 
                operator=REGEXP_EXACT_MATCH, 
                type=INCLUDE, 
                fieldName=name
            }
        ]
    ]
}

HTTP GET 返回的 JSON 数据包含所有字段（完整架构）。

[ { user: 
     { id: 1,
       name: 'Jane Doe',
       email: 'jane@doe.com' } },
  { user: 
     { id: 2,
       name: 'John Doe', 
       email: 'john@doe.com' } }
]

数据经过过滤和转换/转换后，您将获得此结果，该结果由 Google Data Studio 完美显示：

{
    filtersApplied=true, 
    schema=[
        {
            isDefault=true, 
            semantics={
                semanticType=TEXT, 
                conceptType=DIMENSION
            }, 
            label=Name, 
            name=name, 
            dataType=STRING
        }
    ], 
    rows=[
        {values=[Jane Doe]}, 
        {values=[John Doe]}
    ]
}

Answer 3

getData 应该只返回请求字段的数据。在request.fields 中应该有所有请求字段的列表。仅限制这些字段的数据，然后将解析后的数据发回。