Elasticsearch 模板未按预期工作

Question

所以我有一个弹性搜索模板，我通过 Filebeat 将其传递给 ES。我也有 Filebeat 发送到 logstash ......这是相关的东西。

Elasticsearch 模板

{
"template": "filebeat-*",
"mappings": {
    "product__name": {
        "properties": {
            "@timestamp": {
                "type": "date",
                "format": "strict_date_optional_time||epoch_millis"
            },
            "@version": {
                "type": "text"
            },
            "beat": {
                "properties": {
                    "hostname": {
                        "type": "text"
                    },
                    "name": {
                        "type": "text"
                    }
                }
            },
            "class_method": {
                "type": "text"
            },
            "class_name": {
                "type": "text",
                "index": "true",
                "fielddata": "true"
            },
            "clientip": {
                "type": "ip"
            },
            "count": {
                "type": "long"
            },
            "host": {
                "type": "text"
            },
            "input_type": {
                "type": "text"
            },
            "log_level": {
                "type": "text",
                "fielddata": "true",
                "index": "true"
            },
            "log_message": {
                "type": "text",
                "index": "true"
            },
            "log_timestamp": {
                "type": "text"
            },
            "log_ts": {
                "type": "long"
            },
            "message": {
                "type": "text"
            },
            "offset": {
                "type": "long"
            },
            "query_params": {
                "type": "text",
                "fielddata": "true",
                "index": "true"
            },
            "sessionid": {
                "type": "text",
                "index": "true"
            },
            "source": {
                "type": "text"
            },
            "tags": {
                "type": "text"
            },
            "thread": {
                "type": "text",
                "index": "true"
            },
            "type": {
                "type": "text"
            },
            "user_account_combo": {
                "type": "text",
                "index": "true"
            },
            "version": {
                "type": "text"
            }
        }
    },
    "access": {
        "properties": {
            "@timestamp": {
                "type": "date",
                "format": "strict_date_optional_time||epoch_millis"
            },
            "@version": {
                "type": "text"
            },
            "beat": {
                "properties": {
                    "hostname": {
                        "type": "text"
                    },
                    "name": {
                        "type": "text"
                    }
                }
            },
            "clientip": {
                "type": "ip"
            },
            "count": {
                "type": "long"
            },
            "host": {
                "type": "text",
                "index": "true"
            },
            "input_type": {
                "type": "text"
            },
            "log_timestamp": {
                "type": "text"
            },
            "log_ts": {
                "type": "long"
            },
            "message": {
                "type": "text"
            },
            "offset": {
                "type": "long"
            },
            "query_params": {
                "type": "text",
                "fielddata": "true",
                "index": "true"
            },
            "response_time": {
                "type": "long"
            },
            "sessionid": {
                "type": "text",
                "index": "true"
            },
            "source": {
                "type": "text"
            },
            "statuscode": {
                "type": "long"
            },
            "tags": {
                "type": "text"
            },
            "thread": {
                "type": "text",
                "index": "true"
            },
            "type": {
                "type": "text",
                "index": "true"
            },
            "uripath": {
                "type": "text",
                "fielddata": "true",
                "index": "true"
            },
            "user_account_combo": {
                "type": "text",
                "index": "true"
            },
            "verb": {
                "type": "text",
                "fielddata": "true",
                "index": "true"
            }
        }
    }
}
}

filebeat 配置（已修整）

output.elasticsearch:
  hosts: ["10.10.43.210:9200"]
  template:
    name: filebeat
    path: "test-template.json"
    overwrite: true

output.logstash:
    hosts: ["10.10.43.210:5044"]
    worker: 2
    index: filebeat

Logstash 配置（已修整）

output {
  stdout { }
  elasticsearch {
    hosts => "elasticsearch:9200"
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }

最后...这是 Kibana 中显示的内容：

所以问题是：当我非常明确地知道我希望所有内容是什么类型并且不让 logstash 覆盖我的模板时，为什么我会在事物上看到 .keyword 字段？我错过了什么吗？

Answer 1

Kibana 5.x 为所有 string 类型的字段生成 .keyword 字段。这允许您基于整个字段进行聚合。例如，如果您想计算 host 字段的唯一值，您将在 host.keyword 上执行聚合。

所以这种行为并不意味着您的模板被覆盖，它是 Kibana 的有意行为，允许您对字符串字段进行聚合。如果您想检查您的模板是否被 Logstash 以某种方式覆盖，请检查 Elasticsearch curl elasticsearch:9200/{template_name}?pretty 将显示您的索引映射。这可以帮助您验证您的索引是否使用了您期望的映射。