【发布时间】：2016-10-27 17:58:23
【问题描述】：
我正在尝试在 Windows 上从 Logstash 为 Elasticsearch 配置索引模板管理。
我有c:\ulyaoth\logstash-2.3.1\bin\logstash.json 文件：
input {
beats {
port => 5044
type => "log"
}
}
filter {
grok {
match => ["message","%{TIMESTAMP_ISO8601:timestamp_match}"]
remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
}
mutate {
remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
}
date {
match => ["timestamp_match","YYYY-MM-dd HH:mm:ss.SSS"]
target => "timestamp_match"
}
}
output {
elasticsearch {
hosts => "localhost:9200"
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
template => "c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json"
template_name => "elasticsearch-template"
manage_template => true
template_overwrite => true
}
}
和模板文件c:/ulyaoth/logstash-2.3.1/bin/elasticsearch-template.custom.json。我从c:\ulyaoth\logstash-2.3.1\vendor\bundle\jruby\1.9\gems\logstash-output-elasticsearch-2.5.5-java\lib\logstash\outputs\elasticsearch\ 挖出了这个文件并对其进行了编辑，以便：
"source":{"index": "not_analyzed"}
这是整个文件：
{
"template" : "logstash-*",
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true, "omit_norms" : true},
"dynamic_templates" : [ {
"message_field" : {
"match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fielddata" : { "format" : "disabled" }
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fielddata" : { "format" : "disabled" },
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
}
}
}
}, {
"float_fields" : {
"match" : "*",
"match_mapping_type" : "float",
"mapping" : { "type" : "float", "doc_values" : true }
}
}, {
"double_fields" : {
"match" : "*",
"match_mapping_type" : "double",
"mapping" : { "type" : "double", "doc_values" : true }
}
}, {
"byte_fields" : {
"match" : "*",
"match_mapping_type" : "byte",
"mapping" : { "type" : "byte", "doc_values" : true }
}
}, {
"short_fields" : {
"match" : "*",
"match_mapping_type" : "short",
"mapping" : { "type" : "short", "doc_values" : true }
}
}, {
"integer_fields" : {
"match" : "*",
"match_mapping_type" : "integer",
"mapping" : { "type" : "integer", "doc_values" : true }
}
}, {
"long_fields" : {
"match" : "*",
"match_mapping_type" : "long",
"mapping" : { "type" : "long", "doc_values" : true }
}
}, {
"date_fields" : {
"match" : "*",
"match_mapping_type" : "date",
"mapping" : { "type" : "date", "doc_values" : true }
}
}, {
"geo_point_fields" : {
"match" : "*",
"match_mapping_type" : "geo_point",
"mapping" : { "type" : "geo_point", "doc_values" : true }
}
} ],
"properties" : {
"@timestamp": { "type": "date", "doc_values" : true },
"@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
"source":{"index": "not_analyzed"}
"geoip" : {
"type" : "object",
"dynamic": true,
"properties" : {
"ip": { "type": "ip", "doc_values" : true },
"location" : { "type" : "geo_point", "doc_values" : true },
"latitude" : { "type" : "float", "doc_values" : true },
"longitude" : { "type" : "float", "doc_values" : true }
}
}
}
}
}
}
我的问题是模板没有注册。 REST 查询返回空对象，而且我看到该字段仍在 Kibana 中分析。
GET /_template HTTP/1.1
Host: 127.0.0.1:9200
另一个问题是remove_field 也不起作用 - 我仍然看到所有这些字段。
remove_field => ["_id","_index","_score","_type","beat.hostname","beat.name","count","fileds","host","input_type","offset","tags","type"]
我没有看到任何 logstash 日志（讽刺的是：）并且在 ES 日志中我没有看到任何错误或模板问题。
如何解决这些问题？
编辑：
最终的工作配置是：
{
"template" : "filebeat-*",
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"_all" : {"enabled" : true, "omit_norms" : true},
"dynamic_templates" : [ {
"message_field" : {
"match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fielddata" : { "format" : "disabled" }
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "string", "index" : "analyzed", "omit_norms" : true,
"fielddata" : { "format" : "disabled" },
"fields" : {
"raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
}
}
}
}, {
"float_fields" : {
"match" : "*",
"match_mapping_type" : "float",
"mapping" : { "type" : "float", "doc_values" : true }
}
}, {
"double_fields" : {
"match" : "*",
"match_mapping_type" : "double",
"mapping" : { "type" : "double", "doc_values" : true }
}
}, {
"byte_fields" : {
"match" : "*",
"match_mapping_type" : "byte",
"mapping" : { "type" : "byte", "doc_values" : true }
}
}, {
"short_fields" : {
"match" : "*",
"match_mapping_type" : "short",
"mapping" : { "type" : "short", "doc_values" : true }
}
}, {
"integer_fields" : {
"match" : "*",
"match_mapping_type" : "integer",
"mapping" : { "type" : "integer", "doc_values" : true }
}
}, {
"long_fields" : {
"match" : "*",
"match_mapping_type" : "long",
"mapping" : { "type" : "long", "doc_values" : true }
}
}, {
"date_fields" : {
"match" : "*",
"match_mapping_type" : "date",
"mapping" : { "type" : "date", "doc_values" : true }
}
}, {
"geo_point_fields" : {
"match" : "*",
"match_mapping_type" : "geo_point",
"mapping" : { "type" : "geo_point", "doc_values" : true }
}
} ],
"properties" : {
"@timestamp": { "type": "date", "doc_values" : true },
"@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
"source":{ "type": "string", "index": "not_analyzed"}
"geoip" : {
"type" : "object",
"dynamic": true,
"properties" : {
"ip": { "type": "ip", "doc_values" : true },
"location" : { "type" : "geo_point", "doc_values" : true },
"latitude" : { "type" : "float", "doc_values" : true },
"longitude" : { "type" : "float", "doc_values" : true }
}
}
}
}
}
}
- 已更改
"template" : "filebeat-*", 和"source":{ "type": "string", "index": "not_analyzed"}
【问题讨论】：
-
也许您在“整个文件”的第 131 行缺少一个“，”？ （在 geoip 之前）？
-
确实少了一个逗号，但不幸的是它并没有解决问题:(仍然没有模板...谢谢。