在 Kibana 中自定义日志输出

Question

最后，我使用 ELK 堆栈从远程服务器获取一些日志。但是，我想自定义日志的输出。有没有办法删除一些我用黄色突出显示的字段：

我试图从 _source 中删除它们，包括 logstash.conf 中的 remove_field：

input {
  beats {
    port => 5044
    ssl => true
    ssl_certificate => "/..."
    ssl_key => "/..logstash.key"
  }
}

filter {
        grok {
            match => {
                "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}"
            }
            remove_field => [ "tags", "prospector.type", "host.architecture", "host.containerized", "host.id", "host.os.platform", "host.os.family" ]
        }
}

output {
    elasticsearch {
        hosts => "localhost:9200"
        index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    }
}

你知道如何去掉 _source 中来自 filebeat 的日志的黄色字段吗？

基于 Leandro cmets 更新 logstash.conf：

input {
  beats {
    port => 5044
    ssl => true
    ssl_certificate => ".../logstash.crt"
    ssl_key => ".../logstash.key"
  }
}

filter {
        grok {
            match => {
                "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}"
            }
            remove_field => [ "tags","[prospector][type]","[host][architecture]", "[host][containerized]", "[host][id]", "[host][os][platform]", "[host][os][family]", "[beat][hostname]", "[beat][name]", "[beat][version], "[offset]", "[input][type]", "[meta][cloud][provider]", "[meta][cloud][machine_type]", "[meta][cloud][instance_id]"]
        }
}



output {
    elasticsearch {
        hosts => "localhost:9200"
        index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    }
}

在日志中：

019-02-27T17:03:41.637-0800    DEBUG   [input] file/states.go:68       New state added for /logs/api.log
2019-02-27T17:03:41.637-0800    DEBUG   [registrar]     registrar/registrar.go:315      Registrar state updates processed. Count: 1
2019-02-27T17:03:41.637-0800    DEBUG   [registrar]     registrar/registrar.go:400      Write registry file: /filebeat/registry
2019-02-27T17:03:41.637-0800    INFO    log/harvester.go:255    Harvester started for file: /logs/api.log
2019-02-27T17:03:41.647-0800    DEBUG   [publish]       pipeline/processor.go:308       Publish event: {
  "@timestamp": "2019-02-28T01:03:41.647Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "doc",
    "version": "6.6.0"
  },
  "log": {
    "file": {
      "path": "/logs/api.log"
    }
  },
  "input": {
    "type": "log"
  },
  "host": {
    "name": "tomcat",
    "os": {
      "family": "redhat",
      "name": "CentOS Linux",
      "codename": "Core",
      "platform": "centos",
      "version": "7 (Core)"
    },
    "id": "6aaed308aa5a419f880c5e45eea65414",
    "containerized": true,
    "architecture": "x86_64"
  },
  "meta": {
    "cloud": {
      "region": "CanadaCentral",
      "provider": "az",
      "instance_id": "6452bcf4-7f5d-4fc3-9f8e-5ea57f00724b",
      "instance_name": "tomcat",
      "machine_type": "Standard_D8s_v3"
    }
  },
  "message": "2018-09-14 20:23:37 INFO  ContextLoader:272 - Root WebApplicationContext: initialization started",
  "source": "/logs/api.log",
  "offset": 0,
  "prospector": {
    "type": "log"
  },
  "beat": {
    "hostname": "tomcat",
    "version": "6.6.0",
    "name": "tomcat"
  }
}

谢谢

Answer 1

其中一些字段是嵌套字段，在 Logstash 过滤器中访问它们的方法是使用 [field][subfield] 表示法。

你的remove_field应该是这样的：

remove_field => ["tags","[host][architecture]","[meta][cloud][provider]"]

但我认为您不能删除 @version 字段。

更新：

使用 Filebeat 日志中的事件示例，我模拟了一个管道并获得了 _grokparsefailure，即使在 grok 失败时删除字段，您也需要在 mutate 过滤器中使用 remove_field：

filter {
  grok {
     your grok
  }
  mutate {
    remove_field => ["[prospector]","[host][architecture]", "[host][containerized]", "[host][id]", "[host][os][platform]", "[host][os][family]", "[beat]", "[offset]", "[input]", "[meta]"]
  }
}

在修复 groks 之前，请勿删除 tags 字段。

该示例的 logstash 输出为：

{
  "source": "/logs/api.log",
  "tags": [
    "_grokparsefailure"
  ],
  "@timestamp": "2019-02-28T01:03:41.647Z",
  "message": "2018-09-14 20:23:37 INFO  ContextLoader:272 - Root WebApplicationContext: initialization started",
  "log": {
    "file": {
      "path": "/logs/api.log"
    }
  },
  "@version": "1",
  "host": {
    "os": {
      "codename": "Core",
      "version": "7 (Core)",
      "name": "CentOS Linux"
    },
    "name": "tomcat"
  }
}