无法使用 docker logstash 建立索引

Question

我正在使用git@github.com:deviantony/docker-elk.git 存储库的最新代码通过docker-compose up 命令托管ELK 堆栈。 Elastic search 和 kibana 运行良好。

虽然我无法使用我的 logstash.conf 索引到 logstash，如下所示：

input {
    file {
        # Configure your path below
        path => ["C:/Users/matt/Desktop/temp/logs/*.txt*"]
        ignore_older => "141 days"
        start_position => "beginning"
        file_sort_by => "last_modified"
        file_sort_direction => "desc"
        sincedb_path => "NUL"
        type => "appl"
        codec => multiline {
            pattern => "^<log4j:event"
            negate => true
            what => "previous"
        }
    }
}
filter {
    if [type] == "appl" {
        grok {
            add_tag => [ "groked" ]
            match => ["message", ".*"]
            remove_tag => ["_grokparsefailure"]
        }
        mutate {
            gsub => ["message", "log4j:", ""]
        }
        xml {
            source => "message"
            remove_namespaces => true
            target => "log4jevent"
            xpath => [ "//event/@timestamp", "timestamp" ]
            xpath => [ "//event/@level", "loglevel" ]
            xpath => [ "/event/message/text()", "message" ]
            xpath => [ "/event/throwable/text()", "exception" ]
            xpath => [ "//event/properties/data[@name='log4jmachinename']/@value", "machinename" ]
            xpath => [ "//event/properties/data[@name='log4japp']/@value", "app" ]
            xpath => [ "//event/properties/data[@name='log4net:UserName']/@value", "username" ]
            xpath => [ "//event/properties/data[@name='log4net:Identity']/@value", "identity" ]
            xpath => [ "//event/properties/data[@name='log4net:HostName']/@value", "hostname" ]
        }
        mutate {
            remove_field => ["type"]
            gsub => [
            "message", "&amp;", "&",
            "message", "&lt;", "<",
            "message", "&gt;", ">",
            "message", "&quot;", "\"",
            "message", "&apos;", "'"
            ]
        }
        date {
            match => [ "[timestamp][0]","UNIX_MS" ]
            target => "@timestamp"
            remove_field => ["timestamp"]
        }
    }
}
output {
    elasticsearch {
        hosts => ["localhost:9200"]
        index => "log4jevents"
        user => "elastic"
        password => "changeme"
        ecs_compatibility => disabled
    }
    stdout {
        codec => rubydebug
    }
}

我想用我的 logstash 索引的日志文件如下所示

<log4j:event logger="Microsoft.Unity.ApplicationBlocks.Logging.Logger" timestamp="1615025506621" level="DEBUG" thread="13"><log4j:message>SSO-&gt;AccountController-&gt;Login-&gt;Before ClientID Check</log4j:message><log4j:properties><log4j:data name="log4jmachinename" value="hostname01" /><log4j:data name="log4japp" value="/LM/W3SVC/2/ROOT-1-132594985694777790" /><log4j:data name="log4net:UserName" value="IIS APPPOOL\default" /><log4j:data name="log4net:Identity" value="" /><log4j:data name="log4net:HostName" value="hostname01" /></log4j:properties><log4j:locationInfo class="Microsoft.Unity.ApplicationBlocks.Logging.Logger" method="Debug" file="F:\somefolder\Agent\_work\1\s\Unity\Microsoft.Unity.ApplicationBlocks\Logging\Logging.cs" line="353" /></log4j:event>

在启动 docker-compose up 时显示的问题如下所示用于 logstash

Attempted to resurrect connection to dead ES instance, but got an error. {:url=>"http://elastic:xxxxxx@localhost:9200/", :error_type=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError, :error=>"Elasticsearch Unreachable: [http://elastic:xxxxxx@localhost:9200/][Manticore::SocketException] Connection refused (Connection refused)"}

同样的 logstash.conf 在早期的 EK 版本 6.8 中工作。我的 logstash.conf 有什么问题？

Answer 1

在您的输出elasticsearch 插件中，将hosts 属性设置为elasticsearch:9200。

output {
    elasticsearch {
        hosts => ["elasticsearch:9200"]
        index => "log4jevents"
        user => "elastic"
        password => "changeme"
        ecs_compatibility => disabled
    }
    stdout {
        codec => rubydebug
    }
}