在 Dropwizard Application for HealthCheck 中由对等方重置连接

Question

我们正在运行一个放置向导应用程序，并且在从 Prometheus 进行健康检查时收到错误 Connection reset by peer。同样在我们收到此错误后，服务器停止响应，直到我们重新启动服务器。下面是健康检查代码。

Response response = null;
        try {
            Client client = ClientBuilder.newClient();
            WebTarget webTarget = client.target("http://localhost:8080/test");
            Invocation.Builder invocationBuilder =  webTarget.request(MediaType.APPLICATION_JSON);
            response = invocationBuilder.get();
        }catch(Exception e)
        {
            e.printStackTrace();
        }
        if(null != response && response.getStatus() == 302) {
            return Result.healthy();
        }
        return Result.unhealthy("Fail...");

在执行该行时返回 Result.healthy();我们在生产环境中间歇性地低于错误。

org.eclipse.jetty.io.WriteFlusher - write exception
org.eclipse.jetty.io.EofException: null
    at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:192) ~[jetty-io-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.io.WriteFlusher.flush(WriteFlusher.java:408) ~[jetty-io-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.io.WriteFlusher.write(WriteFlusher.java:302) ~[jetty-io-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.io.AbstractEndPoint.write(AbstractEndPoint.java:129) [jetty-io-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpConnection$SendCallback.process(HttpConnection.java:676) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:246) [jetty-util-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:208) [jetty-util-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:471) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:763) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpChannel.write(HttpChannel.java:801) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:135) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpOutput.close(HttpOutput.java:166) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at com.fasterxml.jackson.core.json.UTF8JsonGenerator.close(UTF8JsonGenerator.java:1080) [jackson-core-2.5.1.jar:2.5.1]
    at com.fasterxml.jackson.databind.ObjectWriter._configAndWriteValue(ObjectWriter.java:1055) [jackson-databind-2.5.1.jar:2.5.1]
    at com.fasterxml.jackson.databind.ObjectWriter.writeValue(ObjectWriter.java:889) [jackson-databind-2.5.1.jar:2.5.1]
    at com.codahale.metrics.servlets.HealthCheckServlet.doGet(HealthCheckServlet.java:104) [metrics-servlets-3.1.1.jar:3.1.1]
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:645) [javax.servlet-api-4.0.1.jar:4.0.1]
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:750) [javax.servlet-api-4.0.1.jar:4.0.1]
    at com.codahale.metrics.servlets.AdminServlet.service(AdminServlet.java:101) [metrics-servlets-3.1.1.jar:3.1.1]
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:750) [javax.servlet-api-4.0.1.jar:4.0.1]
    at io.dropwizard.jetty.NonblockingServletHolder.handle(NonblockingServletHolder.java:49) [dropwizard-jetty-0.8.1.jar:0.8.1]
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669) [jetty-servlet-9.2.9.v20150224.jar:9.2.9.v20150224]
    at io.dropwizard.jersey.filter.AllowedMethodsFilter.handle(AllowedMethodsFilter.java:44) [dropwizard-jersey-0.8.1.jar:0.8.1]
    at io.dropwizard.jersey.filter.AllowedMethodsFilter.doFilter(AllowedMethodsFilter.java:39) [dropwizard-jersey-0.8.1.jar:0.8.1]
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) [jetty-servlet-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) [jetty-servlet-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) [jetty-servlet-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at io.dropwizard.jetty.RoutingHandler.handle(RoutingHandler.java:51) [dropwizard-jetty-0.8.1.jar:0.8.1]
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.handler.RequestLogHandler.handle(RequestLogHandler.java:95) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:159) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.Server.handle(Server.java:497) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) [jetty-server-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) [jetty-io-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [jetty-util-9.2.9.v20150224.jar:9.2.9.v20150224]
    at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [jetty-util-9.2.9.v20150224.jar:9.2.9.v20150224]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_302]
Caused by: java.io.IOException: Connection reset by peer
    at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) ~[na:1.8.0_302]
    at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) ~[na:1.8.0_302]
    at sun.nio.ch.IOUtil.write(IOUtil.java:148) ~[na:1.8.0_302]
    at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:503) ~[na:1.8.0_302]
    at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:172) ~[jetty-io-9.2.9.v20150224.jar:9.2.9.v20150224]
    ... 43 common frames omitted

有人可以建议有没有办法可以在本地重现这个？我们也可以修复或有解决方法来避免这种情况。如果我需要提供更多信息，请告诉我。提前致谢！

Answer 1

这仅仅意味着客户端断开连接并且没有完成读取响应。

问题完全出在客户端。

任何定期 HTTP 请求（例如：健康检查）都应使用适当的 HTTP 机制。

• 使用 HTTP/1.1（最简单）
• 客户端发送Connection: close请求头指示它将在交换结束时断开连接。
• 客户端读取整个响应，包括断开连接前的响应正文。

您的代码 sn-p 没有读取响应正文，请先这样做。

然后，如果此后问题仍然存在，请考虑在请求中设置 Connection: close 标头。