HAProxy 和 websockets 从不连接答案

【问题标题】：HAProxy and websockets never connectHAProxy 和 websockets 从不连接
【发布时间】：2021-11-18 18:40:55
【问题描述】：

我认为 HAProxy 2.4 支持基于 h2 的 WebSockets，我终于可以让基于 HAProxy 的 websockets 工作了……但是没有……

当我的浏览器尝试连接时，它会立即放弃“connect?transport=webSockets”并尝试 SSE。这个“连接”也立即被放弃了，但它“start?transport=SSE”就好了。

Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:18.520] www-httpsStrict~ test_backend/werdc2020 557/0/0/2/558 200 704 - - --VN 2/1/0/0/0 0/0 "GET https://myserver.com/signalr/negotiate?clientProtocol=2.1&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D&_=1637143818090 HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.075] www-httpsStrict~ test_backend/werdc2020 52/0/0/1/52 404 1392 - - --VN 2/1/0/0/0 0/0 "GET https://myserver.com/favicon.ico HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.279] www-httpsStrict~ test_backend/werdc2020 96/0/0/7/103 200 339 - - --VN 3/2/1/1/0 0/0 "GET https://myserver.com/signalr/start?transport=serverSentEvents&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D&_=1637143818091 HTTP/2.0"
Nov 17 17:10:19 127.0.0.1 haproxy[26334]: 192.168.126.78:1228 [17/Nov/2021:17:10:19.375] www-httpsStrict~ test_backend/werdc2020 65/0/0/20/84 200 348 - - --VN 3/2/1/1/0 0/0 "POST https://myserver.com/signalr/send?transport=serverSentEvents&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22testservice%22%7D%5D HTTP/2.0"

如果我直接在服务器上连接它就可以了...

HAProxy 配置：

global
    log 127.0.0.1 local0 debug
    log-tag haproxy
    maxconn 10000
    user haproxy
    group haproxy
    daemon
    nbproc 1
    nbthread 8
    #cpu-map auto:all 0-1    
    stats socket /var/run/haproxy.sock mode 600 level admin
    stats timeout 30s
    pidfile /var/run/haproxy.pid
    tune.ssl.default-dh-param 2048 
    tune.ssl.cachesize 100000
    tune.ssl.lifetime 600
    tune.ssl.maxrecord 1460 
    # intermediate configuration
    ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:!MD5:!aNULL:!DH:!RC4    
    ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
  
defaults
    log    global
    mode    http
    option    httplog
    option    dontlognull
    timeout connect 5s
    timeout client    1m
    timeout client-fin    1m
    timeout server    1h
    timeout tunnel 1h

frontend stats 
    bind 0.0.0.0:8080
    mode http
    stats enable
    stats hide-version
    stats realm Haproxy\ Statistics
    stats uri /
    stats auth myuser:1111
    stats refresh 60s

# Force front server MUST use redirect on HTTPS

frontend www-httpStrict
    bind *:80
    option splice-auto
    # Test URI to see if its a letsencrypt request
    acl letsencrypt-acl path_beg /.well-known               
    acl test hdr_sub(cookie) istest=true
    redirect scheme https code 307 if !{ ssl_fc } !letsencrypt-acl  
    use_backend letsencrypt if letsencrypt-acl
    use_backend test_backend if test
    default_backend www-backendStrict    
 
frontend www-httpsStrict   
   bind *:443 tfo ssl crt /etc/ssl/private alpn h2,http/1.1
   option forwardfor
   option splice-auto  
   acl test hdr_sub(cookie) istest=true
   use_backend test_backend if test   
   default_backend www-backendStrict

backend letsencrypt
    server nginx 127.0.0.1:8888    

# this backend require haproxy open SSL tunel to port 443 on webservers    
backend www-backendStrict  
   
   balance roundrobin
   cookie MyWebFarm insert 
   option forwardfor
   option splice-auto
   option tcp-smart-connect
   option httpchk
   http-check connect ssl alpn h2 sni myserver.com
   http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
   http-check expect status 200-399   
   http-request add-header X-Forwarded-Proto https if { ssl_fc }
   retry-on all-retryable-errors
   http-request disable-l7-retry if METH_POST
   default-server ssl tfo verify none alpn h2,http/1.1 check allow-0rtt  
   server werdc01 192.168.1.5:443 cookie werc01 check
   server werdc2020 192.168.1.6:443 cookie werdc2020 check

backend test_backend   
  balance roundrobin
   cookie MyWebFarm insert 
   option forwardfor
   option splice-auto
   option tcp-smart-connect
   option httpchk
   http-check connect ssl alpn h2 sni myserver.com
   http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
   http-check expect status 200-399
   http-request add-header X-Forwarded-Proto https if { ssl_fc }
   retry-on all-retryable-errors
   http-request disable-l7-retry if METH_POST
   default-server ssl verify none alpn h2,http/1.1 check allow-0rtt
   server werdc2020 192.168.1.6:443 cookie werdc2020 check

我将 istest cookie 设置为始终命中 test_backend...

从浏览器发送的标头：

GET /signalr/connect?transport=webSockets&clientProtocol=2.1&connectionToken=%2FZ99uK3N88wIyXYxQlQc4w42M3jcf0Tz6tbLcm7CXhOcTgTu7qgbfsMxn9l6GNP%2FML39o%2BabOZ4GSQDgAxo7oYXIacee8Ku2Nd1QXvWRalisPY%2BTIgddfJsWdE828LaH&connectionData=%5B%7B%22name%22%3A%22catiservice%22%7D%5D&tid=7 undefined
Host: myserver.com
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0
Accept: */*
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate, br
Sec-WebSocket-Version: 13
Origin: https://myserver.com
Sec-WebSocket-Extensions: permessage-deflate
Sec-WebSocket-Key: P1GaqaLxIxBfuXDCtxuQwg==
DNT: 1
Connection: keep-alive, Upgrade
Cookie: _ga=GA1.2.95033637.1627965955; .MyAuth=D758016C45CF1B9BDC38D17DCAF7A3CEE7E528B97352267A1B0D019BFE20964BF2900C655950E126CD7FC08CB421C1A997D1A5FDB0266316B920634FB2C23B63080B271B6331096902B47BF73661240D; MyWebFarm=werdc2020; ASP.NET_SessionId=w1zibnnuht4ohicpntyhn55q; istest=true
Sec-Fetch-Dest: websocket
Sec-Fetch-Mode: websocket
Sec-Fetch-Site: same-origin
Pragma: no-cache
Cache-Control: no-cache
Upgrade: websocket
TE: trailers

为什么 HAProxy 看不到“连接”请求？

为什么不与 WebSockets 连接？

【问题讨论】：

标签： websocket signalr haproxy

【解决方案1】：

这里有一个 github 问题：https://github.com/haproxy/haproxy/issues/162

虽然您的问题不是很有见地，但我建议您将配置精简到最低限度以测试 websocket。

这是一个需要考虑的 sn-p

  option http-server-close 
  option http-use-htx

  acl hdr_connection_upgrade hdr(Connection)  -i upgrade
  acl hdr_upgrade_websocket  hdr(Upgrade)     -i websocket

为什么 HAProxy 看不到“连接”请求？

我想，只是因为 CONNECT 没有成功。如果你使用tcpdump，你肯定会看到一个请求传入。

【讨论】：

注意：我也在努力使用 websocket 到 h2-backend。但我个人并不关心 h2，我只是不想为所需的不同后端 proto 选项编写 2 个单独的后端
谢谢，该链接解释了一切。只需将 alpn 设置为 http/1.1 即可进行 ws 连接

【解决方案2】：

使用来自 @sgohl 的解释，此配置有效（至少在 2.5 修复该问题之前）：

global
    log 127.0.0.1 local0 debug
    log-tag haproxy
    maxconn 10000
    user haproxy
    group haproxy
    daemon
    nbproc 1
    nbthread 8
    #cpu-map auto:all 0-1    
    stats socket /var/run/haproxy.sock mode 600 level admin
    stats timeout 30s
    pidfile /var/run/haproxy.pid
    tune.ssl.default-dh-param 2048
    tune.ssl.cachesize 100000
    tune.ssl.lifetime 600
    tune.ssl.maxrecord 1460
    # intermediate configuration
    ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:!MD5:!aNULL:!DH:!RC4    
    ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
  
defaults all
    log     global
    mode    http
    option  httplog
    option  dontlognull
    timeout connect    5s
    timeout client     1m
    timeout client-fin 1m
    timeout server     1h
    timeout tunnel     4h
    option  splice-auto
    option  tcp-smart-connect
    balance roundrobin
    option  forwardfor
    retry-on all-retryable-errors

frontend stats from all
    bind  *:8080
    stats enable
    stats hide-version
    stats realm Haproxy\ Statistics
    stats uri /
    stats auth Myuser:11111
    stats refresh 60s

listen www from all
    bind *:80
    bind *:443 tfo ssl crt /etc/ssl/private alpn h2,http/1.1 allow-0rtt

    # Test URI to see if its a letsencrypt request
    acs letsencrypt path_beg /.well-known
    http-request redirect scheme https unless { ssl_fc } or letsencrypt
    use_backend letsencrypt if letsencrypt
    
    http-response set-header Strict-Transport-Security "max-age=10886400; includeSubDomains; preload;"

    use_backend ws  if { hdr(Connection) -i upgrade } or { hdr(Upgrade) -i websocket }
    default_backend h2

backend letsencrypt
    server letsencrypt 127.0.0.1:8888

defaults cg from all
    cookie  MyWebFarm insert 
    option  httpchk
    http-check connect ssl alpn h2 sni myserver.com
    http-check send meth HEAD uri /login.aspx ver HTTP/2 hdr Host myserver.com
    http-check expect status 200-399   

backend h2 from cg
    default-server ssl tfo verify none alpn h2 check allow-0rtt
    http-request disable-l7-retry if METH_POST
    use-server werdc2020 if { hdr_sub(cookie) istest=true }
    server werdc01     192.168.1.5:443 cookie werc01 
    server werdc2020   192.168.1.6:443 cookie werdc2020

backend ws from cg
    default-server ssl tfo verify none alpn http/1.1 check allow-0rtt
    http-request disable-l7-retry if METH_POST
    use-server werdc2020 if { hdr_sub(cookie) istest=true }
    server werdc01     192.168.1.5:443 cookie werc01 
    server werdc2020   192.168.1.6:443 cookie werdc2020

【讨论】：