基于 Kafka 的网络中的多个 Orderer 冗余答案

【问题标题】：Multiple Orderer redundancy in Kafka based network基于 Kafka 的网络中的多个 Orderer 冗余
【发布时间】：2018-10-02 10:48:19
【问题描述】：

我们无法配置基于 3 个组织的结构网络，每个组织有 1 个对等方和 2 个基于 kafka 的订购者。对于 kafka 排序，我们使用 4 个 kafka 节点和 3 个 zookeeper。它部署在一些 AWS ec2 实例上，如下所示：

1：组织1

2：组织2

3：组织 3

4：orderer0、orderer1、kafka0、kafka1、kafka2、kafka3、zookeeper0、zookeeper1、zookeeper2

由于连接原因，整个排序节点加上 kafka 集群都位于同一台机器上（在某处读取它们必须在同一台机器上以避免这些问题）

在测试期间，我们尝试使用 docker stop 删除第一个排序节点 (orderer0) 以进行冗余测试。我们希望网络通过 orderer1 继续工作，但它却死了并停止工作。

查看对等方的控制台，我可以看到一些错误。

无法连接到任何端点：[orderer0.example.com:7050, orderer1.example.com:8050]

找到附件中与系统配置相关的文件内容。

orderer + kafka + zk

#
# Copyright IBM Corp. All Rights Reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
version: '2'

services:

  zookeeper0.example.com:
    container_name: zookeeper0.example.com
    extends:
      file: docker-compose-base.yaml
      service: zookeeper0.example.com
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"

  zookeeper1.example.com:
    container_name: zookeeper1.example.com
    extends:
      file: docker-compose-base.yaml
      service: zookeeper1.example.com
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"

  zookeeper2.example.com:
    container_name: zookeeper2.example.com
    extends:
      file: docker-compose-base.yaml
      service: zookeeper2.example.com
    logging:
     driver: "json-file" 
     options:
      max-size: "1m"
      max-file: "3"

  kafka0.example.com:
    container_name: kafka0.example.com
    extends:
      file: docker-compose-base.yaml
      service: kafka0.example.com
    depends_on:
      - zookeeper0.example.com
      - zookeeper1.example.com
      - zookeeper2.example.com
      - orderer0.example.com
      - orderer1.example.com
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"

  kafka1.example.com:
    container_name: kafka1.example.com
    extends:
      file: docker-compose-base.yaml
      service: kafka1.example.com
    depends_on:
      - zookeeper0.example.com
      - zookeeper1.example.com
      - zookeeper2.example.com
      - orderer0.example.com
      - orderer1.example.com
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"

  kafka2.example.com:
    container_name: kafka2.example.com
    extends:
      file: docker-compose-base.yaml
      service: kafka2.example.com
    depends_on:
      - zookeeper0.example.com
      - zookeeper1.example.com
      - zookeeper2.example.com
      - orderer0.example.com
      - orderer1.example.com
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"

  kafka3.example.com:
    container_name: kafka3.example.com
    extends:
      file: docker-compose-base.yaml
      service: kafka3.example.com        
    depends_on:
      - zookeeper0.example.com
      - zookeeper1.example.com
      - zookeeper2.example.com
      - orderer0.example.com
      - orderer1.example.com
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"

  orderer0.example.com:
    container_name: orderer0.example.com
    image: hyperledger/fabric-orderer:x86_64-1.1.0
    environment:
      - ORDERER_GENERAL_LOGLEVEL=debug
      - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
      - ORDERER_LISTEN_PORT=7050
      - ORDERER_GENERAL_GENESISMETHOD=file
      - ORDERER_GENERAL_GENESISFILE=/etc/hyperledger/configtx/genesis.block
      - ORDERER_GENERAL_LOCALMSPID=OrdererMSP
      - ORDERER_GENERAL_LOCALMSPDIR=/etc/hyperledger/crypto/orderer/msp
      - ORDERER_GENERAL_TLS_ENABLED=true
      - ORDERER_GENERAL_TLS_PRIVATEKEY=/etc/hyperledger/crypto/orderer/tls/server.key
      - ORDERER_GENERAL_TLS_CERTIFICATE=/etc/hyperledger/crypto/orderer/tls/server.crt
      - ORDERER_GENERAL_TLS_ROOTCAS=[/etc/hyperledger/crypto/orderer/tls/ca.crt, /etc/hyperledger/crypto/peerOrg1/tls/ca.crt, /etc/hyperledger/crypto/peerOrg2/tls/ca.crt, /etc/hyperledger/crypto/peerOrg3/tls/ca.crt]
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"
    working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderers
    command: orderer
    ports:
      - 7050:7050
    volumes:
        - ./channel:/etc/hyperledger/configtx
        - ./channel/crypto-config/ordererOrganizations/example.com/orderers/orderer0.example.com/:/etc/hyperledger/crypto/orderer
        - ./channel/crypto-config/peerOrganizations/org1.example.com/peers/peer0.org1.example.com/:/etc/hyperledger/crypto/peerOrg1
        - ./channel/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/:/etc/hyperledger/crypto/peerOrg2
        - ./channel/crypto-config/peerOrganizations/org3.example.com/peers/peer0.org3.example.com/:/etc/hyperledger/crypto/peerOrg3
    depends_on:
        - kafka0.example.com
        - kafka1.example.com
        - kafka2.example.com
        - kafka3.example.com

  orderer1.example.com:
    container_name: orderer1.example.com
    image: hyperledger/fabric-orderer:x86_64-1.1.0
    environment:
      - ORDERER_GEN ERAL_LOGLEVEL=debug
      - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
      - ORDERER_LISTEN_PORT=8050
      - ORDERER_GENERAL_GENESISMETHOD=file
      - ORDERER_GENERAL_GENESISFILE=/etc/hyperledger/configtx/genesis.block
      - ORDERER_GENERAL_LOCALMSPID=OrdererMSP
      - ORDERER_GENERAL_LOCALMSPDIR=/etc/hyperledger/crypto/orderer/msp
      - ORDERER_GENERAL_TLS_ENABLED=true
      - ORDERER_GENERAL_TLS_PRIVATEKEY=/etc/hyperledger/crypto/orderer/tls/server.key
      - ORDERER_GENERAL_TLS_CERTIFICATE=/etc/hyperledger/crypto/orderer/tls/server.crt
      - ORDERER_GENERAL_TLS_ROOTCAS=[/etc/hyperledger/crypto/orderer/tls/ca.crt, /etc/hyperledger/crypto/peerOrg1/tls/ca.crt, /etc/hyperledger/crypto/peerOrg2/tls/ca.crt, /etc/hyperledger/crypto/peerOrg3/tls/ca.crt]
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"
    working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderers
    command: orderer
    ports:
      - 8050:7050
    volumes:
        - ./channel:/etc/hyperledger/configtx
        - ./channel/crypto-config/ordererOrganizations/example.com/orderers/orderer1.example.com/:/etc/hyperledger/crypto/orderer
        - ./channel/crypto-config/peerOrganizations/org1.example.com/peers/peer0.org1.example.com/:/etc/hyperledger/crypto/peerOrg1
        - ./channel/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/:/etc/hyperledger/crypto/peerOrg2
        - ./channel/crypto-config/peerOrganizations/org3.example.com/peers/peer0.org3.example.com/:/etc/hyperledger/crypto/peerOrg3
    depends_on:
        - kafka0.example.com
        - kafka1.example.com
        - kafka2.example.com
        - kafka3.example.com

来自 Org2 的 Peer 和 Ca

#
# Copyright IBM Corp. All Rights Reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
version: '2'

services:

  ca.org2.example.com:
    image: hyperledger/fabric-ca:x86_64-1.1.0
    environment:
      - FABRIC_CA_HOME=/etc/hyperledger/fabric-ca-server
      - FABRIC_CA_SERVER_CA_CERTFILE=/etc/hyperledger/fabric-ca-server-config/ca.org2.example.com-cert.pem
      - FABRIC_CA_SERVER_CA_KEYFILE=/etc/hyperledger/fabric-ca-server-config/efa7d0819b7083f6c06eb34da414acbcde79f607b9ce26fb04dee60cf79a389a_sk
      - FABRIC_CA_SERVER_TLS_ENABLED=true
      - FABRIC_CA_SERVER_TLS_CERTFILE=/etc/hyperledger/fabric-ca-server-config/ca.org2.example.com-cert.pem
      - FABRIC_CA_SERVER_TLS_KEYFILE=/etc/hyperledger/fabric-ca-server-config/efa7d0819b7083f6c06eb34da414acbcde79f607b9ce26fb04dee60cf79a389a_sk
    ports:
      - "8054:7054"
    command: sh -c 'fabric-ca-server start -b admin:adminpw -d'
    volumes:
      - ./channel/crypto-config/peerOrganizations/org2.example.com/ca/:/etc/hyperledger/fabric-ca-server-config
    container_name: ca_peerOrg2

  peer0.org2.example.com:
    container_name: peer0.org2.example.com
    extends:
      file:   base.yaml
      service: peer-base
    environment:
      - CORE_PEER_ID=peer0.org2.example.com
      - CORE_PEER_LOCALMSPID=Org2MSP
      - CORE_PEER_ADDRESS=peer0.org2.example.com:7051
    ports:
      - 8051:7051
      - 8053:7053
    volumes:
        - ./channel/crypto-config/peerOrganizations/org2.example.com/peers/peer0.org2.example.com/:/etc/hyperledger/crypto/peer
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"
    extra_hosts:
      - "orderer0.example.com:xxx.xxx.xxx.xxx"
      - "orderer1.example.com:xxx.xxx.xxx.xxx"
      - "kafka0.example.com:xxx.xxx.xxx.xxx"
      - "kafka1.example.com:xxx.xxx.xxx.xxx"
      - "kafka2.example.com:xxx.xxx.xxx.xxx"
      - "kafka3.example.com:xxx.xxx.xxx.xxx"

订货人基础

# Copyright IBM Corp. All Rights Reserved.
#
# SPDX-License-Identifier: Apache-2.0
#

version: '2'

services:

  orderer-base:
    image: hyperledger/fabric-orderer:$IMAGE_TAG
    environment:
      - ORDERER_GENERAL_LOGLEVEL=error
      - ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
      - ORDERER_GENERAL_GENESISMETHOD=file
      - ORDERER_GENERAL_GENESISFILE=/var/hyperledger/orderer/orderer.genesis.block
      - ORDERER_GENERAL_LOCALMSPID=OrdererMSP
      - ORDERER_GENERAL_LOCALMSPDIR=/var/hyperledger/orderer/msp
      # enabled TLS
      - ORDERER_GENERAL_TLS_ENABLED=true
      - ORDERER_GENERAL_TLS_PRIVATEKEY=/var/hyperledger/orderer/tls/server.key
      - ORDERER_GENERAL_TLS_CERTIFICATE=/var/hyperledger/orderer/tls/server.crt
      - ORDERER_GENERAL_TLS_ROOTCAS=[/var/hyperledger/orderer/tls/ca.crt]
      # kafka
      - CONFIGTX_ORDERER_ORDERERTYPE=kafka
      - CONFIGTX_ORDERER_KAFKA_BROKERS=[kafka0.example.com,kafka1.example.com,kafka2.example.com,kafka3.example.com]
      - ORDERER_KAFKA_RETRY_SHORTINTERVAL=1s
      - ORDERER_KAFKA_RETRY_SHORTTOTAL=30s
      - ORDERER_KAFKA_VERBOSE=true
    working_dir: /opt/gopath/src/github.com/hyperledger/fabric
    command: orderer

卡夫卡基地

# Copyright IBM Corp. All Rights Reserved.
#
# SPDX-License-Identifier: Apache-2.0
#

version: '2'

services:

  zookeeper:
    image: hyperledger/fabric-zookeeper
    environment:
      - ZOO_SERVERS=server.1=zookeeper0.example.com:2888:3888 server.2=zookeeper1.example.com:2888:3888 server.3=zookeeper2.example.com:2888:3888
    restart: always

  kafka:
    image: hyperledger/fabric-kafka
    restart: always
    environment:
      - KAFKA_MESSAGE_MAX_BYTES=103809024 # 99 * 1024 * 1024 B
      - KAFKA_REPLICA_FETCH_MAX_BYTES=103809024 # 99 * 1024 * 1024 B
      - KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE=false
      - KAFKA_MIN_INSYNC_REPLICAS=2
      - KAFKA_DEFAULT_REPLICATION_FACTOR=3
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper0.example.com:2181,zookeeper1.example.com:2181,zookeeper2.example.com:2181
    logging:
     driver: "json-file"
     options:
      max-size: "1m"
      max-file: "3"

configtx.yaml

Organizations:


    - &OrdererOrg

        Name: OrdererMSP


        ID: OrdererMSP


        MSPDir: crypto-config/ordererOrganizations/example.com/msp

    - &Org1

        Name: Org1MSP


        ID: Org1MSP

        MSPDir: crypto-config/peerOrganizations/org1.example.com/msp

        AnchorPeers:

            - Host: peer0.org1.example.com
              Port: 7051

    - &Org2

        Name: Org2MSP

        ID: Org2MSP

        MSPDir: crypto-config/peerOrganizations/org2.example.com/msp

        AnchorPeers:

            - Host: peer0.org2.example.com
              Port: 7051

    - &Org3

        Name: Org3MSP


        ID: Org3MSP

        MSPDir: crypto-config/peerOrganizations/org3.example.com/msp

        AnchorPeers:

            - Host: peer0.org3.example.com
              Port: 7051



################################################################################
Orderer: &OrdererDefaults


    OrdererType: kafka

    Addresses:
        - orderer0.example.com:7050
        - orderer1.example.com:7050


    BatchTimeout: 2s


    BatchSize:

        MaxMessageCount: 10

        AbsoluteMaxBytes: 98 MB

        PreferredMaxBytes: 512 KB

    Kafka:
        Brokers:
            - kafka0.example.com:9092
            - kafka1.example.com:9092
            - kafka2.example.com:9092
            - kafka3.example.com:9092

    Organizations:


################################################################################
Application: &ApplicationDefaults

    Organizations:


################################################################################
Profiles:

    ThreeOrgsOrdererGenesis:
        Orderer:
            <<: *OrdererDefaults
            Organizations:
                - *OrdererOrg
        Consortiums:
            SampleConsortium:
                Organizations:
                    - *Org1
                    - *Org2
                    - *Org3
    ThreeOrgsChannel:
        Consortium: SampleConsortium
        Application:
            <<: *ApplicationDefaults
            Organizations:
                - *Org1
                - *Org2
                - *Org3

可能是配置错误？连接问题几乎被丢弃，因为在本地机器上运行相同的网络会产生相同的结果。

提前致谢问候

【问题讨论】：

您正在启动两个 Orderer，它们都具有相同的配置。你确定两个容器都在运行吗？您可以尝试在 CLI 中执行“docker ps”命令并粘贴结果吗？我认为问题在于两个订购者都没有启动
首先，启动两个相同配置的Orderer会不会有问题？只要他们监听不同的端口。其次，我确认第二个订购者 orderer1 正在运行。它写入日志并显示事务。看起来像是 orderer0 的奴隶，当 orderer0 宕机时，它就变得没用了
是的，我同意你的看法。但是，问题是每个节点（Peer、Orderer...）都应该有自己的 MSP 标识符，在您的情况下，它们都有相同的 ORDERER_GENERAL_LOCALMSPID=OrdererMSP。
是的，有道理，谢谢。我将 OrdererMSP 更改为 Orderer0MSP 和 Orderer1MSP，然后我更改了 configtx.yaml 以匹配新定义。现在我从对等端收到一条错误消息：“无法连接到 orderer1:7050，证书对 orderer0 有效，而不是 orderer1”。
我认为您的加密材料不正确。您是否在创建加密材料之前更改了 cryptoconfig.yaml？在 cryptoconfig.yaml 中，您定义要创建的加密材料。在那里你应该定义两个订购者。在 configtx,yaml 中，你也应该做一些改变。

标签： apache-kafka hyperledger-fabric hyperledger

【解决方案1】：

终于可以顺利运行了。原来问题不在于 docker-compose 文件，而在于 web 服务的 fabric sdk 版本。我在 1.1 版上都使用了 fabric-client 和 fabric-ca-client，直到 1.2 版才出现。（更多信息https://jira.hyperledger.org/browse/FABN-90）

澄清一下，由于它们之间的互连，我能够看到两个订购者上发生的交易，但我只是攻击第一个。当那个 orderer 出现故障时，网络就会变暗。

我了解fabric 处理orderer 的方式，它指向列表的第一个orderer，如果它已关闭或无法访问，则将其移动到列表的底部并定位下一个。这是自 1.2 以来发生的情况，在旧版本中，您必须编写自己的错误控制器，以便它更改为下一个排序器。

【讨论】：

【解决方案2】：

我不确定，但可能是因为网络层不同。由于它是一个不同的 compose 文件，docker 会为每个 composer 创建不同的网络层。

另外，我没有看到 yaml 文件中提到的网络。

请使用“docker network list”查看网络层列表。

【讨论】：