javadockerapache-kafkadocker-composekafka-rest

Zookeeper connection failing cp-rest-proxy with spotify kafka image


I have been using the kafka image provided by spotify to run kafka locally. I'm currently trying to use it with cp-kafka-rest and schema-registry images.

I need help resolving this issue:

ERROR (Log Group: kafka_rest_1_609fd108dcf4)

[main-SendThread(zookeeper:2181)] WARN org.apache.zookeeper.ClientCnxn - Session 0x0 for server zookeeper:2181, unexpected error, closing socket connection and attempting reconnect

java.nio.channels.UnresolvedAddressException 
  at sun.nio.ch.Net.checkAddress(Net.java:101)
  at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:622)
  at org.apache.zookeeper.ClientCnxnSocketNIO.registerAndConnect(ClientCnxnSocketNIO.java:277)
  at org.apache.zookeeper.ClientCnxnSocketNIO.connect(ClientCnxnSocketNIO.java:287)
  at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1021)
  at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1064)

Docker Compose

version: '3.5'
services:
  kafka:
    image: 'spotify/kafka'
    hostname: kafka
    environment:
      - ADVERTISED_HOST=kafka
      - ADVERTISED_PORT=9092
    ports:
      - "9092:9092"
      - "2181:2181"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - one
  kafka_rest:
    image: 'confluentinc/cp-kafka-rest:5.1.0'
    hostname: kafka_rest
    environment:
      - KAFKA_REST_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_REST_LISTENERS=http://0.0.0.0:8082
      - KAFKA_REST_SCHEMA_REGISTRY_URL=http:schema-registry:8081
      - KAFKA_REST_HOST_NAME=kafka-rest
    networks:
      - one
  schema_registry:
    hostname: schema-registry
    image: 'confluentinc/cp-schema-registry:5.1.0'
    environment:
      - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
      - SCHEMA_REGISTRY_HOST_NAME=schema-registry
      - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081
    networks:
      - one
networks:
  one:
    name: rest_network

Solution

  • You have no zookepeer container - it is actually your "kafka" service image that includes both Zookeeper and Kafka servers, so zookeeper:2181 should rather be kafka:2181


    However, I would recommend not using the spotify images, as they are significantly outdated

    You can find a fully functional Docker Compose example of the entire Confluent 5.1.0 Platform on Github

    Here is the revelvant configuration you are looking for

    ---
    version: '2'
    services:
      zookeeper:
        image: confluentinc/cp-zookeeper:5.1.0
        hostname: zookeeper
        container_name: zookeeper
        ports:
          - "2181:2181"
        environment:
          ZOOKEEPER_CLIENT_PORT: 2181
          ZOOKEEPER_TICK_TIME: 2000
    
      broker:
        image: confluentinc/cp-enterprise-kafka:5.1.0
        hostname: broker
        container_name: broker
        depends_on:
          - zookeeper
        ports:
          - "9092:9092"
          - "29092:29092"
        environment:
          KAFKA_BROKER_ID: 1
          KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092,PLAINTEXT_HOST://localhost:29092
          KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
          CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:9092
          CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
          CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
          CONFLUENT_METRICS_ENABLE: 'true'
          CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
    
      schema-registry:
        image: confluentinc/cp-schema-registry:5.1.0
        hostname: schema-registry
        container_name: schema-registry
        depends_on:
          - zookeeper
          - broker
        ports:
          - "8081:8081"
        environment:
          SCHEMA_REGISTRY_HOST_NAME: schema-registry
          SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
    
      rest-proxy:
        image: confluentinc/cp-kafka-rest:5.1.0
        depends_on:
          - zookeeper
          - broker
          - schema-registry
        ports:
          - 8082:8082
        hostname: rest-proxy
        container_name: rest-proxy
        environment:
          KAFKA_REST_HOST_NAME: rest-proxy
          KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:9092'
          KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
          KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'