Im new to kafka, and trying to get started with my project. I have this in my docker-compose.yml
version: '3'
services:
zookeeper:
image: confluentinc/cp-zookeeper:7.3.0
container_name: zookeeper
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
broker:
image: confluentinc/cp-kafka:7.3.0
container_name: broker
depends_on:
- zookeeper
ports:
- 9092:9092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
Then im running my main.go file which has both producer and consumer and also some mock topic.
package main
import (
"fmt"
"log"
"time"
"github.com/confluentinc/confluent-kafka-go/kafka"
)
func main() {
topic := "HVSE"
p, err := kafka.NewProducer(&kafka.ConfigMap{
"bootstrap.servers": "localhost:9092",
"client.id": "foo",
"acks": "all",
})
go func() {
consumer, err := kafka.NewConsumer(&kafka.ConfigMap{
"bootstrap.servers": "localhost:9092",
"group.id": "foo",
"auto.offset.reset": "smallest",
})
if err != nil {
log.Fatal(err)
}
err = consumer.Subscribe(topic, nil)
if err != nil {
log.Fatal(err)
}
for {
ev := consumer.Poll(100)
// fmt.Println(ev)
switch e := ev.(type) {
case *kafka.Message:
fmt.Printf("consumed message from queue: %s\n", string(e.Value))
case *kafka.Error:
fmt.Printf("%v\n", e)
// return
// default:
// fmt.Printf("Ignored %v\n", e)
}
}
}()
deliverch := make(chan kafka.Event, 10000)
for {
err = p.Produce(&kafka.Message{
TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny},
Value: []byte("FOO"),
},
deliverch,
)
if err != nil {
log.Fatal(err)
}
<- deliverch
time.Sleep(time.Second * 1)
}
}
If I uncomment the default, I get into it.
Otherwise i got this error in console.
2023/09/26 13:45:05 Broker: Invalid replication factor
exit status 1
My kafka and zookeeper containers are running.
I changed the docker-compose.yml file, but that didn't help. And I found that my consumer.Events() is nil, but I can't understand why this is the case
I replicated your code and it is correct but the problem in the compose.yml only adjusts the ADVERTISED_LISTENERS
Instead of this:
KAFKA_ADVERTISED_LISTENERS:PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
Use this:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092