diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 189433d..dd05287 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,6 @@ jobs: # Runs a set of commands using the runners shell - - uses: docker-practice/actions-setup-docker@master - name: run test single 2 run: ./test.sh zk-single-kafka-single.yml 2 diff --git a/README.md b/README.md index ded686a..4bea24a 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,44 @@ [![Actions Status](https://github.com/conduktor/kafka-stack-docker-compose/workflows/CI/badge.svg)](https://github.com/conduktor/kafka-stack-docker-compose/actions) -# An open-source project by [![Conduktor.io](https://www.conduktor.io/images/logo.svg)](https://conduktor.io/) +![logo](https://raw.githubusercontent.com/conduktor/conduktor.io-public/refs/heads/main/logo/logo-signature.png) +# An open-source project by [Conduktor](https://conduktor.io/) -This project is sponsored by [Conduktor.io](https://www.conduktor.io/), a graphical desktop user interface for Apache Kafka. +This project is sponsored by [Conduktor.io](https://www.conduktor.io/), the Enterprise Data Management +Platform for Streaming. Once you have started your cluster, you can use Conduktor to easily manage it. Just connect against `localhost:9092`. If you are on Mac or Windows and want to connect from another container, use `host.docker.internal:29092` # kafka-stack-docker-compose -This replicates as well as possible real deployment configurations, where you have your zookeeper servers and kafka servers actually all distinct from each other. This solves all the networking hurdles that comes with Docker and docker compose, and is compatible cross platform. +This replicates as well as possible real deployment configurations, where you have your zookeeper servers and Kafka servers distinct from each other. This solves all the networking hurdles that comes with Docker and Docker Compose, and is compatible cross platform. -**UPDATE**: No /etc/hosts file changes are necessary anymore. Explanations at: https://rmoff.net/2018/08/02/kafka-listeners-explained/ +## Stack -## Stack version + - Conduktor Platform + - Zookeeper version + - Kafka version + - Kafka Schema Registry + - Kafka Rest Proxy + - Kafka Connect + - ksqlDB Server + - Zoonavigator - - Conduktor Platform: latest - - Zookeeper version: 3.6.3 (Confluent 7.3.2) - - Kafka version: 3.3.0 (Confluent 7.3.2) - - Kafka Schema Registry: Confluent 7.3.2 - - Kafka Rest Proxy: Confluent 7.3.2 - - Kafka Connect: Confluent 7.3.2 - - ksqlDB Server: Confluent 7.3.2 - - Zoonavigator: 1.1.1 - -For a UI tool to access your local Kafka cluster, use the free version of [Conduktor](https://www.conduktor.io/get-started) +For a UI tool to access your local Kafka cluster, use the free version of [Conduktor](https://www.conduktor.io/get-started). # Requirements -Kafka will be exposed on `127.0.0.1` or `DOCKER_HOST_IP` if set in the environment. -(You probably don't need to set it if you're not using Docker-Toolbox) - -## Docker-Toolbox -Docker toolbox is [deprecated](https://github.com/docker-archive/toolbox) and not maintained anymore for several years. -We can't guarantee this stack will work with Docker Toolbox, but if you want to try anyway, please export your environment before starting the stack: -``` -export DOCKER_HOST_IP=192.168.99.100 -``` -(your docker machine IP is usually `192.168.99.100`) +Kafka will be exposed on `127.0.0.1`. -## Apple M1 support -Confluent platform supports Apple M1 (ARM64) since version `7.2.0`! Basically, this stack will work out of the box. +## Apple M4 Support -If you want to downgrade confluent platform version, there are two ways: +At the time of writing there is an issue with Apple M4 chip machines and running certain Java based Docker images. -1. Add `platform: linux/amd64`. It will work as docker is able to emulate AMD64 instructions. -2. Previous versions have been [built](https://github.com/arm64-compat/confluent-platform) for ARM64 by the community. If you want to use it, just change the image in the corresponding yml. Since it is a not an official image, use it at your own risks. +Modify the `conduktor.yml` file, uncomment the environment variable `CONSOLE_JAVA_OPTS: "-XX:UseSVE=0"`. ## Full stack -To ease you journey with kafka just connect to [localhost:8080](http://localhost:8080/) - -login: `admin@admin.io` -password: `admin` +To ease you journey with Kafka just connect to [localhost:8080](http://localhost:8080/) - Conduktor-platform: `$DOCKER_HOST_IP:8080` - Single Zookeeper: `$DOCKER_HOST_IP:2181` @@ -63,12 +49,11 @@ password: `admin` - KSQL Server: `$DOCKER_HOST_IP:8088` - (experimental) JMX port at `$DOCKER_HOST_IP:9001` - Run with: - ``` - docker compose -f full-stack.yml up - docker compose -f full-stack.yml down - ``` -** Note: if you find that you can not connect to [localhost:8080](http://localhost:8080/) please run `docker compose -f full-stack.yml build` to rebuild the port mappings. +Run with: +``` +docker compose -f full-stack.yml up +docker compose -f full-stack.yml down +``` ## Single Zookeeper / Single Kafka @@ -86,7 +71,7 @@ docker compose -f zk-single-kafka-single.yml down ## Single Zookeeper / Multiple Kafka -If you want to have three brokers and experiment with kafka replication / fault-tolerance. +If you want to have three brokers and experiment with Kafka replication / fault-tolerance. - Zookeeper will be available at `$DOCKER_HOST_IP:2181` - Kafka will be available at `$DOCKER_HOST_IP:9092,$DOCKER_HOST_IP:9093,$DOCKER_HOST_IP:9094` @@ -115,7 +100,7 @@ docker compose -f zk-multiple-kafka-single.yml down ## Multiple Zookeeper / Multiple Kafka -If you want to have three zookeeper nodes and three kafka brokers to experiment with production setup. +If you want to have three zookeeper nodes and three Kafka brokers to experiment with production setup. - Zookeeper will be available at `$DOCKER_HOST_IP:2181,$DOCKER_HOST_IP:2182,$DOCKER_HOST_IP:2183` - Kafka will be available at `$DOCKER_HOST_IP:9092,$DOCKER_HOST_IP:9093,$DOCKER_HOST_IP:9094` @@ -181,9 +166,9 @@ A: yes. This is for testing only!!! Reduce the KAFKA_LOG_SEGMENT_BYTES to 16MB a KAFKA_LOG_RETENTION_BYTES: 134217728 ``` -**Q: How do I expose kafka?** +**Q: How do I expose Kafka?** -A: If you want to expose kafka outside of your local machine, you must set `KAFKA_ADVERTISED_LISTENERS` to the IP of the machine so that kafka is externally accessible. To achieve this you can set `LISTENER_DOCKER_EXTERNAL` to the IP of the machine. +A: If you want to expose Kafka outside of your local machine, you must set `KAFKA_ADVERTISED_LISTENERS` to the IP of the machine so that Kafka is externally accessible. To achieve this you can set `LISTENER_DOCKER_EXTERNAL` to the IP of the machine. For example, if the IP of your machine is `50.10.2.3`, follow the sample mapping below: ``` @@ -195,7 +180,7 @@ For example, if the IP of your machine is `50.10.2.3`, follow the sample mapping KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://50.10.2.3:9093,DOCKER://host.docker.internal:29093 ``` -**Q: How do I add connectors to kafka connect?** +**Q: How do I add connectors to Kafka connect?** Create a `connectors` directory and place your connectors there (usually in a subdirectory) `connectors/example/my.jar` diff --git a/conduktor.yml b/conduktor.yml index 5504f69..7b46a0c 100644 --- a/conduktor.yml +++ b/conduktor.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: postgresql: @@ -28,6 +26,7 @@ services: CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://kafka-schema-registry:8081" CDK_CLUSTERS_0_KAFKACONNECTS_0_URL: "http://kafka-connect:8083" CDK_CLUSTERS_0_KAFKACONNECTS_0_NAME: "full stack kafka connect" + # CONSOLE_JAVA_OPTS: "-XX:UseSVE=0" volumes: pg_data: {} diff --git a/full-stack.yml b/full-stack.yml index b5163bd..4722179 100644 --- a/full-stack.yml +++ b/full-stack.yml @@ -1,8 +1,6 @@ -version: '2.1' - services: zoo1: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +11,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka1 container_name: kafka1 ports: @@ -38,7 +36,7 @@ services: - zoo1 kafka-schema-registry: - image: confluentinc/cp-schema-registry:7.3.2 + image: confluentinc/cp-schema-registry:7.8.0 hostname: kafka-schema-registry container_name: kafka-schema-registry ports: @@ -53,7 +51,7 @@ services: kafka-rest-proxy: - image: confluentinc/cp-kafka-rest:7.3.2 + image: confluentinc/cp-kafka-rest:7.8.0 hostname: kafka-rest-proxy container_name: kafka-rest-proxy ports: @@ -71,7 +69,7 @@ services: kafka-connect: - image: confluentinc/cp-kafka-connect:7.3.2 + image: confluentinc/cp-kafka-connect:7.8.0 hostname: kafka-connect container_name: kafka-connect ports: @@ -113,7 +111,7 @@ services: ksqldb-server: - image: confluentinc/cp-ksqldb-server:7.3.2 + image: confluentinc/cp-ksqldb-server:7.8.0 hostname: ksqldb-server container_name: ksqldb-server ports: diff --git a/zk-multiple-kafka-multiple-schema-registry.yml b/zk-multiple-kafka-multiple-schema-registry.yml index 0dda66b..361c979 100644 --- a/zk-multiple-kafka-multiple-schema-registry.yml +++ b/zk-multiple-kafka-multiple-schema-registry.yml @@ -1,8 +1,6 @@ -version: '2.1' - services: zoo1: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +11,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +22,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo3 container_name: zoo3 ports: @@ -36,7 +34,7 @@ services: kafka1: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka1 container_name: kafka1 ports: @@ -57,7 +55,7 @@ services: - zoo3 kafka2: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka2 container_name: kafka2 ports: @@ -78,7 +76,7 @@ services: - zoo3 kafka3: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka3 container_name: kafka3 ports: @@ -99,7 +97,7 @@ services: - zoo3 kafka-schema-registry: - image: confluentinc/cp-schema-registry:7.3.2 + image: confluentinc/cp-schema-registry:7.8.0 hostname: kafka-schema-registry container_name: kafka-schema-registry depends_on: diff --git a/zk-multiple-kafka-multiple.yml b/zk-multiple-kafka-multiple.yml index 3c6532a..fe44d32 100644 --- a/zk-multiple-kafka-multiple.yml +++ b/zk-multiple-kafka-multiple.yml @@ -1,8 +1,6 @@ -version: '2.1' - services: zoo1: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +11,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +22,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo3 container_name: zoo3 ports: @@ -37,7 +35,7 @@ services: kafka1: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka1 container_name: kafka1 ports: @@ -58,7 +56,7 @@ services: - zoo3 kafka2: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka2 container_name: kafka2 ports: @@ -79,7 +77,7 @@ services: - zoo3 kafka3: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka3 container_name: kafka3 ports: diff --git a/zk-multiple-kafka-single.yml b/zk-multiple-kafka-single.yml index 3aa804f..1ca5dca 100644 --- a/zk-multiple-kafka-single.yml +++ b/zk-multiple-kafka-single.yml @@ -1,8 +1,6 @@ -version: '2.1' - services: zoo1: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +11,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +22,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo3 container_name: zoo3 ports: @@ -36,7 +34,7 @@ services: kafka1: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka1 container_name: kafka1 ports: diff --git a/zk-single-kafka-multiple.yml b/zk-single-kafka-multiple.yml index 01e26e5..787fd4c 100644 --- a/zk-single-kafka-multiple.yml +++ b/zk-single-kafka-multiple.yml @@ -1,8 +1,6 @@ -version: '2.1' - services: zoo1: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo1 container_name: zoo1 ports: @@ -14,7 +12,7 @@ services: kafka1: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka1 container_name: kafka1 ports: @@ -33,7 +31,7 @@ services: - zoo1 kafka2: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka2 container_name: kafka2 ports: @@ -53,7 +51,7 @@ services: kafka3: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka3 container_name: kafka3 ports: diff --git a/zk-single-kafka-single.yml b/zk-single-kafka-single.yml index eace3f6..c6ba4b3 100644 --- a/zk-single-kafka-single.yml +++ b/zk-single-kafka-single.yml @@ -1,8 +1,6 @@ -version: '2.1' - services: zoo1: - image: confluentinc/cp-zookeeper:7.3.2 + image: confluentinc/cp-zookeeper:7.8.0 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +11,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:7.3.2 + image: confluentinc/cp-kafka:7.8.0 hostname: kafka1 container_name: kafka1 ports: