diff --git a/kafka/docker-kraft/README.md b/kafka/docker-kraft/README.md new file mode 100644 index 0000000..4fe2e81 --- /dev/null +++ b/kafka/docker-kraft/README.md @@ -0,0 +1,30 @@ +# kafka kraft setup + +REF: https://github.com/katyagorshkova/kafka-kraft/tree/main + +# Start + +## Pre request +``` +docker compose up kafka-gen +``` + +Generate cluster id + +``` +CLUSTER_ID=$(/bin/kafka-storage random-uui) +``` + +Format configuration file +``` +kafka-storage format --ignore-formatted -t $CLUSTER_ID -c /etc/kafka/kraft/broker.properties +kafka-storage format --ignore-formatted -t $CLUSTER_ID -c /etc/kafka/kraft/controller.properties +``` + + +Check if configure get formatted + +## Start cluster +``` +docker compose up +``` diff --git a/kafka/docker-kraft/compose.yaml b/kafka/docker-kraft/compose.yaml new file mode 100644 index 0000000..ee607e2 --- /dev/null +++ b/kafka/docker-kraft/compose.yaml @@ -0,0 +1,94 @@ +version: "3" + +volumes: + kafka1-data: + kafka2-data: + kafka3-data: + +services: + kafka-gen: + image: confluentinc/cp-kafka:7.4.0 + hostname: kafka-gen + container_name: kafka-gen + volumes: + - ./scripts/create_cluster_id.sh:/tmp/create_cluster_id.sh + - ./clusterID:/tmp/clusterID + command: "bash -c '/tmp/create_cluster_id.sh'" + + # kafka1: + # image: confluentinc/cp-kafka:7.3.3 + # hostname: kafka1 + # container_name: kafka1 + # ports: + # - "39092:39092" + # environment: + # KAFKA_LISTENERS: BROKER://kafka1:19092,EXTERNAL://kafka1:39092,CONTROLLER://kafka1:9093 + # KAFKA_ADVERTISED_LISTENERS: BROKER://kafka1:19092,EXTERNAL://kafka1:39092 + # KAFKA_INTER_BROKER_LISTENER_NAME: BROKER + # KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT + # KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + # KAFKA_PROCESS_ROLES: 'controller,broker' + # KAFKA_NODE_ID: 1 + # KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka1:9093,2@kafka2:9093,3@kafka3:9093' + # KAFKA_METADATA_LOG_SEGMENT_MS: 15000 + # KAFKA_METADATA_MAX_RETENTION_MS: 1200000 + # KAFKA_METADATA_LOG_MAX_RECORD_BYTES_BETWEEN_SNAPSHOTS: 2800 + # KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + # volumes: + # - kafka1-data:/var/lib/kafka/data + # - ./scripts/update_run.sh:/tmp/update_run.sh + # - ./clusterID:/tmp/clusterID + # command: "bash -c '/tmp/update_run.sh && /etc/confluent/docker/run'" + + # kafka2: + # image: confluentinc/cp-kafka:7.3.3 + # hostname: kafka2 + # container_name: kafka2 + # ports: + # - "39093:39093" + # environment: + # KAFKA_LISTENERS: BROKER://kafka2:19093,EXTERNAL://kafka2:39093,CONTROLLER://kafka2:9093 + # KAFKA_ADVERTISED_LISTENERS: BROKER://kafka2:19093,EXTERNAL://kafka2:39093 + # KAFKA_INTER_BROKER_LISTENER_NAME: BROKER + # KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT + # KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + # KAFKA_PROCESS_ROLES: 'controller,broker' + # KAFKA_NODE_ID: 2 + # KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka1:9093,2@kafka2:9093,3@kafka3:9093' + # KAFKA_METADATA_LOG_SEGMENT_MS: 15000 + # KAFKA_METADATA_MAX_RETENTION_MS: 1200000 + # KAFKA_METADATA_LOG_MAX_RECORD_BYTES_BETWEEN_SNAPSHOTS: 2800 + # KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + # volumes: + # - kafka2-data:/var/lib/kafka/data + # - ./scripts/update_run.sh:/tmp/update_run.sh + # - ./clusterID:/tmp/clusterID + # command: "bash -c '/tmp/update_run.sh && /etc/confluent/docker/run'" + + # kafka3: + # image: confluentinc/cp-kafka:7.3.3 + # hostname: kafka3 + # container_name: kafka3 + # ports: + # - "39094:39094" + # environment: + # KAFKA_LISTENERS: BROKER://kafka3:19094,EXTERNAL://kafka3:39094,CONTROLLER://kafka3:9093 + # KAFKA_ADVERTISED_LISTENERS: BROKER://kafka3:19094,EXTERNAL://kafka3:39094 + # KAFKA_INTER_BROKER_LISTENER_NAME: BROKER + # KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT + # KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + # KAFKA_PROCESS_ROLES: 'controller,broker' + # KAFKA_NODE_ID: 3 + # KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka1:9093,2@kafka2:9093,3@kafka3:9093' + # KAFKA_METADATA_LOG_SEGMENT_MS: 15000 + # KAFKA_METADATA_MAX_RETENTION_MS: 1200000 + # KAFKA_METADATA_LOG_MAX_RECORD_BYTES_BETWEEN_SNAPSHOTS: 2800 + # KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + # volumes: + # - kafka3-data:/var/lib/kafka/data + # - ./scripts/update_run.sh:/tmp/update_run.sh + # - ./clusterID:/tmp/clusterID + # command: "bash -c '/tmp/update_run.sh && /etc/confluent/docker/run'"