Task #10
Updated by Fernando Jose Capeletto Neto almost 2 years ago
Migrate Kafka Broker to Local Server :
* to enhance Performance (since CMS components are in the same network) (to be verified)
* The trial period for Kafka broker at Confluence will end in 2 weeks.
== Docker Compose File under Test ==version: '2.1'
services:
zoo1:
image: confluentinc/cp-zookeeper:7.3.2
hostname: zoo1
container_name: zoo1
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_SERVERS: zoo1:2888:3888
kafka1:
image: confluentinc/cp-kafka:7.3.2
hostname: kafka1
container_name: kafka1
ports:
- "9092:9092"
- "29092:29092"
- "9999:9999"
environment:
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://lab.fernando.engineer:9092,DOCKER://host.docker.internal:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
KAFKA_BROKER_ID: 1
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_JMX_PORT: 9999
KAFKA_JMX_HOSTNAME: lab.fernando.engineer
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 1000
depends_on:
- zoo1
init-kafka:
image: confluentinc/cp-kafka:7.3.2
depends_on:
- kafka1
volumes:
- /var/www/vhost/kafka/log:/log
entrypoint: [ '/bin/sh', '-c' ]
command: |
"
# blocks until kafka is reachable
kafka-topics --bootstrap-server kafka1:29092 --list
echo -e 'Creating kafka topics for lab.fernando.engineer'
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic AISMESSAGE_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 3000
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TACTICAL_ALERT_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 5000
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic SYSTEM_ALERT_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 3600000
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_PERFORMANCE_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 1250
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 1250
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic AISMESSAGE_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 3000
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TACTICAL_ALERT_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 5000
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic SYSTEM_ALERT_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 3600000
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_PERFORMANCE_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 1250
kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 1250
#Log each topic configuration during creation for further check
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name AISMESSAGE_TOPIC --describe --all > /log/AISMESSAGE_TOPIC.conf
echo -e 'Configuration for topic AISMESSAGE_TOPIC saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TACTICAL_ALERT_TOPIC --describe --all > /log/TACTICAL_ALERT_TOPIC.conf
echo -e 'Configuration for topic TACTICAL_ALERT_TOPIC saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name SYSTEM_ALERT_TOPIC --describe --all > /log/SYSTEM_ALERT_TOPIC.conf
echo -e 'Configuration for topic SYSTEM_ALERT_TOPIC saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_PERFORMANCE_TOPIC --describe --all > /log/TRACKDAO_PERFORMANCE_TOPIC.conf
echo -e 'Configuration for topic TRACKDAO_PERFORMANCE_TOPIC saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_TOPIC --describe --all > /log/TRACKDAO_TOPIC.conf
echo -e 'Configuration for topic TRACKDAO_TOPIC saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name AISMESSAGE_TOPICTEST --describe --all > /log/AISMESSAGE_TOPICTEST.conf
echo -e 'Configuration for topic AISMESSAGE_TOPICTEST saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TACTICAL_ALERT_TOPICTEST --describe --all > /log/TACTICAL_ALERT_TOPICTEST.conf
echo -e 'Configuration for topic TACTICAL_ALERT_TOPICTEST saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name SYSTEM_ALERT_TOPICTEST --describe --all > /log/SYSTEM_ALERT_TOPICTEST.conf
echo -e 'Configuration for topic SYSTEM_ALERT_TOPICTEST saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_PERFORMANCE_TOPICTEST --describe --all > /log/TRACKDAO_PERFORMANCE_TOPICTEST.conf
echo -e 'Configuration for topic TRACKDAO_PERFORMANCE_TOPICTEST saved'
kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_TOPICTEST --describe --all > /log/TRACKDAO_TOPICTEST.conf
echo -e 'Configuration for topic TRACKDAO_TOPICTEST saved'
echo -e 'Successfully created the following topics:'
kafka-topics --bootstrap-server kafka:29092 --list
"