Task #10
Updated by Fernando Jose Capeletto Neto over 1 year ago
Migrate Kafka Broker to Local Server : * to enhance Performance (since CMS components are in the same network) (to be verified) * The trial period for Kafka broker at Confluence will end in 2 weeks. == Docker Compose File under Test ==version: '2.1' <pre> services: zoo1: image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo1 container_name: zoo1 ports: - "2181:2181" environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_SERVERS: zoo1:2888:3888 kafka1: image: confluentinc/cp-kafka:7.3.2 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" - "29092:29092" - "9999:9999" environment: KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://lab.fernando.engineer:9092,DOCKER://host.docker.internal:29092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_JMX_PORT: 9999 KAFKA_JMX_HOSTNAME: lab.fernando.engineer KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 1000 depends_on: - zoo1 init-kafka: image: confluentinc/cp-kafka:7.3.2 depends_on: - kafka1 volumes: - /var/www/vhost/kafka/log:/log entrypoint: [ '/bin/sh', '-c' ] command: | " # blocks until kafka is reachable kafka-topics --bootstrap-server kafka1:29092 --list echo -e 'Creating kafka topics for lab.fernando.engineer' kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic AISMESSAGE_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 3000 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TACTICAL_ALERT_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 5000 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic SYSTEM_ALERT_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 3600000 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_PERFORMANCE_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 1250 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_TOPIC --replication-factor 1 --partitions 1 --config retention.ms 1250 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic AISMESSAGE_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 3000 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TACTICAL_ALERT_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 5000 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic SYSTEM_ALERT_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 3600000 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_PERFORMANCE_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 1250 kafka-topics --bootstrap-server kafka1:29092 --create --if-not-exists --topic TRACKDAO_TOPICTEST --replication-factor 1 --partitions 1 --config retention.ms 1250 #Log each topic configuration during creation for further check kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name AISMESSAGE_TOPIC --describe --all > /log/AISMESSAGE_TOPIC.conf echo -e 'Configuration for topic AISMESSAGE_TOPIC saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TACTICAL_ALERT_TOPIC --describe --all > /log/TACTICAL_ALERT_TOPIC.conf echo -e 'Configuration for topic TACTICAL_ALERT_TOPIC saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name SYSTEM_ALERT_TOPIC --describe --all > /log/SYSTEM_ALERT_TOPIC.conf echo -e 'Configuration for topic SYSTEM_ALERT_TOPIC saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_PERFORMANCE_TOPIC --describe --all > /log/TRACKDAO_PERFORMANCE_TOPIC.conf echo -e 'Configuration for topic TRACKDAO_PERFORMANCE_TOPIC saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_TOPIC --describe --all > /log/TRACKDAO_TOPIC.conf echo -e 'Configuration for topic TRACKDAO_TOPIC saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name AISMESSAGE_TOPICTEST --describe --all > /log/AISMESSAGE_TOPICTEST.conf echo -e 'Configuration for topic AISMESSAGE_TOPICTEST saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TACTICAL_ALERT_TOPICTEST --describe --all > /log/TACTICAL_ALERT_TOPICTEST.conf echo -e 'Configuration for topic TACTICAL_ALERT_TOPICTEST saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name SYSTEM_ALERT_TOPICTEST --describe --all > /log/SYSTEM_ALERT_TOPICTEST.conf echo -e 'Configuration for topic SYSTEM_ALERT_TOPICTEST saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_PERFORMANCE_TOPICTEST --describe --all > /log/TRACKDAO_PERFORMANCE_TOPICTEST.conf echo -e 'Configuration for topic TRACKDAO_PERFORMANCE_TOPICTEST saved' kafka-configs --bootstrap-server kafka1:29092 --entity-type topics --entity-name TRACKDAO_TOPICTEST --describe --all > /log/TRACKDAO_TOPICTEST.conf echo -e 'Configuration for topic TRACKDAO_TOPICTEST saved' echo -e 'Successfully created the following topics:' kafka-topics --bootstrap-server kafka:29092 --list " </pre>