브로커 장애 후 메시지가 유실됐다

# producer.properties (결제 웹훅 수신기)
acks=1
retries=3
enable.idempotence=false      # acks=1 을 쓰려고 명시적으로 껐음

# 브로커 동적 설정 (클러스터 전역)
unclean.leader.election.enable=true

# 토픽 payment-events
#   min.insync.replicas 미설정 → 기본값 1

INFO [Partition payment-events-7 broker=2] Shrinking ISR from 2,1,3 to 2,1. Leader: (highWatermark: 88210110, endOffset: 88213904). Out of sync replicas: (brokerId: 3, endOffset: 88208442, lastCaughtUpTimeMs: 1761023641118).
INFO [Partition payment-events-4 broker=2] Shrinking ISR from 2,3,1 to 2,1. Leader: (highWatermark: 88109770, endOffset: 88112390). Out of sync replicas: (brokerId: 3, endOffset: 88106001, lastCaughtUpTimeMs: 1761023641131).

INFO [QuorumController id=2] Setting new leader for topicId hR9pQ2vLT4ac-5Wm0eXpZQ, partition 7 to 3 using an unclean election. Previous partition: PartitionRegistration(replicas=[2, 1, 3], isr=[2], leader=2, leaderEpoch=41, ...), change record: PartitionChangeRecord(partitionId=7, topicId=hR9pQ2vLT4ac-5Wm0eXpZQ, isr=[3], leader=3, leaderRecoveryState=1, ...)

WARN [UnifiedLog partition=payment-events-7, dir=/var/lib/kafka/data] Truncating payment-events-7 to offset 88208442 below high watermark 88210110
INFO [UnifiedLog partition=payment-events-7, dir=/var/lib/kafka/data] Truncating to offset 88208442

INFO [Consumer clientId=notifier-1, groupId=payment-notifier] Fetch position FetchPosition{offset=88210500, offsetEpoch=Optional[41], currentLeader=LeaderAndEpoch{leader=Optional[kafka-3:9092 (id: 3 rack: null)], epoch=42}} is out of range for partition payment-events-7, resetting offset

$ kafka-topics.sh --bootstrap-server kafka-1:9092 --describe --topic payment-events

Topic: payment-events	TopicId: hR9pQ2vLT4ac-5Wm0eXpZQ	PartitionCount: 12	ReplicationFactor: 3	Configs: retention.ms=604800000
	Topic: payment-events	Partition: 0	Leader: 1	Replicas: 1,2,3	Isr: 1,2,3	Elr: N/A	LastKnownElr: N/A
	Topic: payment-events	Partition: 4	Leader: 3	Replicas: 2,3,1	Isr: 3	Elr: N/A	LastKnownElr: N/A
	Topic: payment-events	Partition: 7	Leader: 3	Replicas: 2,1,3	Isr: 3	Elr: N/A	LastKnownElr: N/A

$ # 위험 파티션만 골라 보는 옵션
$ kafka-topics.sh --bootstrap-server kafka-1:9092 --describe --under-replicated-partitions
$ kafka-topics.sh --bootstrap-server kafka-1:9092 --describe --unavailable-partitions
$ kafka-topics.sh --bootstrap-server kafka-1:9092 --describe --at-min-isr-partitions

$ kafka-configs.sh --bootstrap-server kafka-1:9092 \
    --entity-type brokers --entity-default --describe

Dynamic configs for broker default are:
  unclean.leader.election.enable=true sensitive=false synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:unclean.leader.election.enable=true}

$ kafka-configs.sh --bootstrap-server kafka-1:9092 \
    --entity-type topics --entity-name payment-events --describe --all

All configs for topic payment-events are:
  min.insync.replicas=1 sensitive=false synonyms={DEFAULT_CONFIG:min.insync.replicas=1}
  unclean.leader.election.enable=true sensitive=false synonyms={DYNAMIC_DEFAULT_BROKER_CONFIG:unclean.leader.election.enable=true, DEFAULT_CONFIG:unclean.leader.election.enable=false}

x-kafka-env: &kenv
  KAFKA_PROCESS_ROLES: 'broker,controller'
  KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
  KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093'
  KAFKA_LISTENERS: 'PLAINTEXT://:19092,CONTROLLER://:9093,PLAINTEXT_HOST://:9092'
  KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
  KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
  CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
  KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
  KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
  KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
  KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
  # 재현 포인트 — 4.x 기본값은 false 입니다. 일부러 켭니다.
  KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE: 'true'

services:
  kafka-1:
    image: apache/kafka:4.3.1
    hostname: kafka-1
    container_name: kafka-1
    ports: ['29092:9092']
    environment:
      <<: *kenv
      KAFKA_NODE_ID: 1
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-1:19092,PLAINTEXT_HOST://localhost:29092'
  kafka-2:
    image: apache/kafka:4.3.1
    hostname: kafka-2
    container_name: kafka-2
    ports: ['39092:9092']
    environment:
      <<: *kenv
      KAFKA_NODE_ID: 2
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-2:19092,PLAINTEXT_HOST://localhost:39092'
  kafka-3:
    image: apache/kafka:4.3.1
    hostname: kafka-3
    container_name: kafka-3
    ports: ['49092:9092']
    environment:
      <<: *kenv
      KAFKA_NODE_ID: 3
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-3:19092,PLAINTEXT_HOST://localhost:49092'

docker compose up -d
K=/opt/kafka/bin

# 1. RF=3, min.insync.replicas 는 기본값 1
docker exec -it kafka-1 $K/kafka-topics.sh --create --topic loss-demo \
  --partitions 1 --replication-factor 3 --bootstrap-server localhost:9092
docker exec -it kafka-1 $K/kafka-topics.sh --describe --topic loss-demo \
  --bootstrap-server localhost:9092
#   → Leader 와 Isr 를 기록합니다. 아래는 Leader=1, Isr=1,2,3 을 가정합니다.

# 2. acks=1 로 5000건 적재
seq 1 5000 | docker exec -i kafka-1 $K/kafka-console-producer.sh --topic loss-demo \
  --bootstrap-server localhost:9092 --producer-property acks=1

# 3. 팔로워 두 대를 내린다 → ISR 이 리더 하나로 줄어든다
docker stop kafka-2 kafka-3
docker exec -it kafka-1 $K/kafka-topics.sh --describe --topic loss-demo \
  --bootstrap-server localhost:9092
#   → Isr: 1

# 4. 리더만 살아 있는 상태에서 4000건 더 적재
#    acks=1 이므로 프로듀서는 전부 성공을 받습니다. 어디에도 복제되지 않습니다.
seq 5001 9000 | docker exec -i kafka-1 $K/kafka-console-producer.sh --topic loss-demo \
  --bootstrap-server localhost:9092 --producer-property acks=1

docker exec -it kafka-1 $K/kafka-get-offsets.sh --topic loss-demo \
  --bootstrap-server localhost:9092
#   → loss-demo:0:9000

# 5. 리더를 내리고 뒤처진 팔로워만 올린다
docker stop kafka-1
docker start kafka-2
sleep 60      # unclean 선출 스레드 주기를 기다립니다

docker logs kafka-2 2>&1 | grep -i "unclean election"
#   → Setting new leader for topicId ..., partition 0 to 2 using an unclean election. ...

# 6. 오프셋을 다시 본다 — 줄어들어 있습니다
docker exec -it kafka-2 $K/kafka-get-offsets.sh --topic loss-demo \
  --bootstrap-server localhost:9092
#   → loss-demo:0:5000   (4000건이 사라졌습니다)

# 7. 예전 리더를 복귀시키면 트렁케이션 로그가 남습니다
docker start kafka-1
docker logs kafka-1 2>&1 | grep -i "Truncating"

# 1. 클러스터 전역 오버라이드를 삭제해 기본값(false)으로 되돌린다
kafka-configs.sh --bootstrap-server kafka-1:9092 \
  --entity-type brokers --entity-default \
  --alter --delete-config unclean.leader.election.enable

# 2. 확인 — DEFAULT_CONFIG:false 만 남아야 합니다
kafka-configs.sh --bootstrap-server kafka-1:9092 \
  --entity-type topics --entity-name payment-events --describe --all

# 3. 중요한 토픽에는 min.insync.replicas 를 명시한다
kafka-configs.sh --bootstrap-server kafka-1:9092 \
  --entity-type topics --entity-name payment-events \
  --alter --add-config min.insync.replicas=2

# 4. 리더 쏠림을 정리한다 (preferred leader 로 복귀)
kafka-leader-election.sh --bootstrap-server kafka-1:9092 \
  --election-type preferred --all-topic-partitions