Kafka Partitioning Strategies for Zero-Data-Loss Architectures

12345678910111213flowchart LR
    P[Event Producer] -->|Hash Partition Key: order_id| K[Kafka Broker Cluster]
    subgraph Partitions [Topic: orders.v1]
      P0[Partition 0: Leader Broker 1]
      P1[Partition 1: Leader Broker 2]
      P2[Partition 2: Leader Broker 3]
    end
    K --> P0
    K --> P1
    K --> P2
    P0 --> C0[Consumer Instance A]
    P1 --> C1[Consumer Instance B]
    P2 --> C2[Consumer Instance C]

1234567891011121314151617181920212223242526272829import { Kafka, CompressionTypes, logLevel } from 'kafkajs';

const kafka = new Kafka({
  clientId: 'order-processing-gateway',
  brokers: ['kafka-1.internal:9092', 'kafka-2.internal:9092', 'kafka-3.internal:9092'],
  logLevel: logLevel.WARN,
});

export const producer = kafka.producer({
  // Idempotent producer prevents duplicate message writes
  idempotent: true,
  maxInFlightRequests: 1,
  transactionTimeout: 30000,
});

export async function publishOrderEvent(orderId: string, payload: Record<string, unknown>) {
  await producer.send({
    topic: 'orders.v1',
    compression: CompressionTypes.GZIP,
    messages: [
      {
        key: orderId, // Guarantees all state transitions for this order land on same partition
        value: JSON.stringify({ ...payload, emittedAt: new Date().toISOString() }),
        headers: { correlationId: crypto.randomUUID() },
      },
    ],
    acks: -1, // acks=all: Wait for full In-Sync Replica (ISR) quorum commit
  });
}