Skip to content

Instantly share code, notes, and snippets.

@almostintuitive
Created September 24, 2026 16:45
Show Gist options
  • Select an option

  • Save almostintuitive/cbff837fce0709ccaa209d1842e957f0 to your computer and use it in GitHub Desktop.

Select an option

Save almostintuitive/cbff837fce0709ccaa209d1842e957f0 to your computer and use it in GitHub Desktop.
Apache Iggy: consumer-group stale member after killing the connection-owning replica (repro for apache/iggy#4273)
#!/usr/bin/env bash
# Starts a 3-node Apache Iggy VSR cluster (IMAGE, default apache/iggy:0.9.0) on a docker
# bridge network with fixed addresses (the host must be able to reach
# 172.29.0.0/24). `cluster.sh down` removes it again.
set -euo pipefail
IMAGE=${IMAGE:-apache/iggy:0.9.0}
NET=iggy-cluster-test
for i in 0 1 2; do docker rm -fv iggy-$i >/dev/null 2>&1 || true; done
docker network rm "$NET" >/dev/null 2>&1 || true
[ "${1:-up}" = down ] && exit 0
docker network create --subnet 172.29.0.0/24 "$NET" >/dev/null
roster=()
for i in 0 1 2; do
roster+=(-e IGGY_CLUSTER_NODES_${i}_NAME=iggy-$i -e IGGY_CLUSTER_NODES_${i}_IP=172.29.0.1$i
-e IGGY_CLUSTER_NODES_${i}_REPLICA_ID=$i -e IGGY_CLUSTER_NODES_${i}_PORTS_TCP=8090
-e IGGY_CLUSTER_NODES_${i}_PORTS_QUIC=8080 -e IGGY_CLUSTER_NODES_${i}_PORTS_HTTP=3000
-e IGGY_CLUSTER_NODES_${i}_PORTS_WEBSOCKET=8092 -e IGGY_CLUSTER_NODES_${i}_PORTS_TCP_REPLICA=9090)
done
for i in 0 1 2; do
# Iggy needs io_uring, which the default seccomp profile blocks.
docker run -d --name iggy-$i --hostname iggy-$i --network "$NET" --ip 172.29.0.1$i \
--memory 2g --security-opt seccomp=unconfined \
-e IGGY_ROOT_USERNAME=iggy -e IGGY_ROOT_PASSWORD=iggy \
-e IGGY_TCP_ADDRESS=172.29.0.1$i:8090 -e IGGY_HTTP_ADDRESS=0.0.0.0:3000 \
-e IGGY_QUIC_ENABLED=false -e IGGY_WEBSOCKET_ENABLED=false \
-e IGGY_SHARDING_CPU_ALLOCATION=1 -e IGGY_MEMORY_POOL_SIZE=512MiB \
-e IGGY_CLUSTER_ENABLED=true -e IGGY_CLUSTER_AUTH_ENABLED=true \
-e IGGY_CLUSTER_AUTH_SHARED_SECRET=0123456789abcdef0123456789abcdef0123456789abcdef \
-e IGGY_CLUSTER_NAME=repro \
"${roster[@]}" "$IMAGE" --replica-id $i >/dev/null
done
module iggyrepro
go 1.25.0
require github.com/apache/iggy/foreign/go v0.9.1-0.20260924080616-85a397d0e5fd
require (
github.com/avast/retry-go/v5 v5.0.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
golang.org/x/sys v0.30.0 // indirect
)
github.com/apache/iggy/foreign/go v0.9.1-0.20260924080616-85a397d0e5fd h1:8NGgmeBHSwVLFWY45+cZtUGKdOQ3gK2PeTPRDQfyGYM=
github.com/apache/iggy/foreign/go v0.9.1-0.20260924080616-85a397d0e5fd/go.mod h1:RSWaRul6swJlvc3k876UHscyCAIlpfrzhMgX9xRpT1I=
github.com/avast/retry-go/v5 v5.0.0 h1:kf1Qc2UsTZ4qq8elDymqfbISvkyMuhgRxuJqX2NHP7k=
github.com/avast/retry-go/v5 v5.0.0/go.mod h1://d+usmKWio1agtZfS1H/ltTqwtIfBnRq9zEwjc3eH8=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
// Reproductions for two Apache Iggy consumer-group issues on a 3-node VSR
// cluster started by cluster.sh (nodes iggy-0..2 at 172.29.0.10..12:8090).
//
// go run . stale-member # (A) a member whose node died never leaves the group
// go run . cancel-drops-membership # (B) a cancelled request silently drops group membership
//
// Only the official Go SDK and the docker CLI are used.
package main
import (
"context"
"fmt"
"log/slog"
"os"
"os/exec"
"strings"
"time"
"github.com/apache/iggy/foreign/go/client"
"github.com/apache/iggy/foreign/go/client/tcp"
iggcon "github.com/apache/iggy/foreign/go/contracts"
)
var (
addrs = []string{"172.29.0.10:8090", "172.29.0.11:8090", "172.29.0.12:8090"}
nodes = []string{"iggy-0", "iggy-1", "iggy-2"}
start = time.Now()
ctx = context.Background()
verbose = os.Getenv("SDK_DEBUG") != ""
)
func logf(format string, args ...any) {
fmt.Printf("[%6.1fs] %s\n", time.Since(start).Seconds(), fmt.Sprintf(format, args...))
}
func must[T any](v T, err error) T {
if err != nil {
logf("FATAL: %v", err)
os.Exit(2)
}
return v
}
func id(s string) iggcon.Identifier { return must(iggcon.NewIdentifier(s)) }
// connect signs in through the given seed, retrying while the cluster boots.
func connect(addr string) iggcon.Client {
level := slog.LevelError + 1
if verbose {
level = slog.LevelDebug
}
for i := 0; ; i++ {
c := must(client.NewIggyClient(
client.WithTcp(tcp.WithServerAddress(addr),
tcp.WithAutoLogin(tcp.NewUsernamePasswordCredentials("iggy", "iggy"))),
client.WithLogger(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))),
))
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
err := c.Connect(cctx)
cancel()
if err == nil {
return c
}
_ = c.Close()
if i == 30 {
logf("FATAL: connect %s: %v", addr, err)
os.Exit(2)
}
time.Sleep(2 * time.Second)
}
}
func leader(c iggcon.Client) int {
md := must(c.GetClusterMetadata(ctx))
for _, n := range md.Nodes {
if n.Role == iggcon.RoleLeader {
for i, a := range addrs {
if strings.HasPrefix(a, n.IP+":") {
return i
}
}
}
}
logf("FATAL: no leader in %+v", md.Nodes)
os.Exit(2)
return -1
}
func docker(args ...string) {
if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil {
logf("FATAL: docker %v: %v %s", args, err, out)
os.Exit(2)
}
}
func setupTopic(c iggcon.Client, partitions uint32) (iggcon.Identifier, iggcon.Identifier, iggcon.Identifier, string) {
if _, err := c.GetStream(ctx, id("repro")); err != nil {
must(c.CreateStream(ctx, "repro"))
}
name := fmt.Sprintf("t%d", time.Now().UnixNano())
must(c.CreateTopic(ctx, id("repro"), name, partitions, iggcon.CompressionAlgorithmNone,
iggcon.IggyExpiryNeverExpire, 64<<20, iggcon.SegmentSizeOption(8<<20)))
must(c.CreateConsumerGroup(ctx, id("repro"), id(name), "g"))
// One message per partition, so offset 0 exists everywhere.
for p := range partitions {
m := must(iggcon.NewIggyMessage([]byte("x")))
must(c.SendMessages(ctx, id("repro"), id(name), iggcon.PartitionId(p), []iggcon.IggyMessage{m}))
}
return id("repro"), id(name), id("g"), name
}
func members(admin iggcon.Client, s, t, g iggcon.Identifier) string {
d, err := admin.GetConsumerGroup(ctx, s, t, g)
if err != nil {
return "error: " + err.Error()
}
return fmt.Sprintf("members=%d %+v", d.MembersCount, d.Members)
}
func sync(c iggcon.Client, s, t, g iggcon.Identifier) string {
a, err := c.SyncConsumerGroup(ctx, s, t, g)
if err != nil {
return "error: " + err.Error()
}
return fmt.Sprintf("generation=%d partitions=%v", a.Generation, a.Partitions)
}
//------------------------------------------------------------------------------
// (A) A consumer joins a group through the metadata leader, the leader is
// killed, and the consumer (reconnected by the SDK under a new identity)
// rejoins. The dead identity stays a member with its partitions.
func staleMember() {
admin := connect(addrs[0])
l := leader(admin)
other := (l + 1) % 3
logf("metadata leader: %s (%s)", nodes[l], addrs[l])
admin = connect(addrs[other]) // an admin session that survives the kill
s, t, g, name := setupTopic(admin, 3)
logf("created topic repro/%s with 3 partitions and consumer group g", name)
consumer := connect(addrs[l])
logf("consumer connected via %s", consumer.GetConnectionInfo().ServerAddress)
must(0, consumer.JoinConsumerGroup(ctx, s, t, g))
logf("consumer joined: sync %s", sync(consumer, s, t, g))
logf("group: %s", members(admin, s, t, g))
logf("docker kill %s", nodes[l])
docker("kill", nodes[l])
time.Sleep(3 * time.Second)
logf("consumer sync after kill: %s (now via %s)", sync(consumer, s, t, g), consumer.GetConnectionInfo().ServerAddress)
if err := consumer.JoinConsumerGroup(ctx, s, t, g); err != nil {
logf("consumer rejoin: %v", err)
} else {
logf("consumer rejoined")
}
logf("consumer sync after rejoin: %s", sync(consumer, s, t, g))
for i := 0; i <= 8; i++ {
logf("group: %s | consumer sync: %s", members(admin, s, t, g), sync(consumer, s, t, g))
if i < 8 {
time.Sleep(15 * time.Second)
}
}
fresh := connect(addrs[other])
must(0, fresh.JoinConsumerGroup(ctx, s, t, g))
logf("a brand-new consumer joined: sync %s", sync(fresh, s, t, g))
must(0, fresh.LeaveConsumerGroup(ctx, s, t, g))
must(0, fresh.Close())
logf("docker start %s", nodes[l])
docker("start", nodes[l])
time.Sleep(40 * time.Second)
logf("after the killed node restarted (40s): group: %s | consumer sync: %s", members(admin, s, t, g), sync(consumer, s, t, g))
d := must(admin.GetConsumerGroup(ctx, s, t, g))
if d.MembersCount > 1 {
logf("REPRODUCED: the killed node's member is still in group g, holding partitions the live consumer never gets")
os.Exit(1)
}
logf("NOT REPRODUCED: the dead member was removed")
}
//------------------------------------------------------------------------------
// (B) A group member's in-flight request is cancelled (a context deadline, or
// shutdown). The SDK tears the connection down to abandon the reply; the next
// request reconnects under a new client identity, which is not a member. The
// SDK does not rejoin the group it joined explicitly, so stores for owned
// partitions are refused and explicit-partition polls return the re-sync
// sentinel, while nothing surfaced the membership loss to the caller.
func cancelDropsMembership() {
admin := connect(addrs[0])
s, t, g, name := setupTopic(admin, 3)
payload := make([]byte, 1024)
var msgs []iggcon.IggyMessage
for range 1000 {
msgs = append(msgs, must(iggcon.NewIggyMessage(payload)))
}
for range 10 {
must(admin.SendMessages(ctx, s, t, iggcon.PartitionId(0), msgs))
}
logf("created topic repro/%s (3 partitions, 10001 x 1 KiB messages in partition 0) and consumer group g", name)
c := connect(addrs[0])
must(0, c.JoinConsumerGroup(ctx, s, t, g))
cons := iggcon.NewGroupConsumer(g)
p0 := uint32(0)
logf("member joined via %s: sync %s", c.GetConnectionInfo().ServerAddress, sync(c, s, t, g))
logf("group: %s", members(admin, s, t, g))
logf("store offset 0 of partition 0 before: err=%v", c.StoreConsumerOffset(ctx, cons, s, t, 0, &p0))
// Cancel a large poll while it is in flight.
var err error
for i := 0; i < 50; i++ {
pctx, cancel := context.WithTimeout(ctx, 500*time.Microsecond)
_, err = c.PollMessages(pctx, s, t, cons, iggcon.OffsetPollingStrategy(0), 10000, false, &p0)
cancel()
if err != nil {
break
}
}
logf("poll with a context cancelled mid-flight: err=%v", err)
if err == nil {
logf("SETUP: could not cancel a poll in flight")
os.Exit(2)
}
reproduced := false
for p := uint32(0); p < 3; p++ {
serr := c.StoreConsumerOffset(ctx, cons, s, t, 0, &p)
logf("StoreConsumerOffset(partition %d, offset 0): err=%v (connection %s)", p, serr, c.GetConnectionInfo().ServerAddress)
if serr != nil {
reproduced = true
}
}
pm, perr := c.PollMessages(ctx, s, t, cons, iggcon.OffsetPollingStrategy(0), 1, false, &p0)
if perr != nil {
logf("group poll of partition 0: err=%v", perr)
} else {
logf("group poll of partition 0: err=nil partition_id=%#x messages=%d", pm.PartitionId, len(pm.Messages))
}
logf("sync: %s", sync(c, s, t, g))
logf("group: %s", members(admin, s, t, g))
if reproduced {
must(0, c.JoinConsumerGroup(ctx, s, t, g))
logf("after an explicit JoinConsumerGroup: store err=%v, sync %s", c.StoreConsumerOffset(ctx, cons, s, t, 0, &p0), sync(c, s, t, g))
logf("REPRODUCED: the explicitly joined membership was dropped by the SDK's reconnect and not restored")
os.Exit(1)
}
logf("NOT REPRODUCED: stores still accepted after the cancelled request")
}
func main() {
if len(os.Args) < 2 {
fmt.Println("usage: go run . stale-member|cancel-drops-membership")
os.Exit(2)
}
switch os.Args[1] {
case "stale-member":
staleMember()
case "cancel-drops-membership":
cancelDropsMembership()
default:
fmt.Println("unknown repro", os.Args[1])
os.Exit(2)
}
}
#!/usr/bin/env bash
# Usage: ./run.sh <stale-member|cancel-drops-membership> [image] [go-sdk-version]
# ./run.sh stale-member apache/iggy:0.9.0 v0.9.0
# ./run.sh stale-member apache/iggy:edge 85a397d0e5fd36262a3bbc203165c7a5e01b9029
# Starts a fresh 3-node cluster, runs the repro, removes the cluster.
# Exit code 1 means reproduced, 0 means not reproduced, 2 means setup failure.
set -euo pipefail
cd "$(dirname "$0")"
REPRO=$1 IMAGE=${2:-apache/iggy:0.9.0} SDK=${3:-v0.9.0}
go get "github.com/apache/iggy/foreign/go@$SDK" >/dev/null 2>&1
go mod tidy >/dev/null 2>&1
echo "image: $IMAGE ($(docker image inspect "$IMAGE" --format '{{index .RepoDigests 0}}'), $(docker run --rm "$IMAGE" --version 2>&1 | tail -1))"
echo "go sdk: $(go list -m github.com/apache/iggy/foreign/go)"
IMAGE=$IMAGE ./cluster.sh
trap './cluster.sh down' EXIT
set +e
go run . "$REPRO"
code=$?
set -e
exit $code
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment