Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 20 additions & 16 deletions internal/controller/etcdcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -481,24 +481,21 @@ func (r *EtcdClusterReconciler) updateConditions(s *reconcileState) {
ObservedGeneration: s.cluster.Generation,
LastTransitionTime: now,
Reason: "ClusterNotReady",
Message: "Etcd cluster is not yet available",
Message: "Cluster is not yet available",
}

if s.memberListResp != nil && len(s.memberListResp.Members) > 0 {
healthyCount := 0
for _, health := range s.memberHealth {
if health.Health {
healthyCount++
}
}

quorum := (len(s.memberListResp.Members) / 2) + 1
if healthyCount >= quorum {
if s.sts != nil && s.memberListResp != nil && len(s.memberListResp.Members) > 0 {
// Issue a linearizable read against the cluster endpoints. A
// linearizable Get requires a Raft leader and quorum acknowledgement,
// so a successful response is sufficient proof that the cluster is
// available — no manual (N/2)+1 arithmetic needed.
eps := clientEndpointsFromStatefulsets(s.sts)
if etcdutils.IsClusterAvailable(eps) {
availableCondition.Status = metav1.ConditionTrue
availableCondition.Reason = "ClusterAvailable"
availableCondition.Message = fmt.Sprintf("Etcd cluster has %d/%d healthy members with quorum", healthyCount, len(s.memberListResp.Members))
availableCondition.Message = fmt.Sprintf("Cluster is available: linearizable read succeeded (%d members)", len(s.memberListResp.Members))
} else {
availableCondition.Message = fmt.Sprintf("Etcd cluster has %d/%d healthy members, quorum requires %d", healthyCount, len(s.memberListResp.Members), quorum)
availableCondition.Message = fmt.Sprintf("Cluster is unavailable: no member responded to a linearizable read (%d members)", len(s.memberListResp.Members))
}
}

Expand All @@ -509,7 +506,7 @@ func (r *EtcdClusterReconciler) updateConditions(s *reconcileState) {
ObservedGeneration: s.cluster.Generation,
LastTransitionTime: now,
Reason: "ClusterStable",
Message: "Etcd cluster is stable",
Message: "Cluster is stable",
}

if s.sts != nil && s.sts.Spec.Replicas != nil {
Expand Down Expand Up @@ -552,8 +549,15 @@ func (r *EtcdClusterReconciler) updateConditions(s *reconcileState) {
if s.memberListResp != nil && len(s.memberHealth) > 0 {
unhealthyMembers := []string{}
for _, health := range s.memberHealth {
if !health.Health && health.Status != nil {
unhealthyMembers = append(unhealthyMembers, fmt.Sprintf("%x", health.Status.Header.MemberId))
if !health.Health {
// Prefer the member ID when Status is available; fall back to
// the endpoint so members that failed Status() are never
// silently omitted from the degraded report.
if health.Status != nil {
unhealthyMembers = append(unhealthyMembers, fmt.Sprintf("%x", health.Status.Header.MemberId))
} else {
unhealthyMembers = append(unhealthyMembers, health.Ep)
}
}
}

Expand Down
51 changes: 50 additions & 1 deletion internal/etcdutils/etcdutils.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func ClusterHealth(eps []string) ([]EpHealth, error) {
}()
startTs := time.Now()
// get a random key. As long as we can get the response
// without an error, the endpoint is health.
// without an error, the endpoint is healthy.
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err = cli.Get(ctx, "health", clientv3.WithSerializable())
Expand Down Expand Up @@ -200,6 +200,55 @@ func ClusterHealth(eps []string) ([]EpHealth, error) {
return healthList, nil
}

// IsClusterAvailable issues a linearizable Get against each provided endpoint
// in turn and returns true as soon as any one responds successfully.
//
// The loop distinguishes two classes of failure:
//
// - Member unreachable (connection refused, pod down): the error comes back
// as rpctypes.ErrNoAvailableEndpoints from the client dial. Skip to the
// next endpoint — this says nothing about cluster health.
//
// - Cluster-state error (no leader, leader not yet elected): the member was
// reachable but Raft reported ErrNoLeader or ErrNotLeader. Every other
// member will report the same thing, so return false immediately rather
// than probing the rest.
//
// ErrPermissionDenied is treated as success: the member completed the full
// linearizable read path (quorum confirmed) before rejecting at the auth layer.
func IsClusterAvailable(eps []string) bool {
for _, ep := range eps {
cfg := clientv3.Config{
Endpoints: []string{ep},
DialTimeout: 2 * time.Second,
DialKeepAliveTime: 2 * time.Second,
DialKeepAliveTimeout: 6 * time.Second,
}
cli, err := clientv3.New(cfg)
if err != nil {
continue
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
_, err = cli.Get(ctx, "health")
cancel()
_ = cli.Close()

if err == nil || errors.Is(err, rpctypes.ErrPermissionDenied) {
// Linearizable read succeeded (or auth denied but quorum confirmed).
return true
}

// No leader elected or leader not yet established, so fail fast.
if errors.Is(err, rpctypes.ErrNoLeader) || errors.Is(err, rpctypes.ErrNotLeader) {
return false
}

// Any other error (e.g. ErrNoAvailableEndpoints, dial timeout) means
// this specific member was unreachable. Try the next one.
}
return false
}

func AddMember(eps []string, peerURLs []string, learner bool) (*clientv3.MemberAddResponse, error) {
cfg := clientv3.Config{
Endpoints: eps,
Expand Down
25 changes: 25 additions & 0 deletions internal/etcdutils/etcdutils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,31 @@ func TestEpHealthString(t *testing.T) {
assert.Equal(t, expected, eh.String())
}

func TestIsClusterAvailable(t *testing.T) {
e := setupEtcdServer(t)
defer e.Close()

t.Run("ReturnsTrueForHealthyCluster", func(t *testing.T) {
eps := []string{"http://localhost:2379"}
assert.True(t, IsClusterAvailable(eps))
})

t.Run("ReturnsFalseForUnreachableEndpoint", func(t *testing.T) {
eps := []string{"http://invalid:2379"}
assert.False(t, IsClusterAvailable(eps))
})

t.Run("ReturnsFalseForEmptyEndpoints", func(t *testing.T) {
assert.False(t, IsClusterAvailable([]string{}))
})

t.Run("SkipsUnreachableMemberAndSucceedsOnHealthyOne", func(t *testing.T) {
// First endpoint is unreachable, second is the real cluster.
eps := []string{"http://invalid:2379", "http://localhost:2379"}
assert.True(t, IsClusterAvailable(eps))
})
}

func TestHealthReportSwap(t *testing.T) {
healthReports := healthReport{
{Ep: "http://localhost:2379", Health: true},
Expand Down