feat: backend quota and auto-compaction spec fields, NOSPACE alarm surfacing

api/v1alpha1/etcdcluster_types.go

@@ -28,6 +28,8 @@ import (
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
 // EtcdClusterSpec defines the desired state of EtcdCluster.
+// +kubebuilder:validation:XValidation:rule="!has(self.autoCompactionMode) || has(self.autoCompactionRetention)",message="autoCompactionRetention must be set when autoCompactionMode is set"
+// +kubebuilder:validation:XValidation:rule="!has(self.autoCompactionMode) || self.autoCompactionMode != 'revision' || (has(self.autoCompactionRetention) && self.autoCompactionRetention.matches('^[0-9]+$'))",message="autoCompactionRetention must be an integer revision count when autoCompactionMode is 'revision'"
 type EtcdClusterSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
@@ -47,6 +49,26 @@ type EtcdClusterSpec struct {
 	TLS *TLSCertificate `json:"tls,omitempty"`
 	// etcd configuration options are passed as command line arguments to the etcd container, refer to etcd documentation for configuration options applicable for the version of etcd being used.
 	EtcdOptions []string `json:"etcdOptions,omitempty"`
+	// QuotaBackendBytes is the etcd backend storage quota (--quota-backend-bytes).
+	// When exceeded etcd raises a NOSPACE alarm and becomes read-only.
+	// Unset means the etcd default (2GiB). etcd recommends at most 8GiB.
+	// Must be at least 100Mi: etcd disables the quota for non-positive
+	// values, and a byte-scale typo ("8" instead of "8Gi") would alarm the
+	// whole cluster read-only on the first pod restart. Lowering the quota
+	// below the current DB size has the same read-only effect.
+	// +kubebuilder:validation:XValidation:rule="!quantity(string(self)).isLessThan(quantity('100Mi'))",message="quotaBackendBytes must be at least 100Mi"
+	// +optional
+	QuotaBackendBytes *resource.Quantity `json:"quotaBackendBytes,omitempty"`
+	// AutoCompactionMode selects the MVCC auto-compaction policy (--auto-compaction-mode).
+	// +kubebuilder:validation:Enum=periodic;revision
+	// +optional
+	AutoCompactionMode string `json:"autoCompactionMode,omitempty"`
+	// AutoCompactionRetention is the retention window for auto compaction
+	// (--auto-compaction-retention): a duration such as "5m"/"1h" in periodic
+	// mode, or a revision count in revision mode. "0" disables auto compaction.
+	// +kubebuilder:validation:Pattern=`^([0-9]+[smh])+$|^[0-9]+$`
+	// +optional
+	AutoCompactionRetention string `json:"autoCompactionRetention,omitempty"`
 	// PodTemplate is the pod template to use for the etcd cluster.
 	PodTemplate *PodTemplate `json:"podTemplate,omitempty"`
 }

config/crd/bases/operator.etcd.io_etcdclusters.yaml

@@ -39,6 +39,20 @@ spec:
           spec:
             description: EtcdClusterSpec defines the desired state of EtcdCluster.
             properties:
+              autoCompactionMode:
+                description: AutoCompactionMode selects the MVCC auto-compaction policy
+                  (--auto-compaction-mode).
+                enum:
+                - periodic
+                - revision
+                type: string
+              autoCompactionRetention:
+                description: |-
+                  AutoCompactionRetention is the retention window for auto compaction
+                  (--auto-compaction-retention): a duration such as "5m"/"1h" in periodic
+                  mode, or a revision count in revision mode. "0" disables auto compaction.
+                pattern: ^([0-9]+[smh])+$|^[0-9]+$
+                type: string
               etcdOptions:
                 description: etcd configuration options are passed as command line
                   arguments to the etcd container, refer to etcd documentation for
@@ -1038,6 +1052,23 @@ spec:
                         type: array
                     type: object
                 type: object
+              quotaBackendBytes:
+                anyOf:
+                - type: integer
+                - type: string
+                description: |-
+                  QuotaBackendBytes is the etcd backend storage quota (--quota-backend-bytes).
+                  When exceeded etcd raises a NOSPACE alarm and becomes read-only.
+                  Unset means the etcd default (2GiB). etcd recommends at most 8GiB.
+                  Must be at least 100Mi: etcd disables the quota for non-positive
+                  values, and a byte-scale typo ("8" instead of "8Gi") would alarm the
+                  whole cluster read-only on the first pod restart. Lowering the quota
+                  below the current DB size has the same read-only effect.
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+                x-kubernetes-validations:
+                - message: quotaBackendBytes must be at least 100Mi
+                  rule: '!quantity(string(self)).isLessThan(quantity(''100Mi''))'
               size:
                 description: Size is the expected size of the etcd cluster.
                 minimum: 1
@@ -1191,6 +1222,14 @@ spec:
             - size
             - version
             type: object
+            x-kubernetes-validations:
+            - message: autoCompactionRetention must be set when autoCompactionMode
+                is set
+              rule: '!has(self.autoCompactionMode) || has(self.autoCompactionRetention)'
+            - message: autoCompactionRetention must be an integer revision count when
+                autoCompactionMode is 'revision'
+              rule: '!has(self.autoCompactionMode) || self.autoCompactionMode != ''revision''
+                || (has(self.autoCompactionRetention) && self.autoCompactionRetention.matches(''^[0-9]+$''))'
           status:
             description: EtcdClusterStatus defines the observed state of EtcdCluster.
             properties:

config/rbac/role.yaml

@@ -61,6 +61,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - events.k8s.io
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 - apiGroups:
   - operator.etcd.io
   resources:

docs/api-references/docs.md

@@ -110,6 +110,9 @@ _Appears in:_
 | `storageSpec` _[StorageSpec](#storagespec)_ | StorageSpec is the name of the StorageSpec to use for the etcd cluster. If not provided, then each POD just uses the temporary storage inside the container. |  |  |
 | `tls` _[TLSCertificate](#tlscertificate)_ | TLS is the TLS certificate configuration to use for the etcd cluster and etcd operator. |  |  |
 | `etcdOptions` _string array_ | etcd configuration options are passed as command line arguments to the etcd container, refer to etcd documentation for configuration options applicable for the version of etcd being used. |  |  |
+| `quotaBackendBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | QuotaBackendBytes is the etcd backend storage quota (--quota-backend-bytes).<br />When exceeded etcd raises a NOSPACE alarm and becomes read-only.<br />Unset means the etcd default (2GiB). etcd recommends at most 8GiB.<br />Must be at least 100Mi: etcd disables the quota for non-positive<br />values, and a byte-scale typo ("8" instead of "8Gi") would alarm the<br />whole cluster read-only on the first pod restart. Lowering the quota<br />below the current DB size has the same read-only effect. |  | Optional: \{\} <br /> |
+| `autoCompactionMode` _string_ | AutoCompactionMode selects the MVCC auto-compaction policy (--auto-compaction-mode). |  | Enum: [periodic revision] <br />Optional: \{\} <br /> |
+| `autoCompactionRetention` _string_ | AutoCompactionRetention is the retention window for auto compaction<br />(--auto-compaction-retention): a duration such as "5m"/"1h" in periodic<br />mode, or a revision count in revision mode. "0" disables auto compaction. |  | Pattern: `^([0-9]+[smh])+$\|^[0-9]+$` <br />Optional: \{\} <br /> |
 | `podTemplate` _[PodTemplate](#podtemplate)_ | PodTemplate is the pod template to use for the etcd cluster. |  |  |
 
 
@@ -153,6 +156,24 @@ _Appears in:_
 | `labels` _object (keys:string, values:string)_ |  |  |  |
 
 
+#### PodSpec
+
+
+
+
+
+
+
+_Appears in:_
+- [PodTemplate](#podtemplate)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#affinity-v1-core)_ |  |  |  |
+| `nodeSelector` _object (keys:string, values:string)_ |  |  |  |
+| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#toleration-v1-core) array_ |  |  |  |
+
+
 #### PodTemplate
 
 
@@ -167,6 +188,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. |  |  |
+| `spec` _[PodSpec](#podspec)_ |  |  |  |
 
 
 #### ProviderAutoConfig

docs/configuring-etcd-options.md

@@ -4,4 +4,13 @@ Additional configuration options can be set using the `.spec.etcdOptions` field.
 
 Options configured via `etcdOptions` have a higher priority than the default configured arguments. For example if one of the default arguments is `--listen-peer-urls=http://0.0.0.0:2380` and you specify `--listen-peer-urls=http://0.0.0.0:3200` using `etcdOptions`, then the argument `--listen-peer-urls=http://0.0.0.0:3200` will be used.
 
-Information about the different configuration options is available from the etcd documentation page here: https://etcd.io/docs/latest/op-guide/configuration/.
+Information about the different configuration options is available from the etcd documentation page here: https://etcd.io/docs/latest/op-guide/configuration/.
+## Database size and compaction
+
+`spec.quotaBackendBytes`, `spec.autoCompactionMode` and `spec.autoCompactionRetention` render the `--quota-backend-bytes`, `--auto-compaction-mode` and `--auto-compaction-retention` flags. When unset, no flag is rendered and etcd defaults apply. A conflicting flag in `etcdOptions` still wins, following the precedence above.
+
+Changing these fields on a running, healthy cluster is not inert: the operator re-renders the StatefulSet template and immediately starts a rolling restart, replacing one pod at a time behind quorum and health gates (all members healthy, revisions caught up, every up-to-date pod Ready). Only apply such changes when a member-by-member restart is acceptable.
+
+While a NOSPACE alarm is active those health gates hold the rollout, and raising `quotaBackendBytes` alone does not clear the alarm: compact, defragment and `etcdctl alarm disarm` first — the rollout then proceeds automatically. If space cannot be freed, delete pods one at a time (`kubectl delete pod`); the StatefulSet uses the OnDelete strategy, so each replacement starts from the already-rendered template carrying the new quota.
+
+Rollback caveat: do not downgrade the operator below the version that introduced these fields while any cluster's DB exceeds etcd's 2GiB default quota. The older operator re-renders the template without `--quota-backend-bytes`, and the restarted pods raise a cluster-wide NOSPACE.

internal/controller/dbsize_cel_test.go

@@ -0,0 +1,114 @@
+package controller
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1"
+)
+
+// TestDBSizeCELValidation drives the apply-time CEL rules on quotaBackendBytes
+// and the autoCompaction fields against the real envtest apiserver. A negative
+// quota disables the quota in etcd, a byte-scale typo alarms the cluster
+// read-only, and a duration retention in revision mode is misread by etcd as a
+// nanosecond revision count — all must be rejected at admission.
+func TestDBSizeCELValidation(t *testing.T) {
+	if k8sClient == nil {
+		t.Skip("envtest apiserver not available")
+	}
+
+	qty := func(s string) *resource.Quantity {
+		q := resource.MustParse(s)
+		return &q
+	}
+
+	tests := []struct {
+		name      string
+		mutate    func(*ecv1alpha1.EtcdClusterSpec)
+		wantApply bool
+	}{
+		{
+			name:      "quota 8Gi accepted",
+			mutate:    func(s *ecv1alpha1.EtcdClusterSpec) { s.QuotaBackendBytes = qty("8Gi") },
+			wantApply: true,
+		},
+		{
+			name:      "quota at the 100Mi floor accepted",
+			mutate:    func(s *ecv1alpha1.EtcdClusterSpec) { s.QuotaBackendBytes = qty("100Mi") },
+			wantApply: true,
+		},
+		{
+			name:      "quota as plain byte integer accepted when large enough",
+			mutate:    func(s *ecv1alpha1.EtcdClusterSpec) { s.QuotaBackendBytes = qty("2147483648") },
+			wantApply: true,
+		},
+		{
+			name:      "negative quota rejected (etcd would disable the quota)",
+			mutate:    func(s *ecv1alpha1.EtcdClusterSpec) { s.QuotaBackendBytes = qty("-1") },
+			wantApply: false,
+		},
+		{
+			name:      "byte-scale typo rejected (\"8\" for \"8Gi\")",
+			mutate:    func(s *ecv1alpha1.EtcdClusterSpec) { s.QuotaBackendBytes = qty("8") },
+			wantApply: false,
+		},
+		{
+			name:      "milli quantity rejected",
+			mutate:    func(s *ecv1alpha1.EtcdClusterSpec) { s.QuotaBackendBytes = qty("500m") },
+			wantApply: false,
+		},
+		{
+			name: "periodic mode with duration retention accepted",
+			mutate: func(s *ecv1alpha1.EtcdClusterSpec) {
+				s.AutoCompactionMode, s.AutoCompactionRetention = "periodic", "5m"
+			},
+			wantApply: true,
+		},
+		{
+			name: "revision mode with integer retention accepted",
+			mutate: func(s *ecv1alpha1.EtcdClusterSpec) {
+				s.AutoCompactionMode, s.AutoCompactionRetention = "revision", "1000"
+			},
+			wantApply: true,
+		},
+		{
+			name: "revision mode with duration retention rejected (etcd reads it as nanoseconds of revisions)",
+			mutate: func(s *ecv1alpha1.EtcdClusterSpec) {
+				s.AutoCompactionMode, s.AutoCompactionRetention = "revision", "5m"
+			},
+			wantApply: false,
+		},
+		{
+			name: "revision mode without retention rejected",
+			mutate: func(s *ecv1alpha1.EtcdClusterSpec) {
+				s.AutoCompactionMode = "revision"
+			},
+			wantApply: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ec := &ecv1alpha1.EtcdCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "dbsize-cel-test-",
+					Namespace:    "default",
+				},
+				Spec: ecv1alpha1.EtcdClusterSpec{Size: 3, Version: "v3.6.12"},
+			}
+			tt.mutate(&ec.Spec)
+			err := k8sClient.Create(t.Context(), ec)
+			if tt.wantApply {
+				require.NoError(t, err, "apiserver should accept a valid spec")
+				_ = k8sClient.Delete(t.Context(), ec, &client.DeleteOptions{})
+			} else {
+				assert.Error(t, err, "apiserver should reject the spec via CEL")
+			}
+		})
+	}
+}

internal/controller/etcdcluster_controller.go

@@ -60,6 +60,7 @@ type reconcileState struct {
 	sts            *appsv1.StatefulSet          // associated StatefulSet for the cluster
 	memberListResp *clientv3.MemberListResponse // member list fetched from the etcd cluster
 	memberHealth   []etcdutils.EpHealth         // health information for each etcd member
+	alarms         []etcdutils.MemberAlarm      // active etcd alarms (e.g. NOSPACE)
 }
 
 // +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdclusters,verbs=get;list;watch;create;update;patch;delete
@@ -69,6 +70,7 @@ type reconcileState struct {
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch;get;list;update
+// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;patch;update;delete
 // +kubebuilder:rbac:groups="cert-manager.io",resources=certificates,verbs=get;list;watch;create;patch;update;delete
 // +kubebuilder:rbac:groups="cert-manager.io",resources=clusterissuers,verbs=get;list;watch
@@ -261,12 +263,52 @@ func (r *EtcdClusterReconciler) performHealthChecks(ctx context.Context, s *reco
 	logger.Info("Now checking health of the cluster members")
 	var err error
 	s.memberListResp, s.memberHealth, err = healthCheck(s.sts, logger)
+
+	// Poll alarms even when the health check errored: a NOSPACE alarm makes
+	// members report unhealthy, and the deferred status update still runs.
+	if s.memberListResp != nil {
+		alarms, aerr := etcdutils.AlarmList(clientEndpointsFromStatefulsets(s.sts))
+		if aerr != nil {
+			logger.Error(aerr, "failed to list etcd alarms")
+		} else {
+			s.alarms = alarms
+		}
+		r.reportNospaceAlarms(s)
+	}
+
 	if err != nil {
 		return fmt.Errorf("health check failed: %w", err)
 	}
 	return nil
 }
 
+// memberNameForID resolves an etcd member ID to its name, falling back to the hex ID.
+func memberNameForID(resp *clientv3.MemberListResponse, id uint64) string {
+	if resp != nil {
+		for _, m := range resp.Members {
+			if m.ID == id && m.Name != "" {
+				return m.Name
+			}
+		}
+	}
+	return fmt.Sprintf("%x", id)
+}
+
+// reportNospaceAlarms emits a Warning event for each member with an active NOSPACE alarm.
+func (r *EtcdClusterReconciler) reportNospaceAlarms(s *reconcileState) {
+	if r.Recorder == nil {
+		return
+	}
+	for _, a := range s.alarms {
+		if a.Type != "NOSPACE" {
+			continue
+		}
+		name := memberNameForID(s.memberListResp, a.MemberID)
+		r.Recorder.Eventf(s.cluster, nil, corev1.EventTypeWarning, "DatabaseQuotaExceeded", "AlarmDetected",
+			"etcd member %s has an active NOSPACE alarm; cluster is read-only. Compact and defragment, then `etcdctl alarm disarm`. A raised spec.quotaBackendBytes rolls out automatically once members report healthy again.", name)
+	}
+}
+
 // reconcileClusterState compares the desired cluster size with the observed
 // etcd member list and StatefulSet replica count. It performs scaling actions
 // and handles learner promotion when needed. A ctrl.Result with a requeue
@@ -564,6 +606,20 @@ func (r *EtcdClusterReconciler) updateConditions(s *reconcileState) {
 		}
 	}
 
+	// NOSPACE takes precedence over the generic UnhealthyMembers reason:
+	// alarmed members are also reported unhealthy by the health check.
+	var nospaceMembers []string
+	for _, a := range s.alarms {
+		if a.Type == "NOSPACE" {
+			nospaceMembers = append(nospaceMembers, memberNameForID(s.memberListResp, a.MemberID))
+		}
+	}
+	if len(nospaceMembers) > 0 {
+		degradedCondition.Status = metav1.ConditionTrue
+		degradedCondition.Reason = "DatabaseQuotaExceeded"
+		degradedCondition.Message = fmt.Sprintf("NOSPACE alarm active on member(s) %s: etcd is read-only until compaction/defragmentation and `etcdctl alarm disarm`", strings.Join(nospaceMembers, ", "))
+	}
+
 	// Update or append conditions
 	meta.SetStatusCondition(&s.cluster.Status.Conditions, availableCondition)
 	meta.SetStatusCondition(&s.cluster.Status.Conditions, progressingCondition)