Add cleanup job support for pre-deletion hooks via Kubernetes Jobs (#48)

* feat: Add cleanup job feature

Signed-off-by: Magnus Ullberg <magnus@ullberg.us>

Author: Copilot

.devcontainer/Dockerfile

@@ -6,6 +6,14 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 # update PATH for local pip installs
 ENV PATH="$PATH:~/.local/bin"
 
+# Install golangci-lint
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl git xz-utils && rm -rf /var/lib/apt/lists/* \
+    && GOLANGCI_LINT_VERSION=2.6.2 \
+    && curl -sSLO "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz" \
+    && tar -xzf golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz -C /tmp \
+    && mv /tmp/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64/golangci-lint /usr/local/bin/golangci-lint \
+    && rm -rf /tmp/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64* golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz
+
 CMD sleep infinity
 
 ENTRYPOINT []

.devcontainer/post-install.sh

@@ -3,3 +3,17 @@ set -eux
 
 # initialize pre-commit
 git config --global --add safe.directory /workspaces
+
+# Install golangci-lint if not present (useful when devcontainer not rebuilt)
+if ! command -v golangci-lint >/dev/null 2>&1; then
+	echo "golangci-lint not found, installing v2 via github release tarball"
+	# We separate update and install to avoid shellcheck SC2015 warnings
+	apt-get update || true
+	apt-get install -y --no-install-recommends ca-certificates curl xz-utils || true
+	GOLANGCI_LINT_VERSION=2.6.2
+	curl -sSLO "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz" || true
+	tar -xzf golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz -C /tmp || true
+	mv /tmp/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64/golangci-lint /usr/local/bin/golangci-lint || true
+	rm -rf /tmp/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64* golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz || true
+	echo "golangci-lint installed"
+fi

.github/workflows/pr-check.yml

@@ -14,7 +14,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install pre-commit
-        run: sudo apt-get update -y && sudo apt-get install -y pre-commit
+        run: sudo apt-get update -y && sudo apt-get install -y pre-commit codespell gitleaks
+
+      - name: Install golangci/golangci-lint
+        run: |
+          export GOLANGCI_LINT_VERSION=2.6.2
+          curl -sSLO "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz"
+          tar -xzf golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz -C /tmp
+          mv /tmp/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64/golangci-lint /usr/local/bin/golangci-lint
+          rm -rf /tmp/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64* golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz
 
       - name: Clone the code
         uses: actions/checkout@v5

.gitleaks.toml

@@ -0,0 +1,8 @@
+title = "gitleaks: project custom allowlist"
+
+useDefault = true
+
+[[allowlists]]
+description = "Ignore known example keys in examples/cleanup"
+paths = [ '''examples/cleanup/backup-to-s3.yaml''' ]
+targetRules = [ "aws-access-token" ]

.pre-commit-config.yaml

@@ -4,28 +4,54 @@ repos:
       hooks:
           - id: trailing-whitespace
           - id: end-of-file-fixer
-          # - id: check-yaml
-          #   args: [--allow-multiple-documents]
+          - id: check-yaml
+            args: [--allow-multiple-documents]
+            # Don't try to validate Helm chart templates — they contain Go
+            # templating and are not valid YAML until rendered.
+            exclude: ^object-lease-operator/helm-charts/
           - id: check-json
-          # - id: check-added-large-files
           - id: detect-private-key
           - id: no-commit-to-branch
-    # - repo: https://github.com/golangci/golangci-lint
-    #   rev: v2.2.2
-    #   hooks:
-    #       - id: golangci-lint
-    #         name: golangci-lint
-    #         description: Fast linters runner for Go. Note that only modified files are linted, so linters like 'unused' that need to scan all files won't work as expected.
-    #         entry: golangci-lint run --fix
-    #         types: [go]
-    #         language: golang
-    #         require_serial: true
-    #         pass_filenames: false
-    #       - id: golangci-lint-fmt
-    #         name: golangci-lint-fmt
-    #         description: Fast linters runner for Go. Note that only modified files are linted, so linters like 'unused' that need to scan all files won't work as expected.
-    #         entry: golangci-lint fmt
-    #         types: [go]
-    #         language: golang
-    #         require_serial: true
-    #         pass_filenames: false
+
+    - repo: local
+      hooks:
+          - id: shellcheck
+            name: Shellcheck
+            entry: shellcheck -x
+            language: system
+            files: ^(scripts/|run.sh$|.*\.sh$)
+
+          - id: gitleaks
+            name: Gitleaks (secrets scanner)
+            entry: gitleaks detect
+            pass_filenames: false
+            always_run: true
+            language: system
+
+          - id: codespell
+            name: Codespell (spell check docs)
+            entry: codespell
+            language: system
+            # Only run on docs and README to avoid noisy suggestions in tests/code
+            files: ^(README.md|docs/|object-lease-console-plugin/)
+
+          - id: make-vet
+            name: Run `go vet`
+            entry: make vet
+            language: system
+            pass_filenames: false
+            always_run: true
+
+          - id: make-fmt
+            name: Run repository formatter (go fmt)
+            entry: make fmt
+            language: system
+            pass_filenames: false
+            always_run: true
+
+          - id: make-lint
+            name: Run repository linter
+            entry: make lint
+            language: system
+            pass_filenames: false
+            always_run: true

Makefile

@@ -36,6 +36,10 @@ fmt: ## Format Go code
 vet: ## Vet Go code
 	go vet ./...
 
+.PHONY: lint
+lint: ## Run golangci-lint (requires golangci-lint installed)
+	golangci-lint run
+
 .PHONY: test
 test: tidy fmt vet ## Run tests with coverage
 	go test ./... -race -coverprofile=coverage.out
@@ -50,11 +54,12 @@ run: build ## Run the application locally
 	./$(BUILD_DIR)/$(BINARY_NAME) \
 		-group startpunkt.ullberg.us \
 		-kind Application \
-		-version v1alpha2 \
+		-version v1alpha4 \
 		-leader-elect \
 		-leader-elect-namespace default \
-		-opt-in-label-key "object-lease-controller.ullberg.io/enabled" \
-		-opt-in-label-value true
+# 		-opt-in-label-key "object-lease-controller.ullberg.io/enabled" \
+# 		-opt-in-label-value true \
+		-zap-log-level debug
 
 # =============================================================================
 # Docker Targets - Main Controller

README.md

@@ -13,8 +13,9 @@ This project implements a Kubernetes operator that allows you to specify a TTL (
 ## Features
 - Deploys as an operator.
 - Dynamically deploys a controller for each configured GVK.
-- Controllers are only managing one GVK each, increasing scaleability.
+- Controllers are only managing one GVK each, increasing scalability.
 - Leader election support for high availability.
+- Custom cleanup scripts via Kubernetes Jobs before object deletion.
 
 ## Architecture
 The operator is designed to be highly extensible and scalable. Once deployed, the operator looks for CRDs and for each GVK specified in a CRD, a dedicated controller is launched.
@@ -134,6 +135,64 @@ Set by the controller. RFC3339 UTC timestamp for when the object will expire. Sa
 
 Set by the controller. Human readable status or validation errors.
 
+### Cleanup Job Annotations
+
+The controller supports running custom cleanup scripts via Kubernetes Jobs before deleting expired objects. This is useful for backing up data, notifying external systems, or cleaning up related resources.
+
+#### object-lease-controller.ullberg.io/on-delete-job
+
+**Required for cleanup jobs**. Specifies the ConfigMap and script key in the format `configmap-name/script-key`.
+
+Example:
+```bash
+kubectl annotate application my-app object-lease-controller.ullberg.io/on-delete-job=cleanup-scripts/backup.sh
+```
+
+#### object-lease-controller.ullberg.io/job-service-account
+
+**Optional** (default: `default`). ServiceAccount to run the cleanup Job as. Use this to grant the cleanup script access to necessary permissions and secrets.
+
+#### object-lease-controller.ullberg.io/job-image
+
+**Optional** (default: `bitnami/kubectl:latest`). Container image for running the cleanup script.
+
+#### object-lease-controller.ullberg.io/job-wait
+
+**Optional** (default: `false`). If `true`, the controller waits for the Job to complete before deleting the object. If `false`, the Job runs in fire-and-forget mode.
+
+#### object-lease-controller.ullberg.io/job-timeout
+
+**Optional** (default: `5m`). Maximum time to wait for Job completion when `job-wait` is `true`. Supports flexible duration format (e.g., `10m`, `1h`, `30s`).
+
+#### object-lease-controller.ullberg.io/job-ttl
+
+**Optional** (default: `300`). TTL in seconds for Job cleanup via `ttlSecondsAfterFinished`.
+
+#### object-lease-controller.ullberg.io/job-backoff-limit
+
+**Optional** (default: `3`). Number of retries for failed Jobs.
+
+### Cleanup Job Environment Variables
+
+Cleanup scripts receive these environment variables:
+
+- `OBJECT_NAME` - Name of the object being deleted
+- `OBJECT_NAMESPACE` - Namespace of the object
+- `OBJECT_KIND` - Kind (e.g., "Application")
+- `OBJECT_GROUP` - API group (e.g., "startpunkt.ullberg.us")
+- `OBJECT_VERSION` - API version (e.g., "v1alpha2")
+- `OBJECT_UID` - UID of the object
+- `OBJECT_RESOURCE_VERSION` - Resource version
+- `LEASE_STARTED_AT` - RFC3339 timestamp when lease started
+- `LEASE_EXPIRED_AT` - RFC3339 timestamp when lease expired
+- `OBJECT_LABELS` - JSON-encoded labels
+- `OBJECT_ANNOTATIONS` - JSON-encoded annotations
+
+See [examples/cleanup/](examples/cleanup/) for complete examples including:
+- Backing up to S3
+- Webhook notifications
+- Cleaning up related Kubernetes resources
+
 ### Removing TTL
 
 Remove `ttl` to stop lease management. The controller clears `lease-start`, `expire-at`, and `lease-status`.
@@ -146,6 +205,12 @@ kubectl annotate pod test object-lease-controller.ullberg.io/ttl-
 - Automatically manage leases for custom resources (e.g., Applications, Databases, Services)
 - Enforce expiration policies
 - Integrate with external systems for lease validation or renewal
+- Execute custom cleanup scripts before object deletion:
+  - Back up data to external storage (S3, GCS, etc.)
+  - Notify external systems or webhooks
+  - Clean up dependent resources not covered by owner references
+  - Archive logs or metrics
+  - Graceful shutdown procedures
 
 ## Usage
 

cmd/main.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -32,6 +33,16 @@ const (
 	AnnLeaseStart = "object-lease-controller.ullberg.io/lease-start" // RFC3339 UTC
 	AnnExpireAt   = "object-lease-controller.ullberg.io/expire-at"
 	AnnStatus     = "object-lease-controller.ullberg.io/lease-status"
+
+	// Cleanup job annotation keys
+	AnnOnDeleteJob       = "object-lease-controller.ullberg.io/on-delete-job"
+	AnnJobServiceAccount = "object-lease-controller.ullberg.io/job-service-account"
+	AnnJobImage          = "object-lease-controller.ullberg.io/job-image"
+	AnnJobWait           = "object-lease-controller.ullberg.io/job-wait"
+	AnnJobTimeout        = "object-lease-controller.ullberg.io/job-timeout"
+	AnnJobTTL            = "object-lease-controller.ullberg.io/job-ttl"
+	AnnJobBackoffLimit   = "object-lease-controller.ullberg.io/job-backoff-limit"
+	AnnJobEnvSecrets     = "object-lease-controller.ullberg.io/job-env-secrets"
 )
 
 // ParseParams holds runtime configuration parsed from flags and environment.
@@ -57,10 +68,19 @@ var statFn = os.Stat
 var readFileFn = os.ReadFile
 
 func main() {
-	ctrl.SetLogger(zap.New())
+	// Bind zap logging flags (e.g., -zap-log-level) to the global flag set
+	// so callers (and the Makefile) can adjust verbosity. Don't set the
+	// logger until after flags are parsed so the selected level is applied.
+	var zapOpts zap.Options
+	zapOpts.BindFlags(flag.CommandLine)
 
 	params := parseParameters()
 
+	// Set logger using the parsed zap options (this reads values parsed by
+	// parseParameters which calls flag.Parse()). This allows callers to pass
+	// flags like -zap-log-level=debug to control verbosity.
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts)))
+
 	enableLeaderElection, leaderElectionNamespace, errE := parseLeaderElectionConfig(params.LeaderElectionEnabled, params.LeaderElectionNamespace)
 	if errE != nil {
 		fmt.Printf("%v\n", errE)
@@ -75,6 +95,7 @@ func main() {
 
 	scheme := runtime.NewScheme()
 	_ = corev1.AddToScheme(scheme)
+	_ = batchv1.AddToScheme(scheme)
 
 	gvk := schema.GroupVersionKind{
 		Group:   params.Group,
@@ -266,7 +287,11 @@ func buildManagerOptions(scheme *runtime.Scheme, group, version, kind string, me
 		Metrics:                       metricsServerOptions,
 		HealthProbeBindAddress:        probeAddr,
 		Cache: cache.Options{
-			DefaultTransform: util.MinimalObjectTransform(AnnTTL, AnnLeaseStart, AnnExpireAt, AnnStatus),
+			DefaultTransform: util.MinimalObjectTransform(
+				AnnTTL, AnnLeaseStart, AnnExpireAt, AnnStatus,
+				AnnOnDeleteJob, AnnJobServiceAccount, AnnJobImage, AnnJobWait,
+				AnnJobTimeout, AnnJobTTL, AnnJobBackoffLimit,
+			),
 		},
 	}
 	if pprofAddr != "" {
@@ -284,10 +309,18 @@ func newLeaseWatcher(mgr ctrl.Manager, gvk schema.GroupVersionKind, leaderElecti
 		GVK:      gvk,
 		Recorder: mgr.GetEventRecorderFor(leaderElectionID),
 		Annotations: controllers.Annotations{
-			TTL:        AnnTTL,
-			LeaseStart: AnnLeaseStart,
-			ExpireAt:   AnnExpireAt,
-			Status:     AnnStatus,
+			TTL:               AnnTTL,
+			LeaseStart:        AnnLeaseStart,
+			ExpireAt:          AnnExpireAt,
+			Status:            AnnStatus,
+			OnDeleteJob:       AnnOnDeleteJob,
+			JobServiceAccount: AnnJobServiceAccount,
+			JobImage:          AnnJobImage,
+			JobWait:           AnnJobWait,
+			JobTimeout:        AnnJobTimeout,
+			JobTTL:            AnnJobTTL,
+			JobBackoffLimit:   AnnJobBackoffLimit,
+			JobEnvSecrets:     AnnJobEnvSecrets,
 		},
 		Metrics: ometrics.NewLeaseMetrics(gvk),
 	}

examples/cleanup/README.md

@@ -0,0 +1,45 @@
+# Cleanup Job Examples
+
+This directory contains example configurations for using cleanup jobs with the object-lease-controller.
+
+## Examples
+
+1. **backup-to-s3.yaml** - Complete example of backing up an object to S3 before deletion
+2. **notify-webhook.yaml** - Example of notifying an external webhook when an object expires
+3. **cleanup-related-resources.yaml** - Example of cleaning up related Kubernetes resources
+
+## Quick Start
+
+1. Create a ConfigMap with your cleanup script
+2. Create a ServiceAccount with necessary permissions
+3. Annotate your resource with cleanup job configuration
+4. When the lease expires, the cleanup job runs automatically
+
+## Annotations
+
+| Annotation | Required | Default | Description |
+|------------|----------|---------|-------------|
+| `object-lease-controller.ullberg.io/on-delete-job` | Yes (only if cleanup is needed) | - | ConfigMap reference in format `configmap-name/script-key` |
+| `object-lease-controller.ullberg.io/job-service-account` | No | `default` | ServiceAccount to run the Job as |
+| `object-lease-controller.ullberg.io/job-image` | No | `bitnami/kubectl:latest` | Container image for running the script |
+| `object-lease-controller.ullberg.io/job-env-secrets` | No | - | Comma-separated list of Secret names to mount as environment variables |
+| `object-lease-controller.ullberg.io/job-wait` | No | `false` | Wait for Job completion before deleting object |
+| `object-lease-controller.ullberg.io/job-timeout` | No | `5m` | Maximum time to wait for Job completion |
+| `object-lease-controller.ullberg.io/job-ttl` | No | `300` | TTL in seconds for Job cleanup |
+| `object-lease-controller.ullberg.io/job-backoff-limit` | No | `3` | Number of retries for failed Jobs |
+
+## Environment Variables
+
+The cleanup script receives these environment variables:
+
+- `OBJECT_NAME` - Name of the object being deleted
+- `OBJECT_NAMESPACE` - Namespace of the object
+- `OBJECT_KIND` - Kind (e.g., "Application")
+- `OBJECT_GROUP` - API group
+- `OBJECT_VERSION` - API version
+- `OBJECT_UID` - UID of the object
+- `OBJECT_RESOURCE_VERSION` - Resource version
+- `LEASE_STARTED_AT` - RFC3339 timestamp when lease started
+- `LEASE_EXPIRED_AT` - RFC3339 timestamp when lease expired
+- `OBJECT_LABELS` - JSON-encoded labels
+- `OBJECT_ANNOTATIONS` - JSON-encoded annotations