Crafting K8s Operators with Finalizers for Stateful App Management

// ManagedDatabaseSpec defines the desired state of ManagedDatabase
type ManagedDatabaseSpec struct {
	// Engine specifies the database engine (e.g., "postgres", "mysql")
	// +kubebuilder:validation:Enum=postgres;mysql
	// +kubebuilder:validation:Required
	Engine string `json:"engine"`

	// Version specifies the engine version.
	// +kubebuilder:validation:Required
	Version string `json:"version"`

	// Username for the database admin user.
	// +kubebuilder:validation:Required
	Username string `json:"username"`
}

// ManagedDatabaseStatus defines the observed state of ManagedDatabase
type ManagedDatabaseStatus struct {
	// Conditions represent the latest available observations of an object's state
	// +optional
	Conditions []metav1.Condition `json:"conditions,omitempty"`

	// InstanceID is the unique identifier for the external database instance.
	// +optional
	InstanceID string `json:"instanceID,omitempty"`

	// Endpoint is the connection address for the database.
	// +optional
	Endpoint string `json:"endpoint,omitempty"`
}

// internal/dbprovider/mock_provider.go

package dbprovider

import (
	"fmt"
	"math/rand"
	"sync"
	"time"
)

// A mock database instance
type DBInstance struct {
	ID       string
	Engine   string
	Version  string
	Endpoint string
	Status   string // PROVISIONING, AVAILABLE, DELETING, DELETED
}

// Mock provider that simulates a cloud database service
type MockProvider struct {
	mu        sync.Mutex
	Instances map[string]*DBInstance
}

func NewMockProvider() *MockProvider {
	return &MockProvider{
		Instances: make(map[string]*DBInstance),
	}
}

// Simulates creating a database
func (p *MockProvider) CreateDB(engine, version string) (*DBInstance, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	id := fmt.Sprintf("db-%d", rand.Intn(10000))
	instance := &DBInstance{
		ID:       id,
		Engine:   engine,
		Version:  version,
		Endpoint: fmt.Sprintf("%s.db.example.com", id),
		Status:   "PROVISIONING",
	}
	p.Instances[id] = instance

	// Simulate provisioning time
	go func() {
		time.Sleep(5 * time.Second)
		p.mu.Lock()
		defer p.mu.Unlock()
		if p.Instances[id] != nil {
			p.Instances[id].Status = "AVAILABLE"
		}
	}()

	return instance, nil
}

// Simulates fetching database status
func (p *MockProvider) GetDB(instanceID string) (*DBInstance, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	instance, ok := p.Instances[instanceID]
	if !ok {
		return nil, fmt.Errorf("instance %s not found", instanceID)
	}
	return instance, nil
}

// Simulates deleting a database
func (p *MockProvider) DeleteDB(instanceID string) error {
	p.mu.Lock()
	defer p.mu.Unlock()

	instance, ok := p.Instances[instanceID]
	if !ok {
        // Idempotency: if it's already gone, that's a success for deletion.
		return nil
	}

	instance.Status = "DELETING"

	// Simulate deletion time
	go func() {
		time.Sleep(5 * time.Second)
		p.mu.Lock()
		defer p.mu.Unlock()
		delete(p.Instances, instanceID)
	}()

	return nil
}

// internal/controller/manageddatabase_controller.go

package controller

import (
	// ... imports
	"context"
	"time"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/log"

	dbV1alpha1 "my-operator/api/v1alpha1"
	"my-operator/internal/dbprovider"
)

const managedDatabaseFinalizer = "db.example.com/finalizer"

type ManagedDatabaseReconciler struct {
	client.Client
	Scheme   *runtime.Scheme
	Provider *dbprovider.MockProvider // Our mock DB provider
}

func (r *ManagedDatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	logger := log.FromContext(ctx)

	// 1. Fetch the ManagedDatabase instance
	dbInstance := &dbV1alpha1.ManagedDatabase{}
	if err := r.Get(ctx, req.NamespacedName, dbInstance); err != nil {
		if apierrors.IsNotFound(err) {
			// Object was deleted, nothing to do. This happens after our finalizer is removed.
			logger.Info("ManagedDatabase resource not found. Ignoring since object must be deleted")
			return ctrl.Result{}, nil
		}
		// Error reading the object - requeue the request.
		logger.Error(err, "Failed to get ManagedDatabase")
		return ctrl.Result{}, err
	}

	// 2. The core finalizer logic
	if dbInstance.ObjectMeta.DeletionTimestamp.IsZero() {
		// The object is NOT being deleted. Let's add our finalizer if it doesn't exist.
		if !controllerutil.ContainsFinalizer(dbInstance, managedDatabaseFinalizer) {
			logger.Info("Adding finalizer for ManagedDatabase")
			controllerutil.AddFinalizer(dbInstance, managedDatabaseFinalizer)
			if err := r.Update(ctx, dbInstance); err != nil {
				return ctrl.Result{}, err
			}
		}
	} else {
		// The object IS being deleted
		if controllerutil.ContainsFinalizer(dbInstance, managedDatabaseFinalizer) {
			logger.Info("Performing finalizer cleanup for ManagedDatabase")
			// Our finalizer is present, so let's handle external dependency cleanup.
			if err := r.cleanupExternalResources(ctx, dbInstance); err != nil {
				// If cleanup fails, we don't remove the finalizer. 
				// The reconciliation will be retried.
				logger.Error(err, "Failed to cleanup external resources")
				return ctrl.Result{}, err
			}

			// Cleanup was successful. Remove our finalizer.
			logger.Info("External resources cleaned up. Removing finalizer")
			controllerutil.RemoveFinalizer(dbInstance, managedDatabaseFinalizer)
			if err := r.Update(ctx, dbInstance); err != nil {
				return ctrl.Result{}, err
			}
		}

		// Stop reconciliation as the item is being deleted
		return ctrl.Result{}, nil
	}

	// 3. The main reconciliation logic (Create/Update)
	// This part runs only if the object is not being deleted.
	return r.reconcileExternalResources(ctx, dbInstance)
}

// internal/controller/manageddatabase_controller.go

func (r *ManagedDatabaseReconciler) reconcileExternalResources(ctx context.Context, dbInstance *dbV1alpha1.ManagedDatabase) (ctrl.Result, error) {
	logger := log.FromContext(ctx)

	// Check if the external DB instance is already created
	if dbInstance.Status.InstanceID == "" {
		// Not created yet, so let's provision it.
		logger.Info("Provisioning new database instance")
		externalDB, err := r.Provider.CreateDB(dbInstance.Spec.Engine, dbInstance.Spec.Version)
		if err != nil {
			logger.Error(err, "Failed to provision external database")
			// TODO: Update status with a failure condition
			return ctrl.Result{}, err
		}

		// Immediately update the status with the new InstanceID.
		// This is critical. If the operator restarts now, it will know the DB exists.
		dbInstance.Status.InstanceID = externalDB.ID
		dbInstance.Status.Endpoint = ""
		// TODO: Set a "Provisioning" condition
		if err := r.Status().Update(ctx, dbInstance); err != nil {
			logger.Error(err, "Failed to update ManagedDatabase status after creation")
			return ctrl.Result{}, err
		}

		// Requeue to check status later, as provisioning is async
		logger.Info("Database is provisioning. Requeueing for status check.")
		return ctrl.Result{RequeueAfter: 15 * time.Second}, nil
	}

	// Instance exists, let's check its status.
	externalDB, err := r.Provider.GetDB(dbInstance.Status.InstanceID)
	if err != nil {
		logger.Error(err, "Failed to get external database status")
		return ctrl.Result{}, err
	}

	switch externalDB.Status {
	case "PROVISIONING":
		logger.Info("Database is still provisioning", "InstanceID", externalDB.ID)
		// TODO: Update status condition to reflect provisioning state
		return ctrl.Result{RequeueAfter: 15 * time.Second}, nil
	case "AVAILABLE":
		if dbInstance.Status.Endpoint != externalDB.Endpoint {
			logger.Info("Database is available. Updating endpoint.", "InstanceID", externalDB.ID)
			dbInstance.Status.Endpoint = externalDB.Endpoint
			// TODO: Set a "Ready" condition
			if err := r.Status().Update(ctx, dbInstance); err != nil {
				logger.Error(err, "Failed to update ManagedDatabase status with endpoint")
				return ctrl.Result{}, err
			}
		}
		logger.Info("Database is in desired state.", "InstanceID", externalDB.ID)
		return ctrl.Result{}, nil // All done!
	default:
		logger.Info("Database in an unknown state", "State", externalDB.Status)
		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
	}
}

// internal/controller/manageddatabase_controller.go

func (r *ManagedDatabaseReconciler) cleanupExternalResources(ctx context.Context, dbInstance *dbV1alpha1.ManagedDatabase) error {
	logger := log.FromContext(ctx)

	if dbInstance.Status.InstanceID == "" {
		// If there's no instance ID, the external resource was likely never created.
		// We can safely remove the finalizer.
		logger.Info("No InstanceID in status. Assuming external resource does not exist.")
		return nil
	}

	// Step 1: Trigger deletion of the external resource.
	logger.Info("Deleting external database instance", "InstanceID", dbInstance.Status.InstanceID)
	if err := r.Provider.DeleteDB(dbInstance.Status.InstanceID); err != nil {
		// This is a critical failure. We must return an error to retry.
		// The finalizer will NOT be removed.
		return fmt.Errorf("failed to trigger deletion of external database %s: %w", dbInstance.Status.InstanceID, err)
	}

	// Step 2: Poll for confirmation of deletion.
	// In a real-world scenario, this is crucial. Deletion is asynchronous.
	externalDB, err := r.Provider.GetDB(dbInstance.Status.InstanceID)
	if err != nil {
		// If we get a "not found" error, it means deletion is complete.
		// Our mock provider's DeleteDB is idempotent, so this works.
		logger.Info("External database successfully deleted (confirmed by GetDB error)")
		return nil
	}

	if externalDB.Status == "DELETING" {
		// The resource is still being deleted. We must requeue and check again later.
		// We return an error to trigger a requeue. A custom error type or specific message
		// could allow for more intelligent backoff in the main loop.
		logger.Info("External database is still deleting. Requeuing.")
		return fmt.Errorf("deletion pending for instance %s", dbInstance.Status.InstanceID)
	}

	logger.Info("External database cleanup complete.")
	return nil
}

    # Get the object in its raw JSON form
    kubectl get manageddatabase my-db-instance -n my-ns -o json > my-db.json

    # Edit my-db.json and remove the "db.example.com/finalizer" entry from the metadata.finalizers array.
    # Then, update the object's finalizers subresource:
    kubectl replace --raw "/api/v1/namespaces/my-ns/manageddatabases/my-db-instance/finalize" -f ./my-db.json

    # A more direct (but potentially racy) approach is patching:
    kubectl patch manageddatabase my-db-instance -n my-ns --type json -p='[{"op": "remove", "path": "/metadata/finalizers"}]'

// ... inside cleanupExternalResources

externalDB, err := r.Provider.GetDB(dbInstance.Status.InstanceID)
if err != nil {
    // Let's assume our provider returns a specific error type for NotFound
    if dbprovider.IsNotFound(err) {
        logger.Info("External resource confirmed deleted.")
        return nil // Success
    }
    // Any other error is a transient failure, retry with backoff
    return err 
}

if externalDB.Status == "DELETING" {
    // This is not an error, but a state requiring a future check.
    // We'll return a specific error that our main Reconcile loop can interpret.
    return &RequeueError{Message: "Deletion pending", RequeueAfter: 30 * time.Second}
}

// A custom error type to control requeue behavior
type RequeueError struct {
    Message      string
    RequeueAfter time.Duration
}

func (e *RequeueError) Error() string {
    return e.Message
}

// In the main Reconcile function:
// ...
if err := r.cleanupExternalResources(ctx, dbInstance); err != nil {
    if requeueErr, ok := err.(*RequeueError); ok {
        logger.Info(requeueErr.Message)
        return ctrl.Result{RequeueAfter: requeueErr.RequeueAfter}, nil
    }
    // It's a real error, use exponential backoff
    logger.Error(err, "Cleanup failed with a transient error")
    return ctrl.Result{}, err
}
// ...

// cmd/main.go

// ...
err = ctrl.NewControllerManagedBy(mgr). 
    For(&dbv1alpha1.ManagedDatabase{}). 
    WithOptions(controller.Options{MaxConcurrentReconciles: 10}). 
    Complete(&dbcontroller.ManagedDatabaseReconciler{ 
        Client: mgr.GetClient(), 
        Scheme: mgr.GetScheme(), 
		Provider: dbprovider.NewMockProvider(),
    })
// ...

import "golang.org/x/time/rate"

// In your provider client setup
limiter := rate.NewLimiter(rate.Limit(10), 1) // 10 requests per second, burst of 1

// In your API call function
if err := limiter.Wait(ctx); err != nil {
    return err // Context cancelled
}
// Proceed with API call