Kubernetes Native Scaling

HPA for OTel Collector

# The OTel Collector is stateless — it can be horizontally scaled based on CPU/memory:
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: otel-collector-hpa
  namespace: observe
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: otel-collector
  minReplicas: 2
  maxReplicas: 10
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
    - type: Resource
      resource:
        name: memory
        target:
          type: Utilization
          averageUtilization: 80

PodDisruptionBudgets for Storage Components

# Ensure Loki ingesters are not all drained simultaneously during node maintenance:
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: loki-ingester-pdb
  namespace: observe
spec:
  maxUnavailable: 1   # At most 1 ingester down at a time
  selector:
    matchLabels:
      app.kubernetes.io/name: loki
      app.kubernetes.io/component: ingester

---
# Same for Tempo ingesters:
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: tempo-ingester-pdb
  namespace: observe
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/component: ingester
      app.kubernetes.io/name: tempo

Resource Requests and Limits

# Recommended resource configuration for production Kubernetes:
# (Adjust based on actual measurements after 1 week of production load)

OTel Collector:
  requests: { cpu: 500m, memory: 512Mi }
  limits:   { cpu: 2000m, memory: 2Gi }

Loki Ingester (per replica):
  requests: { cpu: 1000m, memory: 2Gi }
  limits:   { cpu: 4000m, memory: 8Gi }

Loki Querier (per replica):
  requests: { cpu: 500m, memory: 1Gi }
  limits:   { cpu: 2000m, memory: 4Gi }

Prometheus:
  requests: { cpu: 500m, memory: 2Gi }
  limits:   { cpu: 2000m, memory: 8Gi }

Tempo Ingester (per replica):
  requests: { cpu: 500m, memory: 1Gi }
  limits:   { cpu: 2000m, memory: 4Gi }

Grafana:
  requests: { cpu: 250m, memory: 512Mi }
  limits:   { cpu: 1000m, memory: 2Gi }

Namespace and RBAC Configuration

# Dedicated namespace with resource quotas:
apiVersion: v1
kind: Namespace
metadata:
  name: observe
  labels:
    purpose: observability

---
apiVersion: v1
kind: ResourceQuota
metadata:
  name: observe-quota
  namespace: observe
spec:
  hard:
    requests.cpu: "20"
    requests.memory: 40Gi
    limits.cpu: "40"
    limits.memory: 80Gi
    persistentvolumeclaims: "20"