Kubernetes Helm Deployment

Required Helm Repositories

# Add Grafana and Prometheus Helm repositories
helm repo add grafana https://grafana.github.io/helm-charts
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

Install kube-prometheus-stack (Prometheus + Grafana)

# Install Prometheus + Grafana in bizfirst-observe namespace
helm install kube-prom prometheus-community/kube-prometheus-stack \
  --namespace bizfirst-observe \
  --create-namespace \
  --values prometheus-values.yaml

prometheus-values.yaml (BizFirstGO customization)

grafana:
  enabled: true
  adminPassword: "your-secure-password"
  persistence:
    enabled: true
    size: 10Gi
  grafana.ini:
    feature_toggles:
      enable: traceqlEditor traceToMetrics
  additionalDataSources:
    - name: Loki
      type: loki
      url: http://loki:3100
      access: proxy
    - name: Tempo
      type: tempo
      url: http://tempo:3200
      access: proxy
      uid: tempo-uid

prometheus:
  prometheusSpec:
    retention: 90d
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: standard-ssd
          resources:
            requests:
              storage: 100Gi
    additionalScrapeConfigs:
      - job_name: 'bizfirst-services'
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
            action: keep
            regex: "true"

alertmanager:
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          resources:
            requests:
              storage: 5Gi

Install Loki Stack

helm install loki grafana/loki-stack \
  --namespace bizfirst-observe \
  --values loki-values.yaml

loki-values.yaml

loki:
  enabled: true
  persistence:
    enabled: true
    size: 50Gi
  config:
    limits_config:
      retention_period: 720h  # 30 days
    compactor:
      retention_enabled: true
    storage_config:
      aws:
        s3: s3://your-bucket/loki
        region: us-east-1
      bos_config:
        auth_enabled: false

promtail:
  enabled: true  # Kubernetes pod log collection
  config:
    clients:
      - url: http://loki:3100/loki/api/v1/push

Install Tempo

helm install tempo grafana/tempo \
  --namespace bizfirst-observe \
  --values tempo-values.yaml

tempo-values.yaml

tempo:
  storage:
    trace:
      backend: s3
      s3:
        bucket: your-tempo-bucket
        endpoint: s3.us-east-1.amazonaws.com
        region: us-east-1
  retention: 168h  # 7 days
  resources:
    requests:
      cpu: 500m
      memory: 1Gi
    limits:
      cpu: 2000m
      memory: 4Gi

Verify the Deployment

# Check all pods are running
kubectl get pods -n bizfirst-observe

# Expected output includes:
# alertmanager-kube-prom-alertmanager-0      Running
# kube-prom-grafana-xxxxx                    Running
# kube-prom-prometheus-0                    Running
# loki-0                                    Running
# tempo-0                                   Running

# Port-forward Grafana for initial setup
kubectl port-forward -n bizfirst-observe \
  svc/kube-prom-grafana 3000:80

# Access at http://localhost:3000