Observability & Operations Labs

# sandbox.yaml — the service you will watch for the rest of the track.
# A tiny app that serves 200s on /, 404s on /err and 500s on /internal-err, plus
# two traffic generators. Read the app's routes, not their names: /err is a 404,
# so the endpoint you hammer for 5xx is /internal-err.
apiVersion: v1
kind: Namespace
metadata:
  name: checkout
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: checkout
  namespace: checkout
spec:
  replicas: 2
  selector:
    matchLabels: { app: checkout }
  template:
    metadata:
      labels: { app: checkout }
    spec:
      containers:
        - name: app
          image: quay.io/brancz/prometheus-example-app:v0.5.0
          ports:
            - { name: metrics, containerPort: 8080 }   # NAMED port — the ServiceMonitor needs this name
          resources:
            requests: { cpu: 20m, memory: 32Mi }
            limits:   { memory: 64Mi }
---
apiVersion: v1
kind: Service
metadata:
  name: checkout
  namespace: checkout
  labels:
    app: checkout                # ← the label the ServiceMonitor will select on
spec:
  selector: { app: checkout }
  ports:
    - { name: metrics, port: 8080, targetPort: metrics }
---
# steady, healthy traffic — ~20 good requests/sec
apiVersion: apps/v1
kind: Deployment
metadata:
  name: loadgen
  namespace: checkout
spec:
  replicas: 1
  selector: { matchLabels: { app: loadgen } }
  template:
    metadata:
      labels: { app: loadgen }
    spec:
      containers:
        - name: curl
          image: curlimages/curl:8.10.1
          command: ["sh", "-c"]
          args:
            - "while true; do i=0; while [ $i -lt 20 ]; do curl -s -o /dev/null http://checkout:8080/; i=$((i+1)); done; sleep 1; done"
---
# your fault injector — starts switched OFF. Scale it up to break things on purpose.
# Each replica adds ~20 500s/sec, so 1 replica ≈ a 50% error ratio against loadgen.
apiVersion: apps/v1
kind: Deployment
metadata:
  name: errgen
  namespace: checkout
spec:
  replicas: 0
  selector: { matchLabels: { app: errgen } }
  template:
    metadata:
      labels: { app: errgen }
    spec:
      containers:
        - name: curl
          image: curlimages/curl:8.10.1
          command: ["sh", "-c"]
          args:
            - "while true; do i=0; while [ $i -lt 20 ]; do curl -s -o /dev/null http://checkout:8080/internal-err; i=$((i+1)); done; sleep 1; done"

# servicemonitor.yaml — four things must line up. Get any one wrong and you get
# no error, no event, no warning: just a target that silently never appears.
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: checkout
  namespace: monitoring            # lives beside Prometheus; watches another namespace
  labels:
    release: kube-prometheus-stack # 1. must match Prometheus's serviceMonitorSelector
spec:
  selector:
    matchLabels:
      app: checkout                # 2. must match a label on the SERVICE (not on the pods)
  namespaceSelector:
    matchNames: [ checkout ]       # 3. must name the namespace the Service lives in
  endpoints:
    - port: metrics                # 4. the Service port NAME — never the port number
      path: /metrics
      interval: 15s

# The four golden signals, as real queries. Run each in the Prometheus UI.

# TRAFFIC — requests per second, last 5 minutes
sum(rate(http_requests_total{job="checkout"}[5m]))

# ERRORS — fraction of responses that are 5xx (0 until you scale errgen up)
sum(rate(http_requests_total{job="checkout",code=~"5.."}[5m]))
  / sum(rate(http_requests_total{job="checkout"}[5m]))

# LATENCY — p99 from a histogram. The sandbox app is too simple to export one,
# so practise the function against something that does: the API server.
histogram_quantile(0.99,
  sum(rate(apiserver_request_duration_seconds_bucket{verb="GET"}[5m])) by (le))

# SATURATION — two flavours worth knowing
1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)   # node memory in use
sum by (pod) (rate(container_cpu_cfs_throttled_seconds_total{namespace="checkout"}[5m]))

# recording-rules.yaml — precompute the expensive bits once, reuse them everywhere.
# Naming convention: level:metric:operations
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-recording
  namespace: monitoring
  labels:
    release: kube-prometheus-stack   # same adoption rule as the ServiceMonitor
spec:
  groups:
    - name: checkout.recording
      interval: 30s
      rules:
        - record: job:checkout_requests:rate5m
          expr: sum(rate(http_requests_total{job="checkout"}[5m]))
        - record: job:checkout_errors:rate5m
          expr: sum(rate(http_requests_total{job="checkout",code=~"5.."}[5m]))
        - record: job:checkout_errors:ratio_rate5m
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[5m]))
              / sum(rate(http_requests_total{job="checkout"}[5m]))

# alert-rule.yaml — a symptom-based alert. Note: it consumes the RECORDED series.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-alerts
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  groups:
    - name: checkout.alerts
      rules:
        - alert: CheckoutHighErrorRate
          expr: job:checkout_errors:ratio_rate5m > 0.05
          for: 2m                       # must stay true for 2m — kills flapping
          labels:
            severity: page              # ← Alertmanager routes on this in Lab 5
            team: payments
          annotations:
            summary: "Checkout is serving too many 5xx"
            description: "5xx ratio is {{ $value | humanizePercentage }} over 5m (threshold 5%)."
            runbook_url: "https://example.internal/runbooks/checkout-5xx"
        - alert: CheckoutTargetDown
          expr: up{job="checkout"} == 0
          for: 1m
          labels:
            severity: ticket
            team: payments
          annotations:
            summary: "Prometheus cannot scrape checkout"

# am-values.yaml — apply with:
#   helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
#     -n monitoring --reuse-values -f am-values.yaml
alertmanager:
  config:
    global:
      resolve_timeout: 5m
    route:
      receiver: default              # the catch-all leaf
      group_by: ["alertname", "namespace"]   # one notification per alert per namespace
      group_wait: 10s                # hold the first alert briefly, hoping friends arrive
      group_interval: 1m             # then batch new members of an existing group
      repeat_interval: 12h           # don't re-nag about the same firing group
      routes:
        - matchers: [ "severity = page" ]
          receiver: oncall
        - matchers: [ "severity = ticket" ]
          receiver: default
    receivers:
      - name: default
        webhook_configs:
          - url: http://sink.monitoring.svc/ticket
            send_resolved: true
      - name: oncall
        webhook_configs:
          - url: http://sink.monitoring.svc/page
            send_resolved: true
    inhibit_rules:                   # a page mutes the ticket-level noise it caused
      - source_matchers: [ "severity = page" ]
        target_matchers: [ "severity = ticket" ]
        equal: [ "namespace" ]      # BOTH alerts must carry this label or the rule
                                     # silently never matches — see step 7.

# dashboard-cm.yaml — a dashboard that survives a laptop dying.
# The kube-prometheus-stack Grafana runs a sidecar that watches for this label.
apiVersion: v1
kind: ConfigMap
metadata:
  name: checkout-golden-signals
  namespace: monitoring
  labels:
    grafana_dashboard: "1"        # ← the sidecar's trigger. No label, no dashboard.
# Replace the whole JSON below with your own exported dashboard once you've built it.
# Careful: EVERY line indented under the `|` is part of the file, comments included —
# a stray `# note to self` after the closing brace is invalid JSON and the sidecar
# will refuse the dashboard without telling Grafana why.
data:
  checkout-golden-signals.json: |
    {
      "uid": "checkout-golden",
      "title": "Checkout · Golden Signals",
      "schemaVersion": 39,
      "time": { "from": "now-1h", "to": "now" },
      "panels": [
        {
          "type": "timeseries", "title": "Traffic — req/s",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
          "targets": [ { "refId": "A", "expr": "job:checkout_requests:rate5m" } ]
        },
        {
          "type": "timeseries", "title": "Errors — 5xx ratio",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
          "targets": [ { "refId": "A", "expr": "job:checkout_errors:ratio_rate5m" } ]
        }
      ]
    }

# jaeger.yaml — all-in-one, in-memory storage. Perfect for a lab, never for prod.
apiVersion: v1
kind: Namespace
metadata: { name: tracing }
---
apiVersion: apps/v1
kind: Deployment
metadata: { name: jaeger, namespace: tracing }
spec:
  replicas: 1
  selector: { matchLabels: { app: jaeger } }
  template:
    metadata: { labels: { app: jaeger } }
    spec:
      containers:
        - name: jaeger
          image: jaegertracing/all-in-one:1.62.0
          env:
            - { name: COLLECTOR_OTLP_ENABLED, value: "true" }
          ports:
            - { name: ui,        containerPort: 16686 }
            - { name: otlp-grpc, containerPort: 4317 }
            - { name: otlp-http, containerPort: 4318 }
---
apiVersion: v1
kind: Service
metadata: { name: jaeger, namespace: tracing }
spec:
  selector: { app: jaeger }
  ports:
    - { name: ui,        port: 16686, targetPort: ui }
    - { name: otlp-grpc, port: 4317,  targetPort: otlp-grpc }
    - { name: otlp-http, port: 4318,  targetPort: otlp-http }

# otel-values.yaml — helm upgrade --install otelcol \
#   open-telemetry/opentelemetry-collector -n tracing -f otel-values.yaml
mode: deployment
replicaCount: 1
image:
  repository: otel/opentelemetry-collector-contrib   # the chart makes you choose explicitly
config:
  receivers:
    otlp:
      protocols:
        grpc: { endpoint: 0.0.0.0:4317 }
        http: { endpoint: 0.0.0.0:4318 }
  processors:
    batch: {}
  exporters:
    otlp/jaeger:
      endpoint: jaeger.tracing.svc:4317
      tls: { insecure: true }        # lab only — no TLS inside a throwaway cluster
    debug: { verbosity: normal }     # prints span counts to the collector's own log
  service:
    pipelines:
      traces:
        receivers:  [ otlp ]
        processors: [ batch ]
        exporters:  [ otlp/jaeger, debug ]

# LogQL — it deliberately looks like PromQL. Start with labels, then filter, then parse.

# 1. the stream selector: labels only. Always start here — it's the cheap part.
{namespace="monitoring"}

# 2. line filter: |= contains, != excludes, |~ regex
{namespace="monitoring"} |= "error"

# 3. parser + label filter: this is why structured logging pays off. Pick the parser
#    that matches the format — logfmt for most CNCF components, json for JSON loggers.
{namespace="monitoring"} | logfmt | level = "error"
#    same shape, different parser, for an app that emits JSON lines:
#      {namespace="checkout"} | json | level = "error"

# 4. metrics FROM logs — a range aggregation turns lines into a graph
sum by (pod) (rate({namespace="monitoring"} |= "error" [5m]))
count_over_time({namespace="tracing"} |= "error" [1h])

# 5. THE CORRELATION MOVE: paste the trace ID you copied in Lab 8
{namespace="tracing"} |= "4bf92f3577b34da6a3ce929d0e0e4736"

# slo-burn.yaml — 99.9% availability SLO over 30 days ⇒ 0.1% error budget (≈43 min/month).
# Two windows per alert: the long one says "this is real", the short one says "it's still happening".
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-slo
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  groups:
    - name: checkout.slo
      rules:
        # ratio_rate5m is NOT repeated here — Lab 3 already records it. Recording the
        # same series name in two rule groups writes duplicate samples for one series.
        # Define each recorded series exactly once, then reuse it.
        - record: job:checkout_errors:ratio_rate30m
          expr: sum(rate(http_requests_total{job="checkout",code=~"5.."}[30m]))
                  / sum(rate(http_requests_total{job="checkout"}[30m]))
        - record: job:checkout_errors:ratio_rate1h
          expr: sum(rate(http_requests_total{job="checkout",code=~"5.."}[1h]))
                  / sum(rate(http_requests_total{job="checkout"}[1h]))
        - record: job:checkout_errors:ratio_rate6h
          expr: sum(rate(http_requests_total{job="checkout",code=~"5.."}[6h]))
                  / sum(rate(http_requests_total{job="checkout"}[6h]))

        # FAST BURN — 14.4x budget burn: a month's budget gone in ~2 days ⇒ page a human
        - alert: CheckoutErrorBudgetFastBurn
          expr: job:checkout_errors:ratio_rate1h > (14.4 * 0.001)
                  and job:checkout_errors:ratio_rate5m > (14.4 * 0.001)
          for: 2m
          labels: { severity: page, team: payments }
          annotations:
            summary: "Checkout is burning its error budget 14.4x too fast"

        # SLOW BURN — 6x: real, but not a 3am problem ⇒ open a ticket
        - alert: CheckoutErrorBudgetSlowBurn
          expr: job:checkout_errors:ratio_rate6h > (6 * 0.001)
                  and job:checkout_errors:ratio_rate30m > (6 * 0.001)
          for: 15m
          labels: { severity: ticket, team: payments }
          annotations:
            summary: "Checkout is slowly draining its error budget"

# dora.promql — four delivery metrics, measured from the cluster you already have.
# kube-state-metrics ships with kube-prometheus-stack, so these work with zero extra install.

# 1. DEPLOYMENT FREQUENCY — every change to a Deployment's spec bumps its generation
sum(changes(kube_deployment_metadata_generation{namespace="checkout"}[1d]))

# 2. CHANGE FAILURE RATE — deploys that left the Deployment unavailable / all deploys
sum(changes(kube_deployment_status_condition{namespace="checkout",
      condition="Available",status="false"}[1d]))
  / sum(changes(kube_deployment_metadata_generation{namespace="checkout"}[1d]))

# 3. LEAD TIME — the cluster only knows the last mile (pod created → pod Ready).
#    HEADS UP: kube_pod_status_ready_time is an OPT-IN kube-state-metrics series. If the
#    query returns nothing, that's why. Turn it on with a values file and --reuse-values:
#      kube-state-metrics:
#        extraArgs: [ "--metric-opt-in-list=kube_pod_status_ready_time" ]
max(kube_pod_status_ready_time{namespace="checkout"}
      - kube_pod_created{namespace="checkout"})
#    True commit→prod lead time needs a number only CI knows. Push it from the pipeline:
#      echo "platform_deploy_lead_time_seconds $SECONDS_SINCE_COMMIT" | curl --data-binary @- \
#        http://pushgateway.monitoring:9091/metrics/job/deploy/service/checkout
avg_over_time(platform_deploy_lead_time_seconds{service="checkout"}[7d])

# 4. MTTR — how long the current page has been firing, right now
time() - ALERTS_FOR_STATE{alertname="CheckoutErrorBudgetFastBurn"}