Mock Exam · Set 6

# 1. evidence order — the AnalysisRun tells you what it asked and what came back
kubectl -n checkout get analysisrun -o wide
kubectl -n checkout describe analysisrun $(kubectl -n checkout get analysisrun \
  --sort-by=.metadata.creationTimestamp -o name | tail -1)
# look at status.metricResults[].measurements[].message

# 2. ask Prometheus the same question yourself
kubectl -n monitoring port-forward svc/kube-prometheus-stack-prometheus 9090:9090 &
curl -sG http://localhost:9090/api/v1/query \
  --data-urlencode 'query=job:checkout_success_rate:ratio_rate5m'
# {"status":"success","data":{"resultType":"vector","result":[]}}  <- empty: the rule does not exist

# 3. which rules did Prometheus actually load, and which does it select?
kubectl -n monitoring get prometheus -o jsonpath='{.items[0].spec.ruleSelector}{"\n"}'
curl -s http://localhost:9090/api/v1/rules | grep -o 'checkout' | head

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-slo
  namespace: monitoring
  labels:
    release: kube-prometheus-stack   # MUST match spec.ruleSelector on the Prometheus CR
spec:
  groups:
    - name: checkout.rules
      interval: 30s
      rules:
        - record: job:checkout_success_rate:ratio_rate5m
          expr: |
            sum by (service, namespace) (rate(http_requests_total{job="checkout", code!~"5.."}[5m]))
            /
            sum by (service, namespace) (rate(http_requests_total{job="checkout"}[5m]))

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate
  namespace: checkout
spec:
  args:
    - name: service
    - name: namespace
  metrics:
    - name: success-rate
      initialDelay: 60s          # let the 5m rate window fill before judging
      interval: 30s
      count: 4
      failureLimit: 1
      successCondition: len(result) > 0 && result[0] >= 0.95
      provider:
        prometheus:
          address: http://prometheus-operated.monitoring.svc:9090
          query: |
            job:checkout_success_rate:ratio_rate5m{service="{{args.service}}", namespace="{{args.namespace}}"}
---
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: checkout
  namespace: checkout
spec:
  replicas: 4
  selector:
    matchLabels:
      app: checkout
  template:
    metadata:
      labels:
        app: checkout
    spec:
      containers:
        - name: checkout
          image: ghcr.io/acme/checkout:3.4.0
          ports:
            - name: http
              containerPort: 8080
  strategy:
    canary:
      steps:
        - setWeight: 20
        - pause: { duration: 60s }
        - analysis:
            templates:
              - templateName: success-rate
            args:
              - name: service
                value: checkout
              - name: namespace
                valueFrom:
                  fieldRef:
                    fieldPath: metadata.namespace
        - setWeight: 60
        - pause: { duration: 60s }

kubectl apply -f prometheusrule.yaml
sleep 60                                     # one evaluation interval, then the series exists
curl -sG http://localhost:9090/api/v1/query \
  --data-urlencode 'query=job:checkout_success_rate:ratio_rate5m'

kubectl apply -f analysistemplate.yaml -f rollout.yaml
kubectl argo rollouts retry rollout checkout -n checkout
kubectl argo rollouts get rollout checkout -n checkout --watch
kubectl -n checkout get analysisrun --sort-by=.metadata.creationTimestamp \
  -o jsonpath='{.items[-1:].status.phase}'   # Successful
kubectl argo rollouts status checkout -n checkout

apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
  name: tenant-payments
  namespace: argocd
spec:
  description: Payments tenant — one repo, their own namespaces, no cluster scope.
  sourceRepos:
    - https://github.com/acme/platform-config.git
  destinations:
    - server: https://kubernetes.default.svc
      namespace: 'payments-*'
  clusterResourceWhitelist: []          # empty list = no cluster-scoped kinds at all
  namespaceResourceBlacklist:
    - group: ''
      kind: ResourceQuota
    - group: ''
      kind: LimitRange
  syncWindows:
    - kind: deny
      schedule: '0 9 * * 1-5'
      duration: 8h
      timeZone: Europe/London
      applications:
        - '*'
      manualSync: true                  # a human can still sync during the window

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: rogue
  namespace: argocd
spec:
  project: tenant-payments
  source:
    repoURL: https://github.com/acme/platform-config.git
    targetRevision: main
    path: apps/payments
  destination:
    server: https://kubernetes.default.svc
    namespace: kube-system              # not matched by payments-* — rejected

kubectl apply -f appproject.yaml
argocd proj get tenant-payments
argocd proj windows list tenant-payments

kubectl apply -f rogue-app.yaml
kubectl -n argocd get app rogue -o jsonpath='{.status.conditions[*].type}'
kubectl -n argocd get app rogue -o jsonpath='{.status.conditions[*].message}'
# application destination is not permitted in project 'tenant-payments'

argocd app sync payments-api            # inside the window: blocked unless manual is allowed

# the two image controllers are NOT part of a default `flux install`
flux install --components-extra=image-reflector-controller,image-automation-controller
kubectl api-resources | grep image.toolkit     # confirm the served version before you write YAML

apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImageRepository
metadata:
  name: podinfo
  namespace: flux-system
spec:
  image: ghcr.io/stefanprodan/podinfo
  interval: 5m
---
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImagePolicy
metadata:
  name: podinfo
  namespace: flux-system
spec:
  imageRepositoryRef:
    name: podinfo
  policy:
    semver:
      range: 6.x
---
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImageUpdateAutomation
metadata:
  name: podinfo
  namespace: flux-system
spec:
  interval: 5m
  sourceRef:
    kind: GitRepository
    name: flux-system
  git:
    checkout:
      ref:
        branch: main
    commit:
      author:
        name: fluxcdbot
        email: fluxcdbot@users.noreply.github.com
      messageTemplate: |
        chore(images): {{range .Changed.Changes}}{{print .OldValue}} -> {{println .NewValue}}{{end}}
    push:
      branch: main
  update:
    path: ./apps/podinfo
    strategy: Setters

# apps/podinfo/deployment.yaml — the marker comment is the whole mechanism
spec:
  template:
    spec:
      containers:
        - name: podinfo
          image: ghcr.io/stefanprodan/podinfo:6.5.4 # {"$imagepolicy": "flux-system:podinfo"}

kubectl apply -f image-automation.yaml
flux get images repository -A
flux get images policy -A                  # LATEST IMAGE resolves within one interval
flux reconcile image repository podinfo -n flux-system
flux reconcile image update podinfo -n flux-system

git fetch origin && git log --oneline -1 origin/main
kubectl -n podinfo get deploy podinfo -o jsonpath='{.spec.template.spec.containers[0].image}'

apiVersion: tekton.dev/v1
kind: Task
metadata:
  name: pytest
spec:
  params:
    - name: version
      type: string
  steps:
    - name: test
      image: python:$(params.version)-alpine
      script: |
        #!/bin/sh
        set -eu
        python --version
        echo "running the suite on $(params.version)"
---
apiVersion: tekton.dev/v1
kind: Task
metadata:
  name: cleanup
spec:
  steps:
    - name: cleanup
      image: alpine:3.20
      script: |
        #!/bin/sh
        echo "removing scratch data"
---
apiVersion: tekton.dev/v1
kind: Pipeline
metadata:
  name: verify-and-ship
spec:
  params:
    - name: environment
      type: string
      default: staging
  tasks:
    - name: unit-tests
      taskRef:
        name: pytest
      matrix:
        params:
          - name: version
            value: ["3.11", "3.12", "3.13"]     # one PipelineTask, three TaskRuns
    - name: promote
      runAfter: [unit-tests]
      when:
        - input: $(params.environment)
          operator: in
          values: ["prod"]
      taskRef:
        name: cleanup                            # stand-in for the real promote Task
  finally:
    - name: cleanup
      taskRef:
        name: cleanup
    - name: notify-on-failure
      when:
        - input: $(tasks.status)
          operator: in
          values: ["Failed"]
      taskRef:
        name: cleanup

kubectl apply -f pipeline.yaml
tkn pipeline start verify-and-ship --param environment=staging --showlog

RUN=$(kubectl get pipelinerun --sort-by=.metadata.creationTimestamp -o name | tail -1)
NAME=${RUN#pipelinerun.tekton.dev/}

# scope the label selector to THIS run, or a second sitting counts six TaskRuns
kubectl get taskrun -l tekton.dev/pipelineRun=$NAME,tekton.dev/pipelineTask=unit-tests \
  -o name | wc -l                                                           # 3
kubectl get $RUN -o jsonpath='{.status.skippedTasks[*].name}'               # promote
kubectl get $RUN -o jsonpath='{.status.skippedTasks[*].reason}'
# When Expressions evaluated to false
tkn pipelinerun describe $NAME

# 1. evidence: the resource tree shows Healthy with no message at all
argocd app get platform --show-operation
kubectl -n argocd get app platform -o jsonpath='{range .status.resources[*]}{.kind}{" "}{.health.status}{"\n"}{end}'
# Environment Healthy      <- with no health check registered, unknown kinds default to Healthy

apiVersion: v1
kind: ConfigMap
metadata:
  name: argocd-cm
  namespace: argocd
  labels:
    app.kubernetes.io/name: argocd-cm
    app.kubernetes.io/part-of: argocd
data:
  resource.customizations.health.platform.acme.io_Environment: |
    local hs = {}
    hs.status = "Progressing"
    hs.message = "waiting for the Environment controller"
    if obj.status ~= nil and obj.status.conditions ~= nil then
      for i, condition in ipairs(obj.status.conditions) do
        if condition.type == "Ready" and condition.status == "False" then
          hs.status = "Degraded"
          hs.message = condition.message
          return hs
        end
        if condition.type == "Ready" and condition.status == "True" then
          hs.status = "Healthy"
          hs.message = condition.message
          return hs
        end
      end
    end
    return hs

# 2. test the Lua against a real object BEFORE trusting it in a sync wave
kubectl -n demo get environment demo -o yaml > /tmp/env.yaml
argocd admin settings resource-overrides health /tmp/env.yaml --argocd-cm-path ./argocd-cm.yaml

# merge, never plain apply: argocd-cm holds url, dex and exclusions too, and a
# partial apply would drop every key you left out of your file
kubectl -n argocd patch cm argocd-cm --type merge --patch-file argocd-cm.yaml
kubectl -n argocd rollout restart deploy/argocd-server statefulset/argocd-application-controller

kubectl -n argocd get app platform -o jsonpath='{.status.health.status}'    # Degraded
kubectl -n demo patch environment demo --subresource=status --type=merge \
  -p '{"status":{"conditions":[{"type":"Ready","status":"True","reason":"Provisioned","message":"ready","lastTransitionTime":"2026-07-21T09:00:00Z"}]}}'
kubectl -n argocd get app platform -o jsonpath='{.status.health.status}'    # Healthy

apiVersion: apiextensions.crossplane.io/v1
kind: CompositeResourceDefinition
metadata:
  name: xpostgresinstances.platform.acme.io
spec:
  group: platform.acme.io
  names:
    kind: XPostgresInstance
    plural: xpostgresinstances
  claimNames:
    kind: PostgresInstance
    plural: postgresinstances
  defaultCompositionRef:
    name: postgres-standard
  versions:
    - name: v1alpha1
      served: true
      referenceable: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              required: [parameters]
              properties:
                parameters:
                  type: object
                  required: [size, team]
                  properties:
                    size:
                      type: string
                      enum: [small, medium, large]
                    team:
                      type: string
                      pattern: '^[a-z][a-z0-9-]{1,29}$'
      additionalPrinterColumns:
        - name: Size
          type: string
          jsonPath: .spec.parameters.size
        - name: Team
          type: string
          jsonPath: .spec.parameters.team

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: postgres-claim-guardrails
spec:
  validationFailureAction: Enforce      # per-rule validate.failureAction in newer Kyverno
  background: false
  rules:
    - name: require-cost-centre
      match:
        any:
          - resources:
              kinds:
                - platform.acme.io/v1alpha1/PostgresInstance
      validate:
        message: "a PostgresInstance needs a platform.acme.io/cost-centre annotation like cc-4021"
        pattern:
          metadata:
            annotations:
              platform.acme.io/cost-centre: "cc-?*"

    - name: large-needs-approval
      match:
        any:
          - resources:
              kinds:
                - platform.acme.io/v1alpha1/PostgresInstance
      preconditions:
        all:
          - key: "{{ request.object.spec.parameters.size }}"
            operator: Equals
            value: large
      validate:
        message: "size=large requires a platform.acme.io/approved-by label naming the DBA who signed it off"
        pattern:
          metadata:
            labels:
              platform.acme.io/approved-by: "?*"

# claim-bad.yaml — no cost centre, and large without an approver
apiVersion: platform.acme.io/v1alpha1
kind: PostgresInstance
metadata:
  name: demo
  namespace: team-a
spec:
  parameters:
    size: large
    team: team-a
---
# claim-good.yaml
apiVersion: platform.acme.io/v1alpha1
kind: PostgresInstance
metadata:
  name: demo
  namespace: team-a
  annotations:
    platform.acme.io/cost-centre: cc-4021
  labels:
    platform.acme.io/approved-by: r.okonjo
spec:
  parameters:
    size: large
    team: team-a
  writeConnectionSecretToRef:
    name: demo-postgres

kubectl apply -f xrd.yaml
kubectl get xrd xpostgresinstances.platform.acme.io      # ESTABLISHED and OFFERED both True
kubectl api-resources | grep postgresinstances           # the claim CRD now exists

kubectl apply -f policy.yaml
kubectl get clusterpolicy postgres-claim-guardrails      # READY True before you test

kubectl -n team-a apply -f claim-bad.yaml                # admission webhook denies it
kubectl -n team-a apply -f claim-good.yaml
kubectl -n team-a get postgresinstance demo
kubectl get xpostgresinstance,managed
kubectl -n team-a describe postgresinstance demo         # events explain a stuck composition

apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
  name: golden-service
  title: Golden path service
  description: A service that is attributable, observable and admissible on day one.
  tags: [recommended, golden-path]
spec:
  owner: group:platform
  type: service
  parameters:
    - title: Service details
      required: [name, costCentre, owner]
      properties:
        name:
          title: Name
          type: string
          pattern: '^[a-z][a-z0-9-]{2,29}$'
        costCentre:
          title: Cost centre
          type: string
          description: The finance code this service is billed to.
          pattern: '^cc-[0-9]{4}$'
        owner:
          title: Owner
          type: string
          ui:field: OwnerPicker
          ui:options:
            catalogFilter:
              kind: Group
  steps:
    - id: fetch
      name: Fetch skeleton
      action: fetch:template
      input:
        url: ./skeleton
        values:
          name: ${{ parameters.name }}
          costCentre: ${{ parameters.costCentre }}
          owner: ${{ parameters.owner }}
    - id: publish
      name: Publish the repository
      action: publish:github
      input:
        repoUrl: github.com?owner=acme&repo=${{ parameters.name }}
        description: Scaffolded by the golden path
    - id: register
      name: Register in the catalog
      action: catalog:register
      input:
        repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }}
        catalogInfoPath: /catalog-info.yaml
  output:
    links:
      - title: Repository
        url: ${{ steps.publish.output.remoteUrl }}
      - title: Open in the catalog
        icon: catalog
        entityRef: ${{ steps.register.output.entityRef }}

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
  name: require-cost-centre
spec:
  failurePolicy: Fail
  matchConstraints:
    resourceRules:
      - apiGroups:   ["apps"]
        apiVersions: ["v1"]
        operations:  ["CREATE", "UPDATE"]
        resources:   ["deployments"]
  validations:
    - expression: |
        has(object.spec.template.metadata) &&
        has(object.spec.template.metadata.labels) &&
        'platform.acme.io/cost-centre' in object.spec.template.metadata.labels &&
        object.spec.template.metadata.labels['platform.acme.io/cost-centre'].matches('^cc-[0-9]{4}$')
      message: "every pod template needs a platform.acme.io/cost-centre label of the form cc-NNNN — scaffold from the golden path template"
      reason: Invalid
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
  name: require-cost-centre
spec:
  policyName: require-cost-centre
  validationActions: [Deny, Audit]     # start with [Warn, Audit], then switch
  matchResources:
    namespaceSelector:
      matchLabels:
        platform.acme.io/tenant: "true"

kubectl label ns tenant-a platform.acme.io/tenant=true
kubectl apply -f vap.yaml

# measure before you enforce: Warn returns the message on the client without rejecting
kubectl patch validatingadmissionpolicybinding require-cost-centre --type=merge \
  -p '{"spec":{"validationActions":["Warn","Audit"]}}'
kubectl -n tenant-a create deployment probe --image=nginx     # created, with a warning

kubectl patch validatingadmissionpolicybinding require-cost-centre --type=merge \
  -p '{"spec":{"validationActions":["Deny","Audit"]}}'
kubectl -n tenant-a create deployment nope --image=nginx      # rejected
kubectl -n tenant-a apply -f good-deployment.yaml             # admitted
grep -n 'cc-\[0-9\]' template.yaml                            # same regex in both places

# 1. managedFields is hidden by default — you have to ask for it
kubectl -n shop get deploy web --show-managed-fields -o yaml | head -60
kubectl -n shop get deploy web --show-managed-fields \
  -o jsonpath='{range .metadata.managedFields[*]}{.manager}{"\t"}{.operation}{"\t"}{.subresource}{"\n"}{end}'

# who owns replicas, specifically?
kubectl -n shop get deploy web --show-managed-fields -o json \
  | grep -o '"manager":"[^"]*"\|f:replicas'

# web-resources.yaml — apply ONLY the fields you intend to own.
# Anything you include, you claim; anything you omit later, you release.
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web
  namespace: shop
spec:
  template:
    spec:
      containers:
        - name: web
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              memory: 512Mi

# 2. this conflicts, and the error names the owner and the exact path
kubectl -n shop apply --server-side --field-manager=platform-cli -f web-resources.yaml

# take ONLY these fields — the file has no replicas in it, so nothing else moves
kubectl -n shop apply --server-side --force-conflicts \
  --field-manager=platform-cli -f web-resources.yaml

# 3. prove it: no conflict on a repeat, and replicas still belongs to someone else
kubectl -n shop apply --server-side --field-manager=platform-cli \
  --dry-run=server -f web-resources.yaml && echo "no conflict"
kubectl -n shop get deploy web --show-managed-fields \
  -o jsonpath='{range .metadata.managedFields[*]}{.manager}{" -> "}{.fieldsV1}{"\n"}{end}' | grep replicas

# 1. the logs already contain the answer — grep for the shape, not the symptom
kubectl -n platform-system logs deploy/env-controller --tail=200 | grep -i forbidden
# configmaps is forbidden: User "system:serviceaccount:platform-system:env-controller"
#   cannot create resource "configmaps" in API group "" in the namespace "demo"

# 2. confirm it from the caller's identity rather than your own
SA=system:serviceaccount:platform-system:env-controller
kubectl auth can-i --list --as=$SA -n demo
kubectl auth can-i create configmaps --as=$SA -n demo          # no
kubectl auth can-i patch environments/status --as=$SA -n demo  # no
kubectl auth can-i update environments --as=$SA -n demo        # no  <- finalizers need this

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: platform:env-controller
rules:
  # the resource it owns
  - apiGroups: ["platform.acme.io"]
    resources: ["environments"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: ["platform.acme.io"]
    resources: ["environments/status"]
    verbs: ["get", "update", "patch"]
  - apiGroups: ["platform.acme.io"]
    resources: ["environments/finalizers"]
    verbs: ["update"]
  # the children it creates
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
  # so it can report what it did
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: platform:env-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: platform:env-controller
subjects:
  - kind: ServiceAccount
    name: env-controller
    namespace: platform-system

kubectl apply -f controller-rbac.yaml
kubectl -n platform-system rollout restart deploy/env-controller

kubectl auth can-i create configmaps --as=$SA -n demo          # yes
kubectl -n demo apply -f environment-demo2.yaml
kubectl -n demo wait --for=condition=Ready environment/demo2 --timeout=90s
kubectl -n demo get cm                                         # the child appears
kubectl -n demo get events --field-selector involvedObject.kind=Environment

apiVersion: v1
kind: ResourceQuota
metadata:
  name: platform-objects
  namespace: team-a
spec:
  hard:
    count/workers.platform.acme.io: "5"
    count/postgresinstances.platform.acme.io: "2"
    count/services.loadbalancers: "1"
    count/secrets: "25"
    count/configmaps: "40"

kubectl apply -f quota.yaml
kubectl -n team-a describe quota platform-objects

for i in 1 2 3 4 5 6; do
  kubectl -n team-a create -f - <<EOF
apiVersion: platform.acme.io/v1
kind: Worker
metadata:
  name: etl-$i
spec:
  queue: etl
EOF
done
# the sixth: Error from server (Forbidden): error when creating "STDIN":
#   workers.platform.acme.io "etl-6" is forbidden: exceeded quota: platform-objects,
#   requested: count/workers.platform.acme.io=1, used: 5, limited: 5

kubectl -n team-a get resourcequota platform-objects -o yaml | grep -A6 'used:'

# fixed evidence order. Step one is ALWAYS "what did Prometheus actually load?"
kubectl -n monitoring get secret prometheus-kube-prometheus-stack-prometheus \
  -o jsonpath='{.data.prometheus\.yaml\.gz}' | base64 -d | gunzip | grep -A5 payments
# nothing -> the ServiceMonitor was never selected. It is a discovery problem, not a scrape problem.

# what does the Prometheus CR select, and from where?
kubectl -n monitoring get prometheus -o jsonpath='{.items[0].spec.serviceMonitorSelector}{"\n"}'
kubectl -n monitoring get prometheus -o jsonpath='{.items[0].spec.serviceMonitorNamespaceSelector}{"\n"}'
# {"matchLabels":{"release":"kube-prometheus-stack"}}   and   {}  (all namespaces)

# what does the ServiceMonitor carry, and what does it select?
kubectl -n payments get servicemonitor api \
  -o jsonpath='{.metadata.labels}{"\n"}{.spec.selector}{"\n"}{.spec.endpoints[*].port}{"\n"}'
kubectl -n payments get svc api -o jsonpath='{.metadata.labels}{"\n"}{.spec.ports[*].name}{"\n"}'

# three faults, three one-line fixes
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: api
  namespace: payments
  labels:
    release: kube-prometheus-stack   # (1) was missing: the Prometheus CR never selected it
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: api    # (2) was app=api; the Service is labelled differently
  namespaceSelector:
    matchNames:
      - payments
  endpoints:
    - port: metrics                  # (3) was "8080": this must be the Service PORT NAME
      path: /metrics
      interval: 30s
---
apiVersion: v1
kind: Service
metadata:
  name: api
  namespace: payments
  labels:
    app.kubernetes.io/name: api
spec:
  selector:
    app.kubernetes.io/name: api
  ports:
    - name: metrics                  # a named port, or the ServiceMonitor has nothing to match
      port: 8080
      targetPort: 8080

kubectl apply -f servicemonitor.yaml -f service.yaml

# the operator regenerates the config; watch it appear rather than assuming
kubectl -n monitoring get secret prometheus-kube-prometheus-stack-prometheus \
  -o jsonpath='{.data.prometheus\.yaml\.gz}' | base64 -d | gunzip | grep -A5 payments

kubectl -n monitoring port-forward svc/kube-prometheus-stack-prometheus 9090:9090 &
curl -s 'http://localhost:9090/api/v1/targets?state=active' | grep -o '"job":"api"[^}]*health":"[a-z]*"'
curl -sG http://localhost:9090/api/v1/query --data-urlencode 'query=up{job="api"}'

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  memory_limiter:
    check_interval: 1s
    limit_percentage: 75
    spike_limit_percentage: 15
  k8sattributes:
    auth_type: serviceAccount
    passthrough: false
    extract:
      metadata:
        - k8s.namespace.name
        - k8s.pod.name
        - k8s.node.name
        - k8s.deployment.name
      labels:
        - tag_name: team
          key: platform.acme.io/team
          from: pod
    pod_association:
      - sources:
          - from: resource_attribute
            name: k8s.pod.ip
      - sources:
          - from: connection
  batch:
    timeout: 5s
    send_batch_size: 1024

connectors:
  spanmetrics:
    histogram:
      explicit:
        buckets: [10ms, 50ms, 100ms, 250ms, 500ms, 1s, 5s]
    dimensions:
      - name: k8s.namespace.name
      - name: k8s.deployment.name
      - name: http.route
    metrics_flush_interval: 15s

exporters:
  otlp/jaeger:
    endpoint: jaeger-collector.observability.svc:4317
    tls:
      insecure: true
  prometheus:
    endpoint: 0.0.0.0:8889
    resource_to_telemetry_conversion:
      enabled: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, k8sattributes, batch]
      exporters: [otlp/jaeger, spanmetrics]     # a connector is an exporter here
    metrics/spanmetrics:
      receivers: [spanmetrics]                   # ...and a receiver here
      processors: [batch]
      exporters: [prometheus]

# without this the processor starts, logs "forbidden", and silently enriches nothing
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-collector
rules:
  - apiGroups: [""]
    resources: ["pods", "namespaces", "nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources: ["replicasets"]          # required to derive k8s.deployment.name
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-collector
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: otel-collector
subjects:
  - kind: ServiceAccount
    name: otel-collector
    namespace: observability

kubectl apply -f otel-rbac.yaml
kubectl -n observability create cm otel-collector-conf --from-file=config.yaml \
  --dry-run=client -o yaml | kubectl apply -f -
kubectl -n observability rollout restart deploy/otel-collector
kubectl -n observability logs deploy/otel-collector | grep -i forbidden    # expect nothing

kubectl -n observability port-forward deploy/otel-collector 8889:8889 &
curl -s http://localhost:8889/metrics | grep traces_span_metrics_calls_total
curl -s http://localhost:8889/metrics | grep traces_span_metrics_duration_milliseconds_bucket | head -3

# 1. get the query right interactively before you commit it to a rule file
kubectl -n observability port-forward svc/loki 3100:3100 &
curl -sG http://localhost:3100/loki/api/v1/query_range \
  --data-urlencode 'query=sum by (app) (rate({namespace="payments"} |= `panic:` [5m]))' \
  --data-urlencode 'step=60'
# equivalently, if logcli is on the desktop:
logcli query --since=1h 'sum by (app) (rate({namespace="payments"} |= `panic:` [5m]))'

apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-rules-payments
  namespace: observability
  labels:
    loki_rule: "true"
data:
  # single-tenant Loki (auth_enabled: false) reads rules for the tenant "fake"
  payments.yaml: |
    groups:
      - name: payments-logs
        interval: 1m
        rules:
          - record: payments:log_panics:rate5m
            expr: |
              sum by (namespace, app) (rate({namespace="payments"} |= `panic:` [5m]))

          - alert: PaymentsPanicBurst
            expr: |
              sum by (namespace, app) (rate({namespace="payments"} |= `panic:` [5m])) > 0.1
            for: 5m
            labels:
              severity: critical
              team: payments
            annotations:
              summary: "panics in {{ $labels.app }}"
              description: "more than 0.1 panic lines per second for five minutes in {{ $labels.namespace }}"

# the ruler half of the Loki config — rules are useless if nothing evaluates them
ruler:
  storage:
    type: local
    local:
      directory: /rules
  rule_path: /tmp/loki/rules-temp
  ring:
    kvstore:
      store: inmemory
  enable_api: true
  enable_alertmanager_v2: true
  alertmanager_url: http://kube-prometheus-stack-alertmanager.monitoring.svc:9093

# loki-rules-mount.yaml — a rule file the ruler cannot SEE is a rule file that never runs.
# It must be on disk under ruler.storage.local.directory / <tenant>.
spec:
  template:
    spec:
      containers:
        - name: loki
          volumeMounts:
            - name: loki-rules
              mountPath: /rules/fake      # single-tenant Loki uses the tenant id "fake"
              readOnly: true
      volumes:
        - name: loki-rules
          configMap:
            name: loki-rules-payments

kubectl apply -f loki-rules.yaml
kubectl -n observability patch statefulset loki --patch-file loki-rules-mount.yaml
kubectl -n observability rollout restart statefulset/loki
kubectl -n observability rollout status statefulset/loki

curl -s http://localhost:3100/loki/api/v1/rules                 # the group, as loaded
curl -s http://localhost:3100/prometheus/api/v1/rules | grep PaymentsPanicBurst

# generate the signal, then watch the state machine move
kubectl -n payments exec deploy/api -- sh -c 'for i in 1 2 3 4 5 6 7 8 9 10; do echo "panic: runtime error"; done'
sleep 90
curl -s http://localhost:3100/prometheus/api/v1/rules | grep -o '"state":"[a-z]*"' | sort -u

# 1. a PartiallyFailed backup almost always starts at the storage location
kubectl -n velero get backupstoragelocation
# NAME      PHASE         LAST VALIDATED   ACCESS MODE
# default   Unavailable                    ReadWrite
kubectl -n velero describe backupstoragelocation default | tail -20
velero backup logs payments-firstrun | grep -i error    # the PartiallyFailed one from the story
kubectl -n velero logs deploy/velero | grep -i 'backup storage location'
# fix the bucket / credentials secret, then wait for PHASE Available
kubectl -n velero get backupstoragelocation -w

apiVersion: velero.io/v1
kind: Schedule
metadata:
  name: payments-nightly
  namespace: velero
spec:
  schedule: "0 2 * * *"
  template:
    includedNamespaces:
      - payments
    excludedResources:
      - events
      - events.events.k8s.io
    snapshotVolumes: false
    defaultVolumesToFsBackup: true      # file-level backup: no CSI snapshot support needed
    ttl: 720h0m0s                       # 30 days

kubectl apply -f schedule.yaml
velero schedule get

# 2. take one now rather than waiting for 02:00
velero backup create payments-adhoc --from-schedule payments-nightly --wait
velero backup describe payments-adhoc --details
velero backup get                                  # PHASE Completed, ERRORS 0

# 3. restore into a different namespace so production is never touched
velero restore create payments-dr --from-backup payments-adhoc \
  --namespace-mappings payments:payments-dr --wait
velero restore describe payments-dr --details
velero restore logs payments-dr | grep -i warn

kubectl -n payments-dr get deploy,cm,secret,pvc
kubectl -n payments-dr get pods

# 1. the PDB status is the whole diagnosis in one line
kubectl -n shop get pdb
# NAME   MIN AVAILABLE   MAX UNAVAILABLE   ALLOWED DISRUPTIONS   AGE
# api    3               N/A               0                     40d
kubectl -n shop get deploy api -o jsonpath='{.spec.replicas}{"\n"}'      # 3
# minAvailable == replicas -> zero disruptions are ever allowed. The drain can never finish.
kubectl -n shop get pdb api -o jsonpath='{.status}{"\n"}'
kubectl -n shop get pods -l app=api -o wide

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api
  namespace: shop
spec:
  maxUnavailable: 1                       # expressed relatively: survives replica changes
  selector:
    matchLabels:
      app: api
  unhealthyPodEvictionPolicy: AlwaysAllow  # a broken pod can no longer deadlock the drain
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api
  namespace: shop
spec:
  replicas: 4
  selector:
    matchLabels:
      app: api
  template:
    metadata:
      labels:
        app: api
    spec:
      terminationGracePeriodSeconds: 45
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: ScheduleAnyway
          labelSelector:
            matchLabels:
              app: api
      containers:
        - name: api
          image: ghcr.io/acme/api:5.1.0
          lifecycle:
            preStop:
              exec:
                # let endpoint removal propagate before the process starts shutting down
                command: ["/bin/sh", "-c", "sleep 10"]
          readinessProbe:
            httpGet:
              path: /healthz
              port: 8080
            periodSeconds: 5

kubectl apply -f pdb.yaml -f deployment.yaml
kubectl -n shop get pdb api -o jsonpath='{.status.disruptionsAllowed}{"\n"}'   # 1

kubectl cordon <node>
kubectl drain <node> --ignore-daemonsets --delete-emptydir-data --timeout=300s
echo $?                                                    # 0
kubectl -n shop get pods -l app=api -o wide                # rescheduled, never below 3 ready
kubectl uncordon <node>

apiVersion: v1
kind: Service
metadata:
  name: ledger-db
  namespace: platform
spec:
  clusterIP: None              # headless: gives each pod a stable DNS name
  selector:
    app: ledger-db
  ports:
    - name: pg
      port: 5432
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: ledger-db
  namespace: platform
spec:
  serviceName: ledger-db       # must name the headless Service above
  replicas: 3
  podManagementPolicy: OrderedReady
  persistentVolumeClaimRetentionPolicy:
    whenScaled: Delete         # scale down -> the surplus claim goes with it
    whenDeleted: Retain        # delete the set -> the data survives
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      partition: 2             # only ordinals >= 2 take the new template
  selector:
    matchLabels:
      app: ledger-db
  template:
    metadata:
      labels:
        app: ledger-db
    spec:
      terminationGracePeriodSeconds: 30
      containers:
        - name: db
          image: postgres:16-alpine
          env:
            - name: POSTGRES_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: ledger-db
                  key: password
            - name: PGDATA
              value: /var/lib/postgresql/data/pgdata
          ports:
            - name: pg
              containerPort: 5432
          volumeMounts:
            - name: data
              mountPath: /var/lib/postgresql/data
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: standard
        resources:
          requests:
            storage: 1Gi

kubectl -n platform create secret generic ledger-db --from-literal=password=s3cr3t
kubectl apply -f statefulset.yaml
kubectl -n platform rollout status sts/ledger-db

# stable identity
kubectl -n platform get pvc            # data-ledger-db-0 .. -2
kubectl -n platform exec ledger-db-0 -- nslookup ledger-db-1.ledger-db   # busybox is always there

# whenScaled: Delete
kubectl -n platform scale sts ledger-db --replicas=2
kubectl -n platform get pvc            # data-ledger-db-2 is gone

# whenDeleted: Retain
kubectl -n platform delete sts ledger-db
kubectl -n platform get pvc            # data-ledger-db-0 and -1 still there

# 1. sign, then verify from the CLI before you ever involve admission control
cosign generate-key-pair                       # cosign.key + cosign.pub
# keep signing and verification consistent with the policy's rekor.ignoreTlog below:
# sign without a transparency-log entry, and tell verify not to demand one
cosign sign --key cosign.key --tlog-upload=false -y ghcr.io/acme/checkout:3.4.0
cosign verify --key cosign.pub --insecure-ignore-tlog=true ghcr.io/acme/checkout:3.4.0 | head
cat cosign.pub                                 # this goes into publicKeys, verbatim

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: verify-images
spec:
  validationFailureAction: Enforce
  background: false
  webhookTimeoutSeconds: 30
  failurePolicy: Fail
  rules:
    - name: verify-signature
      match:
        any:
          - resources:
              kinds:
                - Pod
      verifyImages:
        - imageReferences:
            - "ghcr.io/acme/*"
          required: true          # unsigned is a rejection, not a shrug
          mutateDigest: true      # rewrite tag -> digest so the verified bytes are what runs
          verifyDigest: true
          attestors:
            - count: 1
              entries:
                - keys:
                    publicKeys: |-
                      -----BEGIN PUBLIC KEY-----
                      MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEXAMPLEEXAMPLEEXAMPLEEXAMPLE
                      EXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLEEXAMPLE==
                      -----END PUBLIC KEY-----
                    rekor:
                      ignoreTlog: true      # key-based signing without a transparency log

apiVersion: kyverno.io/v2
kind: PolicyException
metadata:
  name: allow-legacy-batch
  namespace: kyverno
spec:
  exceptions:
    - policyName: verify-images
      ruleNames:
        - verify-signature
        - autogen-verify-signature     # the auto-generated Deployment/Job variant
  match:
    any:
      - resources:
          kinds:
            - Pod
          namespaces:
            - legacy-batch

kubectl apply -f verify-images.yaml
kubectl get clusterpolicy verify-images                  # READY True before testing

# exceptions must be switched on, and the namespace holding them allow-listed
kubectl -n kyverno get deploy kyverno-admission-controller \
  -o jsonpath='{.spec.template.spec.containers[0].args}' | tr ',' '\n' | grep -i exception
kubectl apply -f policy-exception.yaml

kubectl -n shop run signed --image=ghcr.io/acme/checkout:3.4.0 --restart=Never
kubectl -n shop get pod signed -o jsonpath='{.spec.containers[0].image}{"\n"}'   # ...@sha256:...

kubectl -n shop run bad --image=ghcr.io/acme/checkout:tampered --restart=Never
# error: failed to verify image ghcr.io/acme/checkout:tampered: .attestors[0].entries[0].keys: no matching signatures

kubectl -n legacy-batch run bad --image=ghcr.io/acme/checkout:tampered --restart=Never   # allowed
kubectl get policyreport -A | grep verify-images

# 1. audit from the identity, never from your own admin context
SA=system:serviceaccount:payments:deploy-bot
kubectl auth can-i --list --as=$SA
kubectl auth can-i create clusterrolebindings --as=$SA          # yes  <- full escalation
kubectl auth can-i get secrets --as=$SA -A                      # yes  <- every secret

# which binding is responsible?
kubectl get clusterrolebinding -o jsonpath='{range .items[?(@.roleRef.name=="cluster-admin")]}{.metadata.name}{"\t"}{.subjects[*].name}{"\n"}{end}'
# anyone holding escalate or bind can grant themselves anything, so check those too
kubectl get clusterroles -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.rules[*].verbs}{"\n"}{end}' | grep -E 'escalate|bind|\*'

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: deploy-bot
  namespace: payments
rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    verbs: ["get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: deploy-bot
  namespace: payments
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: deploy-bot
subjects:
  - kind: ServiceAccount
    name: deploy-bot
    namespace: payments

apiVersion: v1
kind: ServiceAccount
metadata:
  name: default
  namespace: payments
automountServiceAccountToken: false     # nothing in this namespace gets a token by accident
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: deploy-bot
  namespace: payments
spec:
  replicas: 1
  selector:
    matchLabels:
      app: deploy-bot
  template:
    metadata:
      labels:
        app: deploy-bot
    spec:
      serviceAccountName: deploy-bot
      automountServiceAccountToken: false    # no ambient API token...
      containers:
        - name: bot
          image: ghcr.io/acme/deploy-bot:2.0.1
          volumeMounts:
            - name: api-token
              mountPath: /var/run/secrets/tokens
              readOnly: true
      volumes:
        - name: api-token
          projected:
            sources:
              - serviceAccountToken:
                  path: api-token              # ...only this narrow, expiring one
                  audience: vault
                  expirationSeconds: 3600

kubectl delete clusterrolebinding deploy-bot-admin
kubectl apply -f rbac.yaml -f serviceaccount.yaml -f deployment.yaml

kubectl auth can-i create clusterrolebindings --as=$SA           # no
kubectl auth can-i get secrets --as=$SA -A                       # no
kubectl auth can-i patch deployments --as=$SA -n payments        # yes — the job still works
kubectl auth can-i --list --as=$SA -n payments

# the projected token is short-lived and bound to one audience
kubectl -n payments exec deploy/deploy-bot -- cat /var/run/secrets/tokens/api-token \
  | cut -d. -f2 | base64 -d 2>/dev/null
# {"aud":["vault"],"exp":...,"kubernetes.io":{"namespace":"payments",...}}

# and no long-lived token Secret is left lying around
kubectl -n payments get secrets --field-selector type=kubernetes.io/service-account-token