Mock Exam · Set 4

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: platform-addons
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://github.com/acme/platform-config.git
    targetRevision: main
    path: infrastructure/observability
  destination:
    server: https://kubernetes.default.svc
    namespace: observability
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    managedNamespaceMetadata:          # Argo CD owns these on the namespace it creates
      labels:
        pod-security.kubernetes.io/enforce: baseline
        platform.acme.io/managed: "true"
      annotations:
        platform.acme.io/owner: platform-team
    syncOptions:
      - CreateNamespace=true
      - ServerSideApply=true
      - ApplyOutOfSyncOnly=true

# per-resource override, when only one object needs it:
metadata:
  annotations:
    argocd.argoproj.io/sync-options: ServerSideApply=true

kubectl apply -f platform-addons-app.yaml
argocd app sync platform-addons
argocd app get platform-addons

kubectl get ns observability -o jsonpath='{.metadata.labels}{"\n"}'
kubectl get crd prometheuses.monitoring.coreos.com \
  -o jsonpath='{.metadata.managedFields[*].manager}{"\n"}'
kubectl get crd prometheuses.monitoring.coreos.com \
  -o jsonpath='{.metadata.managedFields[?(@.manager=="argocd-controller")].operation}{"\n"}'   # Apply

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: checkout
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://github.com/acme/platform-config.git
    targetRevision: main
    path: apps/checkout/overlays/prod
  destination:
    server: https://kubernetes.default.svc
    namespace: checkout
  ignoreDifferences:
    # 1. drop everything a named field manager owns — the right tool for a second controller
    - group: apps
      kind: Deployment
      namespace: checkout
      managedFieldsManagers:
        - kube-controller-manager     # the HPA writes spec.replicas through this manager
    # 2. and match the injected array element by name, not by index
    - group: admissionregistration.k8s.io
      kind: ValidatingWebhookConfiguration
      name: checkout-validate
      jqPathExpressions:
        - '.webhooks[] | select(.name == "validate.checkout.acme.io") | .clientConfig.caBundle'
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - RespectIgnoreDifferences=true   # without this, self-heal reverts anyway

kubectl apply -f checkout-app.yaml
argocd app get checkout
argocd app diff checkout; echo $?        # 0 means no diff at all

kubectl -n argocd get app checkout -o jsonpath='{.status.sync.status}{"\n"}'
kubectl -n checkout get deploy checkout -o jsonpath='{.spec.replicas}{"\n"}'   # the HPA's number
kubectl get validatingwebhookconfiguration checkout-validate \
  -o jsonpath='{.webhooks[0].clientConfig.caBundle}{"\n"}' | head -c 40; echo
kubectl -n checkout get deploy checkout \
  -o jsonpath='{range .metadata.managedFields[*]}{.manager}{" "}{.operation}{"\n"}{end}'   # who owns what

apiVersion: v1
kind: ServiceAccount
metadata:
  name: tekton-triggers-sa
  namespace: ci
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: tekton-triggers-eventlistener
  namespace: ci
subjects:
  - kind: ServiceAccount
    name: tekton-triggers-sa
    namespace: ci
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: tekton-triggers-eventlistener-roles    # shipped by the Triggers install
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding                       # ALSO required: interceptors are cluster-scoped
metadata:
  name: tekton-triggers-eventlistener-clusterbinding
subjects:
  - kind: ServiceAccount
    name: tekton-triggers-sa
    namespace: ci
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: tekton-triggers-eventlistener-clusterroles
---
apiVersion: triggers.tekton.dev/v1beta1
kind: TriggerBinding
metadata:
  name: github-push
  namespace: ci
spec:
  params:
    - name: gitrevision
      value: $(body.head_commit.id)
    - name: gitrepositoryurl
      value: $(body.repository.clone_url)
---
apiVersion: triggers.tekton.dev/v1beta1
kind: TriggerTemplate
metadata:
  name: build-template
  namespace: ci
spec:
  params:
    - name: gitrevision
    - name: gitrepositoryurl
  resourcetemplates:
    - apiVersion: tekton.dev/v1
      kind: PipelineRun
      metadata:
        generateName: build-
      spec:
        pipelineRef:
          name: build-and-deploy
        params:
          - name: revision
            value: $(tt.params.gitrevision)
          - name: repo-url
            value: $(tt.params.gitrepositoryurl)
        workspaces:
          - name: shared
            volumeClaimTemplate:
              spec:
                accessModes: ["ReadWriteOnce"]
                resources:
                  requests:
                    storage: 1Gi
---
apiVersion: triggers.tekton.dev/v1beta1
kind: EventListener
metadata:
  name: build-listener
  namespace: ci
spec:
  serviceAccountName: tekton-triggers-sa
  triggers:
    - name: github-push
      interceptors:
        - ref:
            name: github
          params:
            - name: secretRef
              value:
                secretName: github-webhook-secret
                secretKey: secretToken
            - name: eventTypes
              value: ["push"]
      bindings:
        - ref: github-push
      template:
        ref: build-template

kubectl -n ci create secret generic github-webhook-secret \
  --from-literal=secretToken="$WEBHOOK_TOKEN"
kubectl apply -f triggers.yaml
kubectl -n ci get eventlistener build-listener
kubectl -n ci get pods -l eventlistener=build-listener      # the sink runs as a pod

kubectl -n ci port-forward svc/el-build-listener 8080:8080 &

BODY='{"head_commit":{"id":"a1b2c3d4e5f6"},"repository":{"clone_url":"https://github.com/acme/checkout.git"}}'
SIG=$(printf '%s' "$BODY" | openssl dgst -sha256 -hmac "$WEBHOOK_TOKEN" | awk '{print $2}')

curl -sS -X POST http://localhost:8080 \
  -H 'Content-Type: application/json' \
  -H 'X-GitHub-Event: push' \
  -H "X-Hub-Signature-256: sha256=$SIG" \
  -d "$BODY"

tkn pipelinerun list -n ci
kubectl -n ci get pipelinerun --sort-by=.metadata.creationTimestamp \
  -o jsonpath='{.items[-1:].spec.params[?(@.name=="revision")].value}{"\n"}'
kubectl -n ci logs -l eventlistener=build-listener --tail=40   # where silent failures show up

apiVersion: tekton.dev/v1
kind: Task
metadata:
  name: kaniko-build
  namespace: ci
spec:
  params:
    - name: IMAGE
      type: string
    - name: DOCKERFILE
      type: string
      default: ./Dockerfile
    - name: CONTEXT
      type: string
      default: ./
  workspaces:
    - name: source
    - name: dockerconfig
      optional: true
      mountPath: /kaniko/.docker      # where the credential chain looks
  results:
    - name: IMAGE_DIGEST
      description: Digest of the image that was pushed
    - name: IMAGE_URL
      description: Fully qualified image reference
  steps:
    - name: build-and-push
      image: gcr.io/kaniko-project/executor:v1.23.2
      workingDir: $(workspaces.source.path)
      env:
        - name: DOCKER_CONFIG
          value: /kaniko/.docker
      args:
        - --dockerfile=$(params.DOCKERFILE)
        - --context=$(workspaces.source.path)/$(params.CONTEXT)
        - --destination=$(params.IMAGE)
        - --digest-file=$(results.IMAGE_DIGEST.path)
      securityContext:
        runAsUser: 0                  # root INSIDE the container, not on the node
        privileged: false
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
          # least privilege, not zero privilege: unpacking a base image
          # restores file ownership and modes, so these are still required
          add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
    - name: record-url
      image: alpine:3.20
      script: |
        #!/bin/sh
        printf '%s' "$(params.IMAGE)" > $(results.IMAGE_URL.path)

apiVersion: tekton.dev/v1
kind: TaskRun
metadata:
  generateName: kaniko-build-
  namespace: ci
spec:
  taskRef:
    name: kaniko-build
  params:
    - name: IMAGE
      value: ghcr.io/acme/checkout:2.7.0
  workspaces:
    - name: source
      persistentVolumeClaim:
        claimName: source-cache
    - name: dockerconfig
      secret:
        secretName: ghcr-push          # type kubernetes.io/dockerconfigjson
        items:
          - key: .dockerconfigjson     # the key a docker-registry secret uses
            path: config.json          # the filename kaniko/docker expects

kubectl -n ci create secret docker-registry ghcr-push \
  --docker-server=ghcr.io --docker-username=acme-ci --docker-password="$GHCR_TOKEN"

kubectl create -f kaniko-taskrun.yaml
tkn taskrun logs -f -n ci --last

kubectl -n ci get taskrun --sort-by=.metadata.creationTimestamp \
  -o jsonpath='{.items[-1:].status.results[?(@.name=="IMAGE_DIGEST")].value}{"\n"}'
kubectl -n ci get pods -l tekton.dev/taskRun \
  -o jsonpath='{.items[*].spec.containers[*].securityContext.privileged}{"\n"}'
kubectl -n ci get pods -l tekton.dev/taskRun \
  -o jsonpath='{.items[*].spec.containers[*].securityContext.allowPrivilegeEscalation}{"\n"}'

apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: preview-error-rate
  namespace: checkout
spec:
  args:
    - name: service
  metrics:
    - name: error-ratio
      interval: 30s
      count: 4                       # four samples before a verdict
      failureLimit: 1
      successCondition: result[0] <= 0.01
      provider:
        prometheus:
          address: http://prometheus-operated.monitoring.svc:9090
          query: |
            sum(rate(http_requests_total{service="{{args.service}}",code=~"5.."}[2m]))
            /
            sum(rate(http_requests_total{service="{{args.service}}"}[2m]))
---
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: checkout
  namespace: checkout
spec:
  replicas: 8
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      app: checkout
  template:
    metadata:
      labels:
        app: checkout
    spec:
      containers:
        - name: checkout
          image: ghcr.io/acme/checkout:3.4.0
          ports:
            - containerPort: 8080
  strategy:
    blueGreen:
      activeService: checkout-active
      previewService: checkout-preview
      previewReplicaCount: 2          # 25% of production while under test
      autoPromotionEnabled: false
      scaleDownDelaySeconds: 120      # old stack stays warm for instant rollback
      abortScaleDownDelaySeconds: 30
      prePromotionAnalysis:
        templates:
          - templateName: preview-error-rate
        args:
          - name: service
            value: checkout-preview

kubectl -n checkout get svc checkout-active \
  -o jsonpath='{.spec.selector.rollouts-pod-template-hash}{"\n"}'   # note this first

# the plugin, not `kubectl set image` — that only understands built-in workload kinds
kubectl argo rollouts set image checkout checkout=ghcr.io/acme/checkout:3.5.0 -n checkout
kubectl argo rollouts get rollout checkout -n checkout --watch

kubectl -n checkout get analysisrun
kubectl -n checkout get analysisrun -o jsonpath='{.items[-1:].status.phase}{"\n"}'
kubectl -n checkout get rollout checkout -o jsonpath='{.status.phase}{"\n"}'   # Degraded

kubectl -n checkout get svc checkout-active \
  -o jsonpath='{.spec.selector.rollouts-pod-template-hash}{"\n"}'   # unchanged
kubectl argo rollouts abort checkout -n checkout
kubectl argo rollouts undo checkout -n checkout

# 1. EVIDENCE — a Terminating object always tells you both halves
kubectl -n demo get environment demo \
  -o jsonpath='{.metadata.deletionTimestamp}{"\n"}{.metadata.finalizers}{"\n"}'

# the namespace names its own blockers in status.conditions
kubectl get ns demo -o jsonpath='{range .status.conditions[*]}{.type}{": "}{.message}{"\n"}{end}'

# is the controller that owns that finalizer even running?
kubectl get crd environments.platform.acme.io
kubectl -n platform-system get deploy,pods

# what else is still alive in the namespace?
kubectl api-resources --verbs=list --namespaced -o name \
  | xargs -n1 kubectl get -n demo --show-kind --ignore-not-found

# 2. remove ONLY the orphaned finalizer, on the object that carries it
kubectl -n demo patch environment demo --type=json \
  -p '[{"op":"remove","path":"/metadata/finalizers/0"}]'

# same thing for every remaining object, once you have confirmed each is safe
for e in $(kubectl get environments -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}'); do
  kubectl -n "${e%%/*}" patch environment "${e##*/}" --type=merge \
    -p '{"metadata":{"finalizers":null}}'
done

# 3. the namespace finishes terminating by itself — no force, no /finalize
kubectl get environments -A
kubectl get ns demo

func (r *EnvironmentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
    var env platformv1.Environment
    if err := r.Get(ctx, req.NamespacedName, &env); err != nil {
        // gone: stop cleanly, never requeue a NotFound
        return ctrl.Result{}, client.IgnoreNotFound(err)
    }

    // ... do the actual work, which must itself be idempotent ...

    ready := metav1.Condition{
        Type:               "Ready",
        Status:             metav1.ConditionTrue,
        Reason:             "Provisioned",
        Message:            "all child resources are healthy",
        ObservedGeneration: env.Generation,
    }

    // SetStatusCondition reports whether it changed anything (apimachinery v0.27+)
    if meta.SetStatusCondition(&env.Status.Conditions, ready) {
        if err := r.Status().Update(ctx, &env); err != nil {
            // a conflict is normal; controller-runtime requeues with backoff
            return ctrl.Result{}, err
        }
    }

    // periodic resync, not a busy loop
    return ctrl.Result{RequeueAfter: 5 * time.Minute}, nil
}

func (r *EnvironmentReconciler) SetupWithManager(mgr ctrl.Manager) error {
    return ctrl.NewControllerManagedBy(mgr).
        For(&platformv1.Environment{},
            builder.WithPredicates(predicate.GenerationChangedPredicate{})).
        Complete(r)
}

# the CRD side: a status subresource, a typed conditions array, a printer column
      subresources:
        status: {}
      additionalPrinterColumns:
        - name: Ready
          type: string
          jsonPath: .status.conditions[?(@.type=="Ready")].status
        - name: Age
          type: date
          jsonPath: .metadata.creationTimestamp
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              properties:
                tier: { type: string, enum: [dev, staging, prod] }
            status:
              type: object
              properties:
                conditions:
                  type: array
                  x-kubernetes-list-type: map
                  x-kubernetes-list-map-keys: [type]
                  items:
                    type: object
                    required: [type, status, lastTransitionTime, reason]
                    properties:
                      type:               { type: string }
                      status:             { type: string, enum: ["True", "False", "Unknown"] }
                      reason:             { type: string }
                      message:            { type: string }
                      observedGeneration: { type: integer, format: int64 }
                      lastTransitionTime: { type: string, format: date-time }

kubectl -n platform-system port-forward deploy/environment-controller-manager 8080:8080 &
curl -s localhost:8080/metrics | grep -E 'controller_runtime_reconcile_total|workqueue_depth'
sleep 120
curl -s localhost:8080/metrics | grep controller_runtime_reconcile_total   # barely moved

kubectl get environment demo -n demo
kubectl wait --for=condition=Ready environment/demo -n demo --timeout=60s
kubectl get environment demo -n demo \
  -o jsonpath='{.status.conditions[?(@.type=="Ready")].observedGeneration}{"\n"}'

# 1. FOLLOW THE CHAIN — the claim names its composite, the composite names its children
kubectl -n team-a get postgresinstance db-1 \
  -o jsonpath='{.spec.resourceRef.kind}{" "}{.spec.resourceRef.name}{"\n"}'

XR=$(kubectl -n team-a get postgresinstance db-1 -o jsonpath='{.spec.resourceRef.name}')
kubectl get xpostgresinstance "$XR" -o yaml | sed -n '/status:/,$p'
kubectl describe xpostgresinstance "$XR" | sed -n '/Events/,$p'   # composition errors land here

kubectl get managed          # the SYNCED / READY columns point at the culprit

# 2. THE TWO CONDITIONS SAY DIFFERENT THINGS
kubectl get instance.rds.aws.upbound.io "$XR-db" \
  -o jsonpath='{.status.conditions[?(@.type=="Synced")].message}{"\n"}'
kubectl get instance.rds.aws.upbound.io "$XR-db" \
  -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}{"\n"}'
kubectl describe instance.rds.aws.upbound.io "$XR-db" | sed -n '/Events/,$p'

# the usual culprits behind Synced=False
kubectl get providers
kubectl get providerconfigs.aws.upbound.io
kubectl -n crossplane-system logs -l pkg.crossplane.io/provider --tail=50

# ROOT CAUSE: the Composition points at a ProviderConfig name that does not exist,
# so every composed resource sits at Synced=False, "cannot get referenced ProviderConfig".
apiVersion: aws.upbound.io/v1beta1
kind: ProviderConfig
metadata:
  name: default                  # the exact name the Composition references
spec:
  credentials:
    source: Secret
    secretRef:
      namespace: crossplane-system
      name: aws-creds
      key: creds
---
# and the base in the Composition must reference it explicitly
            base:
              apiVersion: rds.aws.upbound.io/v1beta1
              kind: Instance
              spec:
                forProvider:
                  region: eu-west-1
                  engine: postgres
                  instanceClass: db.t3.micro
                  allocatedStorage: 20
                providerConfigRef:
                  name: default

kubectl apply -f providerconfig.yaml
kubectl apply -f composition.yaml            # no need to touch the claim

kubectl get managed -w
kubectl -n team-a get postgresinstance db-1 \
  -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}{"\n"}'

apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
  name: node-service
  title: Node.js service (golden path)
  description: Creates a Node.js service repo with CI and a registered catalog entry.
  tags: [recommended, nodejs]
spec:
  type: service
  owner: group:default/platform-team
  parameters:
    - title: Service
      required: [name, owner]
      properties:
        name:
          title: Name
          type: string
          description: Lowercase DNS label; becomes the repo and component name.
          pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$'
          maxLength: 30
          ui:autofocus: true
        owner:
          title: Owner
          type: string
          description: The team that will be paged for this service.
          ui:field: OwnerPicker
          ui:options:
            catalogFilter:
              kind: Group
    - title: Repository
      required: [repoUrl]
      properties:
        repoUrl:
          title: Location
          type: string
          ui:field: RepoUrlPicker
          ui:options:
            allowedHosts: [github.com]
            allowedOwners: [acme]
  steps:
    - id: fetch
      name: Render the skeleton
      action: fetch:template
      input:
        url: ./skeleton
        values:
          name: ${{ parameters.name }}
          owner: ${{ parameters.owner }}

    - id: publish
      name: Publish to GitHub
      action: publish:github
      input:
        repoUrl: ${{ parameters.repoUrl }}
        description: ${{ parameters.name }}
        defaultBranch: main
        repoVisibility: private

    - id: register
      name: Register in the catalog
      action: catalog:register
      input:
        repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }}
        catalogInfoPath: /catalog-info.yaml

  output:
    links:
      - title: Repository
        url: ${{ steps.publish.output.remoteUrl }}
      - title: Open in catalog
        icon: catalog
        entityRef: ${{ steps.register.output.entityRef }}

# skeleton/catalog-info.yaml — rendered by fetch:template, then registered
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
  name: ${{ values.name }}
  annotations:
    github.com/project-slug: acme/${{ values.name }}
    backstage.io/techdocs-ref: dir:.
spec:
  type: service
  lifecycle: experimental
  owner: ${{ values.owner }}
  system: checkout

# app-config.yaml — the template only exists once the catalog is told about it
catalog:
  locations:
    - type: url
      target: https://github.com/acme/software-templates/blob/main/node-service/template.yaml
      rules:
        - allow: [Template]

curl -s 'http://localhost:7007/api/catalog/entities?filter=kind=template,metadata.name=node-service' \
  | jq -r '.[].metadata.name'

# after running the template from the UI:
curl -s 'http://localhost:7007/api/catalog/entities?filter=kind=component,metadata.name=orders-api' \
  | jq -r '.[] | .metadata.name + " owned by " + .spec.owner'

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
  name: env-tier-guard
spec:
  failurePolicy: Fail
  matchConstraints:
    resourceRules:
      - apiGroups:   ["platform.acme.io"]
        apiVersions: ["v1beta1"]
        operations:  ["CREATE", "UPDATE"]
        resources:   ["environments"]
  matchConditions:
    - name: only-prod-requests            # cheap short-circuit: skip everything else
      # tier is optional in the schema, so probe before you read it — a CEL error
      # here is a request failure, and failurePolicy: Fail would deny it
      expression: 'has(object.spec.tier) && object.spec.tier == "prod"'
  variables:
    - name: approved
      expression: >-
        has(namespaceObject.metadata.labels) &&
        'platform.acme.io/prod-approved' in namespaceObject.metadata.labels &&
        namespaceObject.metadata.labels['platform.acme.io/prod-approved'] == 'true'
  validations:
    - expression: variables.approved
      reason: Forbidden
      message: "tier=prod is only allowed in namespaces labelled platform.acme.io/prod-approved=true"
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
  name: env-tier-guard
spec:
  policyName: env-tier-guard
  validationActions: [Deny, Audit]
  matchResources:
    namespaceSelector:
      matchExpressions:
        - key: platform.acme.io/tenant
          operator: Exists

kubectl apply -f env-tier-guard.yaml
kubectl get validatingadmissionpolicy env-tier-guard -o jsonpath='{.status.typeChecking}{"\n"}'

kubectl label ns team-a          platform.acme.io/tenant=acme-a
kubectl label ns team-approved   platform.acme.io/tenant=acme-b platform.acme.io/prod-approved=true

kubectl -n team-a apply -f prod-env.yaml           # denied, with your message
kubectl -n team-approved apply -f prod-env.yaml    # created
kubectl -n team-approved get environment prod-1

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-slo
  namespace: monitoring
  labels:
    release: kube-prometheus-stack     # must match the Prometheus CR's ruleSelector
spec:
  groups:
    - name: checkout-slo-recording
      interval: 30s
      rules:
        - record: job:slo_errors:ratio_rate5m
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[5m]))
            /
            sum(rate(http_requests_total{job="checkout"}[5m]))
        - record: job:slo_errors:ratio_rate30m
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[30m]))
            /
            sum(rate(http_requests_total{job="checkout"}[30m]))
        - record: job:slo_errors:ratio_rate1h
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[1h]))
            /
            sum(rate(http_requests_total{job="checkout"}[1h]))
        - record: job:slo_errors:ratio_rate6h
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[6h]))
            /
            sum(rate(http_requests_total{job="checkout"}[6h]))

    - name: checkout-slo-alerting
      rules:
        - alert: CheckoutErrorBudgetBurnFast
          # 14.4x burn on a 0.1% budget = the whole 30-day budget gone in ~2 days
          expr: |
            job:slo_errors:ratio_rate1h > (14.4 * 0.001)
            and
            job:slo_errors:ratio_rate5m > (14.4 * 0.001)
          for: 2m
          labels:
            severity: critical
            team: checkout
          annotations:
            summary: "Checkout is burning its error budget 14.4x too fast"
            description: "At this rate the 30-day budget is gone in about 2 days."

        - alert: CheckoutErrorBudgetBurnSlow
          # 6x burn = budget gone in ~5 days. A ticket, not a page.
          expr: |
            job:slo_errors:ratio_rate6h > (6 * 0.001)
            and
            job:slo_errors:ratio_rate30m > (6 * 0.001)
          for: 15m
          labels:
            severity: warning
            team: checkout
          annotations:
            summary: "Checkout is burning its error budget 6x too fast"

# slo-tests.yaml — promtool unit tests: 5% errors is a fast burn and must page
rule_files:
  - slo-rules.yaml

evaluation_interval: 1m

tests:
  - interval: 1m
    input_series:
      - series: 'http_requests_total{job="checkout",code="200"}'
        values: '0+95x180'
      - series: 'http_requests_total{job="checkout",code="500"}'
        values: '0+5x180'
    alert_rule_test:
      - eval_time: 90m
        alertname: CheckoutErrorBudgetBurnFast
        exp_alerts:
          - exp_labels:
              severity: critical
              team: checkout
            exp_annotations:
              summary: "Checkout is burning its error budget 14.4x too fast"
              description: "At this rate the 30-day budget is gone in about 2 days."

# promtool wants a plain rules file, so lift .spec out of the PrometheusRule
kubectl -n monitoring get prometheusrule checkout-slo -o jsonpath='{.spec}' | yq -P > slo-rules.yaml

promtool check rules slo-rules.yaml
promtool test rules slo-tests.yaml

kubectl apply -f checkout-slo.yaml
kubectl -n monitoring get prometheusrule checkout-slo -o jsonpath='{.spec.groups[*].name}{"\n"}'
curl -s localhost:9090/api/v1/rules | jq -r '.data.groups[].name'
curl -s --get localhost:9090/api/v1/query \
  --data-urlencode 'query=job:slo_errors:ratio_rate1h' | jq '.data.result'

kubectl -n observability port-forward svc/jaeger-query 16686:16686 &

# 1. the slowest traces for one operation in the last hour
curl -s "http://localhost:16686/api/traces?service=checkout&operation=%2Fcheckout&lookback=1h&minDuration=2s&limit=20" \
  | jq -r '.data[] | [.traceID, (.spans | length), ([.spans[].duration] | max / 1000)] | @tsv'

# 2a. ONE SLOW SPAN? sort the spans of a single trace by duration
TRACE=<traceID-from-above>
curl -s "http://localhost:16686/api/traces/$TRACE" \
  | jq -r '.data[0].spans[] | [(.duration/1000|floor), .operationName] | @tsv' \
  | sort -nr | head -20

# 2b. MANY REPEATED SPANS? count operation names in the same trace
curl -s "http://localhost:16686/api/traces/$TRACE" \
  | jq -r '.data[0].spans[].operationName' | sort | uniq -c | sort -nr | head

#    -> 124  db.query   ... each ~15ms, none individually slow. An N+1.
#    which service owns them:
curl -s "http://localhost:16686/api/traces/$TRACE" \
  | jq -r '.data[0].spans[] | select(.operationName=="db.query")
           | .tags[] | select(.key=="db.statement") | .value' | sort -u | head

# 3. the guardrail: alert when the per-request fan-out ratio regresses.
#    traces_span_metrics_calls_total comes from the collector's spanmetrics connector.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-fanout
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  groups:
    - name: checkout-fanout
      rules:
        - alert: CheckoutDatabaseFanoutRegression
          expr: |
            sum(rate(traces_span_metrics_calls_total{service_name="checkout",span_name="db.query"}[10m]))
            /
            sum(rate(traces_span_metrics_calls_total{service_name="checkout",span_name="/checkout"}[10m]))
            > 10
          for: 10m
          labels:
            severity: warning
            team: checkout
          annotations:
            summary: "checkout issues more than 10 database calls per request"
            description: "A fan-out regression: individual calls are fast, the count per request is not."

# after the fix (batch the per-line-item query into one IN (...) query):
curl -s "http://localhost:16686/api/traces?service=checkout&operation=%2Fcheckout&lookback=15m&minDuration=2s" \
  | jq '.data | length'          # 0
curl -s "http://localhost:16686/api/traces?service=checkout&operation=%2Fcheckout&lookback=15m&limit=1" \
  | jq '[.data[0].spans[] | select(.operationName=="db.query")] | length'   # single digits

# THE CHAIN, IN ORDER. Do not skip a link; each one fails silently.

# 1. is the pod serving, and does the SERVICE name that port?
kubectl -n payments exec deploy/payments -- wget -qO- localhost:9102/metrics | head -3
kubectl -n payments get svc payments -o jsonpath='{.spec.ports[*].name}{"\n"}'
kubectl -n payments get svc payments -o jsonpath='{.metadata.labels}{"\n"}'
kubectl -n payments get endpointslices -l kubernetes.io/service-name=payments

# 2. does the ServiceMonitor select that SERVICE (not the pods) and name the PORT NAME?
kubectl -n payments get servicemonitor payments -o yaml

# 3. does the Prometheus CR select that ServiceMonitor, in that namespace?
kubectl -n monitoring get prometheus -o jsonpath='{.items[0].spec.serviceMonitorSelector}{"\n"}'
kubectl -n monitoring get prometheus -o jsonpath='{.items[0].spec.serviceMonitorNamespaceSelector}{"\n"}'

# 4. can Prometheus's ServiceAccount even list endpoints there?
kubectl auth can-i list endpoints -n payments \
  --as=system:serviceaccount:monitoring:kube-prometheus-stack-prometheus

# 5. ground truth — what config did the operator actually generate?
kubectl -n monitoring get secret prometheus-kube-prometheus-stack-prometheus \
  -o jsonpath='{.data.prometheus\.yaml\.gz}' | base64 -d | gunzip | grep -A6 payments

# the fixed ServiceMonitor — three corrections, all of them common
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: payments
  namespace: payments
  labels:
    release: kube-prometheus-stack     # (a) matched by the Prometheus ruleSelector-equivalent
spec:                                  #     spec.serviceMonitorSelector
  selector:
    matchLabels:
      app.kubernetes.io/name: payments # (b) must match the SERVICE's labels, not the pods'
  namespaceSelector:
    matchNames: [payments]
  endpoints:
    - port: metrics                    # (c) the Service PORT NAME, never a number
      path: /metrics
      interval: 30s

# if the Prometheus CR was only watching its own namespace:
kubectl -n monitoring patch prometheus kube-prometheus-stack-prometheus --type=merge -p \
  '{"spec":{"serviceMonitorNamespaceSelector":{},"serviceMonitorSelector":{"matchLabels":{"release":"kube-prometheus-stack"}}}}'

kubectl apply -f servicemonitor.yaml
kubectl -n monitoring port-forward svc/prometheus-operated 9090:9090 &

curl -s localhost:9090/api/v1/targets \
  | jq -r '.data.activeTargets[] | select(.labels.job=="payments") | [.health, .scrapeUrl, .lastError] | @tsv'
curl -s --get localhost:9090/api/v1/query \
  --data-urlencode 'query=up{job="payments"}' | jq -r '.data.result[0].value[1]'

apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: batch
spec:
  template:
    metadata:
      labels:
        workload-class: batch
    spec:
      nodeClassRef:
        group: karpenter.k8s.aws
        kind: EC2NodeClass
        name: default
      expireAfter: 168h                 # replace nodes weekly: this is your patching story
      taints:
        - key: workload-class
          value: batch
          effect: NoSchedule
      requirements:
        - key: karpenter.sh/capacity-type
          operator: In
          values: ["spot"]
        - key: kubernetes.io/arch
          operator: In
          values: ["amd64", "arm64"]
        - key: karpenter.k8s.aws/instance-family
          operator: In
          values: ["c6i", "c6g", "m6i", "m6g"]
        - key: karpenter.k8s.aws/instance-size
          operator: NotIn
          values: ["nano", "micro", "small"]
  limits:                                # the hard ceiling for the whole pool
    cpu: "200"
    memory: 800Gi
  disruption:
    consolidationPolicy: WhenEmptyOrUnderutilized
    consolidateAfter: 5m
    budgets:
      - nodes: "10%"                     # never churn more than a tenth at once
      - nodes: "0"                       # and nothing during weekday working hours
        schedule: "0 9 * * mon-fri"
        duration: 9h

# the workload has to opt in: the taint keeps everything else off the pool
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cruncher
  namespace: batch
spec:
  replicas: 30
  selector:
    matchLabels: { app: cruncher }
  template:
    metadata:
      labels: { app: cruncher }
    spec:
      nodeSelector:
        workload-class: batch
      tolerations:
        - key: workload-class
          operator: Equal
          value: batch
          effect: NoSchedule
      containers:
        - name: cruncher
          image: ghcr.io/acme/cruncher:2.2.0
          resources:
            requests: { cpu: "2", memory: 4Gi }

kubectl apply -f nodepool.yaml
kubectl get nodepool batch -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}{"\n"}'
kubectl get nodepool batch -o jsonpath='{.spec.limits}{"\n"}{.spec.disruption.budgets}{"\n"}'

kubectl apply -f cruncher.yaml
kubectl get nodeclaims -w
kubectl get nodes -L karpenter.sh/capacity-type,node.kubernetes.io/instance-type,workload-class
kubectl -n kube-system logs -l app.kubernetes.io/name=karpenter --tail=50 | grep -i launch

apiVersion: v1
kind: ResourceQuota
metadata:
  name: team-a-compute
  namespace: team-a
spec:
  hard:
    requests.cpu: "20"
    requests.memory: 40Gi
    limits.cpu: "40"
    limits.memory: 80Gi
    count/deployments.apps: "30"
    persistentvolumeclaims: "10"
---
apiVersion: v1
kind: ResourceQuota
metadata:
  name: team-a-batch
  namespace: team-a
spec:
  hard:
    pods: "20"
    requests.cpu: "4"
    requests.memory: 8Gi
  scopeSelector:                 # only pods with this PriorityClass count here
    matchExpressions:
      - scopeName: PriorityClass
        operator: In
        values: ["batch-background"]
---
apiVersion: v1
kind: LimitRange
metadata:
  name: team-a-defaults
  namespace: team-a
spec:
  limits:
    - type: Container
      default:                   # becomes limits when the pod omits them
        cpu: 500m
        memory: 512Mi
      defaultRequest:            # becomes requests when the pod omits them
        cpu: 100m
        memory: 128Mi
      min:
        cpu: 10m
        memory: 32Mi
      max:
        cpu: "4"
        memory: 8Gi
      maxLimitRequestRatio:      # a limit may not exceed 4x its request
        cpu: "4"
        memory: "4"

# wide-pod.yaml — deliberately over-committed: ratio 2000m / 100m = 20
apiVersion: v1
kind: Pod
metadata:
  name: wide
  namespace: team-a
spec:
  containers:
    - name: app
      image: nginx:1.27-alpine
      resources:
        requests: { cpu: 100m }
        limits:   { cpu: "2" }

kubectl apply -f tenant-limits.yaml
kubectl -n team-a describe quota
kubectl -n team-a describe limitrange team-a-defaults

# defaults are injected, so the ticket goes away
kubectl -n team-a run defaults --image=nginx:1.27-alpine --restart=Never
kubectl -n team-a get pod defaults -o jsonpath='{.spec.containers[0].resources}{"\n"}'

# the over-committed pod is refused at admission
kubectl -n team-a apply -f wide-pod.yaml
# Error from server (Forbidden): ... cpu max limit to request ratio per Container is 4,
# but provided ratio is 20.000000

kubectl -n team-a get quota team-a-batch -o jsonpath='{.status.used}{"\n"}'

apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
  name: rabbitmq-auth
  namespace: media
spec:
  secretTargetRef:
    - parameter: host              # the parameter the rabbitmq scaler expects
      name: rabbitmq-conn          # an existing Secret; never inline the URL
      key: host
---
apiVersion: keda.sh/v1alpha1
kind: ScaledJob
metadata:
  name: transcode
  namespace: media
spec:
  jobTargetRef:
    parallelism: 1
    completions: 1
    backoffLimit: 2
    template:
      spec:
        restartPolicy: Never
        containers:
          - name: worker
            image: ghcr.io/acme/transcode:1.8.0
            args: ["--once"]        # process exactly one item, then exit
            resources:
              requests: { cpu: "1", memory: 2Gi }
  pollingInterval: 15
  maxReplicaCount: 20               # concurrency cap
  successfulJobsHistoryLimit: 5
  failedJobsHistoryLimit: 5
  rollout:
    strategy: gradual               # spec changes do not nuke running Jobs
  scalingStrategy:
    strategy: accurate              # subtract in-flight Jobs from the queue length
  triggers:
    - type: rabbitmq
      metadata:
        protocol: amqp
        queueName: transcode
        mode: QueueLength
        value: "1"                  # one Job per queued message
      authenticationRef:
        name: rabbitmq-auth

kubectl -n media delete scaledobject transcode --ignore-not-found
kubectl apply -f scaledjob.yaml

kubectl -n media get scaledjob transcode
kubectl -n media get scaledjob transcode \
  -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}{"\n"}'

# publish five messages, then watch
kubectl -n media get jobs -w
kubectl -n media get jobs -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.succeeded}{"\n"}{end}'
kubectl -n media get pods
kubectl -n media describe scaledjob transcode | sed -n '/Events/,$p'

apiVersion: external-secrets.io/v1          # external-secrets.io/v1beta1 on older installs
kind: ClusterSecretStore
metadata:
  name: vault-kv
spec:
  provider:
    vault:
      server: https://vault.platform.svc:8200
      path: kv                              # the KV mount point
      version: v2
      auth:
        kubernetes:
          mountPath: kubernetes             # the Vault auth mount
          role: external-secrets            # a Vault role scoped to payments/* only
          serviceAccountRef:
            name: external-secrets
            namespace: external-secrets
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
  name: payments-db
  namespace: payments
spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-kv
  target:
    name: payments-db
    creationPolicy: Owner                   # ESO owns it: delete it and it comes back
    deletionPolicy: Retain
    template:
      engineVersion: v2
      type: Opaque
      data:
        DATABASE_URL: "postgres://{{ .username }}:{{ .password }}@pg.payments.svc:5432/payments?sslmode=require"
  data:
    - secretKey: username
      remoteRef:
        key: payments/db
        property: username
    - secretKey: password
      remoteRef:
        key: payments/db
        property: password

kubectl apply -f vault-store.yaml
kubectl get clustersecretstore vault-kv \
  -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}{"\n"}'

kubectl -n payments get externalsecret payments-db
kubectl -n payments get secret payments-db -o jsonpath='{.metadata.ownerReferences[0].kind}{"\n"}'
kubectl -n payments get secret payments-db -o jsonpath='{.data.DATABASE_URL}' | base64 -d; echo

# self-healing: ESO recreates what it owns
kubectl -n payments delete secret payments-db
kubectl -n payments get secret payments-db

# force an immediate resync instead of waiting out refreshInterval
kubectl -n payments annotate externalsecret payments-db force-sync="$(date +%s)" --overwrite
kubectl -n payments get secret payments-db -o jsonpath='{.data.DATABASE_URL}' | base64 -d; echo

kubectl -n external-secrets logs deploy/external-secrets --tail=30   # auth failures land here

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: tenant-bootstrap
spec:
  background: true                       # also applies to namespaces that already exist
  rules:
    - name: clone-platform-templates
      match:
        any:
          - resources:
              kinds: [Namespace]
              selector:
                matchLabels:
                  platform.acme.io/tenant: "?*"
      generate:
        namespace: "{{request.object.metadata.name}}"
        synchronize: true                # updates to the source flow to every clone
        cloneList:
          namespace: platform-templates
          kinds:
            - v1/Secret
            - v1/ConfigMap
          selector:
            matchLabels:
              platform.acme.io/clone: "true"

    - name: default-resource-requests
      match:
        any:
          - resources:
              kinds: [Pod]
              namespaceSelector:
                matchLabels:
                  platform.acme.io/tenant: "?*"
      mutate:
        patchStrategicMerge:
          spec:
            containers:
              - (name): "*"              # conditional anchor: every container
                resources:
                  requests:
                    +(cpu): "100m"       # add anchor: only when the key is absent
                    +(memory): "128Mi"

# the single source of truth, maintained in one place
kubectl create ns platform-templates
kubectl -n platform-templates create secret docker-registry ghcr-pull \
  --docker-server=ghcr.io --docker-username=acme-ci --docker-password="$GHCR_TOKEN"
kubectl -n platform-templates create cm platform-defaults --from-literal=REGION=eu-west-1
kubectl -n platform-templates label secret ghcr-pull      platform.acme.io/clone=true
kubectl -n platform-templates label cm     platform-defaults platform.acme.io/clone=true

kubectl apply -f tenant-bootstrap.yaml
kubectl get clusterpolicy tenant-bootstrap          # READY True before you test anything

kubectl create ns tenant-y
kubectl label ns tenant-y platform.acme.io/tenant=acme-y
kubectl -n tenant-y get secret,cm

kubectl -n tenant-y run bare --image=nginx:1.27-alpine --restart=Never
kubectl -n tenant-y get pod bare -o jsonpath='{.spec.containers[0].resources.requests}{"\n"}'

# the source is live, not a snapshot
kubectl -n platform-templates patch cm platform-defaults -p '{"data":{"REGION":"eu-central-1"}}'
sleep 30 && kubectl -n tenant-y get cm platform-defaults -o jsonpath='{.data.REGION}{"\n"}'

kubectl get policyreport -A                          # what fired, and where