Mock Exam · Set 5

# clusters are registered as Secrets in the argocd namespace
kubectl -n argocd get secrets -l argocd.argoproj.io/secret-type=cluster
kubectl -n argocd label secret cluster-prod-eu   platform.acme.io/tier=prod
kubectl -n argocd label secret cluster-prod-us   platform.acme.io/tier=prod
kubectl -n argocd label secret cluster-prod-apac platform.acme.io/tier=prod

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: platform-addons
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ["missingkey=error"]
  generators:
    - clusters:
        selector:
          matchLabels:
            platform.acme.io/tier: prod
        values:
          revision: main
  template:
    metadata:
      name: '{{.nameNormalized}}-addons'
    spec:
      project: platform
      source:
        repoURL: https://github.com/acme/platform-config.git
        targetRevision: '{{.values.revision}}'
        path: 'clusters/{{.name}}/addons'
      destination:
        server: '{{.server}}'
        namespace: platform-system
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

kubectl apply -f platform-addons-appset.yaml
kubectl -n argocd get applicationset platform-addons -o jsonpath='{.status.conditions[?(@.type=="ResourcesUpToDate")].status}'
kubectl -n argocd get applications -l argocd.argoproj.io/application-set-name=platform-addons
kubectl -n argocd get applications -l argocd.argoproj.io/application-set-name=platform-addons \
  -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status

# remove the hand-written ones only after the generated ones are Healthy
kubectl -n argocd delete application prod-eu-addons-manual prod-us-addons-manual

apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: infrastructure
  namespace: flux-system
spec:
  interval: 10m
  retryInterval: 1m
  timeout: 5m
  sourceRef:
    kind: GitRepository
    name: platform-config
  path: ./infrastructure/controllers
  prune: true
  wait: true              # Ready means every applied object is healthy
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: apps
  namespace: flux-system
spec:
  interval: 10m
  dependsOn:
    - name: infrastructure
  sourceRef:
    kind: GitRepository
    name: platform-config
  path: ./apps/production
  prune: true
  wait: false             # not waiting, so name the checks that matter
  healthChecks:
    - apiVersion: apps/v1
      kind: Deployment
      name: cert-manager
      namespace: cert-manager
    - apiVersion: apps/v1
      kind: Deployment
      name: kube-prometheus-stack-operator
      namespace: monitoring

kubectl apply -f flux-kustomizations.yaml
flux get kustomizations
flux tree kustomization apps

# prove the gate — break the dependency for real so its Ready condition flips to False
kubectl -n flux-system patch kustomization infrastructure --type=merge \
  -p '{"spec":{"path":"./infrastructure/does-not-exist"}}'
flux reconcile kustomization infrastructure --with-source     # fails: path not found
flux get kustomizations infrastructure                        # READY False

flux reconcile kustomization apps                             # refuses to run
kubectl -n flux-system get kustomization apps \
  -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}{"\n"}'
#   dependency 'flux-system/infrastructure' is not ready

# put it back
kubectl -n flux-system patch kustomization infrastructure --type=merge \
  -p '{"spec":{"path":"./infrastructure/controllers"}}'
flux reconcile kustomization infrastructure --with-source

# 1. evidence first — what does the controller think is happening?
kubectl argo rollouts get rollout checkout -n checkout
kubectl -n checkout get rollout checkout \
  -o jsonpath='{.status.phase}{" / "}{.status.message}{"\n"}'
kubectl -n checkout get rollout checkout \
  -o jsonpath='{.status.stableRS}{" stable | canary "}{.status.currentPodHash}{"\n"}'

# why did the analysis fail? the measurements are on the AnalysisRun, not the Rollout
kubectl -n checkout get analysisrun
kubectl -n checkout describe analysisrun checkout-6d4f9c7b8-4-1 | grep -A12 'Metric Results'

# 2. abort: traffic goes back to stable immediately, rollout stays Degraded on purpose
kubectl argo rollouts abort checkout -n checkout
kubectl argo rollouts status checkout -n checkout --timeout 60s   # Degraded

# 3. the spec still asks for 4.7.0, so revert the pod template to the last good revision
kubectl argo rollouts undo checkout -n checkout --to-revision=4
#   equivalent, and clearer in a git-managed world — change it in Git and let Argo CD sync:
#   kubectl argo rollouts set image checkout checkout=ghcr.io/acme/checkout:4.6.3 -n checkout

kubectl argo rollouts status checkout -n checkout   # Healthy
kubectl -n checkout get rollout checkout -o jsonpath='{.status.phase}{"\n"}'
kubectl argo rollouts get rollout checkout -n checkout

# the guardrail that would have made 07:10 unnecessary:
spec:
  progressDeadlineSeconds: 600
  progressDeadlineAbort: true      # a stuck rollout aborts itself instead of hanging Degraded
  strategy:
    canary:
      abortScaleDownDelaySeconds: 30   # keep the canary RS briefly for forensics
      steps:
        - setWeight: 25
        - pause: { duration: 5m }
        - analysis:
            templates:
              - templateName: error-rate
      analysis:
        templates:
          - templateName: error-rate
        startingStep: 1

apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
  name: payments
  namespace: argocd
spec:
  description: Payments tenant — their repo, their namespaces, nothing cluster-scoped.
  sourceRepos:
    - https://github.com/acme/payments-config.git
  destinations:
    - server: https://kubernetes.default.svc
      namespace: 'payments-*'
  clusterResourceWhitelist: []          # empty list = no cluster-scoped kinds permitted
  namespaceResourceBlacklist:
    - group: ''
      kind: ResourceQuota
    - group: ''
      kind: LimitRange
    - group: rbac.authorization.k8s.io
      kind: Role
    - group: rbac.authorization.k8s.io
      kind: RoleBinding
  orphanedResources:
    warn: true

kubectl apply -f payments-appproject.yaml
kubectl -n argocd patch app payments --type=merge -p '{"spec":{"project":"payments"}}'
kubectl -n argocd get app payments -o jsonpath='{.spec.project}{"\n"}'

argocd proj get payments
argocd app sync payments            # fails on the ClusterRoleBinding
kubectl -n argocd get app payments -o jsonpath='{.status.conditions[*].message}{"\n"}'
argocd proj allow-cluster-resource payments '' Namespace   # if they genuinely need one kind

apiVersion: tekton.dev/v1
kind: Pipeline
metadata:
  name: build-release
spec:
  params:
    - name: branch
      type: string
      default: main
  workspaces:
    - name: shared
  tasks:
    - name: unit-tests
      taskRef:
        name: run-tests
      workspaces:
        - name: source
          workspace: shared
    - name: publish
      runAfter: [unit-tests]
      when:
        - input: "$(params.branch)"
          operator: in
          values: ["main"]
      taskRef:
        name: publish-image
      workspaces:
        - name: source
          workspace: shared
  finally:
    - name: cleanup
      taskRef:
        name: cleanup-workspace
      workspaces:
        - name: source
          workspace: shared
    - name: report
      params:
        - name: status
          value: "$(tasks.status)"          # Succeeded | Failed | Completed | None
        - name: tests
          value: "$(tasks.unit-tests.status)"
      taskRef:
        name: post-status

kubectl apply -f pipeline.yaml

tkn pipeline start build-release -p branch=feature-x \
  --workspace name=shared,volumeClaimTemplateFile=vct.yaml --showlog
kubectl get pipelinerun --sort-by=.metadata.creationTimestamp -o name | tail -1
kubectl get pipelinerun <run> -o jsonpath='{.status.skippedTasks[*].name}{"\n"}'   # publish
tkn pipelinerun describe <run>                                                    # cleanup Succeeded

tkn pipeline start build-release -p branch=main \
  --workspace name=shared,volumeClaimTemplateFile=vct.yaml --showlog
kubectl get pipelinerun <run2> -o jsonpath='{.status.skippedTasks[*].name}{"\n"}'  # empty

apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: environments.platform.acme.io
spec:
  group: platform.acme.io
  names:
    kind: Environment
    plural: environments
    singular: environment
    shortNames: [env]
    categories: [platform]        # kubectl get platform lists it alongside the rest
  scope: Namespaced
  versions:
    - name: v1beta1
      served: true
      storage: true
      subresources:
        status: {}
      additionalPrinterColumns:
        - name: Tier
          type: string
          jsonPath: .spec.tier
        - name: Ready
          type: string
          jsonPath: .status.conditions[?(@.type=="Ready")].status
        - name: Reason
          type: string
          priority: 1             # priority > 0 = only shown with -o wide
          jsonPath: .status.conditions[?(@.type=="Ready")].reason
        - name: Message
          type: string
          priority: 1
          jsonPath: .status.conditions[?(@.type=="Ready")].message
        - name: Age
          type: date
          jsonPath: .metadata.creationTimestamp
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              required: [tier]
              properties:
                tier:
                  type: string
                  enum: [dev, staging, prod]
            status:
              type: object
              properties:
                conditions:
                  type: array
                  items:
                    type: object
                    required: [type, status, lastTransitionTime, reason]
                    properties:
                      type:               { type: string }
                      status:             { type: string, enum: ["True", "False", "Unknown"] }
                      reason:             { type: string }
                      message:            { type: string }
                      observedGeneration: { type: integer, format: int64 }
                      lastTransitionTime: { type: string, format: date-time }
                  x-kubernetes-list-type: map
                  x-kubernetes-list-map-keys: [type]

kubectl apply -f environments-crd.yaml
kubectl get environments
kubectl get environments -o wide

# status is a separate endpoint now — this is how a controller writes it
kubectl patch environment demo --subresource=status --type=merge -p '{
  "status": {"conditions": [{
    "type": "Ready", "status": "True", "reason": "Provisioned",
    "message": "namespace, quota and rolebinding created",
    "lastTransitionTime": "2026-07-21T09:14:00Z"
  }]}}'

kubectl get environments          # READY now True
kubectl patch environment demo --type=merge -p '{"status":{"phase":"hacked"}}'
kubectl get environment demo -o jsonpath='{.status}{"\n"}'   # unchanged — status ignored

# 1. the garbage collector announces itself — this event is the whole diagnosis
kubectl get events -A --field-selector reason=OwnerRefInvalidNamespace
kubectl -n platform get cm demo-settings -o jsonpath='{.metadata.ownerReferences}{"\n"}'
kubectl -n team-a get environment demo -o jsonpath='{.metadata.uid}{"\n"}'

# 2. the bug: the child was created in `platform`, its owner lives in `team-a`.
#    Owner references may NOT cross namespaces. The GC treats a namespaced
#    dependent whose owner is in another namespace as having an unresolvable
#    owner and deletes it — immediately, every time.

# 3. the fix: create the child in the owner's namespace and use the live uid
apiVersion: v1
kind: ConfigMap
metadata:
  name: demo-settings
  namespace: team-a                      # same namespace as the Environment
  ownerReferences:
    - apiVersion: platform.acme.io/v1beta1
      kind: Environment
      name: demo
      uid: 8c1f4a2e-77b1-4f0a-9d2c-3b6e5a4c1f90    # the LIVE object's uid
      controller: true
      blockOwnerDeletion: true
data:
  TIER: dev

UID=$(kubectl -n team-a get environment demo -o jsonpath='{.metadata.uid}')
kubectl -n team-a create cm demo-settings --from-literal=TIER=dev
kubectl -n team-a patch cm demo-settings --type=merge -p "{\"metadata\":{\"ownerReferences\":[{\"apiVersion\":\"platform.acme.io/v1beta1\",\"kind\":\"Environment\",\"name\":\"demo\",\"uid\":\"$UID\",\"controller\":true,\"blockOwnerDeletion\":true}]}}"

sleep 120 && kubectl -n team-a get cm demo-settings        # still there
kubectl -n team-a delete environment demo
kubectl -n team-a get cm demo-settings                     # NotFound — collected

# 1. walk the chain: Provider -> ProviderRevision -> Deployment -> Pod
kubectl get providers
kubectl describe provider provider-aws-s3 | tail -25
kubectl get providerrevisions
kubectl describe providerrevision provider-aws-s3-9f2c1ab | tail -25
kubectl -n crossplane-system get pods
kubectl -n crossplane-system describe pod -l pkg.crossplane.io/revision=provider-aws-s3-9f2c1ab | grep -A8 Events
#   Failed to pull image ... manifest unknown  -> the package tag does not exist

# 2a. fix the package reference (a real, existing tag) and let the new revision take over
apiVersion: pkg.crossplane.io/v1
kind: Provider
metadata:
  name: provider-aws-s3
spec:
  package: xpkg.upbound.io/upbound/provider-aws-s3:v1.21.0
  revisionActivationPolicy: Automatic
  revisionHistoryLimit: 1
---
# 2b. the second failure: no credentials for the ProviderConfig the composition names
apiVersion: aws.upbound.io/v1beta1
kind: ProviderConfig
metadata:
  name: default
spec:
  credentials:
    source: Secret
    secretRef:
      namespace: crossplane-system
      name: aws-creds
      key: creds

kubectl create secret generic aws-creds -n crossplane-system --from-file=creds=./aws-creds.txt
kubectl apply -f provider.yaml -f providerconfig.yaml

kubectl get providers -w                  # INSTALLED True, HEALTHY True
kubectl get providerrevisions             # old revision Inactive, new one Active

kubectl apply -f claim.yaml
kubectl get claim,composite,managed
kubectl describe bucket <name> | grep -A10 Conditions   # Synced/Ready and the real cause if not

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: provision-namespace
  namespace: platform
spec:
  serviceAccountName: namespace-provisioner
  entrypoint: provision
  arguments:
    parameters:
      - name: team
      - name: quotaCPU
        value: "8"
  templates:
    - name: provision
      inputs:
        parameters:
          - name: team
          - name: quotaCPU
      steps:
        - - name: namespace
            template: apply-namespace
            arguments:
              parameters: [{ name: team, value: "{{inputs.parameters.team}}" }]
        - - name: quota
            template: apply-quota
            arguments:
              parameters:
                - { name: team,     value: "{{inputs.parameters.team}}" }
                - { name: quotaCPU, value: "{{inputs.parameters.quotaCPU}}" }
        - - name: rolebinding
            template: apply-rolebinding
            arguments:
              parameters: [{ name: team, value: "{{inputs.parameters.team}}" }]

    - name: apply-namespace
      inputs:
        parameters: [{ name: team }]
      retryStrategy:
        limit: "3"
        retryPolicy: OnError            # transient API errors only, not logic failures
        backoff:
          duration: "5s"
          factor: "2"
      resource:
        action: apply                   # idempotent: create-or-update, never "already exists"
        successCondition: status.phase == Active
        failureCondition: status.phase == Terminating
        manifest: |
          apiVersion: v1
          kind: Namespace
          metadata:
            name: team-{{inputs.parameters.team}}
            labels:
              platform.acme.io/team: "{{inputs.parameters.team}}"
              platform.acme.io/managed: "true"

    - name: apply-quota
      inputs:
        parameters: [{ name: team }, { name: quotaCPU }]
      resource:
        action: apply
        manifest: |
          apiVersion: v1
          kind: ResourceQuota
          metadata:
            name: team-quota
            namespace: team-{{inputs.parameters.team}}
          spec:
            hard:
              requests.cpu: "{{inputs.parameters.quotaCPU}}"
              requests.memory: 16Gi
              count/pods: "60"

    - name: apply-rolebinding
      inputs:
        parameters: [{ name: team }]
      resource:
        action: apply
        manifest: |
          apiVersion: rbac.authorization.k8s.io/v1
          kind: RoleBinding
          metadata:
            name: team-admins
            namespace: team-{{inputs.parameters.team}}
          roleRef:
            apiGroup: rbac.authorization.k8s.io
            kind: ClusterRole
            name: admin
          subjects:
            - kind: Group
              apiGroup: rbac.authorization.k8s.io
              name: "{{inputs.parameters.team}}-admins"

kubectl apply -f provision-namespace.yaml

argo submit -n platform --from workflowtemplate/provision-namespace -p team=research --wait
argo submit -n platform --from workflowtemplate/provision-namespace -p team=research --wait
argo list -n platform                       # both Succeeded

kubectl get ns team-research --show-labels
kubectl -n team-research get resourcequota team-quota
kubectl -n team-research get rolebinding team-admins

# if a step hangs, it is nearly always RBAC on the workflow service account:
kubectl auth can-i create namespaces --as=system:serviceaccount:platform:namespace-provisioner

apiVersion: flowcontrol.apiserver.k8s.io/v1     # v1beta3 on clusters older than 1.29
kind: PriorityLevelConfiguration
metadata:
  name: platform-controllers
spec:
  type: Limited
  limited:
    nominalConcurrencyShares: 10
    lendablePercent: 50
    limitResponse:
      type: Queue
      queuing:
        queues: 64
        handSize: 6
        queueLengthLimit: 50
---
apiVersion: flowcontrol.apiserver.k8s.io/v1
kind: FlowSchema
metadata:
  name: platform-controllers
spec:
  matchingPrecedence: 700          # lower number wins; must beat the catch-all at 10000
  priorityLevelConfiguration:
    name: platform-controllers
  distinguisherMethod:
    type: ByUser                   # one queue set per identity, so one bad SA cannot starve its peers
  rules:
    - subjects:
        - kind: ServiceAccount
          serviceAccount:
            name: environment-controller
            namespace: platform
      resourceRules:
        - verbs: ["*"]
          apiGroups: ["platform.acme.io"]
          resources: ["*"]
          clusterScope: true
          namespaces: ["*"]

kubectl api-resources | grep flowcontrol          # confirm the served version first
kubectl apply -f apf.yaml

kubectl get flowschema platform-controllers \
  -o jsonpath='{.status.conditions[?(@.type=="Dangling")].status}{"\n"}'      # False = it resolves

kubectl get flowschema platform-controllers -o jsonpath='{.metadata.uid}{"\n"}'
kubectl get environments -A -v=8 \
  --as=system:serviceaccount:platform:environment-controller 2>&1 | grep -i 'Pf-'

# and the load-shedding evidence, once it is under pressure:
kubectl get --raw /metrics | grep apiserver_flowcontrol_current_inqueue_requests
kubectl get --raw /debug/api_priority_and_fairness/dump_priority_levels

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: checkout-slo
  namespace: monitoring
  labels:
    release: kube-prometheus-stack      # must match the Prometheus ruleSelector
spec:
  groups:
    - name: checkout-sli
      interval: 30s
      rules:
        - record: sli:checkout_errors:ratio_rate5m
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[5m]))
              /
            sum(rate(http_requests_total{job="checkout"}[5m]))
        - record: sli:checkout_errors:ratio_rate30m
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[30m]))
              /
            sum(rate(http_requests_total{job="checkout"}[30m]))
        - record: sli:checkout_errors:ratio_rate1h
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[1h]))
              /
            sum(rate(http_requests_total{job="checkout"}[1h]))
        - record: sli:checkout_errors:ratio_rate6h
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[6h]))
              /
            sum(rate(http_requests_total{job="checkout"}[6h]))
        - record: sli:checkout_errors:ratio_rate3d
          expr: |
            sum(rate(http_requests_total{job="checkout",code=~"5.."}[3d]))
              /
            sum(rate(http_requests_total{job="checkout"}[3d]))

    - name: checkout-slo-burn
      rules:
        - alert: CheckoutErrorBudgetBurnFast
          expr: |
            sli:checkout_errors:ratio_rate1h  > (14.4 * 0.001)
            and
            sli:checkout_errors:ratio_rate5m  > (14.4 * 0.001)
          for: 2m
          labels:
            severity: critical
            team: checkout
          annotations:
            summary: "checkout is burning its 30-day error budget 14.4x"
            description: "2% of the 30-day budget consumed in the last hour. At this rate the whole budget is gone in ~50 hours."
        - alert: CheckoutErrorBudgetBurnSlow
          expr: |
            sli:checkout_errors:ratio_rate6h  > (6 * 0.001)
            and
            sli:checkout_errors:ratio_rate30m > (6 * 0.001)
          for: 15m
          labels:
            severity: warning
            team: checkout
          annotations:
            summary: "checkout is burning its error budget 6x — ticket, do not page"

# promtool validates a plain Prometheus rules file, not the PrometheusRule wrapper —
# lift the spec (the groups: block) into its own file before checking it
yq '.spec' checkout-slo.yaml > /tmp/checkout-rules.yaml
promtool check rules /tmp/checkout-rules.yaml

kubectl apply -f checkout-slo.yaml
kubectl -n monitoring get prometheusrule checkout-slo

kubectl -n monitoring port-forward svc/prometheus-operated 9090 &
curl -s 'http://localhost:9090/api/v1/query?query=sli:checkout_errors:ratio_rate5m'
curl -s 'http://localhost:9090/api/v1/rules' | grep -c CheckoutErrorBudget
curl -s --get --data-urlencode 'query=ALERTS{alertname="CheckoutErrorBudgetBurnFast"}' \
  http://localhost:9090/api/v1/query      # empty result vector in steady state

# 1. inject a known trace id at the edge and follow it. W3C format:
#    00-<32 hex trace id>-<16 hex span id>-01   (01 = sampled)
TP='00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01'
kubectl -n checkout run probe --rm -it --restart=Never --image=curlimages/curl --command -- \
  curl -s -H "traceparent: $TP" -o /dev/null -w '%{http_code}\n' http://gateway.edge:8080/api/checkout

# does the next hop even receive the header? echo it back from the receiving service
kubectl -n payments logs deploy/payments | grep -i traceparent | tail -5

# what does each service think it is configured to do?
kubectl -n payments set env deploy/payments --list | grep OTEL
kubectl -n checkout  set env deploy/checkout  --list | grep OTEL

# 2. the two usual causes, both fixed in the workload, not the collector

# (a) a service configured for a different propagation format: it emits and
#     expects b3 headers, its neighbours emit W3C traceparent. Nothing matches,
#     so every service starts a brand new trace.
env:
  - name: OTEL_PROPAGATORS
    value: tracecontext,baggage        # the same list in EVERY service
  - name: OTEL_SERVICE_NAME
    value: payments
  - name: OTEL_EXPORTER_OTLP_ENDPOINT
    value: http://otel-collector.observability.svc:4318
  - name: OTEL_TRACES_SAMPLER
    value: parentbased_always_on        # never let a child re-decide sampling

# (b) a proxy or gateway that does not forward the header. For an Envoy/Istio
#     gateway the app must still pass it through on the outbound call; for an
#     nginx ingress, stop stripping unknown headers.

kubectl -n payments rollout restart deploy/payments
kubectl -n checkout  rollout restart deploy/checkout

kubectl -n observability port-forward svc/jaeger-query 16686 &
curl -s "http://localhost:16686/api/traces/4bf92f3577b34da6a3ce929d0e0e4736" \
  | tr ',' '\n' | grep -c '"serviceName"'
curl -s "http://localhost:16686/api/services"

# 1. evidence, in a fixed order. Start with why the crashing pod died LAST time.
kubectl -n ledger get pods -o wide
kubectl -n ledger get pod ledger-7d9c-abcde \
  -o jsonpath='{.status.containerStatuses[0].lastState.terminated.reason}{" exit "}{.status.containerStatuses[0].lastState.terminated.exitCode}{"\n"}'
#   OOMKilled exit 137   -> the kernel killed it; this is a limits problem, not a bug

kubectl -n ledger get pod ledger-7d9c-abcde \
  -o jsonpath='{.spec.containers[0].resources}{"\n"}'
kubectl top pods -n ledger

# then why the third pod cannot start at all
kubectl -n ledger describe pod ledger-7d9c-fghij | grep -A6 Events
#   0/3 nodes are available: 3 Insufficient memory  -> nothing to do with the app
kubectl describe nodes | grep -A6 'Allocated resources'
kubectl get events -n ledger --sort-by=.lastTimestamp | tail -20

# the chain: traffic up 30% -> working set exceeded the 256Mi limit -> container
# OOMKilled -> restart backoff -> CrashLoopBackOff. The 8Gi "fix" then made the
# replacement pod unschedulable, because no node has 8Gi free. Two symptoms,
# one cause, and the second symptom is self-inflicted.

spec:
  template:
    spec:
      containers:
        - name: ledger
          image: ghcr.io/acme/ledger:4.6.3
          resources:
            requests:
              cpu: 200m
              memory: 512Mi      # observed working set ~430Mi under the new load
            limits:
              memory: 768Mi      # headroom, and small enough to schedule
              # no cpu limit: CPU is compressible, throttling is worse than bursting

kubectl -n ledger apply -f ledger-deployment.yaml
kubectl -n ledger rollout status deploy/ledger

kubectl -n ledger get deploy ledger -o jsonpath='{.status.availableReplicas}{"\n"}'
kubectl -n ledger get pods -o jsonpath='{.items[*].status.containerStatuses[*].restartCount}{"\n"}'

# five minutes later: no container has died since the change
kubectl -n ledger get pods \
  -o jsonpath='{.items[*].status.containerStatuses[*].lastState.terminated.reason}{"\n"}'
# and no node-level OOM either — the kubelet reports those against the Node as SystemOOM
kubectl get events -A --field-selector reason=SystemOOM

apiVersion: apps/v1
kind: Deployment
metadata:
  name: ingress
  namespace: platform
spec:
  replicas: 6
  selector:
    matchLabels:
      app: ingress
  template:
    metadata:
      labels:
        app: ingress
    spec:
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule       # zones: enforce
          labelSelector:
            matchLabels:
              app: ingress
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: ScheduleAnyway      # nodes: prefer, never block
          labelSelector:
            matchLabels:
              app: ingress
      containers:
        - name: ingress
          image: ghcr.io/acme/ingress:1.9.2
          resources:
            requests: { cpu: 200m, memory: 256Mi }
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: ingress
  namespace: platform
spec:
  maxUnavailable: 1          # NOT minAvailable: 6 — that permits zero disruptions
  selector:
    matchLabels:
      app: ingress
  unhealthyPodEvictionPolicy: AlwaysAllow   # policy/v1, GA in newer clusters

kubectl apply -f ingress.yaml
kubectl -n platform rollout status deploy/ingress

# how many zones are the pods actually in?
for n in $(kubectl -n platform get pods -l app=ingress -o jsonpath='{.items[*].spec.nodeName}'); do
  kubectl get node "$n" -o jsonpath='{.metadata.labels.topology\.kubernetes\.io/zone}{"\n"}'
done | sort | uniq -c

kubectl -n platform get pdb ingress -o jsonpath='{.status.disruptionsAllowed}{"\n"}'
kubectl drain kind-worker2 --ignore-daemonsets --delete-emptydir-data --timeout=120s
kubectl uncordon kind-worker2

apiVersion: security.istio.io/v1
kind: PeerAuthentication
metadata:
  name: default
  namespace: payments
spec:
  mtls:
    mode: STRICT              # no selector = the whole namespace
---
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata:
  name: deny-all
  namespace: payments
spec: {}                      # no rules, default action ALLOW = nothing is allowed
---
apiVersion: security.istio.io/v1
kind: AuthorizationPolicy
metadata:
  name: allow-checkout-to-charges
  namespace: payments
spec:
  selector:
    matchLabels:
      app: payments-api
  action: ALLOW
  rules:
    - from:
        - source:
            principals: ["cluster.local/ns/checkout/sa/checkout"]
      to:
        - operation:
            methods: ["GET", "POST"]
            paths: ["/v1/charges", "/v1/charges/*"]

kubectl apply -f payments-mtls.yaml
istioctl analyze -n payments

# allowed caller -> 200
kubectl -n checkout exec deploy/checkout -c checkout -- \
  curl -s -o /dev/null -w '%{http_code}\n' http://payments-api.payments:8080/v1/charges

# meshed but wrong identity -> 403 (RBAC: access denied)
kubectl -n analytics exec deploy/reporter -c reporter -- \
  curl -s -o /dev/null -w '%{http_code}\n' http://payments-api.payments:8080/v1/charges

# no sidecar, plaintext -> connection reset by the STRICT listener
kubectl -n default run probe --rm -it --restart=Never --image=curlimages/curl --command -- \
  curl -sv --max-time 5 http://payments-api.payments:8080/v1/charges

kubectl -n payments get peerauthentication,authorizationpolicy

kubectl -n opencost get deploy opencost
kubectl -n opencost port-forward svc/opencost 9003:9003 &

# 1. by namespace, one row per namespace for the whole window
curl -s "http://localhost:9003/allocation/compute?window=7d&aggregate=namespace&accumulate=true" \
  | tr '}' '}\n' | grep -E '"name"|totalCost' | head -40

# 2. by team label, with idle reported as its own line rather than shared out
curl -s "http://localhost:9003/allocation/compute?window=7d&aggregate=label:team&accumulate=true&includeIdle=true"

# 3. the same window shared instead, for a chargeback number that sums to the bill
curl -s "http://localhost:9003/allocation/compute?window=7d&aggregate=namespace&accumulate=true&shareIdle=true"

# label-based aggregation only works if the labels exist. This is the platform's job:
apiVersion: v1
kind: Namespace
metadata:
  name: analytics
  labels:
    team: data-platform
    platform.acme.io/cost-centre: "CC-4402"

# right-sizing evidence for the follow-up conversation: requested vs actually used
kubectl -n analytics get pods \
  -o custom-columns='POD:.metadata.name,CPU_REQ:.spec.containers[*].resources.requests.cpu,MEM_REQ:.spec.containers[*].resources.requests.memory'
kubectl top pods -n analytics

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: payments
spec:
  podSelector: {}                 # every pod in the namespace
  policyTypes: [Ingress, Egress]  # both directions, no rules = deny both
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns-egress
  namespace: payments
spec:
  podSelector: {}
  policyTypes: [Egress]
  egress:
    - to:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: kube-system
          podSelector:
            matchLabels:
              k8s-app: kube-dns       # AND, not OR: both selectors in ONE list item
      ports:
        - protocol: UDP
          port: 53
        - protocol: TCP
          port: 53                    # TCP 53 too — truncated answers and some resolvers
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-metrics-scrape
  namespace: payments
spec:
  podSelector:
    matchLabels:
      app: payments-api
  policyTypes: [Ingress]
  ingress:
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: monitoring
      ports:
        - protocol: TCP
          port: 9090
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-gateway-egress
  namespace: payments
spec:
  podSelector:
    matchLabels:
      app: payments-api
  policyTypes: [Egress]
  egress:
    - to:
        - ipBlock:
            cidr: 203.0.113.0/24
            except:
              - 203.0.113.7/32
      ports:
        - protocol: TCP
          port: 443

kubectl apply -f payments-netpol.yaml
kubectl -n payments get netpol

kubectl -n payments exec deploy/payments-api -- nslookup payments-db.data.svc.cluster.local
kubectl -n payments exec deploy/payments-api -- \
  timeout 5 wget -qO- --timeout=4 https://203.0.113.10/health ; echo "exit=$?"
kubectl -n payments exec deploy/payments-api -- \
  timeout 5 wget -qO- --timeout=4 https://example.com ; echo "exit=$? (expect non-zero)"

# lateral movement still denied
kubectl -n analytics run probe --rm -it --restart=Never --image=curlimages/curl --command -- \
  curl -s --max-time 4 http://payments-api.payments:8080/ ; echo "exit=$?"

# 1. what is running, and what is wrong with it — from the cluster, not from memory
kubectl get vulnerabilityreports -A \
  -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,CRIT:.report.summary.criticalCount,HIGH:.report.summary.highCount'
kubectl -n checkout get vulnerabilityreport replicaset-checkout-7d9c -o yaml | grep -B2 -A6 CRITICAL
kubectl -n checkout get vulnerabilityreport replicaset-checkout-7d9c \
  -o jsonpath='{.report.artifact.repository}:{.report.artifact.tag}{"\n"}'

# 2. the SBOM answers "which version of the package is actually in this image"
trivy image --format cyclonedx --output sbom.cdx.json ghcr.io/acme/checkout:4.6.3
grep -o '"name":"libxyz"[^}]*' sbom.cdx.json | head

# scan the SBOM itself — no registry pull, and it is the artifact you keep for audit
trivy sbom sbom.cdx.json --severity CRITICAL --exit-code 1 ; echo "exit=$?"

# the same gate in CI, and the flag that decides whether the gate is honest:
trivy image --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 \
  ghcr.io/acme/checkout:4.6.4

# 3. roll forward to the rebuilt image and let the operator re-scan
kubectl -n checkout set image deploy/checkout checkout=ghcr.io/acme/checkout:4.6.4
kubectl -n checkout rollout status deploy/checkout

kubectl get vulnerabilityreports -A \
  -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,CRIT:.report.summary.criticalCount'
kubectl -n checkout get vulnerabilityreport -o jsonpath='{.items[*].report.summary.criticalCount}{"\n"}'