Mock Exam · Set 3

apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
  name: tenant-payments
  namespace: argocd
spec:
  description: Payments tenant — one repo, one namespace, no cluster-scoped resources
  sourceRepos:
    - https://github.com/acme/platform-config.git
  destinations:
    - server: https://kubernetes.default.svc
      namespace: payments
  clusterResourceWhitelist: []          # empty list = no cluster-scoped kinds at all
  namespaceResourceBlacklist:           # platform owns these, tenants may not set them
    - group: ""
      kind: ResourceQuota
    - group: ""
      kind: LimitRange
  orphanedResources:
    warn: true                          # flag live resources the tenant never declared
  syncWindows:
    - kind: deny
      schedule: "0 18 * * 5"            # Friday 18:00 …
      duration: 62h                     # … until Monday 08:00
      timeZone: "Europe/London"
      applications: ["*"]
      manualSync: false                 # even a human cannot force a sync in the window
  roles:
    - name: read-only
      description: Tenant engineers may look, not sync
      policies:
        - p, proj:tenant-payments:read-only, applications, get, tenant-payments/*, allow

kubectl apply -f appproject.yaml
argocd proj get tenant-payments
argocd proj allow-cluster-resource tenant-payments '' Namespace   # if you must permit one

kubectl -n argocd patch app payments --type=merge -p '{"spec":{"project":"tenant-payments"}}'
kubectl -n argocd get app payments -o jsonpath='{.spec.project}{"\n"}'

# now commit a ClusterRole into apps/payments/ and sync:
argocd app sync payments
# FATA[0001] rpc error: ... Resource rbac.authorization.k8s.io:ClusterRole is not
#   permitted in project tenant-payments
kubectl -n argocd get app payments -o jsonpath='{.status.conditions[*].message}{"\n"}'

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: root
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io   # cascade: delete root, delete children
spec:
  project: default
  source:
    repoURL: https://github.com/acme/platform-config.git
    targetRevision: main
    path: bootstrap/prod
    directory:
      recurse: true
      include: '{*.yaml,*/*.yaml}'
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd            # the children are Application objects — they live here
  syncPolicy:
    automated:
      prune: true                # an Application deleted from Git is deleted for real
      selfHeal: true
    syncOptions:
      - CreateNamespace=true

# bootstrap/prod/30-monitoring.yaml — one of the fourteen children, ordered by wave
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: monitoring
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "30"
  finalizers:
    - resources-finalizer.argocd.argoproj.io
spec:
  project: platform
  source:
    repoURL: https://github.com/acme/platform-config.git
    targetRevision: main
    path: infrastructure/monitoring
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring
  syncPolicy:
    automated: { prune: true, selfHeal: true }
    syncOptions: [ CreateNamespace=true ]

kubectl apply -f root-app.yaml
kubectl -n argocd get applications                 # root + 14 children
argocd app get root
argocd app resources root                          # the children, as resources of root

# cascade check — on a scratch cluster only
kubectl -n argocd delete app root
kubectl -n argocd get applications                 # empty

apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImageRepository
metadata:
  name: api
  namespace: flux-system
spec:
  image: ghcr.io/acme/api
  interval: 5m
  secretRef:
    name: ghcr-auth              # docker-registry secret; omit for a public repo
---
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImagePolicy
metadata:
  name: api
  namespace: flux-system
spec:
  imageRepositoryRef:
    name: api
  policy:
    semver:
      range: ">=1.4.0 <2.0.0"    # patch and minor flow; a major never does
---
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImageUpdateAutomation
metadata:
  name: api
  namespace: flux-system
spec:
  interval: 5m
  sourceRef:
    kind: GitRepository
    name: flux-system            # must have WRITE credentials, not a read-only key
  git:
    checkout:
      ref:
        branch: main
    commit:
      author:
        name: fluxcdbot
        email: fluxcdbot@users.noreply.github.com
      messageTemplate: |
        chore(images): automated update

        {{ range .Updated.Images }}- {{ . }}
        {{ end }}
    push:
      branch: main               # set a different branch here to get a PR flow instead
  update:
    path: ./apps/api
    strategy: Setters

# apps/api/deployment.yaml — the marker is the whole contract
    spec:
      containers:
        - name: api
          image: ghcr.io/acme/api:1.4.2 # {"$imagepolicy": "flux-system:api"}

kubectl apply -f image-automation.yaml
flux get image repository api
flux get image policy api            # LATEST IMAGE  ghcr.io/acme/api:1.4.9

flux reconcile image repository api  # force a scan
flux reconcile image update api      # force a commit

git pull && git log --oneline -1
git show --stat HEAD
kubectl -n apps get deploy api -o jsonpath='{.spec.template.spec.containers[0].image}{"\n"}'

# these APIs have moved between beta versions — check the cluster in front of you
kubectl api-resources | grep image.toolkit

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: nightly-reconcile
  namespace: argo
spec:
  entrypoint: main
  onExit: notify                     # runs on success, failure and error
  podGC:
    strategy: OnWorkflowSuccess      # keep the pods of a failed run for debugging
  ttlStrategy:
    secondsAfterSuccess: 3600
    secondsAfterFailure: 86400       # failures stick around a day
  templates:
    - name: main
      dag:
        tasks:
          - name: drift-scan
            template: run
            arguments:
              parameters: [{ name: cmd, value: "drift-scan" }]
          - name: publish
            template: run
            dependencies: [drift-scan]
            arguments:
              parameters: [{ name: cmd, value: "publish" }]

    - name: run
      inputs:
        parameters: [{ name: cmd }]
      retryStrategy:
        limit: "3"
        retryPolicy: "OnTransientError"
        backoff:
          duration: "10s"
          factor: "2"
          maxDuration: "2m"
      container:
        image: alpine:3.20
        command: [sh, -c]
        args: ["echo running {{inputs.parameters.cmd}}; sleep 5"]

    - name: notify
      container:
        image: alpine:3.20
        command: [sh, -c]
        args:
          - "echo workflow {{workflow.name}} finished: {{workflow.status}} ({{workflow.duration}}s)"

apiVersion: argoproj.io/v1alpha1
kind: CronWorkflow
metadata:
  name: nightly-reconcile
  namespace: argo
spec:
  schedule: "0 2 * * *"              # newer Argo Workflows: schedules: ["0 2 * * *"]
  timezone: "Europe/London"
  concurrencyPolicy: Forbid          # never start a second copy on top of the first
  startingDeadlineSeconds: 300       # a missed slot is skipped, not run at 09:00
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  workflowSpec:
    workflowTemplateRef:
      name: nightly-reconcile

kubectl apply -f nightly.yaml
argo cron list -n argo
argo cron get nightly-reconcile -n argo

# test it now instead of waiting for 02:00
argo submit --from cronwf/nightly-reconcile -n argo --watch
argo get @latest -n argo             # the notify node appears at the end of the tree
argo logs @latest -n argo

kubectl -n argo get pods             # cleaned up after a successful run
argo cron suspend nightly-reconcile -n argo   # and resume, during a freeze

apiVersion: flagger.app/v1beta1
kind: MetricTemplate
metadata:
  name: checkout-error-ratio
  namespace: prod
spec:
  provider:
    type: prometheus
    address: http://prometheus-operated.monitoring.svc:9090
  query: |
    100 * (
      sum(
        rate(
          http_requests_total{
            namespace="{{ namespace }}",
            job="{{ target }}-canary",
            status=~"5.."
          }[{{ interval }}]
        )
      )
      /
      sum(
        rate(
          http_requests_total{
            namespace="{{ namespace }}",
            job="{{ target }}-canary"
          }[{{ interval }}]
        )
      )
    )

apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: checkout
  namespace: prod
spec:
  provider: istio                  # must match the mesh/ingress actually installed
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: checkout
  service:
    port: 8080
    targetPort: 8080
  analysis:
    interval: 30s
    threshold: 5                   # consecutive failed checks before rollback
    maxWeight: 50
    stepWeight: 10
    metrics:
      - name: checkout-error-ratio
        templateRef:
          name: checkout-error-ratio
          namespace: prod
        thresholdRange:
          max: 1                   # more than 1% 5xx on the canary halts it
        interval: 1m
    webhooks:
      - name: "human approval"
        type: confirm-rollout      # blocks BEFORE any traffic shifts
        url: http://flagger-loadtester.prod/gate/check   # closed until someone opens it
      - name: load-test
        type: rollout              # generates traffic so the metric has data
        url: http://flagger-loadtester.prod/
        timeout: 15s
        metadata:
          cmd: "hey -z 1m -q 10 -c 2 http://checkout-canary.prod:8080/"

kubectl apply -f canary.yaml
kubectl -n prod set image deploy/checkout checkout=ghcr.io/acme/checkout:3.4.0

kubectl -n prod get canary checkout -w         # STATUS Waiting — held at the gate
kubectl -n prod describe canary checkout | tail -20

# open the gate from inside the cluster (the loadtester serves the gate API)
kubectl -n prod exec -it deploy/flagger-loadtester -- \
  curl -s -d '{"name": "checkout", "namespace": "prod"}' http://localhost:8080/gate/open

kubectl -n prod get canary checkout -w         # Progressing → Promoting → Succeeded
kubectl -n prod exec -it deploy/flagger-loadtester -- \
  curl -s -d '{"name": "checkout", "namespace": "prod"}' http://localhost:8080/gate/close

apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: databases.platform.acme.io
  annotations:
    # cainjector copies the CA from this Certificate's secret into spec.conversion
    cert-manager.io/inject-ca-from: platform-system/database-conversion-tls
spec:
  group: platform.acme.io
  names:
    kind: Database
    plural: databases
    singular: database
  scope: Namespaced
  conversion:
    strategy: Webhook
    webhook:
      conversionReviewVersions: ["v1"]
      clientConfig:
        service:
          namespace: platform-system
          name: database-conversion
          path: /convert
          port: 443
  versions:
    - name: v1alpha1
      served: true
      storage: false
      deprecated: true
      deprecationWarning: "platform.acme.io/v1alpha1 Database is deprecated; use v1"
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              required: [size]
              properties:
                size: { type: string, enum: [small, medium, large] }
    - name: v1
      served: true
      storage: true                     # what actually gets persisted in etcd
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              required: [resources]
              properties:
                resources:
                  type: object
                  required: [cpu, memory]
                  properties:
                    cpu:    { type: string }
                    memory: { type: string }

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: database-conversion-tls
  namespace: platform-system
spec:
  secretName: database-conversion-tls
  dnsNames:
    - database-conversion.platform-system.svc
    - database-conversion.platform-system.svc.cluster.local
  issuerRef:
    name: platform-ca
    kind: ClusterIssuer
    group: cert-manager.io

kubectl apply -f database-crd.yaml
kubectl get crd databases.platform.acme.io -o jsonpath='{.spec.conversion.strategy}{"\n"}'
kubectl get crd databases.platform.acme.io -o jsonpath='{.spec.conversion.webhook.clientConfig.caBundle}' | head -c 40

kubectl -n demo apply -f database-v1.yaml
kubectl -n demo get databases.v1.platform.acme.io demo -o yaml         # spec.resources
kubectl -n demo get databases.v1alpha1.platform.acme.io demo -o yaml   # spec.size

# when it is broken, the error is always at read time:
#   conversion webhook for platform.acme.io/v1, Kind=Database failed:
#   Post "https://database-conversion.platform-system.svc:443/convert?timeout=30s":
#   x509: certificate signed by unknown authority
kubectl -n platform-system logs deploy/database-conversion --tail=20

apiVersion: pkg.crossplane.io/v1
kind: Function
metadata:
  name: function-go-templating
spec:
  package: xpkg.upbound.io/crossplane-contrib/function-go-templating:v0.9.2
---
apiVersion: pkg.crossplane.io/v1
kind: Function
metadata:
  name: function-auto-ready
spec:
  package: xpkg.upbound.io/crossplane-contrib/function-auto-ready:v0.4.1

apiVersion: apiextensions.crossplane.io/v1
kind: Composition
metadata:
  name: xpostgres
spec:
  compositeTypeRef:
    apiVersion: platform.acme.io/v1alpha1
    kind: XPostgres
  mode: Pipeline
  pipeline:
    - step: render
      functionRef:
        name: function-go-templating
      input:
        apiVersion: gotemplating.fn.crossplane.io/v1beta1
        kind: GoTemplate
        source: Inline
        inline:
          template: |
            {{ $params := .observed.composite.resource.spec.parameters }}
            apiVersion: kubernetes.crossplane.io/v1alpha2
            kind: Object
            metadata:
              annotations:
                # this annotation is the resource's stable identity across reconciles
                gotemplating.fn.crossplane.io/composition-resource-name: connection-config
            spec:
              forProvider:
                manifest:
                  apiVersion: v1
                  kind: ConfigMap
                  metadata:
                    name: {{ .observed.composite.resource.metadata.name }}-db
                    namespace: {{ $params.namespace }}
                  data:
                    size: {{ $params.size | quote }}
                    region: {{ $params.region | quote }}
              providerConfigRef:
                name: kubernetes-provider

    - step: ready
      functionRef:
        name: function-auto-ready       # last step: derive XR readiness from children

kubectl apply -f functions.yaml
kubectl get functions -w                 # both INSTALLED True / HEALTHY True

# render the pipeline locally before you touch the cluster — no claim, no waiting
crossplane render xr.yaml composition.yaml functions.yaml

kubectl apply -f composition.yaml
kubectl apply -f claim.yaml
kubectl get claim,composite,managed
kubectl get composition xpostgres -o jsonpath='{.spec.pipeline[*].step}{"\n"}'
kubectl describe xpostgres              # events name the failing pipeline step

# billing-api/catalog-info.yaml
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
  name: billing-api
  description: Issues and reconciles customer invoices
  annotations:
    backstage.io/techdocs-ref: dir:.
    backstage.io/kubernetes-id: billing-api        # matches the k8s label selector
    argocd/app-name: billing-api
    github.com/project-slug: acme/billing-api
  tags: [go, tier-1]
  links:
    - url: https://grafana.acme.io/d/billing
      title: Dashboard
      icon: dashboard
spec:
  type: service
  lifecycle: production
  owner: group:default/payments        # creates the ownedBy relation
  system: billing                      # creates the partOf relation
  providesApis:
    - billing-api-http                 # creates the providesApi relation
---
apiVersion: backstage.io/v1alpha1
kind: API
metadata:
  name: billing-api-http
spec:
  type: openapi
  lifecycle: production
  owner: group:default/payments
  system: billing
  definition:
    $text: ./openapi.yaml
---
apiVersion: backstage.io/v1alpha1
kind: System
metadata:
  name: billing
spec:
  owner: group:default/payments

# app-config.yaml — register the location and allow the kinds it introduces
catalog:
  rules:
    - allow: [Component, System, API, Group, User, Resource, Location]
  locations:
    - type: url
      target: https://github.com/acme/billing-api/blob/main/catalog-info.yaml
  # going forward, discover them automatically instead of listing forty files:
  providers:
    github:
      acme:
        organization: acme
        catalogPath: /catalog-info.yaml
        schedule:
          frequency: { minutes: 30 }
          timeout: { minutes: 3 }

# validate the location before you commit to it — dryRun parses and reports errors
#   without adding anything to the catalog
curl -s -X POST 'http://localhost:7007/api/catalog/locations?dryRun=true' \
  -H 'Content-Type: application/json' \
  -d '{"type":"url","target":"https://github.com/acme/billing-api/blob/main/catalog-info.yaml"}' | jq

curl -s http://localhost:7007/api/catalog/entities/by-name/component/default/billing-api | jq '.spec.owner, .relations'
curl -s "http://localhost:7007/api/catalog/entities/by-name/api/default/billing-api-http" | jq '.kind'
curl -s http://localhost:7007/api/catalog/locations | jq '.[].data.target'

# platform-base.yaml — the platform owns resources and securityContext only
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api
  namespace: apps
spec:
  template:
    spec:
      containers:
        - name: api
          resources:
            requests: { cpu: 100m, memory: 128Mi }
            limits:   { cpu: "1",  memory: 512Mi }
---
# app-overlay.yaml — the app team owns the image and the replica count only
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api
  namespace: apps
spec:
  replicas: 4
  selector:
    matchLabels: { app: api }
  template:
    metadata:
      labels: { app: api }
    spec:
      containers:
        - name: api
          image: ghcr.io/acme/api:1.4.9

# order matters on a fresh cluster: the overlay is the only one of the two that carries
# spec.selector and the pod template labels, and a Deployment cannot be CREATED without
# them. Apply it first (or start from the Deployment that already exists), then layer
# the platform's fields on top — after that either may apply in any order.
kubectl apply --server-side --field-manager=app-team           -f app-overlay.yaml
kubectl apply --server-side --field-manager=platform-controller -f platform-base.yaml

# who owns what, field by field
kubectl -n apps get deploy api --show-managed-fields -o json \
  | jq '.metadata.managedFields[] | {manager, operation, time}'

# the conflict, reproduced: app-team tries to set a field platform-controller owns
kubectl apply --server-side --field-manager=app-team -f app-with-limits.yaml
# error: Apply failed with 1 conflict: conflict with "platform-controller"
#   using apps/v1: .spec.template.spec.containers[name="api"].resources.limits.memory

# deliberate handover — this TAKES ownership, it does not merge
kubectl apply --server-side --force-conflicts --field-manager=app-team -f app-with-limits.yaml

mkdir -p platform-operator && cd platform-operator
kubebuilder init --domain acme.io --repo github.com/acme/platform-operator
kubebuilder create api --group platform --version v1alpha1 --kind Runner --resource --controller
# → api/v1alpha1/runner_types.go

// api/v1alpha1/runner_types.go

type RunnerSpec struct {
    // Pool selects the machine class the runners are placed on.
    // +kubebuilder:validation:Enum=small;large
    // +kubebuilder:default=small
    Pool string `json:"pool"`

    // Replicas is how many runners to keep warm.
    // +kubebuilder:validation:Minimum=0
    // +kubebuilder:validation:Maximum=50
    // +kubebuilder:default=1
    Replicas int32 `json:"replicas"`

    // +kubebuilder:validation:MinLength=1
    Repository string `json:"repository"`
}

type RunnerStatus struct {
    ReadyReplicas int32              `json:"readyReplicas,omitempty"`
    Conditions    []metav1.Condition `json:"conditions,omitempty"`
}

// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:shortName=rn,categories=platform
// +kubebuilder:printcolumn:name="Pool",type=string,JSONPath=`.spec.pool`
// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.readyReplicas`
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
type Runner struct {
    metav1.TypeMeta   `json:",inline"`
    metav1.ObjectMeta `json:"metadata,omitempty"`

    Spec   RunnerSpec   `json:"spec,omitempty"`
    Status RunnerStatus `json:"status,omitempty"`
}

make manifests generate                       # controller-gen writes the CRD + deepcopy
ls config/crd/bases/                          # platform.acme.io_runners.yaml
make install                                  # kubectl apply -f config/crd/bases

kubectl get crd runners.platform.acme.io -o jsonpath='{.spec.versions[0].subresources}{"\n"}'
kubectl explain runner.spec.pool
kubectl get runners                           # POOL / READY / AGE columns

kubectl apply -f - <<'EOF'
apiVersion: platform.acme.io/v1alpha1
kind: Runner
metadata: { name: ci, namespace: default }
spec: { pool: huge, replicas: 99, repository: acme/api }
EOF
# The Runner "ci" is invalid:
#   spec.pool: Unsupported value: "huge": supported values: "small", "large"
#   spec.replicas: Invalid value: 99: spec.replicas in body should be less than or equal to 50

# 1. log query — stream selector, then line filter, then parser, then label filter
{namespace="prod", app="checkout"}
  |= "error"
  | json
  | level = "error"
  | code =~ "5.."
  | line_format "{{.ts}} {{.code}} {{.msg}}"

# 2a. rate of errors per response code
sum by (code) (
  rate({namespace="prod", app="checkout"} |= "error" | json | level = "error" [5m])
)

# 2b. p95 of a numeric field lifted out of the line with unwrap
quantile_over_time(0.95,
  {namespace="prod", app="checkout"}
    | json
    | unwrap duration_ms [5m]
) by (route)

# loki ruler rule — Prometheus rule format, LogQL expression
groups:
  - name: checkout
    interval: 1m
    rules:
      - alert: CheckoutErrorLogRateHigh
        expr: |
          sum(
            rate({namespace="prod", app="checkout"} |= "error" | json | level = "error" [5m])
          ) > 5
        for: 5m
        labels:
          severity: critical
          team: payments
        annotations:
          summary: "checkout is logging more than 5 errors/s"
          runbook_url: "https://runbooks.acme.io/checkout-errors"

export LOKI_ADDR=http://loki-gateway.loki

logcli labels                                   # which labels exist at all
logcli labels namespace                         # and their values
logcli series '{namespace="prod"}'              # which streams exist

logcli query --since=15m --limit=200 \
  '{namespace="prod", app="checkout"} | json | level = "error"'

logcli query --since=1h \
  'sum by (code) (rate({namespace="prod", app="checkout"} | json | level = "error" [5m]))'

kubectl -n loki get cm loki-rules -o yaml
curl -s $LOKI_ADDR/prometheus/api/v1/rules | jq '.data.groups[].rules[].name'

receivers:
  otlp:
    protocols:
      grpc: { endpoint: 0.0.0.0:4317 }
      http: { endpoint: 0.0.0.0:4318 }

connectors:
  spanmetrics:
    namespace: traces.span.metrics       # → traces_span_metrics_calls_total
    histogram:
      explicit:
        buckets: [10ms, 50ms, 100ms, 250ms, 500ms, 1s, 2s, 5s]
    dimensions:
      - name: http.route
      - name: http.response.status_code
    exclude_dimensions: [span.kind]
    metrics_flush_interval: 15s

processors:
  memory_limiter:
    check_interval: 1s
    limit_percentage: 75
    spike_limit_percentage: 15
  k8sattributes:
    auth_type: serviceAccount
    passthrough: false
    extract:
      metadata:
        - k8s.namespace.name
        - k8s.pod.name
        - k8s.deployment.name
        - k8s.node.name
    pod_association:
      - sources:
          - from: resource_attribute
            name: k8s.pod.ip
      - sources:
          - from: connection            # fallback: the sender's source IP
  batch:
    timeout: 5s
    send_batch_size: 1024

exporters:
  otlp/tempo:
    endpoint: tempo.observability.svc:4317
    tls: { insecure: true }
  prometheus:
    endpoint: 0.0.0.0:8889
    resource_to_telemetry_conversion:
      enabled: true                     # resource attrs become metric labels

service:
  telemetry:
    metrics:
      level: detailed                   # collector self-metrics on :8888
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, k8sattributes, batch]
      exporters: [otlp/tempo, spanmetrics]     # connector as an EXPORTER here
    metrics/spanmetrics:
      receivers: [spanmetrics]                 # …and as a RECEIVER here
      processors: [batch]
      exporters: [prometheus]

# k8sattributes cannot work without this — the silent-failure trap
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-collector-k8sattributes
rules:
  - apiGroups: [""]
    resources: ["pods", "namespaces", "nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources: ["replicasets"]          # deployment name is derived via the ReplicaSet
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-collector-k8sattributes
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: otel-collector-k8sattributes
subjects:
  - kind: ServiceAccount
    name: otel-collector
    namespace: observability

kubectl -n observability create cm otel-collector-conf --from-file=config.yaml \
  --dry-run=client -o yaml | kubectl apply -f -
kubectl apply -f collector-rbac.yaml
kubectl -n observability rollout restart deploy/otel-collector
kubectl -n observability logs deploy/otel-collector | tail -30

kubectl -n observability port-forward deploy/otel-collector 8889:8889 8888:8888 &
curl -s http://localhost:8889/metrics | grep traces_span_metrics_calls_total | head
curl -s http://localhost:8888/metrics | grep -E 'otelcol_(processor_dropped|exporter_send_failed)'

export PROM=http://prometheus-operated.monitoring.svc:9090
export AM=http://alertmanager-operated.monitoring.svc:9093

# HOP 1 — is the rule loaded and evaluating? (a rule the operator never picked up
#          simply does not exist, and nothing anywhere says so)
curl -s $PROM/api/v1/rules \
  | jq '.data.groups[].rules[] | select(.name=="KubePodCrashLooping") | {state, health, lastError}'
kubectl -n monitoring get prometheus -o jsonpath='{.items[0].spec.ruleSelector}{"\n"}'
kubectl get prometheusrule -A --show-labels

# HOP 2 — does Prometheus know about any Alertmanager?
curl -s $PROM/api/v1/alertmanagers | jq '.data.activeAlertmanagers'
kubectl -n monitoring get prometheus -o jsonpath='{.items[0].spec.alerting}{"\n"}'

# HOP 3 — did the alert actually arrive?
amtool --alertmanager.url=$AM alert query
amtool --alertmanager.url=$AM alert query alertname=KubePodCrashLooping

# HOP 4 — where would it route, and is anything swallowing it?
amtool config routes test --config.file=alertmanager.yaml severity=critical team=platform
amtool --alertmanager.url=$AM silence query
kubectl -n monitoring logs alertmanager-main-0 -c alertmanager | grep -i 'notify.*error'

# the four root causes this evidence order separates, and the smallest fix for each:

# (a) the PrometheusRule was never loaded — its labels do not match spec.ruleSelector
kubectl -n monitoring label prometheusrule platform-alerts release=kube-prometheus-stack

# (b) Prometheus discovered no Alertmanager — the alerting block points at nothing
kubectl -n monitoring patch prometheus k8s --type=merge -p '{"spec":{"alerting":{"alertmanagers":[{"namespace":"monitoring","name":"alertmanager-operated","port":"web"}]}}}'

# (c) a silence from a previous incident, opened "for an hour", never expired
amtool --alertmanager.url=$AM silence query --expired=false
amtool --alertmanager.url=$AM silence expire 4f8f0c1e-1e4b-4c2a-9a5d-0f8a1c7b2d33

# (d) it routed to a receiver with no working integration — the logs are the only tell
kubectl -n monitoring logs alertmanager-main-0 -c alertmanager | grep -i 'notify'

# prove the whole path end to end, without waiting for a real failure
amtool --alertmanager.url=$AM alert add \
  alertname=SmokeTest severity=critical team=platform \
  --annotation=summary="pipeline smoke test" --end="$(date -u -d '+5 min' +%Y-%m-%dT%H:%M:%SZ)"
amtool --alertmanager.url=$AM alert query alertname=SmokeTest

# fixed evidence order — the events almost always name the cause outright
kubectl -n data describe pvc data-timescale-0 | tail -20
#   Warning  ProvisioningFailed  persistentvolume-controller
#     storageclass.storage.k8s.io "fast" not found
kubectl -n data describe pod timescale-0 | grep -A8 Events
#   Warning  FailedScheduling  pod has unbound immediate PersistentVolumeClaims

kubectl get sc
kubectl get sc -o custom-columns='NAME:.metadata.name,PROV:.provisioner,BINDING:.volumeBindingMode,DEFAULT:.metadata.annotations.storageclass\.kubernetes\.io/is-default-class'
kubectl get pv
kubectl -n data get pvc data-timescale-0 -o jsonpath='{.spec.storageClassName}{" "}{.spec.accessModes}{" "}{.spec.resources.requests.storage}{"\n"}'
kubectl -n kube-system get pods -l app=local-path-provisioner   # is the provisioner alive?

# (a) the named StorageClass does not exist on this cluster
#     storageClassName is IMMUTABLE on a claim, and volumeClaimTemplates are immutable
#     on a StatefulSet — so you recreate, keeping the pods alive with --cascade=orphan
kubectl -n data delete sts timescale --cascade=orphan
kubectl -n data delete pvc data-timescale-0
# …edit volumeClaimTemplates to a class that exists, then:
kubectl apply -f timescale-sts.yaml

# (b) the claim omitted storageClassName and there is no default class
kubectl patch sc standard -p \
  '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
# (only one class may hold this annotation — clear it from the old one first)

# (c) a hand-made PV exists but does not match: capacity too small, wrong accessModes,
#     or a stale claimRef from a deleted claim
kubectl get pv pv-timescale -o jsonpath='{.spec.capacity.storage}{" "}{.spec.accessModes}{" "}{.spec.claimRef.name}{"\n"}'
kubectl patch pv pv-timescale --type=json -p '[{"op":"remove","path":"/spec/claimRef"}]'

kubectl -n data get pvc data-timescale-0 -o jsonpath='{.status.phase}{"\n"}'   # Bound
kubectl -n data get pod timescale-0
kubectl get pv

# 1. what priority and policy does the PENDING pod actually have?
kubectl -n platform get pod -l app=ingress \
  -o jsonpath='{.items[0].spec.priorityClassName}{" prio="}{.items[0].spec.priority}{" policy="}{.items[0].spec.preemptionPolicy}{"\n"}'
kubectl get priorityclass

# 2. what priority do the candidate victims have? ties are never preempted
kubectl get pods -A --sort-by=.spec.priority \
  -o custom-columns='NS:.metadata.namespace,POD:.metadata.name,PRIO:.spec.priority,NODE:.spec.nodeName' | tail -20

# 3. is a PodDisruptionBudget standing in the way?
kubectl get pdb -A
kubectl -n batch get pdb cruncher -o jsonpath='{.status.disruptionsAllowed}{"\n"}'

# 4. did the scheduler already pick a victim and is simply waiting?
kubectl -n platform get pod -l app=ingress -o jsonpath='{.items[0].status.nominatedNodeName}{"\n"}'
kubectl -n platform describe pod -l app=ingress | grep -A10 Events

# the four causes and their smallest fixes:

# (a) the incoming pod's class says preemptionPolicy: Never — it queues first but
#     may never evict anything. Give the platform component a preempting class.
kubectl get priorityclass platform-critical -o jsonpath='{.preemptionPolicy}{"\n"}'
kubectl -n platform patch deploy ingress --type=merge \
  -p '{"spec":{"template":{"spec":{"priorityClassName":"platform-critical"}}}}'

# (b) every candidate victim has the SAME priority as the incoming pod — a tie is
#     never a victim. Lower the batch class so a gap exists.
kubectl delete priorityclass batch-background
kubectl apply -f - <<'EOF'
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: batch-background
value: 100
globalDefault: false
preemptionPolicy: Never
description: "Best-effort batch. Waits for room; never evicts anything."
EOF
kubectl -n batch rollout restart deploy/cruncher     # priority is set at admission

# (c) a PDB with disruptionsAllowed 0 makes every victim set violate a budget
kubectl -n batch patch pdb cruncher --type=merge -p '{"spec":{"minAvailable":"50%"}}'

# (d) the request is larger than any node can free — the pod, not the cluster, is wrong
kubectl -n platform get deploy ingress -o jsonpath='{.spec.template.spec.containers[0].resources.requests}{"\n"}'
kubectl describe node | grep -A6 'Allocated resources'

kubectl -n platform get pod -l app=ingress -w
kubectl get events -A --field-selector reason=Preempted
kubectl -n platform get pod -l app=ingress -o jsonpath='{.items[0].spec.priority}{"\n"}'

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: selfsigned-root
spec:
  selfSigned: {}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: platform-ca
  namespace: cert-manager        # a ClusterIssuer reads its secret from
spec:                            # --cluster-resource-namespace (default: cert-manager)
  isCA: true
  commonName: platform-ca
  secretName: platform-ca-tls
  duration: 43800h               # 5 years
  renewBefore: 720h              # 30 days
  privateKey:
    algorithm: ECDSA
    size: 256
  issuerRef:
    name: selfsigned-root
    kind: ClusterIssuer
    group: cert-manager.io
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: platform-ca
spec:
  ca:
    secretName: platform-ca-tls
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: webhook-tls
  namespace: platform-system
spec:
  secretName: webhook-tls
  duration: 2160h                # 90 days
  renewBefore: 720h              # renew a third of the way out, not the night before
  dnsNames:
    - platform-webhook.platform-system.svc
    - platform-webhook.platform-system.svc.cluster.local
  usages: [server auth]
  issuerRef:
    name: platform-ca
    kind: ClusterIssuer
    group: cert-manager.io

apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: platform
  annotations:
    cert-manager.io/inject-ca-from: platform-system/webhook-tls   # cainjector fills caBundle
webhooks:
  - name: inject.platform.acme.io
    admissionReviewVersions: ["v1"]
    sideEffects: None
    failurePolicy: Fail
    clientConfig:
      service:
        name: platform-webhook
        namespace: platform-system
        path: /mutate
        port: 443
    rules:
      - apiGroups: [""]
        apiVersions: ["v1"]
        operations: ["CREATE"]
        resources: ["pods"]
        scope: Namespaced
    namespaceSelector:
      matchLabels:
        platform.acme.io/inject: "true"    # never match every namespace

kubectl apply -f tls.yaml
kubectl -n platform-system get certificate,certificaterequest
kubectl -n platform-system describe certificate webhook-tls | tail -15

kubectl -n platform-system get secret webhook-tls \
  -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -noout -dates -subject -ext subjectAltName

kubectl get mutatingwebhookconfiguration platform \
  -o jsonpath='{.webhooks[0].clientConfig.caBundle}' | head -c 40

# force a renewal and watch the secret change without touching the webhook config
cmctl renew webhook-tls -n platform-system
cmctl status certificate webhook-tls -n platform-system

# 1. without this, data.inventory is empty and the policy silently never fires
apiVersion: config.gatekeeper.sh/v1alpha1
kind: Config
metadata:
  name: config
  namespace: gatekeeper-system
spec:
  sync:
    syncOnly:
      - group: "networking.k8s.io"
        version: "v1"
        kind: "Ingress"

apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
  name: k8suniqueingresshost
spec:
  crd:
    spec:
      names:
        kind: K8sUniqueIngressHost
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8suniqueingresshost

        identical(obj, review) {
          obj.metadata.namespace == review.object.metadata.namespace
          obj.metadata.name == review.object.metadata.name
        }

        violation[{"msg": msg}] {
          input.review.kind.kind == "Ingress"
          re_match("^(extensions|networking.k8s.io)$", input.review.kind.group)
          host := input.review.object.spec.rules[_].host
          other := data.inventory.namespace[ns][otherapiversion]["Ingress"][name]
          re_match("^(extensions|networking.k8s.io)/.+$", otherapiversion)
          not identical(other, input.review)
          other.spec.rules[_].host == host
          msg := sprintf("ingress host %v is already claimed by %v/%v", [host, ns, name])
        }

apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sUniqueIngressHost
metadata:
  name: unique-ingress-host
spec:
  enforcementAction: dryrun        # measure first — flip to deny only when clean
  match:
    kinds:
      - apiGroups: ["networking.k8s.io"]
        kinds: ["Ingress"]
    excludedNamespaces: ["kube-system", "gatekeeper-system"]

kubectl apply -f gatekeeper-config.yaml
kubectl apply -f constrainttemplate.yaml
kubectl get constrainttemplate k8suniqueingresshost -o jsonpath='{.status.created}{"\n"}'   # true
kubectl apply -f constraint.yaml

# audit runs on a timer (60s by default) and writes what it found into status
kubectl get k8suniqueingresshost unique-ingress-host \
  -o jsonpath='{.status.totalViolations}{"\n"}'
kubectl get k8suniqueingresshost unique-ingress-host -o yaml | grep -A12 violations

# only now, enforce
kubectl patch k8suniqueingresshost unique-ingress-host --type=merge \
  -p '{"spec":{"enforcementAction":"deny"}}'
kubectl -n team-b apply -f duplicate-ingress.yaml
# Error from server (Forbidden): admission webhook "validation.gatekeeper.sh" denied
#   the request: [unique-ingress-host] ingress host shop.acme.io is already claimed by team-a/shop

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: verify-acme-images
spec:
  validationFailureAction: Audit    # Enforce only after the reports come back clean
  background: false                 # image verification is admission-time only
  failurePolicy: Fail               # Kyverno down = no unverified images admitted
  webhookTimeoutSeconds: 30         # registry round-trips are not instant
  rules:
    - name: verify-signature
      match:
        any:
          - resources:
              kinds: [Pod]
              namespaces: [prod, staging]
      verifyImages:
        - imageReferences:
            - "ghcr.io/acme/*"
          required: true            # an image with no matching attestor is REJECTED
          mutateDigest: true        # rewrite the tag to the digest we just verified
          verifyDigest: true
          attestors:
            - count: 1
              entries:
                - keys:
                    publicKeys: |-
                      -----BEGIN PUBLIC KEY-----
                      MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEabc123...replace...with...yours
                      -----END PUBLIC KEY-----
                    rekor:
                      url: https://rekor.sigstore.dev

# keyless variant — no key to rotate, identity comes from the CI workload itself
          attestors:
            - count: 1
              entries:
                - keyless:
                    subject: "https://github.com/acme/*/.github/workflows/*@refs/heads/main"
                    issuer: "https://token.actions.githubusercontent.com"
                    rekor:
                      url: https://rekor.sigstore.dev

# prove the signature exists before you blame the policy
cosign verify --key cosign.pub ghcr.io/acme/api:1.4.9

kubectl apply -f verify-acme-images.yaml
kubectl get clusterpolicy verify-acme-images        # READY True before testing anything

# audit first: create both Pods while nothing is blocked, and read what it WOULD have done
kubectl -n prod run signed   --image=ghcr.io/acme/api:1.4.9        --restart=Never
kubectl -n prod run unsigned --image=ghcr.io/acme/scratchpad:dev   --restart=Never
kubectl -n prod get policyreport -o wide            # one pass, one fail
kubectl -n prod get policyreport -o yaml | grep -B4 -A6 'result: fail'
kubectl -n prod delete pod signed unsigned

# then enforce
kubectl patch clusterpolicy verify-acme-images --type=merge \
  -p '{"spec":{"validationFailureAction":"Enforce"}}'

kubectl -n prod run signed --image=ghcr.io/acme/api:1.4.9 --restart=Never
kubectl -n prod get pod signed -o jsonpath='{.spec.containers[0].image}{"\n"}'
#   ghcr.io/acme/api:1.4.9@sha256:9b2c…

kubectl -n prod run unsigned --image=ghcr.io/acme/scratchpad:dev --restart=Never
#   Error from server: admission webhook "mutate.kyverno.svc-fail" denied the request:
#   failed to verify image ghcr.io/acme/scratchpad:dev: .attestors[0].entries[0].keys:
#   no matching signatures found