Evidence-first troubleshooting Kubernetes

date -u
kubectl config current-context
kubectl get nodes -o wide
kubectl get --raw='/readyz?verbose'
kubectl get events -A --sort-by=.metadata.creationTimestamp | tail -30
kubectl top node

mkdir -p /tmp/k8s-m18
cat > /tmp/k8s-m18/lab.yaml <<'YAML'
apiVersion: v1
kind: Namespace
metadata: {name: k8s-m18}
---
apiVersion: v1
kind: PersistentVolume
metadata: {name: k8s-m18-data}
spec:
  capacity: {storage: 128Mi}
  accessModes: [ReadWriteOnce]
  persistentVolumeReclaimPolicy: Retain
  storageClassName: m18-static
  hostPath: {path: /var/lib/k8s-m18-data, type: DirectoryOrCreate}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: broken-app, namespace: k8s-m18}
spec:
  replicas: 1
  selector: {matchLabels: {app: broken-app}}
  template:
    metadata: {labels: {app: broken-app}}
    spec:
      containers:
      - name: app
        image: busybox:1.37
        command: ["sh","-c","echo MODE=$MODE; test "$MODE" = production && exec httpd -f -p 8080"]
        env: [{name: MODE, value: staging}]
        ports: [{name: http, containerPort: 8080}]
---
apiVersion: v1
kind: Service
metadata: {name: broken-app, namespace: k8s-m18}
spec:
  selector: {app: wrong-label}
  ports: [{port: 80, targetPort: http}]
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata: {name: broken-data, namespace: k8s-m18}
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: class-does-not-exist
  resources: {requests: {storage: 128Mi}}
---
apiVersion: v1
kind: Pod
metadata: {name: unschedulable, namespace: k8s-m18}
spec:
  nodeSelector: {course.goretskiy.pro/disk: nvme-impossible}
  containers: [{name: app, image: busybox:1.37, command: ["sh","-c","sleep 3600"]}]
YAML
kubectl apply --dry-run=server -f /tmp/k8s-m18/lab.yaml
kubectl apply -f /tmp/k8s-m18/lab.yaml
sleep 15
kubectl -n k8s-m18 get deploy,pod,service,endpointslice,pvc -o wide
kubectl -n k8s-m18 get events --sort-by=.metadata.creationTimestamp

pod=$(kubectl -n k8s-m18 get pod -l app=broken-app -o jsonpath='{.items[0].metadata.name}')
kubectl -n k8s-m18 get pod "$pod" -o jsonpath='phase={.status.phase}{" waiting="}{.status.containerStatuses[0].state.waiting.reason}{" restarts="}{.status.containerStatuses[0].restartCount}{" exit="}{.status.containerStatuses[0].lastState.terminated.exitCode}{"\n"}'
kubectl -n k8s-m18 describe pod "$pod"
kubectl -n k8s-m18 logs "$pod" --previous --timestamps || kubectl -n k8s-m18 logs "$pod" --timestamps
kubectl -n k8s-m18 set env deployment/broken-app MODE=production
kubectl -n k8s-m18 rollout status deployment/broken-app --timeout=120s

kubectl -n k8s-m18 get service broken-app -o yaml
kubectl -n k8s-m18 get endpointslice -l kubernetes.io/service-name=broken-app
kubectl -n k8s-m18 get pods --show-labels
kubectl -n k8s-m18 patch service broken-app --type=merge -p '{"spec":{"selector":{"app":"broken-app"}}}'
kubectl -n k8s-m18 run client --image=curlimages/curl:8.17.0 --restart=Never --command -- sleep 3600
kubectl -n k8s-m18 wait --for=condition=Ready pod/client --timeout=120s
kubectl -n k8s-m18 exec client -- curl -fsS http://broken-app

kubectl -n k8s-m18 describe pvc broken-data
kubectl get pv k8s-m18-data; kubectl get storageclass
kubectl -n k8s-m18 delete pvc broken-data
sed 's/class-does-not-exist/m18-static/' /tmp/k8s-m18/lab.yaml | kubectl apply -f -
kubectl -n k8s-m18 wait --for=jsonpath='{.status.phase}'=Bound pvc/broken-data --timeout=60s
kubectl -n k8s-m18 get pvc; kubectl get pv k8s-m18-data

kubectl -n k8s-m18 get pod unschedulable -o jsonpath='{range .status.conditions[*]}{.type}{"="}{.status}{" reason="}{.reason}{" message="}{.message}{"\n"}{end}'
kubectl -n k8s-m18 describe pod unschedulable
kubectl -n k8s-m18 get events --field-selector involvedObject.name=unschedulable
kubectl -n k8s-m18 patch pod unschedulable --type=json -p='[{"op":"remove","path":"/spec/nodeSelector"}]' || true
kubectl -n k8s-m18 delete pod unschedulable
kubectl -n k8s-m18 run unschedulable --image=busybox:1.37 --restart=Never --command -- sleep 3600
kubectl -n k8s-m18 wait --for=condition=Ready pod/unschedulable --timeout=120s

dir=/tmp/k8s-m18/evidence-$(date -u +%Y%m%dT%H%M%SZ); mkdir -p "$dir"
kubectl -n k8s-m18 get all,pvc -o yaml > "$dir/objects.yaml"
kubectl -n k8s-m18 get events --sort-by=.metadata.creationTimestamp > "$dir/events.txt"
kubectl -n k8s-m18 logs deployment/broken-app --all-pods=true --prefix --tail=100 > "$dir/app.log" 2>&1 || true
kubectl get nodes -o wide > "$dir/nodes.txt"
kubectl get --raw='/readyz?verbose' > "$dir/readyz.txt"
find "$dir" -type f -printf '%f %s bytes\n' | sort
kubectl -n k8s-m18 get pod,service,endpointslice,pvc
kubectl -n k8s-m18 exec client -- curl -fsS http://broken-app

kubectl delete namespace k8s-m18 --wait=true
kubectl delete pv k8s-m18-data --ignore-not-found
sudo rm -rf /var/lib/k8s-m18-data
rm -rf /tmp/k8s-m18
kubectl get nodes
kubectl get apiservice v1beta1.metrics.k8s.io