Debugging Guide

docker exec -it <container-name> /bin/bash

ls -la
ps aux
df -h
nvidia-smi
env

docker logs <container-name>
docker logs <container-name> --tail=100 --follow

docker inspect <container-name>

docker network ls
docker network inspect <network-name>
docker exec <container> ping license-proxy
docker exec <container> curl http://license-proxy:3369/health

kubectl debug <pod-name> -it --image=ubuntu --target=<container-name>

kubectl cp ./debug-script.sh <pod-name>:/tmp/debug.sh
kubectl exec -it <pod-name> -- bash /tmp/debug.sh

kubectl debug -it <pod-name> --image=nicolaka/netshoot --target=lightning-asr

nslookup license-proxy
curl http://api-server:7100/health
tcpdump -i eth0

kubectl logs <pod-name> --previous
kubectl logs <pod-name> -c <container-name> --previous

kubectl run netdebug --rm -it --restart=Never \
  --image=nicolaka/netshoot \
  --namespace=smallest \
  -- bash

nslookup api-server
nslookup license-proxy
nslookup lightning-asr

curl http://api-server:7100/health
curl http://license-proxy:3369/health

traceroute api-server
ping -c 3 lightning-asr

kubectl run dnstest --rm -it --restart=Never \
  --image=busybox \
  -- nslookup kubernetes.default

kubectl logs -n kube-system -l k8s-app=kube-dns

kubectl get networkpolicy -n smallest
kubectl describe networkpolicy <policy-name> -n smallest

kubectl delete networkpolicy <policy-name> -n smallest

kubectl top pods -n smallest
kubectl top pods -n smallest --sort-by=memory
kubectl top pods -n smallest --sort-by=cpu

kubectl top nodes
kubectl describe node <node-name> | grep -A 10 "Allocated resources"

kubectl exec -it <lightning-asr-pod> -- nvidia-smi

kubectl exec -it <lightning-asr-pod> -- nvidia-smi dmon

kubectl exec -it <lightning-asr-pod> -- watch -n 1 nvidia-smi

kubectl exec -it <lightning-asr-pod> -- nvidia-smi -q -d MEMORY,UTILIZATION,POWER,CLOCK,PERFORMANCE

kubectl exec -it <pod> -- sh -c 'apt-get update && apt-get install -y python3-pip && pip3 install py-spy'

kubectl exec -it <pod> -- py-spy top --pid 1

kubectl exec -it <pod> -- sh -c 'cat /proc/1/status | grep -i mem'

kubectl logs <pod> | grep -i "error\|exception\|failed"

kubectl logs <pod> | grep -i "error" | wc -l

kubectl logs <pod> | grep -B 5 -A 5 "error"

kubectl logs -l app=lightning-asr -n smallest --tail=100 --all-containers=true

kubectl logs -l app=lightning-asr -f --max-log-requests=10

kubectl logs <pod> | jq 'select(.level=="error")'
kubectl logs <pod> | jq 'select(.duration > 1000)'
kubectl logs <pod> | jq '.message' -r

kubectl exec -it <redis-pod> -- redis-cli

AUTH your-password
INFO
DBSIZE
KEYS *
GET some_key
MONITOR

INFO memory

SLOWLOG GET 10

kubectl port-forward svc/api-server 7100:7100
curl http://localhost:7100/health

curl -X POST http://localhost:7100/v1/listen \
  -H "Authorization: Token ${LICENSE_KEY}" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://www2.cs.uic.edu/~i101/SoundFiles/StarWars60.wav"}' \
  -v

curl -X POST http://localhost:7100/v1/listen \
  -H "Authorization: Token ${LICENSE_KEY}" \
  -H "X-Request-ID: debug-123" \
  -H "Content-Type: application/json" \
  -d '{"url": "..."}' \
  -v

kubectl logs -l app=api-server | grep "debug-123"
kubectl logs -l app=lightning-asr | grep "debug-123"

kubectl exec -it <pod> -- apt-get update && apt-get install -y tcpdump

kubectl exec -it <pod> -- tcpdump -i any -w /tmp/capture.pcap port 7100

kubectl cp <pod>:/tmp/capture.pcap ./capture.pcap

tcpdump -r capture.pcap -A

kubectl get events -n smallest --watch

kubectl get events -n smallest --field-selector type=Warning

kubectl get events -n smallest --sort-by='.lastTimestamp'

kubectl get events -n smallest -o json | jq '.items | group_by(.reason) | map({reason: .[0].reason, count: length})'

kubectl port-forward -n default svc/smallest-prometheus-stack-prometheus 9090:9090

asr_active_requests
rate(asr_total_requests[5m])
asr_gpu_utilization

kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq .

kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/smallest/pods/*/asr_active_requests" | jq .

lightningAsr:
  env:
    - name: LOG_LEVEL
      value: "DEBUG"

kubectl delete pod <pod-name>
kubectl drain <node-name> --ignore-daemonsets

kubectl run load-test --rm -it --image=williamyeh/hey \
  -- -z 5m -c 50 http://api-server:7100/health

apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: pod-failure
spec:
  action: pod-failure
  mode: one
  selector:
    namespaces:
      - smallest
    labelSelectors:
      app: lightning-asr
  duration: "30s"