05、cilium故障排查

参考文档：https://docs.cilium.io/en/stable/operations/troubleshooting/

可批量执行cilium pod命令的脚本(1.11~1.13)

vim k8s-cilium-exec.sh
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# Copyright Authors of Cilium
# 在所有cilium容器中执行命令
trap cleanup EXIT
function kill_jobs {
        j=$(jobs -p)
        if [ ! -z "$j" ]; then
                kill -$1 $j 2> /dev/null
        fi
}
function cleanup {
        kill_jobs INT
        sleep 2s
        kill_jobs TERM
}
function get_cilium_pods {
    kubectl -n "${K8S_NAMESPACE}" get pods -l k8s-app=cilium -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName | \
       grep cilium
}
K8S_NAMESPACE="${K8S_NAMESPACE:-kube-system}"
CONTAINER="${CONTAINER:-cilium-agent}"
while read -r podName nodeName ; do
        (
                title="==== detail from pod $podName , on node $nodeName "
                msg=$( kubectl -n "${K8S_NAMESPACE}" exec -c "${CONTAINER}" "${podName}" -- "${@}" 2>&1 )
                echo -e "$title \n$msg\n"
        )&
done <<< "$(get_cilium_pods)"
wait

查看所有节点状态：

./k8s-cilium-exec.sh cilium status

查看cilium pod状态：

./k8s-cilium-exec.sh cilium-health status

获取cilium pod脚本

vim k8s-get-cilium-pod.sh
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# Copyright Authors of Cilium
# 在所有cilium容器中执行命令
trap cleanup EXIT
function kill_jobs {
        j=$(jobs -p)
        if [ ! -z "$j" ]; then
                kill -$1 $j 2> /dev/null
        fi
}
function cleanup {
        kill_jobs INT
        sleep 2s
        kill_jobs TERM
}
function get_cilium_pods {
    kubectl -n "${K8S_NAMESPACE}" get pods -l k8s-app=cilium -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName | \
       grep cilium
}
K8S_NAMESPACE="${K8S_NAMESPACE:-kube-system}"
CONTAINER="${CONTAINER:-cilium-agent}"
while read -r podName nodeName ; do
        (
                title="==== detail from pod $podName , on node $nodeName "
                msg=$( kubectl -n "${K8S_NAMESPACE}" exec -c "${CONTAINER}" "${podName}" -- "${@}" 2>&1 )
                echo -e "$title \n$msg\n"
        )&
done <<< "$(get_cilium_pods)"
wait
root@tke-cilium-node-0003.novalocal:/root/cilium-scripts#^C
root@tke-cilium-node-0003.novalocal:/root/cilium-scripts#./k8s-cilium-exec.sh cilium-health status^C
root@tke-cilium-node-0003.novalocal:/root/cilium-scripts#ll
total 8
-rwxr-xr-x 1 root root 813 Apr 11 10:44 k8s-cilium-exec.sh
-rwxr-xr-x 1 root root 428 Apr 11 10:56 k8s-get-cilium-pod.sh
root@tke-cilium-node-0003.novalocal:/root/cilium-scripts#cat k8s-get-cilium-pod.sh
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# Copyright Authors of Cilium
# Given an app pod and namespace; get corresponding cilium pod
if [ $# -ne 2  ]
then
        echo "Usage: get_cilium_pod.sh <pod> <namespace>"
        exit 1
fi
K8S_NAMESPACE="${K8S_NAMESPACE:-kube-system}"
kubectl get pods -n "${K8S_NAMESPACE}" -owide | grep cilium | grep `kubectl get pods $1 -owide -n $2 | awk '{print $7}' | tail -n1` | awk '{print $1}'

列出 Cilium不为其提供网络的集群中的所有 Kubernetes pod 。这包括在主机网络模式下运行的 pod 和在部署 Cilium 之前启动的 pod。

vim k8s-unmanaged.sh
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# Copyright Authors of Cilium
function all_ceps { kubectl get cep --all-namespaces -o json | jq -r '.items[].metadata | .namespace + "/" + .name'; }
function all_pods { kubectl get pods --all-namespaces -o json | jq -r '.items[] | select((.status.phase=="Running" or .status.phase=="Pending") and (.spec.hostNetwork==true | not)) | .metadata | .namespace + "/" + .name'; }
echo "Skipping pods with host networking enabled or with status not in Running or Pending phase..."
sort <(all_ceps) <(all_pods) | uniq -u

查看:

./k8s-unmanaged.sh

执行命令从您的 Kubernetes 集群中收集故障排除信息

cilium-cli sysdump