kubenetes iptables/dns not works stable.
sometimes it can parsing(netcat) correctly. sometimes not.
i belive it related to network(iptables) not work stable.
PS: those 3 servers(cluster0/1/2) they are all vm on same esxi server, same ubuntu vm image.
root@cluster2:~# kubectl exec -i -t dnsutils -- nslookup kubernetes.default.svc
;; connection timed out; no servers could be reached
command terminated with exit code 1
root@cluster2:~#
root@cluster2:~#
root@cluster2:~# kubectl exec -i -t dnsutils -- nslookup kubernetes.default.svc
Server: 10.96.0.10
Address: 10.96.0.10#53
Name: kubernetes.default.svc.cluster.local
Address: 10.96.0.1
Connection to 10.96.0.10 53 port [tcp/domain] succeeded!
root@cluster2:~# nc -vzw 3 10.96.0.10 53
Connection to 10.96.0.10 53 port [tcp/domain] succeeded!
root@cluster2:~# nc -vzw 3 10.96.0.10 53
nc: connect to 10.96.0.10 port 53 (tcp) timed out: Operation now in progress
root@cluster2:~#
root@cluster2:~#
root@cluster2:~#
here is the kubunetes config
root@cluster1:~# kubectl get pod,svc -o wide --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
default pod/dnsutils 1/1 Running 0 14h 10.85.0.19 cluster2 <none> <none>
kube-system pod/coredns-7ccf44d5bc-dq28b 1/1 Running 142 (16h ago) 38h 10.85.1.247 cluster0 <none> <none>
kube-system pod/coredns-7ccf44d5bc-v8gxf 1/1 Running 142 (16h ago) 38h 10.85.0.18 cluster2 <none> <none>
kube-system pod/etcd-cluster0 1/1 Running 276 2d21h 192.168.10.45 cluster0 <none> <none>
kube-system pod/kube-apiserver-cluster0 1/1 Running 0 2d21h 192.168.10.45 cluster0 <none> <none>
kube-system pod/kube-controller-manager-cluster0 1/1 Running 0 2d21h 192.168.10.45 cluster0 <none> <none>
kube-system pod/kube-proxy-2cg7v 1/1 Running 0 2d20h 192.168.10.245 cluster2 <none> <none>
kube-system pod/kube-proxy-rs775 1/1 Running 0 2d20h 192.168.10.44 cluster1 <none> <none>
kube-system pod/kube-proxy-zrrmw 1/1 Running 0 2d21h 192.168.10.45 cluster0 <none> <none>
kube-system pod/kube-scheduler-cluster0 1/1 Running 0 2d21h 192.168.10.45 cluster0 <none> <none>
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
default service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d21h <none>
kube-system service/kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 2d21h k8s-app=kube-dns
here is iptable-save
# Generated by iptables-save v1.8.7 on Mon Nov 14 11:29:01 2022
*mangle
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:KUBE-IPTABLES-HINT - [0:0]
:KUBE-KUBELET-CANARY - [0:0]
:KUBE-PROXY-CANARY - [0:0]
COMMIT
# Completed on Mon Nov 14 11:29:01 2022
# Generated by iptables-save v1.8.7 on Mon Nov 14 11:29:01 2022
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:KUBE-EXTERNAL-SERVICES - [0:0]
:KUBE-FORWARD - [0:0]
:KUBE-NODEPORTS - [0:0]
:KUBE-PROXY-FIREWALL - [0:0]
:KUBE-SERVICES - [0:0]
-A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes load balancer firewall" -j KUBE-PROXY-FIREWALL
-A INPUT -m comment --comment "kubernetes health check service ports" -j KUBE-NODEPORTS
-A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES
-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes load balancer firewall" -j KUBE-PROXY-FIREWALL
-A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD
-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES
-A OUTPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes load balancer firewall" -j KUBE-PROXY-FIREWALL
-A OUTPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
COMMIT
# Completed on Mon Nov 14 11:29:01 2022
# Generated by iptables-save v1.8.7 on Mon Nov 14 11:29:01 2022
*nat
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:CNI-549cb17de39efc34b0fb079e - [0:0]
:CNI-ee4ee4c8f8e1f07f7fd1ba09 - [0:0]
:KUBE-KUBELET-CANARY - [0:0]
:KUBE-MARK-DROP - [0:0]
:KUBE-MARK-MASQ - [0:0]
:KUBE-NODEPORTS - [0:0]
:KUBE-POSTROUTING - [0:0]
:KUBE-PROXY-CANARY - [0:0]
:KUBE-SEP-2RXCSE2GBPGJ5JEN - [0:0]
:KUBE-SEP-3CJFI6TGWLB42ULE - [0:0]
:KUBE-SEP-ANS6YOJVY7RAKHWE - [0:0]
:KUBE-SEP-FIVS6NG556TR7BUA - [0:0]
:KUBE-SEP-G7D5ETT6NKIIQ57G - [0:0]
:KUBE-SEP-HE7266DQZ4PI3CCF - [0:0]
:KUBE-SEP-Z4FXB2QUTX4XGLJ6 - [0:0]
:KUBE-SERVICES - [0:0]
:KUBE-SVC-ERIFXISQEP7F7OF4 - [0:0]
:KUBE-SVC-JD5MR3NA4I4DYORP - [0:0]
:KUBE-SVC-NPX46M4PTMTKRN6Y - [0:0]
:KUBE-SVC-TCOU7JCQXEZGVUNU - [0:0]
-A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING
-A POSTROUTING -s 10.85.0.18/32 -m comment --comment "name: \"crio\" id: \"92825a3d13dbe8d020cdcdc352d8976c066f11e10291ca127eca2ffeb2b8bc90\"" -j CNI-549cb17de39efc34b0fb079e
-A POSTROUTING -s 10.85.0.19/32 -m comment --comment "name: \"crio\" id: \"f3ca19b84a825810b63449ca8261d82a14c321d7929455ddf1516e7db083c657\"" -j CNI-ee4ee4c8f8e1f07f7fd1ba09
-A CNI-549cb17de39efc34b0fb079e -d 10.85.0.0/16 -m comment --comment "name: \"crio\" id: \"92825a3d13dbe8d020cdcdc352d8976c066f11e10291ca127eca2ffeb2b8bc90\"" -j ACCEPT
-A CNI-549cb17de39efc34b0fb079e ! -d 224.0.0.0/4 -m comment --comment "name: \"crio\" id: \"92825a3d13dbe8d020cdcdc352d8976c066f11e10291ca127eca2ffeb2b8bc90\"" -j MASQUERADE
-A CNI-ee4ee4c8f8e1f07f7fd1ba09 -d 10.85.0.0/16 -m comment --comment "name: \"crio\" id: \"f3ca19b84a825810b63449ca8261d82a14c321d7929455ddf1516e7db083c657\"" -j ACCEPT
-A CNI-ee4ee4c8f8e1f07f7fd1ba09 ! -d 224.0.0.0/4 -m comment --comment "name: \"crio\" id: \"f3ca19b84a825810b63449ca8261d82a14c321d7929455ddf1516e7db083c657\"" -j MASQUERADE
-A KUBE-MARK-MASQ -j MARK --set-xmark 0x4000/0x4000
-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
-A KUBE-POSTROUTING -j MARK --set-xmark 0x4000/0x0
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --random-fully
-A KUBE-SEP-2RXCSE2GBPGJ5JEN -s 10.85.1.247/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ
-A KUBE-SEP-2RXCSE2GBPGJ5JEN -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.85.1.247:53
-A KUBE-SEP-3CJFI6TGWLB42ULE -s 192.168.10.45/32 -m comment --comment "default/kubernetes:https" -j KUBE-MARK-MASQ
-A KUBE-SEP-3CJFI6TGWLB42ULE -p tcp -m comment --comment "default/kubernetes:https" -m tcp -j DNAT --to-destination 192.168.10.45:6443
-A KUBE-SEP-ANS6YOJVY7RAKHWE -s 10.85.0.18/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ
-A KUBE-SEP-ANS6YOJVY7RAKHWE -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.85.0.18:53
-A KUBE-SEP-FIVS6NG556TR7BUA -s 10.85.1.247/32 -m comment --comment "kube-system/kube-dns:metrics" -j KUBE-MARK-MASQ
-A KUBE-SEP-FIVS6NG556TR7BUA -p tcp -m comment --comment "kube-system/kube-dns:metrics" -m tcp -j DNAT --to-destination 10.85.1.247:9153
-A KUBE-SEP-G7D5ETT6NKIIQ57G -s 10.85.0.18/32 -m comment --comment "kube-system/kube-dns:metrics" -j KUBE-MARK-MASQ
-A KUBE-SEP-G7D5ETT6NKIIQ57G -p tcp -m comment --comment "kube-system/kube-dns:metrics" -m tcp -j DNAT --to-destination 10.85.0.18:9153
-A KUBE-SEP-HE7266DQZ4PI3CCF -s 10.85.1.247/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ
-A KUBE-SEP-HE7266DQZ4PI3CCF -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.85.1.247:53
-A KUBE-SEP-Z4FXB2QUTX4XGLJ6 -s 10.85.0.18/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ
-A KUBE-SEP-Z4FXB2QUTX4XGLJ6 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.85.0.18:53
-A KUBE-SERVICES -d 10.96.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-SVC-TCOU7JCQXEZGVUNU
-A KUBE-SERVICES -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-SVC-ERIFXISQEP7F7OF4
-A KUBE-SERVICES -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:metrics cluster IP" -m tcp --dport 9153 -j KUBE-SVC-JD5MR3NA4I4DYORP
-A KUBE-SERVICES -d 10.96.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-SVC-NPX46M4PTMTKRN6Y
-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp -> 10.85.0.18:53" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-Z4FXB2QUTX4XGLJ6
-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp -> 10.85.1.247:53" -j KUBE-SEP-HE7266DQZ4PI3CCF
-A KUBE-SVC-JD5MR3NA4I4DYORP -m comment --comment "kube-system/kube-dns:metrics -> 10.85.0.18:9153" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-G7D5ETT6NKIIQ57G
-A KUBE-SVC-JD5MR3NA4I4DYORP -m comment --comment "kube-system/kube-dns:metrics -> 10.85.1.247:9153" -j KUBE-SEP-FIVS6NG556TR7BUA
-A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https -> 192.168.10.45:6443" -j KUBE-SEP-3CJFI6TGWLB42ULE
-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns -> 10.85.0.18:53" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-ANS6YOJVY7RAKHWE
-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns -> 10.85.1.247:53" -j KUBE-SEP-2RXCSE2GBPGJ5JEN
COMMIT
# Completed on Mon Nov 14 11:29:01 2022
root@cluster2:~#
