Kubernetes 之 kubectl exec 指令工作原理 实现copy和webshell以及file browser

March 10, 2021 默认分类

# Kubernetes 之 kubectl exec 指令工作原理

Kubectl exec 的工作原理

kubectl exec 工作原理

kubectl cp 命令行执行详细输出

I0309 17:43:57.371546    9890 loader.go:379] Config loaded from file:  /root/.kube/config
I0309 17:43:57.375227    9890 round_trippers.go:425] curl -k -v -XGET  -H "Accept: application/json, */*" -H "User-Agent: kubectl/v1.20.0 (linux/amd64) kubernetes/af46c47" 'https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4'
I0309 17:43:57.395829    9890 round_trippers.go:445] GET https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4 200 OK in 20 milliseconds
I0309 17:43:57.395862    9890 round_trippers.go:451] Response Headers:
I0309 17:43:57.395874    9890 round_trippers.go:454]     Date: Tue, 09 Mar 2021 09:43:57 GMT
I0309 17:43:57.395885    9890 round_trippers.go:454]     Cache-Control: no-cache, private
I0309 17:43:57.395896    9890 round_trippers.go:454]     Content-Type: application/json
I0309 17:43:57.395906    9890 round_trippers.go:454]     X-Kubernetes-Pf-Flowschema-Uid: adfbb80d-70c3-4b0b-918d-0d98b7f957e1
I0309 17:43:57.395917    9890 round_trippers.go:454]     X-Kubernetes-Pf-Prioritylevel-Uid: 7aaef20e-1b25-46c2-b653-86e7fd6cb388
I0309 17:43:57.396109    9890 request.go:1107] Response Body: {"kind":"Pod","apiVersion":"v1","metadata":{"name":"nginx-test-76996486df-8rnb4","generateName":"nginx-test-76996486df-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4","uid":"25581d24-1f5f-4103-a494-77b78c4de5c5","resourceVersion":"191426","creationTimestamp":"2021-03-03T06:51:42Z","labels":{"app":"nginx-test","pod-template-hash":"76996486df"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"ReplicaSet","name":"nginx-test-76996486df","uid":"0d4f29b9-9fe9-48f5-a418-697f4ab95bba","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2021-03-03T06:51:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"0d4f29b9-9fe9-48f5-a418-697f4ab95bba\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx-0\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":80,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:protocol":{}}},"f:resources":{".":{},"f:limits":{".":{},"f:memory":{}},"f:requests":{".":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:imagePullSecrets":{".":{},"k:{\"name\":\"docker-secret\"}":{".":{},"f:name":{}}},"f:nodeSelector":{".":{},"f:beta.kubernetes.io/os":{}},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2021-03-03T06:51:44Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.99\"}":{".":{},"f:ip":{}}},"f:startTime":{}}}}]},"spec":{"volumes":[{"name":"default-token-2wzcd","secret":{"secretName":"default-token-2wzcd","defaultMode":420}}],"containers":[{"name":"nginx-0","image":"nginx:1.19.6","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{"limits":{"memory":"200Mi"},"requests":{"memory":"100Mi"}},"volumeMounts":[{"name":"default-token-2wzcd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"beta.kubernetes.io/os":"linux"},"serviceAccountName":"default","serviceAccount":"default","nodeName":"linux-master","securityContext":{},"imagePullSecrets":[{"name":"docker-secret"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:42Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:44Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:44Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:42Z"}],"hostIP":"172.16.127.45","podIP":"10.244.0.99","podIPs":[{"ip":"10.244.0.99"}],"startTime":"2021-03-03T06:51:42Z","containerStatuses":[{"name":"nginx-0","state":{"running":{"startedAt":"2021-03-03T06:51:43Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"docker.io/library/nginx:1.19.6","imageID":"docker.io/library/nginx@sha256:4cf620a5c81390ee209398ecc18e5fb9dd0f5155cd82adcbae532fec94006fb9","containerID":"containerd://d45c10a132a5b777583b3468739018113a46e7025a6d58099ab523ec54accae7","started":true}],"qosClass":"Burstable"}}
I0309 17:43:57.409856    9890 round_trippers.go:425] curl -k -v -XPOST  -H "X-Stream-Protocol-Version: v4.channel.k8s.io" -H "X-Stream-Protocol-Version: v3.channel.k8s.io" -H "X-Stream-Protocol-Version: v2.channel.k8s.io" -H "X-Stream-Protocol-Version: channel.k8s.io" -H "User-Agent: kubectl/v1.20.0 (linux/amd64) kubernetes/af46c47" 'https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4/exec?command=test&command=-d&command=%2Froot&container=nginx-0&stderr=true&stdout=true'
I0309 17:43:57.444467    9890 round_trippers.go:445] POST https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4/exec?command=test&command=-d&command=%2Froot&container=nginx-0&stderr=true&stdout=true 101 Switching Protocols in 34 milliseconds
I0309 17:43:57.444501    9890 round_trippers.go:451] Response Headers:
I0309 17:43:57.444521    9890 round_trippers.go:454]     Upgrade: SPDY/3.1
I0309 17:43:57.444533    9890 round_trippers.go:454]     X-Stream-Protocol-Version: v4.channel.k8s.io
I0309 17:43:57.444543    9890 round_trippers.go:454]     Date: Tue, 09 Mar 2021 09:43:57 GMT
I0309 17:43:57.444554    9890 round_trippers.go:454]     Connection: Upgrade
I0309 17:43:57.539486    9890 round_trippers.go:425] curl -k -v -XGET  -H "Accept: application/json, */*" -H "User-Agent: kubectl/v1.20.0 (linux/amd64) kubernetes/af46c47" 'https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4'
I0309 17:43:57.542487    9890 round_trippers.go:445] GET https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4 200 OK in 2 milliseconds
I0309 17:43:57.542534    9890 round_trippers.go:451] Response Headers:
I0309 17:43:57.542547    9890 round_trippers.go:454]     Date: Tue, 09 Mar 2021 09:43:57 GMT
I0309 17:43:57.542559    9890 round_trippers.go:454]     Cache-Control: no-cache, private
I0309 17:43:57.542571    9890 round_trippers.go:454]     Content-Type: application/json
I0309 17:43:57.542582    9890 round_trippers.go:454]     X-Kubernetes-Pf-Flowschema-Uid: adfbb80d-70c3-4b0b-918d-0d98b7f957e1
I0309 17:43:57.542592    9890 round_trippers.go:454]     X-Kubernetes-Pf-Prioritylevel-Uid: 7aaef20e-1b25-46c2-b653-86e7fd6cb388
I0309 17:43:57.542807    9890 request.go:1107] Response Body: {"kind":"Pod","apiVersion":"v1","metadata":{"name":"nginx-test-76996486df-8rnb4","generateName":"nginx-test-76996486df-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4","uid":"25581d24-1f5f-4103-a494-77b78c4de5c5","resourceVersion":"191426","creationTimestamp":"2021-03-03T06:51:42Z","labels":{"app":"nginx-test","pod-template-hash":"76996486df"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"ReplicaSet","name":"nginx-test-76996486df","uid":"0d4f29b9-9fe9-48f5-a418-697f4ab95bba","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2021-03-03T06:51:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"0d4f29b9-9fe9-48f5-a418-697f4ab95bba\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx-0\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":80,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:protocol":{}}},"f:resources":{".":{},"f:limits":{".":{},"f:memory":{}},"f:requests":{".":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:imagePullSecrets":{".":{},"k:{\"name\":\"docker-secret\"}":{".":{},"f:name":{}}},"f:nodeSelector":{".":{},"f:beta.kubernetes.io/os":{}},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2021-03-03T06:51:44Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.99\"}":{".":{},"f:ip":{}}},"f:startTime":{}}}}]},"spec":{"volumes":[{"name":"default-token-2wzcd","secret":{"secretName":"default-token-2wzcd","defaultMode":420}}],"containers":[{"name":"nginx-0","image":"nginx:1.19.6","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{"limits":{"memory":"200Mi"},"requests":{"memory":"100Mi"}},"volumeMounts":[{"name":"default-token-2wzcd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"beta.kubernetes.io/os":"linux"},"serviceAccountName":"default","serviceAccount":"default","nodeName":"linux-master","securityContext":{},"imagePullSecrets":[{"name":"docker-secret"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:42Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:44Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:44Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:42Z"}],"hostIP":"172.16.127.45","podIP":"10.244.0.99","podIPs":[{"ip":"10.244.0.99"}],"startTime":"2021-03-03T06:51:42Z","containerStatuses":[{"name":"nginx-0","state":{"running":{"startedAt":"2021-03-03T06:51:43Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"docker.io/library/nginx:1.19.6","imageID":"docker.io/library/nginx@sha256:4cf620a5c81390ee209398ecc18e5fb9dd0f5155cd82adcbae532fec94006fb9","containerID":"containerd://d45c10a132a5b777583b3468739018113a46e7025a6d58099ab523ec54accae7","started":true}],"qosClass":"Burstable"}}
I0309 17:43:57.543968    9890 round_trippers.go:425] curl -k -v -XPOST  -H "User-Agent: kubectl/v1.20.0 (linux/amd64) kubernetes/af46c47" -H "X-Stream-Protocol-Version: v4.channel.k8s.io" -H "X-Stream-Protocol-Version: v3.channel.k8s.io" -H "X-Stream-Protocol-Version: v2.channel.k8s.io" -H "X-Stream-Protocol-Version: channel.k8s.io" 'https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4/exec?command=tar&command=-xmf&command=-&command=-C&command=%2Froot&container=nginx-0&stderr=true&stdin=true&stdout=true'
I0309 17:43:57.577687    9890 round_trippers.go:445] POST https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4/exec?command=tar&command=-xmf&command=-&command=-C&command=%2Froot&container=nginx-0&stderr=true&stdin=true&stdout=true 101 Switching Protocols in 33 milliseconds
I0309 17:43:57.578212    9890 round_trippers.go:451] Response Headers:
I0309 17:43:57.578228    9890 round_trippers.go:454]     Connection: Upgrade
I0309 17:43:57.578236    9890 round_trippers.go:454]     Upgrade: SPDY/3.1
I0309 17:43:57.578244    9890 round_trippers.go:454]     X-Stream-Protocol-Version: v4.channel.k8s.io
I0309 17:43:57.578252    9890 round_trippers.go:454]     Date: Tue, 09 Mar 2021 09:43:57 GMT

kubectl exec -it 命令行执行详细输出

kubectl -v=9 exec -it nginx-test-76996486df-8rnb4 -- bash         
I0309 17:44:34.259082   12133 loader.go:379] Config loaded from file:  /root/.kube/config
I0309 17:44:34.268695   12133 round_trippers.go:425] curl -k -v -XGET  -H "Accept: application/json, */*" -H "User-Agent: kubectl/v1.20.0 (linux/amd64) kubernetes/af46c47" 'https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4'
I0309 17:44:34.295256   12133 round_trippers.go:445] GET https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4 200 OK in 26 milliseconds
I0309 17:44:34.295342   12133 round_trippers.go:451] Response Headers:
I0309 17:44:34.295365   12133 round_trippers.go:454]     Cache-Control: no-cache, private
I0309 17:44:34.295383   12133 round_trippers.go:454]     Content-Type: application/json
I0309 17:44:34.295587   12133 round_trippers.go:454]     X-Kubernetes-Pf-Flowschema-Uid: adfbb80d-70c3-4b0b-918d-0d98b7f957e1
I0309 17:44:34.295621   12133 round_trippers.go:454]     X-Kubernetes-Pf-Prioritylevel-Uid: 7aaef20e-1b25-46c2-b653-86e7fd6cb388
I0309 17:44:34.295654   12133 round_trippers.go:454]     Date: Tue, 09 Mar 2021 09:44:34 GMT
I0309 17:44:34.295874   12133 request.go:1107] Response Body: {"kind":"Pod","apiVersion":"v1","metadata":{"name":"nginx-test-76996486df-8rnb4","generateName":"nginx-test-76996486df-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4","uid":"25581d24-1f5f-4103-a494-77b78c4de5c5","resourceVersion":"191426","creationTimestamp":"2021-03-03T06:51:42Z","labels":{"app":"nginx-test","pod-template-hash":"76996486df"},"ownerReferences":[{"apiVersion":"apps/v1","kind":"ReplicaSet","name":"nginx-test-76996486df","uid":"0d4f29b9-9fe9-48f5-a418-697f4ab95bba","controller":true,"blockOwnerDeletion":true}],"managedFields":[{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2021-03-03T06:51:42Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"0d4f29b9-9fe9-48f5-a418-697f4ab95bba\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx-0\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":80,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:protocol":{}}},"f:resources":{".":{},"f:limits":{".":{},"f:memory":{}},"f:requests":{".":{},"f:memory":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:imagePullSecrets":{".":{},"k:{\"name\":\"docker-secret\"}":{".":{},"f:name":{}}},"f:nodeSelector":{".":{},"f:beta.kubernetes.io/os":{}},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2021-03-03T06:51:44Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.244.0.99\"}":{".":{},"f:ip":{}}},"f:startTime":{}}}}]},"spec":{"volumes":[{"name":"default-token-2wzcd","secret":{"secretName":"default-token-2wzcd","defaultMode":420}}],"containers":[{"name":"nginx-0","image":"nginx:1.19.6","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{"limits":{"memory":"200Mi"},"requests":{"memory":"100Mi"}},"volumeMounts":[{"name":"default-token-2wzcd","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","nodeSelector":{"beta.kubernetes.io/os":"linux"},"serviceAccountName":"default","serviceAccount":"default","nodeName":"linux-master","securityContext":{},"imagePullSecrets":[{"name":"docker-secret"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:42Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:44Z"},{"type":"ContainersReady","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:44Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2021-03-03T06:51:42Z"}],"hostIP":"172.16.127.45","podIP":"10.244.0.99","podIPs":[{"ip":"10.244.0.99"}],"startTime":"2021-03-03T06:51:42Z","containerStatuses":[{"name":"nginx-0","state":{"running":{"startedAt":"2021-03-03T06:51:43Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"docker.io/library/nginx:1.19.6","imageID":"docker.io/library/nginx@sha256:4cf620a5c81390ee209398ecc18e5fb9dd0f5155cd82adcbae532fec94006fb9","containerID":"containerd://d45c10a132a5b777583b3468739018113a46e7025a6d58099ab523ec54accae7","started":true}],"qosClass":"Burstable"}}
I0309 17:44:34.313710   12133 round_trippers.go:425] curl -k -v -XPOST  -H "X-Stream-Protocol-Version: v4.channel.k8s.io" -H "X-Stream-Protocol-Version: v3.channel.k8s.io" -H "X-Stream-Protocol-Version: v2.channel.k8s.io" -H "X-Stream-Protocol-Version: channel.k8s.io" -H "User-Agent: kubectl/v1.20.0 (linux/amd64) kubernetes/af46c47" 'https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4/exec?command=bash&container=nginx-0&stdin=true&stdout=true&tty=true'
I0309 17:44:34.361436   12133 round_trippers.go:445] POST https://172.16.127.45:6443/api/v1/namespaces/default/pods/nginx-test-76996486df-8rnb4/exec?command=bash&container=nginx-0&stdin=true&stdout=true&tty=true 101 Switching Protocols in 47 milliseconds
I0309 17:44:34.361487   12133 round_trippers.go:451] Response Headers:
I0309 17:44:34.361506   12133 round_trippers.go:454]     Connection: Upgrade
I0309 17:44:34.361519   12133 round_trippers.go:454]     Upgrade: SPDY/3.1
I0309 17:44:34.361531   12133 round_trippers.go:454]     X-Stream-Protocol-Version: v4.channel.k8s.io
I0309 17:44:34.361589   12133 round_trippers.go:454]     Date: Tue, 09 Mar 2021 09:44:34 GMT
                                                                                                                                                                                                                                 root@ngiroot@nginx-test-76996486df-8rnb4:/# 

这里有两个重要的 HTTP 请求:

  • GET 请求用来获取Pod信息。
  • POST 请求调用Pod的子资源exec在容器内执行命令。

最后 API Server 返回了 101 Ugrade 响应,向客户端表示已切换到 SPDY 协议

  • SPDY协议是google开发的TCP会话层协议, SPDY协议中将Http的Request/Response称为Stream,并支持TCP的链接复用,同时多个stream之间通过Stream-id来进行标记,简单来说就是支持在单个链接同时进行多个请求响应的处理,并且互不影响,k8s中的命令执行主要也就是通过stream来进行消息传递的

API Server 收到请求后,找到需要转发的 node 地址,即 nodeip:10255端口,然后开始连接

// GetConnectionInfo 从节点API对象的状态检索连接信息
func (k *NodeConnectionInfoGetter) GetConnectionInfo(ctx context.Context, nodeName types.NodeName) (*ConnectionInfo, error) {
        node, err := k.nodes.Get(ctx, string(nodeName), metav1.GetOptions{})
        if err != nil {
                return nil, err
        }

       ....

        return &ConnectionInfo{
                Scheme:    k.scheme,
                Hostname:  host,
                Port:      strconv.Itoa(port),
                Transport: k.transport,
        }, nil
}
....

location, transport, err := pod.ExecLocation(r.Store, r.KubeletConn, ctx, name, execOpts)
        if err != nil {
                return nil, err
        }
        return newThrottledUpgradeAwareProxyHandler(location, transport, false, true, true, responder), nil

kubelet 接到 apiserver 的请求后,调用各运行时的RuntimeServiceServer的实现,包括 exec 接口的实现,给 apiserver返回一个连接端点

// Exec 准备一个流入口点以在容器中执行命令。
Exec(context.Context, *ExecRequest) (*ExecResponse, error)

最后,容器运行时在工作节点上执行命令,如:

 if v, found := os.LookupEnv("XDG_RUNTIME_DIR"); found {
                execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", v))
        }
        var cmdErr, copyError error
        if tty {
                cmdErr = ttyCmd(execCmd, stdin, stdout, resize)
        } else {
                if stdin != nil {
                    ......

exec 流程

kubectl exec 流程完成

Exec 核心源码(非官方)

    // 初始化pod所在的corev1资源组,发送请求
    // PodExecOptions struct 包括Container stdout stdout  Command 等结构
    // scheme.ParameterCodec 应该是pod 的GVK (GroupVersion & Kind)之类的

    req := e.K8sClient.CoreV1().RESTClient().Post().
        Resource("pods").
        Name(e.PodName).
        Namespace(e.Namespace).
        SubResource("exec").
        VersionedParams(&coreV1.PodExecOptions{
            Command:   e.Command,
            Container: e.ContainerName,
            Stdin:     e.Stdin != nil,
            Stdout:    e.Stdout != nil,
            Stderr:    e.Stderr != nil,
            TTY:       e.Tty,
        }, scheme.ParameterCodec)

    // remotecommand 主要实现了http 转 SPDY 添加X-Stream-Protocol-Version相关header 并发送请求
    exec, err := remotecommand.NewSPDYExecutor(e.RESTConfig, "POST", req.URL())
    if err != nil {
        return err
    }

    // 建立链接之后从请求的sream中发送、读取数据
    var sizeQueue remotecommand.TerminalSizeQueue
    err = exec.Stream(remotecommand.StreamOptions{
        Stdin:             e.Stdin,
        Stdout:            e.Stdout,
        Stderr:            e.Stderr,
        Tty:               e.Tty,
        TerminalSizeQueue: sizeQueue,
    })
    if err != nil {
        return err
    }

kubectl cp 原理

kubectl cp 指令是将容器当中的数据拷贝到当前机器上; 同理,也可以将当前机器上的文件或文件目录拷贝到容器当中; 当然也可从pod拷贝到pod
查看 kubernetes/pkg/kubectl/cmd/cp/cp.go 目录下的代码,可以看到具体实现.
其实现是将文件或文件夹抽象成一个数据流 Stream,然后通过 kube-apiserver 访问容器,对进行文件传输。

tar

使用 tar -cf - 将具有文件夹结构的数据转换成数据流,再通过 linux 管道接收这个数据流;通过 tar -xf - 将数据流转换成 linux 文件系统。

-f, --file=ARCHIVE use archive file or device ARCHIVE

从本地拷贝到容器内

tar file ---> io.Writer ---> kubernetes api ---> io.Reader ---> unTar file

完整源码

func (o *CopyOptions) copyToPod(src, dest fileSpec, options *exec.ExecOptions) error {
    if len(src.File) == 0 || len(dest.File) == 0 {
        return errFileCannotBeEmpty
    }
    if _, err := os.Stat(src.File); err != nil {
        return fmt.Errorf("%s doesn't exist in local filesystem", src.File)
    }
    reader, writer := io.Pipe()

    // strip trailing slash (if any)
    if dest.File != "/" && strings.HasSuffix(string(dest.File[len(dest.File)-1]), "/") {
        dest.File = dest.File[:len(dest.File)-1]
    }

    if err := o.checkDestinationIsDir(dest); err == nil {
        // If no error, dest.File was found to be a directory.
        // Copy specified src into it
        dest.File = dest.File + "/" + path.Base(src.File)
    }

    go func() {
        defer writer.Close()
        // 使用tar压缩输出到writer
        err := makeTar(src.File, dest.File, writer)
        cmdutil.CheckErr(err)
    }()
    var cmdArr []string

    // TODO: Improve error messages by first testing if 'tar' is present in the container?
    if o.NoPreserve {
        cmdArr = []string{"tar", "--no-same-permissions", "--no-same-owner", "-xmf", "-"}
    } else {
        cmdArr = []string{"tar", "-xmf", "-"}
    }
    destDir := path.Dir(dest.File)
    if len(destDir) > 0 {
        cmdArr = append(cmdArr, "-C", destDir)
    }

    options.StreamOptions = exec.StreamOptions{
        IOStreams: genericclioptions.IOStreams{
            In:     reader,
            Out:    o.Out,
            ErrOut: o.ErrOut,
        },
        Stdin: true,

        Namespace: dest.PodNamespace,
        PodName:   dest.PodName,
    }

    options.Command = cmdArr
    options.Executor = &exec.DefaultRemoteExecutor{}
    return o.execute(options)
}

从容器内拷贝到本地

tar file ---> io.Writer ---> kubernetes api ---> io.Reader ---> unTar file

完整源码

func (o *CopyOptions) copyFromPod(src, dest fileSpec) error {
    if len(src.File) == 0 || len(dest.File) == 0 {
        return errFileCannotBeEmpty
    }

    reader, outStream := io.Pipe()
    options := &exec.ExecOptions{
        StreamOptions: exec.StreamOptions{
            IOStreams: genericclioptions.IOStreams{
                In:     nil,
                Out:    outStream,
                ErrOut: o.Out,
            },

            Namespace: src.PodNamespace,
            PodName:   src.PodName,
        },

        // TODO: Improve error messages by first testing if 'tar' is present in the container?
        Command:  []string{"tar", "cf", "-", src.File},
        Executor: &exec.DefaultRemoteExecutor{},
    }

    go func() {
        defer outStream.Close()
        err := o.execute(options)
        cmdutil.CheckErr(err)
    }()
    prefix := getPrefix(src.File)
    prefix = path.Clean(prefix)
    // remove extraneous path shortcuts - these could occur if a path contained extra "../"
    // and attempted to navigate beyond "/" in a remote filesystem
    prefix = stripPathShortcuts(prefix)
    // 使用tar把拷贝出来的解压
    return o.untarAll(src, reader, dest.File, prefix)
}

kubernetes webshell实现

k8s.io/client-go/tools/remotecommand kubernetes client-go 提供的 remotecommand 包,提供了方法与集群中的容器建立长连接,并设置容器的 stdinstdout 等。
remotecommand 包提供基于 SPDY 协议的 Executor interface,进行和 pod 终端的流的传输。初始化一个 Executor 很简单,只需要调用 remotecommandNewSPDYExecutor 并传入对应参数。
ExecutorStream 方法,会建立一个流传输的连接,直到服务端和调用端一端关闭连接,才会停止传输。常用的做法是定义一个如下 PtyHandlerinterface,然后使用你想用的客户端实现该 interface 对应的Read(p []byte) (int, error)Write(p []byte) (int, error)方法即可,调用 Stream 方法时,只要将 StreamOptionsStdin Stdout 都设置为 ptyHandlerExecutor 就会通过你定义的 writeread 方法来传输数据。

// PtyHandler
type PtyHandler interface {
    io.Reader
    io.Writer
    remotecommand.TerminalSizeQueue
}

// NewSPDYExecutor
req := kubeClient.CoreV1().RESTClient().Post().
        Resource("pods").
        Name(podName).
        Namespace(namespace).
        SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
    Container: containerName,
    Command:   cmd,
    Stdin:     true,
    Stdout:    true,
    Stderr:    true,
    TTY:       true,
}, scheme.ParameterCodec)
executor, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL())
if err != nil {
    log.Printf("NewSPDYExecutor err: %v", err)
    return err
}

// Stream
err = executor.Stream(remotecommand.StreamOptions{
        Stdin:             ptyHandler,
        Stdout:            ptyHandler,
        Stderr:            ptyHandler,
        TerminalSizeQueue: ptyHandler,
        Tty:               true,
    })

websocket

效果图

github.com/gorilla/websocketgo 的一个 websocket 实现,提供了全面的 websocket 相关的方法,这里使用它来实现上面所说的PtyHandler接口。
首先定义一个 TerminalSession 类,该类包含一个 *websocket.Conn,通过 websocket 连接实现PtyHandler接口的读写方法,Next 方法在 remotecommand 执行过程中会被调用。

// TerminalSession
type TerminalSession struct {
    wsConn   *websocket.Conn
    sizeChan chan remotecommand.TerminalSize
    doneChan chan struct{}
}

// from remotecommand executor回调获取web是否resize
func (t *TerminalSession) Next() *remotecommand.TerminalSize {
    select {
    case size := <-t.sizeChan:
        return &size
    case <-t.doneChan:
        return nil
    }
}
// remotecommand executor回调读取web端的输入
func (t *TerminalSession) Read(p []byte) (int, error) {
    _, message, err := t.wsConn.ReadMessage()
    if err != nil {
        log.Printf("read message err: %v", err)
        return copy(p, webshell.EndOfTransmission), err
    }
    var msg webshell.TerminalMessage
    if err := json.Unmarshal([]byte(message), &msg); err != nil {
        log.Printf("read parse message err: %v", err)
        // return 0, nil
        return copy(p, webshell.EndOfTransmission), err
    }
    switch msg.Operation {
    case "stdin":
        return copy(p, msg.Data), nil
    case "resize":
        t.sizeChan <- remotecommand.TerminalSize{Width: msg.Cols, Height: msg.Rows}
        return 0, nil
    default:
        log.Printf("unknown message type '%s'", msg.Operation)
        // return 0, nil
        return copy(p, webshell.EndOfTransmission), fmt.Errorf("unknown message type '%s'", msg.Operation)
    }
}

// remotecommand executor回调向web端输出
func (t *TerminalSession) Write(p []byte) (int, error) {
    msg, err := json.Marshal(webshell.TerminalMessage{
        Operation: "stdout",
        Data:      string(p),
    })
    if err != nil {
        log.Printf("write parse message err: %v", err)
        return 0, err
    }
    if err := t.wsConn.WriteMessage(websocket.TextMessage, msg); err != nil {
        log.Printf("write message err: %v", err)
        return 0, err
    }
    return len(p), nil
}

// Close close session
func (t *TerminalSession) Close() error {
    return t.wsConn.Close()
}

file browser

效果图

关键代码

func FileBrowser(c *gin.Context) {
    render := apis.Gin{C: c}
    // {"ls", "-lQ", "--color=never", "--full-time", "/"}
    var query = &FileBrowserQuery{
        Path: "/",
    }
    if err := c.ShouldBindQuery(query); err != nil {
        render.SetError(utils.CODE_ERR_PARAM, err)
        return
    }
    // check namespace
    _, err := configs.RestClient.CoreV1().Namespaces().
        Get(context.TODO(), query.Namespace, metaV1.GetOptions{})
    if err != nil {
        logs.Error(err)
        render.SetError(utils.CODE_ERR_APP, err)
        return
    }

    // check pod
    pod, err := configs.RestClient.CoreV1().Pods(query.Namespace).
        Get(context.TODO(), query.Pods, metaV1.GetOptions{})
    if err != nil {
        logs.Error(err)
        render.SetError(utils.CODE_ERR_APP, err)
        return
    }
    var isUnix = true
    for _, value := range pod.Spec.NodeSelector {
        if strings.Contains(value, "windows") {
            isUnix = false
            break
        }
    }

    lsPath := "/ls_linux_amd64"
    command := []string{"/ls", query.Path}
    if !isUnix {
        lsPath = "/ls_windows_amd64.exe"
    }
    resByte, err := query.exec(command)
    if err != nil {
        logs.Error(err)
        if strings.Contains(err.Error(), "ls") ||
            err.Error() == "command terminated with exit code 126" {
            if isUnix {
                _, err = query.exec([]string{"sh"})
            } else {
                _, err = query.exec([]string{"cmd"})
            }
            if err != nil {
                logs.ErrorWithFields(err, logs.Fields{
                    "annotation": "test container terminal shell",
                })
                render.SetError(utils.CODE_ERR_APP, err)
                return
            }
            err = query.copyLsTar(lsPath)
            if err != nil {
                logs.Error(err)
                render.SetError(utils.CODE_ERR_APP, err)
                return
            }
            if isUnix {
                _cmd := []string{"chmod", "+x", "/ls"}
                _, err = query.exec(_cmd)
                if err != nil {
                    logs.Error(err)
                    render.SetError(utils.CODE_ERR_APP, err)
                    return
                }
            }
            resByte, err = query.exec(command)
            if err != nil {
                logs.Error(err)
                render.SetError(utils.CODE_ERR_APP, err)
                return
            }
        } else {
            render.SetError(utils.CODE_ERR_APP, err)
            return
        }
    }
    var res []utils.File
    if err := json.Unmarshal(resByte, &res); err != nil {
        logs.Error(err)
        render.SetError(utils.CODE_ERR_APP, err)
        return
    }
    render.SetRes(res, nil, 0)
}

func (query *FileBrowserQuery) exec(command []string) ([]byte, error) {
    var stdout, stderr bytes.Buffer
    exec := execer.NewExec(query.Namespace, query.Pods, query.Container, configs.KuBeResConf, configs.RestClient)
    exec.Command = command
    exec.Tty = false
    exec.Stdout = &stdout
    exec.Stderr = &stderr
    err := exec.Exec()
    if err != nil {
        logs.Error(stderr.String())
        return nil, err
    }
    if len(stderr.Bytes()) != 0 {
        return nil, fmt.Errorf(stderr.String())
    }

    return stdout.Bytes(), nil
}

func (query *FileBrowserQuery) copyLsTar(lsPath string) error {
    reader, writer := io.Pipe()
    cp := copyer.NewCopyer(query.Namespace, query.Pods, query.Container, configs.KuBeResConf, configs.RestClient)
    cp.Stdin = reader

    go func() {
        defer writer.Close()
        err := utils.TarLs(lsPath, writer)
        if err != nil {
            logs.Error(err)
        }
    }()
    return cp.CopyToPod(lsPath)
}

参考文献

源码

kubefilerbrowser

添加新评论