containerd 를 사용하는 윈도우 노드 추가
지난 '쿠버네티스 윈도우 워커 노드 추가(with Calico CNI)' 에서 Docker EE 를 사용하여 Windows 워커 노드를 추가했습니다. Docker EE 가 deprecated 됨에 따라 containerd 를 활용할 필요가 있어서 추가로 containerd 를 이용해 Windows 워커노드를 추가하는 절차를 기록했습니다.
아래 'containerd 시작하기' 문서를 참고하여 containerd 를 설치합니다.
PS C:\Users\Administrator> $Version="1.6.4"
PS C:\Users\Administrator> curl.exe -L https://github.com/containerd/containerd/releases/download/v$Version/containerd-$Version-windows-amd64.tar.gz -o containerd-windows-amd64.tar.gz
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 30.0M 100 30.0M 0 0 1393k 0 0:00:22 0:00:22 --:--:-- 1630k
PS C:\Users\Administrator> tar.exe xvf .\containerd-windows-amd64.tar.gz
x bin/
x bin/containerd.exe
x bin/containerd-shim-runhcs-v1.exe
x bin/containerd-stress.exe
x bin/ctr.exe
PS C:\Users\Administrator> Copy-Item -Path ".\bin\" -Destination "$Env:ProgramFiles\containerd" -Recurse -Force
PS C:\Users\Administrator> cd $Env:ProgramFiles\containerd\
PS C:\Program Files\containerd> .\containerd.exe config default | Out-File config.toml -Encoding ascii
PS C:\Program Files\containerd> Get-Content config.toml
disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "C:\\ProgramData\\containerd\\root"
state = "C:\\ProgramData\\containerd\\state"
temp = ""
version = 2
[cgroup]
path = ""
[debug]
address = ""
format = ""
gid = 0
level = ""
uid = 0
[grpc]
address = "\\\\.\\pipe\\containerd-containerd"
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
tcp_address = ""
tcp_tls_ca = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0
[metrics]
address = ""
grpc_histogram = false
[plugins]
[plugins."io.containerd.gc.v1.scheduler"]
deletion_threshold = 0
mutation_threshold = 100
pause_threshold = 0.02
schedule_delay = "0s"
startup_delay = "100ms"
[plugins."io.containerd.grpc.v1.cri"]
device_ownership_from_security_context = false
disable_apparmor = false
disable_cgroup = false
disable_hugetlb_controller = false
disable_proc_mount = false
disable_tcp_service = true
enable_selinux = false
enable_tls_streaming = false
enable_unprivileged_icmp = false
enable_unprivileged_ports = false
ignore_image_defined_volumes = false
max_concurrent_downloads = 3
max_container_log_line_size = 16384
netns_mounts_under_state_dir = false
restrict_oom_score_adj = false
sandbox_image = "k8s.gcr.io/pause:3.6"
selinux_category_range = 0
stats_collect_period = 10
stream_idle_timeout = "4h0m0s"
stream_server_address = "127.0.0.1"
stream_server_port = "0"
systemd_cgroup = false
tolerate_missing_hugetlb_controller = false
unset_seccomp_profile = ""
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "C:\\Program Files\\containerd\\cni\\bin"
conf_dir = "C:\\Program Files\\containerd\\cni\\conf"
conf_template = ""
ip_pref = ""
max_conf_num = 1
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runhcs-wcow-process"
disable_snapshot_annotations = false
discard_unpacked_layers = false
ignore_rdt_not_enabled_errors = false
no_pivot = false
snapshotter = "windows"
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = "io.containerd.runhcs.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process.options]
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
base_runtime_spec = ""
cni_conf_dir = ""
cni_max_conf_num = 0
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_path = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = "node"
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = ""
[plugins."io.containerd.grpc.v1.cri".registry.auths]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.headers]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.internal.v1.opt"]
path = "C:\\ProgramData\\containerd\\root\\opt"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.internal.v1.tracing"]
sampling_ratio = 1.0
service_name = "containerd"
[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"
[plugins."io.containerd.runtime.v2.task"]
platforms = ["windows/amd64", "linux/amd64"]
sched_core = false
[plugins."io.containerd.service.v1.diff-service"]
default = ["windows", "windows-lcow"]
[plugins."io.containerd.service.v1.tasks-service"]
rdt_config_file = ""
[plugins."io.containerd.tracing.processor.v1.otlp"]
endpoint = ""
insecure = false
protocol = ""
[proxy_plugins]
[stream_processors]
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar"
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar+gzip"
[timeouts]
"io.containerd.timeout.bolt.open" = "0s"
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"
[ttrpc]
address = ""
gid = 0
uid = 0
PS C:\Program Files\containerd> .\containerd.exe --register-service
PS C:\Program Files\containerd> Start-Service containerd
PS C:\Program Files\containerd> Get-Service containerd
Status Name DisplayName
------ ---- -----------
Running containerd containerd
containerd 가 윈도우의 서비스 형태로 실행됩니다.
docker 가 설치되지 않기 때문에 별도의 CLI를 설치합니다. crictl 명령을 수행해보면 각각의 컨테이너 런타임의 endpont를 찾기 때문에, 아래를 참고하여 user profile 쪽에 crictl.config을 생성했습니다.
https://github.com/kubernetes-sigs/cri-tools/blob/master/docs/crictl.md
PS C:\Users\Administrator> $VERSION="v1.24.2"
PS C:\Users\Administrator> curl.exe -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-windows-amd64.tar.gz -o crictl-windows-amd64.tar.gz
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 14.1M 100 14.1M 0 0 1275k 0 0:00:11 0:00:11 --:--:-- 1705k
PS C:\Users\Administrator> tar.exe xvf .\crictl-windows-amd64.tar.gz -C $ENV:WINDIR\system32
x crictl.exe
PS C:\Users\Administrator> crictl ps
time="2022-07-12T14:35:37+09:00" level=warning msg="runtime connect using default endpoints: [npipe:////./pipe/dockershim npipe:////./pipe/containerd-containerd npipe:////./pipe/cri-dockerd]. As the default settings are now deprecated, you should set the endpoint instead."
time="2022-07-12T14:35:37+09:00" level=error msg="unable to determine runtime API version: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing open //./pipe/dockershim: The system cannot find the file specified.\""
time="2022-07-12T14:35:37+09:00" level=warning msg="image connect using default endpoints: [npipe:////./pipe/dockershim npipe:////./pipe/containerd-containerd npipe:////./pipe/cri-dockerd]. As the default settings are now deprecated, you should set the endpoint instead."
time="2022-07-12T14:35:37+09:00" level=error msg="unable to determine image API version: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing open //./pipe/dockershim: The system cannot find the file specified.\""
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
PS C:\Users\Administrator> mkdir $Env:UserProfile\.crictl
Directory: C:\Users\Administrator
Mode LastWriteTime Length Name
---- ------------- ------ ----
d----- 2022-07-12 오후 2:40 .crictl
PS C:\Users\Administrator> notepad $Env:UserProfile\.crictl\crictl.yaml
PS C:\Users\Administrator> type $Env:UserProfile\.crictl\crictl.yaml
runtime-endpoint: npipe:\\\\.\\pipe\\containerd-containerd
image-endpoint: npipe:\\\\.\\pipe\\containerd-containerd
timeout: 2
debug: true
pull-image-on-create: false
PS C:\Users\Administrator> crictl ps
time="2022-07-12T14:43:19+09:00" level=debug msg="get runtime connection"
time="2022-07-12T14:43:19+09:00" level=debug msg="get image connection"
time="2022-07-12T14:43:19+09:00" level=debug msg="ListContainerResponse: []"
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID PODCalico 를 설치하려고 진행하니 아래와 같이 에러가 발생합니다.
PS C:\Users\Administrator> mkdir c:\k
Directory: C:\
Mode LastWriteTime Length Name
---- ------------- ------ ----
d----- 2022-07-12 오후 2:51 k
PS C:\Users\Administrator> scp root@172.16.3.170:~/.kube/config c:\k\
root@172.16.3.170's password:
config 100% 5640 5.5KB/s 00:00
PS C:\Users\Administrator> Invoke-WebRequest https://projectcalico.docs.tigera.io/scripts/install-calico-windows.ps1 -OutFile c:\install-calico-windows.ps1
PS C:\Users\Administrator> c:\install-calico-windows.ps1 -KubeVersion 1.22.6 -ServiceCidr 10.96.0.0/12 -DNSServerIPs 10.96.0.10
WARNING: The names of some imported commands from the module 'helper' include unapproved verbs that might make them less discoverable. To find the commands with unapproved verbs, run the Import-Module command again with the Verbose
parameter. For a list of approved verbs, type Get-Verb.
c:\calico-windows.zip not found, downloading Calico for Windows release...
Downloaded [https://github.com/projectcalico/calico/releases/download/v3.23.2//calico-windows-v3.23.2.zip] => [c:\calico-windows.zip]
C:\install-calico-windows.ps1 : The term 'Get-HnsNetwork' is not recognized as the name of a cmdlet, function, script file, or operable program. Check the spelling of the name, or if a path was included, verify that the path is correct and t
ry again.
At line:1 char:1
+ c:\install-calico-windows.ps1 -KubeVersion 1.22.6 -ServiceCidr 10.96. ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : ObjectNotFound: (Get-HnsNetwork:String) [install-calico-windows.ps1], CommandNotFoundException
+ FullyQualifiedErrorId : CommandNotFoundException,install-calico-windows.ps1
Docker EE를 사용할 때와 다르게, containerd 를 설치하는 것 자체만으로 Windows 의 컨테이너 feature를 설치하지 않기 때문에 HNS와 같은 서비스 및 기타 cmdlet이 설치되어 있지 않습니다.
The term 'Get-HnsNetwork' is not recognized as the name of a cmdlet, function, script file, or operable program.
윈도우 서버에서 containers fature를 먼서 설치합니다. (재시작으로 수행결과가 없지만 아래와 같이 진행하면 됩니다)
PS C:\Users\Administrator> Install-WindowsFeature -Name containers
PS C:\Users\Administrator> Restart-Computer -Force재시작후 다시 확인해보면 Host Network Service 가 조회됩니다.
PS C:\Users\Administrator> Get-Service hns
Status Name DisplayName
------ ---- -----------
Stopped hns Host Network Service
containerd 로 변경한 이후 한 가지 에러가 더 발생합니다.
PS C:\Users\Administrator> c:\install-calico-windows.ps1 -KubeVersion 1.22.6 -ServiceCidr 10.96.0.0/12 -DNSServerIPs 10.96.0.10
WARNING: The names of some imported commands from the module 'helper' include unapproved verbs that might make them less discoverable. To find the commands with unapproved verbs, run the Import-Module command again with the Verbose
parameter. For a list of approved verbs, type Get-Verb.
Unzip Calico for Windows release...
Creating CNI directory
<생략>
Validating configuration...
CNI binary directory C:\Program Files\containerd\cni\bin doesn't exist. Please create it and ensure kubelet is configured with matching --cni-bin-dir.
At C:\CalicoWindows\libs\calico\calico.psm1:35 char:13
+ throw "CNI binary directory $env:CNI_BIN_DIR doesn't exis ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : OperationStopped: (CNI binary dire... --cni-bin-dir.:String) [], RuntimeException
+ FullyQualifiedErrorId : CNI binary directory C:\Program Files\containerd\cni\bin doesn't exist. Please create it and ensure kubelet is configured with matching --cni-bin-dir.containerd로 변경되면서 cni\bin 위치를 제대로 생성하지 않고, 참조하는 것 같습니다만.. 이 부분은 설치 스크립트를 더 분석해야 확인가능 할 것 같습니다.
일단 해당 디렉터리를 수동으로 생성해주고 시작합니다.
PS C:\Program Files\containerd\cni\conf> mkdir "C:\Program Files\containerd\cni\bin"
Directory: C:\Program Files\containerd\cni
Mode LastWriteTime Length Name
---- ------------- ------ ----
d----- 2022-07-12 오후 3:51 bin
PS C:\Program Files\containerd\cni> c:\install-calico-windows.ps1 -KubeVersion 1.22.6 -ServiceCidr 10.96.0.0/12 -DNSServerIPs 10.96.0.10
WARNING: The names of some imported commands from the module 'helper' include unapproved verbs that might make them less discoverable. To find the commands with unapproved verbs, run the Import-Module command again with the Verbose
parameter. For a list of approved verbs, type Get-Verb.
Unzip Calico for Windows release...
Creating CNI directory
Downloading Windows Kubernetes scripts
[DownloadFile] File c:\k\hns.psm1 already exists.
WARNING: The names of some imported commands from the module 'hns' include unapproved verbs that might make them less discoverable. To find the commands with unapproved verbs, run the Import-Module command again with the Verbose
parameter. For a list of approved verbs, type Get-Verb.
Downloaded [https://dl.k8s.io/v1.22.6/kubernetes-node-windows-amd64.tar.gz] => [C:\Users\Administrator\AppData\Local\Temp\2\tmp3B38.tar.gz]
Setup Calico for Windows...
Error from server (NotFound): namespaces "calico-system" not found
Calico running in kube-system namespace
Backend networking is vxlan
Start Calico for Windows install...
Setting environment variables if not set...
Environment variable KUBE_NETWORK is already set: Calico.*
Environment variable CALICO_NETWORKING_BACKEND is already set: vxlan
Environment variable K8S_SERVICE_CIDR is already set: 10.96.0.0/12
Environment variable DNS_NAME_SERVERS is already set: 10.96.0.10
Environment variable DNS_SEARCH is already set: svc.cluster.local
Environment variable CALICO_DATASTORE_TYPE is already set: kubernetes
Environment variable KUBECONFIG is already set: c:\k\config
Environment variable ETCD_ENDPOINTS is not set. Setting it to the default value:
Environment variable ETCD_KEY_FILE is not set. Setting it to the default value:
Environment variable ETCD_CERT_FILE is not set. Setting it to the default value:
Environment variable ETCD_CA_CERT_FILE is not set. Setting it to the default value:
Environment variable CNI_BIN_DIR is already set: C:\Program Files\containerd\cni\bin
Environment variable CNI_CONF_DIR is already set: C:\Program Files\containerd\cni\conf
Environment variable CNI_CONF_FILENAME is already set: 10-calico.conf
Environment variable CNI_IPAM_TYPE is already set: calico-ipam
Environment variable VXLAN_VNI is already set: 4096
Environment variable VXLAN_MAC_PREFIX is already set: 0E-2A
Environment variable VXLAN_ADAPTER is not set. Setting it to the default value:
Environment variable NODENAME is already set: k8s-ww2
Environment variable CALICO_K8S_NODE_REF is already set: k8s-ww2
Environment variable STARTUP_VALID_IP_TIMEOUT is already set: 90
Environment variable IP is already set: autodetect
Environment variable CALICO_LOG_DIR is already set: C:\CalicoWindows\logs
Environment variable FELIX_LOGSEVERITYFILE is already set: none
Environment variable FELIX_LOGSEVERITYSYS is already set: none
Validating configuration...
Installing node startup service...
Hive: HKEY_LOCAL_MACHINE\Software
Name Property
---- --------
Tigera
Hive: HKEY_LOCAL_MACHINE\Software\Tigera
Name Property
---- --------
Calico
Service "CalicoNode" installed successfully!
Set parameter "AppParameters" for service "CalicoNode".
Set parameter "AppDirectory" for service "CalicoNode".
Set parameter "DisplayName" for service "CalicoNode".
Set parameter "Description" for service "CalicoNode".
Set parameter "Start" for service "CalicoNode".
Reset parameter "ObjectName" for service "CalicoNode" to its default.
Set parameter "Type" for service "CalicoNode".
Reset parameter "AppThrottle" for service "CalicoNode" to its default.
Creating log directory.
PSPath : Microsoft.PowerShell.Core\FileSystem::C:\CalicoWindows\logs
PSParentPath : Microsoft.PowerShell.Core\FileSystem::C:\CalicoWindows
PSChildName : logs
PSDrive : C
PSProvider : Microsoft.PowerShell.Core\FileSystem
PSIsContainer : True
Name : logs
FullName : C:\CalicoWindows\logs
Parent : CalicoWindows
Exists : True
Root : C:\
Extension :
CreationTime : 2022-07-12 오후 3:53:55
CreationTimeUtc : 2022-07-12 오전 6:53:55
LastAccessTime : 2022-07-12 오후 3:53:55
LastAccessTimeUtc : 2022-07-12 오전 6:53:55
LastWriteTime : 2022-07-12 오후 3:53:55
LastWriteTimeUtc : 2022-07-12 오전 6:53:55
Attributes : Directory
Mode : d-----
BaseName : logs
Target : {}
LinkType :
Set parameter "AppStdout" for service "CalicoNode".
Set parameter "AppStderr" for service "CalicoNode".
Set parameter "AppRotateFiles" for service "CalicoNode".
Set parameter "AppRotateOnline" for service "CalicoNode".
Set parameter "AppRotateSeconds" for service "CalicoNode".
Set parameter "AppRotateBytes" for service "CalicoNode".
Done installing startup service.
Installing Felix service...
Service "CalicoFelix" installed successfully!
Set parameter "AppParameters" for service "CalicoFelix".
Set parameter "AppDirectory" for service "CalicoFelix".
Set parameter "DependOnService" for service "CalicoFelix".
Set parameter "DisplayName" for service "CalicoFelix".
Set parameter "Description" for service "CalicoFelix".
Set parameter "Start" for service "CalicoFelix".
Reset parameter "ObjectName" for service "CalicoFelix" to its default.
Set parameter "Type" for service "CalicoFelix".
Reset parameter "AppThrottle" for service "CalicoFelix" to its default.
Set parameter "AppStdout" for service "CalicoFelix".
Set parameter "AppStderr" for service "CalicoFelix".
Set parameter "AppRotateFiles" for service "CalicoFelix".
Set parameter "AppRotateOnline" for service "CalicoFelix".
Set parameter "AppRotateSeconds" for service "CalicoFelix".
Set parameter "AppRotateBytes" for service "CalicoFelix".
Done installing Felix service.
Copying CNI binaries to C:\Program Files\containerd\cni\bin
Writing CNI configuration to C:\Program Files\containerd\cni\conf\10-calico.conf.
Wrote CNI configuration.
Calico for Windows installed
Starting Calico...
This may take several seconds if the vSwitch needs to be created.
Waiting for Calico initialisation to finish...
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Waiting for Calico initialisation to finish...StoredLastBootTime , CurrentLastBootTime 2022-07-12 오후 3:09:43
Calico initialisation finished.
Done, the Calico services are running:
Status : Running
Name : CalicoFelix
DisplayName : Calico Windows Agent
Status : Running
Name : CalicoNode
DisplayName : Calico Windows Startup
Caption :
Description : Enable kubectl exec and log
ElementName : kubectl exec 10250
InstanceID : KubectlExec10250
CommonName :
PolicyKeywords :
Enabled : True
PolicyDecisionStrategy : 2
PolicyRoles :
ConditionListType : 3
CreationClassName : MSFT|FW|FirewallRule|KubectlExec10250
ExecutionStrategy : 2
Mandatory :
PolicyRuleName :
Priority :
RuleUsage :
SequencedActions : 3
SystemCreationClassName :
SystemName :
Action : Allow
Direction : Inbound
DisplayGroup :
DisplayName : kubectl exec 10250
EdgeTraversalPolicy : Block
EnforcementStatus : NotApplicable
LocalOnlyMapping : False
LooseSourceMapping : False
Owner :
Platforms : {}
PolicyStoreSource : PersistentStore
PolicyStoreSourceType : Local
PrimaryStatus : OK
Profiles : 0
RuleGroup :
Status : The rule was parsed successfully from the store. (65536)
StatusCode : 65536
PSComputerName :
Name : KubectlExec10250
ID : KubectlExec10250
Group :
Profile : Any
Platform : {}
LSM : False
Calico Node와 Calico Felix 가 정상 실행되었습니다.
PS C:\Program Files\containerd\cni\bin> Get-Service -Name CalicoNode
Status Name DisplayName
------ ---- -----------
Running CalicoNode Calico Windows Startup
PS C:\Program Files\containerd\cni\bin> Get-Service -Name CalicoFelix
Status Name DisplayName
------ ---- -----------
Running CalicoFelix Calico Windows Agent
추가 스크립트를 수행하여 kubelet과 kube-proxy 를 설치하고 서비스를 실행합니다.
PS C:\Program Files\containerd\cni\bin> C:\CalicoWindows\kubernetes\install-kube-services.ps1
Installing kubelet service...
Service "kubelet" installed successfully!
Set parameter "AppParameters" for service "kubelet".
Set parameter "AppDirectory" for service "kubelet".
Set parameter "DisplayName" for service "kubelet".
Set parameter "Description" for service "kubelet".
Set parameter "Start" for service "kubelet".
Reset parameter "ObjectName" for service "kubelet" to its default.
Set parameter "Type" for service "kubelet".
Reset parameter "AppThrottle" for service "kubelet" to its default.
Set parameter "AppStdout" for service "kubelet".
Set parameter "AppStderr" for service "kubelet".
Set parameter "AppRotateFiles" for service "kubelet".
Set parameter "AppRotateOnline" for service "kubelet".
Set parameter "AppRotateSeconds" for service "kubelet".
Set parameter "AppRotateBytes" for service "kubelet".
Done installing kubelet service.
Installing kube-proxy service...
Service "kube-proxy" installed successfully!
Set parameter "AppParameters" for service "kube-proxy".
Set parameter "AppDirectory" for service "kube-proxy".
Set parameter "DisplayName" for service "kube-proxy".
Set parameter "Description" for service "kube-proxy".
Set parameter "Start" for service "kube-proxy".
Reset parameter "ObjectName" for service "kube-proxy" to its default.
Set parameter "Type" for service "kube-proxy".
Reset parameter "AppThrottle" for service "kube-proxy" to its default.
Set parameter "AppStdout" for service "kube-proxy".
Set parameter "AppStderr" for service "kube-proxy".
Set parameter "AppRotateFiles" for service "kube-proxy".
Set parameter "AppRotateOnline" for service "kube-proxy".
Set parameter "AppRotateSeconds" for service "kube-proxy".
Set parameter "AppRotateBytes" for service "kube-proxy".
Done installing kube-proxy service.
PS C:\Program Files\containerd\cni\bin> Get-Service kubelet
Status Name DisplayName
------ ---- -----------
Stopped kubelet kubelet service
PS C:\Program Files\containerd\cni\bin> Get-Service kube-proxy
Status Name DisplayName
------ ---- -----------
Stopped kube-proxy kube-proxy service
PS C:\Program Files\containerd\cni\bin> Start-Service kubelet
PS C:\Program Files\containerd\cni\bin> start-Service kube-proxy
PS C:\Program Files\containerd\cni\bin> Get-Service kubelet
Status Name DisplayName
------ ---- -----------
Running kubelet kubelet service
PS C:\Program Files\containerd\cni\bin> Get-Service kube-proxy
Status Name DisplayName
------ ---- -----------
Running kube-proxy kube-proxy service
kubelet과 kube-proxy가 실행되면 워커 노드가 Join 된 것으로 확인됩니다. 기존에 조인되었던 k8s-ww 서버가 다운된 상태라, 해당 노드에 스케줄되었던 pod가 신규로 조인된 서버로 스케줄링이 되었습니다.
root@k8s-m:~# kubectl get no
NAME STATUS ROLES AGE VERSION
k8s-lw Ready <none> 120d v1.22.6
k8s-m Ready control-plane,master 120d v1.22.6
k8s-ww NotReady <none> 120d v1.22.6
root@k8s-m:~# kubectl get no
NAME STATUS ROLES AGE VERSION
k8s-lw Ready <none> 120d v1.22.6
k8s-m Ready control-plane,master 120d v1.22.6
k8s-ww NotReady <none> 120d v1.22.6
k8s-ww2 Ready <none> 49s v1.22.6
root@k8s-m:~# kubectl get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
iis-7dfbf869dd-4472b 0/1 ContainerCreating 0 5h46m <none> k8s-ww2 <none> <none>
iis-7dfbf869dd-brqrc 1/1 Terminating 0 120d 192.168.208.5 k8s-ww <none> <none>
netshoot 1/1 Running 11 (6h51m ago) 120d 192.168.114.5 k8s-lw <none> <none>
nginx-868547d6bf-kv858 1/1 Running 1 (6h51m ago) 120d 192.168.114.4 k8s-lw <none> <none>
root@k8s-m:~# kubectl get no
NAME STATUS ROLES AGE VERSION
k8s-lw Ready <none> 120d v1.22.6
k8s-m Ready control-plane,master 120d v1.22.6
k8s-ww NotReady <none> 120d v1.22.6
k8s-ww2 Ready <none> 85m v1.22.6
root@k8s-m:~# kubectl get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
iis-7dfbf869dd-4472b 1/1 Running 0 7h11m 192.168.112.4 k8s-ww2 <none> <none>
iis-7dfbf869dd-brqrc 1/1 Terminating 0 120d 192.168.208.5 k8s-ww <none> <none>
netshoot 1/1 Running 11 (8h ago) 120d 192.168.114.5 k8s-lw <none> <none>
nginx-868547d6bf-kv858 1/1 Running 1 (8h ago) 120d 192.168.114.4 k8s-lw <none> <none>윈도우 워커노드에서도 crictl 로 컨테이너를 확인할 수 있습니다.
C:\Users\Administrator>crictl ps
time="2022-07-12T22:45:46+09:00" level=debug msg="get runtime connection"
time="2022-07-12T22:45:46+09:00" level=debug msg="get image connection"
time="2022-07-12T22:45:46+09:00" level=debug msg="ListContainerResponse: [&Container{Id:077ab4df6b9972af7806c017a7e85e3b437475de5f3a8c9316c983240af73af7,PodSandboxId:f994462e02b028c6cf9e0f38eab984bf0158734db64aaf4379c326224fe53c87,Metadata:&ContainerMetadata{Name:iis,Attempt:0,},Image:&ImageSpec{Image:sha256:cf88a43a7460e1a84be0a24ee22042f73069346710907e702dadb1a7e8a39eaf,Annotations:map[string]string{},},ImageRef:sha256:cf88a43a7460e1a84be0a24ee22042f73069346710907e702dadb1a7e8a39eaf,State:CONTAINER_RUNNING,CreatedAt:1657614998696454400,Labels:map[string]string{io.kubernetes.container.name: iis,io.kubernetes.pod.name: iis-7dfbf869dd-4472b,io.kubernetes.pod.namespace: default,io.kubernetes.pod.uid: 8aecf61b-2e61-4f55-80a0-551b146b4cc6,},Annotations:map[string]string{io.kubernetes.container.hash: 42b06ae0,io.kubernetes.container.restartCount: 0,io.kubernetes.container.terminationMessagePath: /dev/termination-log,io.kubernetes.container.terminationMessagePolicy: File,io.kubernetes.pod.terminationGracePeriod: 30,},}]"
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
077ab4df6b997 cf88a43a7460e 5 hours ago Running iis 0 f994462e02b02 iis-7dfbf869dd-4472b
calico 에서 제공하는 스크립트를 통해 워커노드 조인이 진행되어 과정이 많이 생략되어 있습니다. 상세한 과정을 이해하기위해서는 스크립트를 전반적으로 살펴봐야 할 것 같습다.
현재 calico 의 QuickStart 가이드에는 'Install Calico for Windows using HostProcess containers' 라는 방식을 추가로 제공하고 있습니다. 2022/7/12일 현재 GA가 안된 상태이긴 한데, 이후 이과정도 한번 살펴보겠습니다.
'Kubernetes' 카테고리의 다른 글
| Jenkins와 Argo CD를 활용한 Kubernetes 환경 CI/CD 구성 (0) | 2025.03.30 |
|---|---|
| CNI(Container Network Interface)란? (0) | 2025.02.19 |
| 쿠버네티스 윈도우 워커 노드 추가 (with Calico CNI) (2) | 2022.03.14 |
| Kubernetes 업그레이드 (K8S v1.21.x → v1.22.x) (0) | 2022.02.05 |