kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/tigera-operator.yaml


wget https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/custom-resources.yaml

# update https://docs.tigera.io/calico/latest/networking/ipam/ip-autodetection#change-the-autodetection-method
      cidr: 10.10.0.0/16
    nodeAddressAutodetectionV4:
#      interface: en.*
      cidrs:
      - 172.xxx.xxxx.0/24

kubectl create -f tigera-operator.yaml
kubectl create -f custom-resources.yaml


# 安装 calicoctl 
cd /usr/local/bin/
curl -L https://github.com/projectcalico/calico/releases/download/v3.26.1/calicoctl-linux-amd64 -o kubectl-calico
chmod +x kubectl-calico
kubectl calico -h
kubectl calico

# node.kubernetes.io/disk-pressure:NoSchedule
# docker system prune
# kubectl get pods -A -owide
# kubectl get events -owide

# kubectl cordon k8s-node1
# kubectl uncordon k8s-node1

# kubectl delete event --all
vim /var/lib/kubelet/config.yaml

#https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-thresholds
#https://kubernetes.io/zh-cn/docs/reference/config-api/kubelet-config.v1beta1/

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.1/cert-manager.yaml

# 等上面初始化完成

# kubectl get pods,svc,ingress -A -o wide

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v3.0.0-alpha0/charts/kubernetes-dashboard.yaml

# 直接用 portainer 修改 ingress host -> k8s.example.com

# https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md
kubectl apply -f dashboard-adminuser.yaml
kubectl apply -f dashboard-ClusterRoleBinding.yaml
kubectl apply -f dashboard-Secret.yaml

# 获取token
kubectl get secret admin-user -n kubernetes-dashboard -o jsonpath={".data.token"} | base64 -d
eyJhbGciOiJSUzIxxxxxxxxxxxxxxxxxlciJ9.Sa8JxxxxxxxxxxxxxxxxxxxxxxxxxxxYA7xOA

$ kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.14.1/nvidia-device-plugin.yml

wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# 镜像加速
sed -i 's/registry.k8s.io\/metrics-server/registry.cn-hangzhou.aliyuncs.com\/google_containers/g' components.yaml

# 添加参数
- --secure-port=4443

kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

kubectl get nodes -A -owide
NAME     STATUS   ROLES           AGE     VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME
node-1   Ready    control-plane   19d     v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 22.04.3 LTS   5.15.0-84-generic   docker://24.0.6
node-2   Ready    <none>          56m     v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 18.04.6 LTS   5.4.0-90-generic    docker://24.0.2
node-3   Ready    <none>          5m19s   v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 20.04.2 LTS   5.8.0-43-generic    docker://24.0.6
node-4   Ready    control-plane   19d     v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 22.04.3 LTS   5.15.0-84-generic   docker://24.0.6



kubectl label nodes node-1 node-role.kubernetes.io/worker=Master
kubectl label nodes node-4 node-role.kubernetes.io/worker=Master

kubectl label nodes node-2 node-role.kubernetes.io/worker=Worker
kubectl label nodes node-3 node-role.kubernetes.io/worker=Worker


kubectl get nodes -A -owide
NAME     STATUS   ROLES                  AGE     VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME
node-1   Ready    control-plane,worker   19d     v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 22.04.3 LTS   5.15.0-84-generic   docker://24.0.6
node-2   Ready    worker                 59m     v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 18.04.6 LTS   5.4.0-90-generic    docker://24.0.2
node-3   Ready    worker                 8m40s   v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 20.04.2 LTS   5.8.0-43-generic    docker://24.0.6
node-4   Ready    control-plane,worker   19d     v1.28.2   172.xxx.xxx.x   <none>        Ubuntu 22.04.3 LTS   5.15.0-84-generic   docker://24.0.6

kubectl drain app-node --delete-emptydir-data --force --ignore-daemonsets

kubectl delete node app-node

systemctl disable kubelet
systemctl stop kubelet

root@app-node:~# kubectl get nodes
NAME               STATUS   ROLES                  AGE    VERSION
xxxxx1   Ready    control-plane,master   442d   v1.23.9
xxxxx2   Ready    worker                 437d   v1.23.9
xxxxx3   Ready    worker                 420d    v1.23.9

## 创建证书到k8s
kubectl create secret tls tls-secret --cert=/harbor/cert/registry.home.com.crt --key=/harbor/cert/registry.home.com.key

kubectl get secret tls tls-secret -o yaml > tls-secret.yaml

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.9.0/deploy/static/provider/cloud/deploy.yaml

sed -i 's/registry.k8s.io/registry.home.com/g' deploy.yaml
sed -i 's/@sha256:[a-z0-9]*//g' deploy.yaml

# cat deploy.yaml | grep image: | uniq 
        image: registry.home.com/ingress-nginx/controller:v1.9.0
        image: registry.home.com/ingress-nginx/kube-webhook-certgen:v20230407
        
# sudo docker pull registry.k8s.io/ingress-nginx/controller:v1.9.0
# sudo docker pull registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20230407


kubectl apply -f deploy.yaml

# 没有 EXTERNAL-IP  ingress 没有 ADDRESS 配置 ingress-nginx 所在pod的IP
kubectl edit svc ingress-nginx-controller -n ingress-nginx
spec:
  allocateLoadBalancerNodePorts: true
  externalIPs:
  - 172.xxx.xxx.xxx

# 修改配置
# https://github.com/kubernetes/ingress-nginx/blob/controller-v1.9.0/docs/user-guide/nginx-configuration/configmap.md
 kubectl edit configmap ingress-nginx-controller -n ingress-nginx
 
 data:
  allow-snippet-annotations: "false"
  hsts: "false"
  ssl-redirect: "false"
  
            
# 查看
kubectl edit configmap ingress-nginx-controller -n ingress-nginx