一、实验环境

---

1、k8s环境

版本v1.26.5，容器为containerd
二进制安装Kubernetes(K8s)集群(基于containerd)—从零安装教程（带证书）

主机名	IP	系统版本	安装服务
master01	10.10.10.21	rhel7.5	nginx、etcd、api-server、scheduler、controller-manager、kubelet、proxy
master02	10.10.10.22	rhel7.5	nginx、etcd、api-server、scheduler、controller-manager、kubelet、proxy
master03	10.10.10.23	rhel7.5	nginx、etcd、api-server、scheduler、controller-manager、kubelet、proxy
node01	10.10.10.24	rhel7.5	nginx、kubelet、proxy
node02	10.10.10.25	rhel7.5	nginx、kubelet、proxy

2、jenkins环境

jenkins入门与安装
容器为docker

主机	IP	系统版本
jenkins	10.10.10.10	rhel7.5

二、docker-compose安装

---

jenkins服务器上面安装

1、下载

https://github.com/docker/compose/releases/
下载版本：v2.18.0

2、安装

[root@jenkins ~]# cp docker-compose-linux-x86_64 /usr/local/bin/docker-compose
[root@jenkins ~]# chmod +x /usr/local/bin/docker-compose

3、查看版本

[root@jenkins ~]# docker-compose --version
Docker Compose version v2.18.0

三、cfssl证书生成

---

此处记录使用cfssl工具生成harbor私有证书，并使用证书搭建Harbor仓库，此证书使用按照kubernetes时使用的ca证书安装

1、安装cfssl

https://imroc.cc/kubernetes/trick/certs/sign-certs-with-cfssl.html
安装包下载地址：https://github.com/cloudflare/cfssl/releases

[root@jenkins ~]# ls cfssl*
cfssl_1.6.2_linux_amd64  cfssl-certinfo_1.6.2_linux_amd64  cfssljson_1.6.2_linux_amd64
[root@jenkins ~]# mv cfssl_1.6.2_linux_amd64 /usr/bin/cfssl
[root@jenkins ~]# mv cfssl-certinfo_1.6.2_linux_amd64  /usr/bin/cfssl-certinfo
[root@jenkins ~]# mv cfssljson_1.6.2_linux_amd64 /usr/bin/cfssljson
[root@jenkins ~]# chmod +x /usr/bin/cfssl*

2、ca生成证书

[root@jenkins ~]# mkdir -p pki &&  cd pki
[root@jenkins pki]# cat > ca-csr.json   << EOF 
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "Kubernetes",
      "OU": "Kubernetes-manual"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF

[root@jenkins pki]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
[root@jenkins pki]# ls
ca.csr  ca-csr.json  ca-key.pem  ca.pem

3、过期时间查看

[root@jenkins pki]# openssl x509 -noout -text -in ca.pem|grep -A 5 Validity
        Validity
            Not Before: Jun  4 12:32:00 2023 GMT
            Not After : May 11 12:32:00 2123 GMT
        Subject: C=CN, ST=Beijing, L=Beijing, O=Kubernetes, OU=Kubernetes-manual, CN=kubernetes
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption

4、创建Harbor证书

[root@jenkins pki]# cat > ca-config.json << EOF 
{
  "signing": {
    "default": {
      "expiry": "438000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "438000h"
      }
    }
  }
}
EOF

[root@jenkins pki]# cat > harbor-csr.json  << EOF 
{
  "CN": "harbor",
  "hosts": [
    "127.0.0.1",
    "10.10.10.10",  
    "harbor.wielun.com"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "Kubernetes",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF

[root@jenkins pki]# cfssl gencert \
-ca=ca.pem \
-ca-key=ca-key.pem \
-config=ca-config.json \
-profile=kubernetes harbor-csr.json | cfssljson -bare harbor

[root@jenkins pki]# mkdir -p /etc/harbor/pki
[root@jenkins pki]# cp harbor.pem harbor-key.pem /etc/harbor/pki

四、安装harbor

---

jenkins服务器上面安装

1、下载

下载地址：https://github.com/goharbor/harbor/releases#install
安装官网：https://goharbor.io/docs/2.8.0/install-config/

2、安装

（1）解压文件

[root@jenkins ~]#  tar xf harbor-offline-installer-v2.8.1.tgz -C /usr/local

（2）修改harbor.yml

[root@jenkins ~]# cd /usr/local/harbor/
[root@jenkins harbor]# cp harbor.yml.tmpl harbor.yml
[root@jenkins harbor]# vim harbor.yml

（3）启动

[root@jenkins harbor]# docker load -i harbor.v2.8.1.tar.gz
[root@jenkins harbor]# ./prepare
[root@jenkins harbor]# ./install.sh
[root@jenkins harbor]# docker-compose up -d     #手动启动命令

3、创建登录证书

[root@jenkins ~]# mkdir -p /etc/docker/certs.d/10.10.10.10
[root@jenkins ~]# mkdir -p /etc/docker/certs.d/harbor.wielun.com
[root@jenkins ~]# cp pki/ca.pem /etc/docker/certs.d/10.10.10.10
[root@jenkins ~]# cp pki/ca.pem /etc/docker/certs.d/harbor.wielun.com

4、修改daemon.json

[root@jenkins ~]# cat /etc/docker/daemon.json
{
 "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"],
  "insecure-registries": [
    "10.10.10.10",
    "harbor.wielun.com"
  ]
}

[root@jenkins ~]# systemctl restart docker

5、添加hosts

[root@jenkins ~]# vim /etc/hosts
10.10.10.10 harbor.wielun.com

6、登录验证

账号密码：admin/Harbor12345

（1）docker login

[root@jenkins ~]# docker login 10.10.10.10
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

[root@jenkins ~]# docker login harbor.wielun.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

（2）浏览器登录

https://10.10.10.10/

7、docker上传镜像测试

（1）拉取镜像

[root@jenkins ~]# docker pull nginx
[root@jenkins ~]# docker images|grep nginx
nginx                           latest              f9c14fe76d50        10 days ago         143MB
goharbor/nginx-photon           v2.8.1              cea1bb2450ee        3 weeks ago         127MB

（2）打包上传

[root@jenkins ~]# docker tag nginx:latest harbor.wielun.com/library/nginx:latest
[root@jenkins ~]# docker push harbor.wielun.com/library/nginx

（3）浏览器中查看

五、K8s（containerd）拉取镜像（每台机器）

---

选择一种即可，这边我使用得是跳过证书

1、K8s（containerd）拉取镜像（跳过证书）

（1）删除之前containerd配置

[root@master01 ~]# rm -rf /etc/containerd/config.toml
[root@master01 ~]# containerd config default | tee /etc/containerd/config.toml
[root@master01 ~]# sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
[root@master01 ~]# sed -i "s#registry.k8s.io#registry.cn-hangzhou.aliyuncs.com/chenby#g" /etc/containerd/config.toml
[root@master01 ~]# sed -i "s#config_path\ \=\ \"\"#config_path\ \=\ \"/etc/containerd/certs.d\"#g" /etc/containerd/config.toml

（2）配置hosts.toml

[root@master01 ~]# mkdir -p /etc/containerd/certs.d/harbor.wielun.com
[root@master01 ~]# cat > /etc/containerd/certs.d/harbor.wielun.com/hosts.toml << EOF
server = "https://harbor.wielun.com"
[host."https://harbor.wielun.com"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
EOF

（3）重启containerd

[root@master01 ~]# systemctl restart containerd

（4）添加hosts

[root@master01 ~]# cat /etc/hosts
10.10.10.10 harbor.wielun.com

2、K8s（containerd）拉取镜像（通过证书）

（1）证书配置

[root@master01 ~]# mkdir -p /etc/containerd/certs.d/harbor.wielun.com

[root@jenkins ~]# cd pki/
[root@jenkins pki]# scp ca.pem harbor.pem harbor-key.pem root@10.10.10.21:/etc/containerd/certs.d/harbor.wielun.com

（2）删除之前containerd配置

[root@master01 ~]# rm -rf /etc/containerd/config.toml
[root@master01 ~]# containerd config default | tee /etc/containerd/config.toml
[root@master01 ~]# sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
[root@master01 ~]# sed -i "s#registry.k8s.io#registry.cn-hangzhou.aliyuncs.com/chenby#g" /etc/containerd/config.toml

（3）配置config.toml

[root@master01 ~]# vim /etc/containerd/config.toml
    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = "node"

    [plugins."io.containerd.grpc.v1.cri".registry]
      config_path = ""

      [plugins."io.containerd.grpc.v1.cri".registry.auths]

      [plugins."io.containerd.grpc.v1.cri".registry.configs]
        [plugin."io.containerd.grpc.v1.cri".registry.configs."harbor.wielun.com".tls]
          ca_file = "/etc/containerd/certs.d/harbor.wielun.com/ca.pem"
          cert_file = "/etc/containerd/certs.d/harbor.wielun.com/harbor.pem"
          key_file = "/etc/containerd/certs.d/harbor.wielun.com/harbor-key.pem"
        [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.wielun.com".auth]
          username = "admin"
          password = "Harbor12345"

      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.wielun.com"]
          endpoint = ["https://harbor.wielun.com"]

    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""

（4）重启containerd

[root@master01 ~]# systemctl restart containerd

（5）添加hosts

[root@master01 ~]# cat /etc/hosts
10.10.10.10 harbor.wielun.com

3、测试拉取镜像

（1）拉取镜像

# -k：跳过证书认证
[root@master01 ~]# ctr -n harbor.wielun.com  images  pull harbor.wielun.com/library/nginx:latest -k
harbor.wielun.com/library/nginx:latest:                                           resolved       |++++++++++++++++++++++++++++++++++++++|
manifest-sha256:6b06964cdbbc517102ce5e0cef95152f3c6a7ef703e4057cb574539de91f72e6: done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:831f51541d386c6d0d86f6799fcfabb48e91e9e5aea63c726240dd699179f495:    done           |++++++++++++++++++++++++++++++++++++++|
config-sha256:f9c14fe76d502861ba0939bc3189e642c02e257f06f4c0214b1f8ca329326cda:   done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:f03b40093957615593f2ed142961afb6b540507e0b47e3f7626ba5e02efbbbf1:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:eed12bbd64949353649476b59d486ab4c5b84fc5ed2b2dc96384b0b892b6bf7e:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:fa7eb8c8eee8792b8db1c0043092b817376f096e3cc8feeea623c6e00211dad1:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:7ff3b2b12318a41d4b238b643d7fcf1fe6da400ca3e02aa61766348f90455354:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:0f67c7de5f2c7e0dc408ce685285419c1295f24b7a01d554517c7a72374d4aeb:    done           |++++++++++++++++++++++++++++++++++++++|
elapsed: 0.1 s                                                                    total:   0.0 B (0.0 B/s)
unpacking linux/amd64 sha256:6b06964cdbbc517102ce5e0cef95152f3c6a7ef703e4057cb574539de91f72e6...

[root@master01 ~]# ctr -n harbor.wielun.com images pull harbor.wielun.com/library/nginx:latest --tlscacert  /etc/containerd/certs.d/harbor.wielun.com/ca.pem --tlscert   /etc/containerd/certs.d/harbor.wielun.com/harbor.pem  --tlskey  /etc/containerd/certs.d/harbor.wielun.com/harbor-key.pem
harbor.wielun.com/library/nginx:latest:                                           resolved       |++++++++++++++++++++++++++++++++++++++|
manifest-sha256:6b06964cdbbc517102ce5e0cef95152f3c6a7ef703e4057cb574539de91f72e6: done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:831f51541d386c6d0d86f6799fcfabb48e91e9e5aea63c726240dd699179f495:    done           |++++++++++++++++++++++++++++++++++++++|
config-sha256:f9c14fe76d502861ba0939bc3189e642c02e257f06f4c0214b1f8ca329326cda:   done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:f03b40093957615593f2ed142961afb6b540507e0b47e3f7626ba5e02efbbbf1:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:eed12bbd64949353649476b59d486ab4c5b84fc5ed2b2dc96384b0b892b6bf7e:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:fa7eb8c8eee8792b8db1c0043092b817376f096e3cc8feeea623c6e00211dad1:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:7ff3b2b12318a41d4b238b643d7fcf1fe6da400ca3e02aa61766348f90455354:    done           |++++++++++++++++++++++++++++++++++++++|
layer-sha256:0f67c7de5f2c7e0dc408ce685285419c1295f24b7a01d554517c7a72374d4aeb:    done           |++++++++++++++++++++++++++++++++++++++|

（2）查看镜像

[root@master01 ~]# ctr -n harbor.wielun.com images ls
REF                                    TYPE                                                 DIGEST                                                                  SIZE     PLATFORMS   LABELS
harbor.wielun.com/library/nginx:latest application/vnd.docker.distribution.manifest.v2+json sha256:6b06964cdbbc517102ce5e0cef95152f3c6a7ef703e4057cb574539de91f72e6 54.5 MiB linux/amd64 -

（3）crictl拉取镜像

[root@master01 ~]# crictl  pull harbor.wielun.com/library/nginx:latest
Image is up to date for sha256:f9c14fe76d502861ba0939bc3189e642c02e257f06f4c0214b1f8ca329326cda

[root@master01 ~]# crictl images ls|grep harbor.wielun.com/library/nginx
harbor.wielun.com/library/nginx                                                latest              f9c14fe76d502       57.2MB

六、jenkins发布到K8S

---

1、推送tomcat到harbor

这里用Java项目做演示，一般项目打包成jar包，我们这里打包成war包

[root@jenkins ~]# docker pull tomcat:8.5.59
[root@jenkins ~]# docker tag tomcat:8.5.59 harbor.wielun.com/library/tomcat:8.5.59
[root@jenkins ~]# docker push harbor.wielun.com/library/tomcat:8.5.59

2、创建项目

这里我们使用Jenkinsfile

3、查看项目文件

（1）配置Dockerfile

FROM harbor.wielun.com/library/tomcat:8.5.59
MAINTAINER Wielun
RUN rm -rf /usr/local/tomcat/webapps/*
ADD target/*.war /usr/local/tomcat/webapps/ROOT.war

（2）配置Jenkinsfile

pipeline {
    agent any
	environment {
		harborUser = 'admin'
		harborPasswd = 'Harbor12345'
		HarborAddress = 'harbor.wielun.com'
		harborRepo = 'library'
	}
    stages {
        stage('git拉取代码') {
            steps {
				git credentialsId: '0c71c0f9-8277-493b-xxxx-540a9324cf08', url: 'https://jihulab.com/xxxx/java-demo.git'
            }
        }
    
        stage('maven编译') { 
           steps {
                    sh '''JAVA_HOME=/usr/local/jdk
                    PATH=$PATH:$JAVA_HOME/bin
                    /usr/local/maven/bin/mvn clean package -Dmaven.test.skip=true'''
                }
        }
        stage('生成自定义镜像') { 
           steps {
                    sh '''docker build -t ${JOB_NAME}:latest .'''
                }
        }
        stage('上传自定义镜像到harbor') { 
           steps {
                    sh '''docker login -u ${harborUser} -p ${harborPasswd} ${HarborAddress}
                    docker tag ${JOB_NAME}:latest ${HarborAddress}/${harborRepo}/${JOB_NAME}:latest
                    docker push ${HarborAddress}/${harborRepo}/${JOB_NAME}:latest'''
                }
        }
        stage('发送yaml到k8s-master并部署') { 
           steps {
					sshPublisher(publishers: [sshPublisherDesc(configName: 'k8s-master', transfers: [sshTransfer(cleanRemote: false, excludes: '', execCommand: '''
					/usr/local/bin/kubectl apply -f /tmp/${JOB_NAME}/pipeline.yaml''', execTimeout: 120000, flatten: false, makeEmptyDirs: false, noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: '${JOB_NAME}', remoteDirectorySDF: false, removePrefix: '', sourceFiles: 'pipeline.yaml')], usePromotionTimestamp: false, useWorkspaceInPromotion: false, verbose: false)])
                }
        } 
    }
}

（3）配置pipeline.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: test
  name: pipeline
  labels:
    app: pipeline
spec:
  replicas: 2
  selector:
    matchLabels:
      app: pipeline
  template:
    metadata:
      labels:
        app: pipeline
    spec:
      containers:
      - name: pipeline
        image: harbor.wielun.com/library/java-k8s:latest
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
  namespace: test
  name: pipeline
  labels:
    app: pipeline
spec:
  ports:
  - port: 8081
    targetPort: 8080
  selector:
    app: pipeline
  type: NodePort
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  namespace: test
  name: pipeline
spec:
  ingressClassName: nginx
  rules:
  - host: "harbor.wielun.com"
    http:
      paths:
      - pathType: Prefix
        path: /
        backend:
          service:
            name: pipeline
            port:
              number: 8081

4、创建namespace

[root@master01 ~]# kubectl create ns test

5、构建并查看结果

[root@master01 ~]# kubectl get pod -n test
NAME                        READY   STATUS    RESTARTS   AGE
pipeline-556759f7b4-7x8ml   1/1     Running   0          11s
pipeline-556759f7b4-zwdgr   1/1     Running   0          11s