1. 说明

k8s带来的灵活性，使我们在部署上面有了很多花样选择，这里我希望做到以下方案：

Plan:

- namespace为iot-age, nginx 与 php分别部署到不同的pods, 它们共用相同的pvc存储（即wwwroot目录相同，应用部署在里面）, php pods 访问相同的数据库。

- 使用buildkit作为image构建工具，通过指定镜像部署定制化的php pods。

PlanB: namespace为ia2, nginx与php及其应用联合到一个deployment.yaml部署，有自己对应的存储目录, pods 共用相同的数据库。

2. 开始前的准备工作

2.1 docker 验证用户信息设置

因为将使用BuildKit来访问docker仓库（buildkit使用~/.docker/config.json）, 需要验证用户信息，先行进行设置，以免push时出现验证错误：

# 生成base64编码的auth值, 其中user, pw为hub.docker.com的账密

echo -n "{user}:{pw}" | base64

vi ~/.docker/config.json
# example:
{
  "auths": {
          "https://index.docker.io/v1/": {
               "auth": "{auth}"
          }
  }
}

2.2 安装BuildKit

法一：在指定主机上安装使用

# amd64用这个
wget https://github.com/moby/buildkit/releases/download/v0.9.0/buildkit-v0.9.0.linux-amd64.tar.gz
tar xvf buildkit-v0.9.0.linux-amd64.tar.gz -C /usr/local/bin/

# arm64用这个
wget https://github.com/moby/buildkit/releases/download/v0.9.0/buildkit-v0.9.0.linux-arm64.tar.gz
tar xvf buildkit-v0.9.0.linux-arm64.tar.gz -C /usr/local/bin/

# 创建服务配置
cat >/etc/systemd/system/buildkitd.service <<EOF
[Unit]
Description=BuildKit
Documentation=https://github.com/moby/buildkit
After=containerd.service

[Service]
Type=notify
ExecStart=/usr/local/bin/buildkitd  --oci-worker=false --containerd-worker=true 

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl restart buildkitd 
systemctl enable buildkitd
systemctl status buildkitd

法2：创建 buildkitd和buildkit-cli pods，并在buildkit-cli里调用服务构建，需要说明的是: 我尚未能成功构建并推送上去仓库，可参考这篇文章Building Docker Images with BuildKit | Kubernetes Course Labs

# 安装buildkit server
kubectl apply -f https://kubernetes.courselabs.co/labs/buildkit/specs/buildkitd/buildkitd.yaml
kubectl logs -l app=buildkitd
# 删除buildkit
kubectl delete deploy,svc,pod -l kubernetes.courselabs.co=buildkit

kubectl create secret docker-registry registry-creds --docker-server=docker.io --docker-username={user} --docker-password={pw}
kubectl create configmap build-config --from-literal=REGISTRY=docker.io  --from-literal=REPOSITORY=bennybi

kubectl apply -f https://kubernetes.courselabs.co/labs/buildkit/specs/buildkit-cli/buildkit-cli.yaml

# 进入buildkit cli pod
kubectl exec -it buildkit-cli -- sh

# 在buildkit-cli pod内调用
buildctl --addr tcp://buildkitd:1234 build --frontend=dockerfile.v0 --local context=. --local dockerfile=. --output type=image,name=${REGISTRY}/${REPOSITORY}/simple,push=true

3. 安装步骤

3.1 申请一个pvc存储区 (wwwroot-pvc.yaml)

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: wwwroot
  namespace: iot-age
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2G

3.2 Nginx

文件清单: nginx.yaml, nginx-configmap.yaml

iot-age中所有的nginx pods将使用统一的配置，ConfigMap (nginx-configmap.yaml)

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-conf
  namespace: iot-age
data:
  default.conf: |
    server {
        listen       80;
        listen  [::]:80;
        server_name  localhost;
        
        index index.php index.html;
        error_log  /var/log/nginx/error.log;
        access_log /var/log/nginx/access.log;
        root /var/www/html;
        location / {
            try_files $uri $uri/ /index.php?$query_string;
        }
        
        location ~ \.php$ {
            try_files $uri =404;
            fastcgi_split_path_info ^(.+\.php)(/.+)$;
            fastcgi_pass   php:9000;
            fastcgi_index  index.php;
            include        fastcgi_params;
            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
            fastcgi_param PATH_INFO $fastcgi_path_info;
        }

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }
    }

kubectl apply -f nginx-configmap.yaml

nginx.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nginx
  namespace: iot-age
spec:
  selector:
    matchLabels:
      app: nginx
  serviceName: "nginx"
  replicas: 2
  #volumeClaimTemplates:
  #  - kind: PersistentVolumeClaim
  #    apiVersion: v1
  #    metadata:
  #      name: wwwroot
  #    spec:
  #      accessModes:
  #        - ReadWriteOnce
  #      resources:
  #        requests:
  #          storage: 2Gi
  #      volumeMode: Filesystem
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 80
        volumeMounts:
        - name: wwwroot
          mountPath: /var/www/html
        - name: nginx-conf
          mountPath: /etc/nginx/conf.d/
      volumes:
      - name: wwwroot
        persistentVolumeClaim:
          claimName: wwwroot
      - name: nginx-conf
        configMap:
         name: nginx-conf
status: {}

---
apiVersion: v1
kind: Service
metadata:
  name: nginx
  namespace: iot-age
spec:
  type: NodePort
  ports:
  - name: nginx
    port: 80
    protocol: TCP
    targetPort: 80
    nodePort: 30010
  selector:
    app: nginx

kubectl apply -f nginx.yaml

3.3 php-fpm

文件清单: Dockerfile, php.yaml

Dockerfile，该文件参考了php/Dockerfile at master · khs1994-docker/php · GitHub

# syntax=docker/dockerfile:labs
ARG PHP_VERSION=8.2.5

FROM --platform=$TARGETPLATFORM php:${PHP_VERSION}-fpm-alpine3.17 as php

LABEL maintainer="khs1994-docker/lnmp <khs1994@khs1994.com>"

ARG PHP_EXTENSION_EXTRA

ARG PECL_EXTENSION_EXTRA

ARG APK_EXTRA

ARG APK_DEV_EXTRA

ENV TZ=Asia/Shanghai \
    APP_ENV=development

ENV PHP_EXTENSION \
      bcmath \
      bz2 \
      calendar \
      enchant \
      exif \
      gd \
      gettext \
      gmp \
      imap \
      intl \
      mysqli \
      pcntl \
      pdo_pgsql \
      pdo_mysql \
      pgsql \
      sockets \
      sysvmsg \
      sysvsem \
      sysvshm \
      # tidy \
      # xsl \
      zip \
      shmop \
      ffi \
      ${PHP_EXTENSION_EXTRA:-}

ENV PECL_EXTENSION \
      # mongodb \
      # https://github.com/mongodb/mongo-php-driver/archive/master.tar.gz \
      # igbinary \
      https://github.com/igbinary/igbinary/archive/master.tar.gz \
      # redis \
      https://github.com/phpredis/phpredis/archive/develop.tar.gz \
      memcached \
# 安装测试版的扩展，可以在扩展名后加 -beta
      # yaml \
      # https://github.com/php/pecl-file_formats-yaml/archive/php7.tar.gz \
      https://github.com/kjdev/php-ext-zstd/archive/0.11.0.tar.gz \
      ${PECL_EXTENSION_EXTRA:-}

ARG ALPINE_URL=dl-cdn.alpinelinux.org

RUN sed -i "s/dl-cdn.alpinelinux.org/${ALPINE_URL}/g" /etc/apk/repositories \
      && set -xe \
      \
      # && apk add --no-cache patch \
      # && cd / \
      # && { \
      #   echo "--- a/usr/local/lib/php/build/php.m4"; \
      #    } | tee 1.diff \
      # && patch -p1 < 1.diff \
      # && apk del --no-network patch \
      # && rm -rf 1.diff \
      \
# 不要删除
      && PHP_FPM_RUN_DEPS=" \
                         bash \
                         tzdata \
                         libmemcached-libs \
                         libpq \
                         libzip \
                         zlib \
                         libpng \
                         freetype \
                         libjpeg-turbo \
                         libxpm \
                         libwebp \
                         libbz2 \
                         libexif \
                         gmp \
                         enchant2 \
                         c-client \
                         icu-libs \
                         zstd-libs \
                         linux-headers \
                         libavif \
                         ${APK_EXTRA:-} \
                         " \
                         # tidyhtml-libs \
                         # libxslt \
                         # yaml \
# *-dev 编译之后删除
      && PHP_FPM_BUILD_DEPS=" \
                         openssl-dev \
                         libmemcached-dev \
                         cyrus-sasl-dev \
                         postgresql-dev \
                         libzip-dev \
                         zlib-dev \
                         libpng-dev \
                         freetype-dev \
                         libjpeg-turbo-dev \
                         libxpm-dev \
                         libwebp-dev \
                         libexif-dev \
                         gmp-dev \
                         bzip2-dev \
                         enchant2-dev \
                         imap-dev \
                         gettext-dev \
                         libwebp-dev \
                         icu-dev \
                         zstd-dev \
                         libavif-dev \
                         ${APK_DEV_EXTRA:-} \
                         " \
                         # tidyhtml-dev \
                         # libxslt-dev \
                         # yaml-dev \
        && apk add --no-cache --virtual .php-fpm-run-deps $PHP_FPM_RUN_DEPS \
        && apk add --no-cache --virtual .php-fpm-build-deps $PHP_FPM_BUILD_DEPS \
        && apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS \
        && curl -fsSL -o /usr/local/bin/pickle \
           https://github.com/khs1994-php/pickle/releases/download/nightly/pickle-debug.phar \
        && chmod +x /usr/local/bin/pickle \
# 安装内置扩展
        && docker-php-source extract \
        && docker-php-ext-configure zip \
                                    --with-zip \
        && docker-php-ext-install zip \
        && strip --strip-all $(php-config --extension-dir)/zip.so \
        # && docker-php-ext-configure gd \
        && echo " \
                                        --disable-gd-jis-conv \
                                        --with-freetype \
                                        --with-jpeg \
                                        --with-webp \
                                        --with-xpm \
                                        --with-avif" > /tmp/gd.configure.options \
        # && docker-php-ext-install $PHP_EXTENSION \
        && pickle install $PHP_EXTENSION -n --defaults --strip \
        && docker-php-source delete \
# 安装 PECL 扩展
        && echo "--enable-redis-igbinary --enable-redis-zstd" > /tmp/redis.configure.options \
        && echo "--enable-memcached-igbinary" > /tmp/memcached.configure.options \
        && echo "--with-libzstd" > /tmp/zstd.configure.options \
        && pickle install $PECL_EXTENSION -n --defaults \
           --strip --cleanup \
# 默认不启用的扩展
        && pickle install \
# https://github.com/tideways/php-xhprof-extension.git
             https://github.com/tideways/php-xhprof-extension/archive/master.tar.gz \
             -n --defaults --strip --cleanup --no-write \
        && pickle install opcache \
        # && docker-php-ext-enable opcache \
        && apk del --no-network .phpize-deps .php-fpm-build-deps \
        && rm -rf /tmp/* \
# 创建日志文件夹
        && mkdir -p /var/log/php-fpm \
        && ln -sf /dev/stdout /var/log/php-fpm/access.log \
        && ln -sf /dev/stderr /var/log/php-fpm/error.log \
        && chmod -R 777 /var/log/php-fpm \
        && rm -rf /usr/local/lib/php/.registry/.channel.pecl.php.net/* \
        && php -m \
        && ls -la $(php-config --extension-dir) \
        && php -d error_reporting=22527 -d display_errors=1 -r 'var_dump(iconv("UTF-8", "UTF-8//IGNORE", "This is the Euro symbol '\''€'\''."));' \
# smoke test
        && php --version

WORKDIR /app

ARG VCS_REF="unknow"

LABEL org.opencontainers.image.revision=$VCS_REF \
      org.opencontainers.image.source="https://github.com/khs1994-docker/php"

3.3.1 构建并推送镜像

buildctl build \
--frontend=dockerfile.v0 \
--local context=. \
--local dockerfile=. \
--export-cache type=inline \
--output type=image,name=docker.io/bennybi/php-fpm:v1,push=true

php.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: php
  namespace: iot-age
spec:
  selector:
    matchLabels:
      app: php
      tier: backend
  replicas: 2
  template:
    metadata:
      labels:
        app: php
        tier: backend
    spec:
      containers:
      - name: php
        #image: php:8.2-fpm
        image: bennybi/php-fpm:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9000
        volumeMounts:
        - name: wwwroot
          mountPath: /var/www/html/
 
      volumes:
      - name: wwwroot
        persistentVolumeClaim:
          claimName: wwwroot
 
---
apiVersion: v1
kind: Service
metadata:
  name: php
  namespace: iot-age
spec:
  ports:
  - name: php
    port: 9000
    protocol: TCP
    targetPort: 9000
  selector:
    app: php