openEuler 22.03 LTS x86_64 cephadm 部署ceph 16.2.14 未完成笔记

环境

准备三台虚拟机

10.47.76.94 node-1
10.47.76.95 node-2
10.47.76.96 node-3

下载cephadm

[root@node-1 ~]# yum install cephadm
Last metadata expiration check: 0:11:31 ago on Tue 21 Nov 2023 10:00:20 AM CST.
Dependencies resolved.
================================================================================================================================================================================
Package Architecture Version Repository Size
================================================================================================================================================================================
Installing:
cephadm noarch 2:16.2.7-18.oe2203 update 79 k
Installing dependencies:
conmon x86_64 2:2.0.2-4.oe2203 everything 36 k
container-selinux noarch 2:2.138-4.oe2203 OS 37 k
containernetworking-plugins x86_64 1.0.1-2.oe2203 OS 18 M
containers-common x86_64 1:1.5.2-3.oe2203 update 9.5 k
docker-runc x86_64 1.0.0.rc3-310.oe2203 update 2.0 M
ostree x86_64 2021.6-1.oe2203 OS 460 k
Installing weak dependencies:
podman x86_64 1:0.10.1-12.oe2203 everything 6.7 M

Transaction Summary
================================================================================================================================================================================
Install 8 Packages

Total download size: 27 M
Installed size: 94 M
Is this ok [y/N]: y
Downloading Packages:
(1/8): container-selinux-2.138-4.oe2203.noarch.rpm 114 kB/s | 37 kB 00:00
(2/8): conmon-2.0.2-4.oe2203.x86_64.rpm 252 kB/s | 36 kB 00:00
(3/8): ostree-2021.6-1.oe2203.x86_64.rpm 633 kB/s | 460 kB 00:00
(4/8): cephadm-16.2.7-18.oe2203.noarch.rpm 440 kB/s | 79 kB 00:00
(5/8): containers-common-1.5.2-3.oe2203.x86_64.rpm 128 kB/s | 9.5 kB 00:00
(6/8): docker-runc-1.0.0.rc3-310.oe2203.x86_64.rpm 1.7 MB/s | 2.0 MB 00:01
(7/8): podman-0.10.1-12.oe2203.x86_64.rpm 2.2 MB/s | 6.7 MB 00:03
(8/8): containernetworking-plugins-1.0.1-2.oe2203.x86_64.rpm 1.5 MB/s | 18 MB 00:11 =--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Total 2.3 MB/s | 27 MB 00:11
Running transaction check
Transaction check succeeded.
Running transaction test
Transaction test succeeded.
Running transaction
Preparing : 1/1
Installing : docker-runc-1.0.0.rc3-310.oe2203.x86_64 1/8
Installing : containers-common-1:1.5.2-3.oe2203.x86_64 2/8
Installing : conmon-2:2.0.2-4.oe2203.x86_64 3/8
Installing : ostree-2021.6-1.oe2203.x86_64 4/8
Running scriptlet: ostree-2021.6-1.oe2203.x86_64 4/8
Installing : containernetworking-plugins-1.0.1-2.oe2203.x86_64 5/8
Installing : container-selinux-2:2.138-4.oe2203.noarch 6/8
Running scriptlet: container-selinux-2:2.138-4.oe2203.noarch 6/8
Installing : podman-1:0.10.1-12.oe2203.x86_64 7/8
Running scriptlet: cephadm-2:16.2.7-18.oe2203.noarch 8/8
Installing : cephadm-2:16.2.7-18.oe2203.noarch 8/8
Running scriptlet: cephadm-2:16.2.7-18.oe2203.noarch 8/8
Verifying : container-selinux-2:2.138-4.oe2203.noarch 1/8
Verifying : containernetworking-plugins-1.0.1-2.oe2203.x86_64 2/8
Verifying : ostree-2021.6-1.oe2203.x86_64 3/8
Verifying : conmon-2:2.0.2-4.oe2203.x86_64 4/8
Verifying : podman-1:0.10.1-12.oe2203.x86_64 5/8
Verifying : cephadm-2:16.2.7-18.oe2203.noarch 6/8
Verifying : containers-common-1:1.5.2-3.oe2203.x86_64 7/8
Verifying : docker-runc-1.0.0.rc3-310.oe2203.x86_64 8/8

Installed:
cephadm-2:16.2.7-18.oe2203.noarch conmon-2:2.0.2-4.oe2203.x86_64 container-selinux-2:2.138-4.oe2203.noarch containernetworking-plugins-1.0.1-2.oe2203.x86_64
containers-common-1:1.5.2-3.oe2203.x86_64 docker-runc-1.0.0.rc3-310.oe2203.x86_64 ostree-2021.6-1.oe2203.x86_64 podman-1:0.10.1-12.oe2203.x86_64

Complete!

故障

Failed command: /usr/bin/podman version --format {{.Client.Version}}

[root@node-1 ~]# cephadm bootstrap --mon-ip 10.47.76.94
Non-zero exit code 125 from /usr/bin/podman version --format {{.Client.Version}}
/usr/bin/podman: stdout Incorrect Usage: flag provided but not defined: -format
/usr/bin/podman: stdout
/usr/bin/podman: stdout NAME:
/usr/bin/podman: stdout podman version - Display the PODMAN Version Information
/usr/bin/podman: stdout
/usr/bin/podman: stdout USAGE:
/usr/bin/podman: stdout podman version [arguments...]
/usr/bin/podman: stderr flag provided but not defined: -format
Traceback (most recent call last):
File "/usr/sbin/cephadm", line 8571, in <module>
main()
File "/usr/sbin/cephadm", line 8557, in main
check_container_engine(ctx)
File "/usr/sbin/cephadm", line 2014, in check_container_engine
engine.get_version(ctx)
File "/usr/sbin/cephadm", line 196, in get_version
out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}'])
File "/usr/sbin/cephadm", line 1467, in call_throws
raise RuntimeError('Failed command: %s' % ' '.join(command))
RuntimeError: Failed command: /usr/bin/podman version --format {{.Client.Version}}

原因 podman版本不匹配！

[root@node-1 ~]# /usr/bin/podman version --format {{.Client.Version}}
Incorrect Usage: flag provided but not defined: -format

NAME:
podman version - Display the PODMAN Version Information

USAGE:
podman version [arguments...]
flag provided but not defined: -format

Compatibility and Stability — Ceph Documentation

axel -n 50 https://gh-proxy.com/https://github.com/containers/podman/releases/download/v3.4.4/podman-remote-static.tar.gz -k

[root@node-1 ~]# tar -xvf podman-remote-static.tar.gz
podman-remote-static
[root@node-1 ~]# ./podman-remote-static -v
podman-remote-static version 3.4.4
[root@node-1 ~]# cp podman-remote-static /usr/bin/podman
[root@node-1 ~]# podman -v
podman version 3.4.4

[root@node-1 ~]# vim /usr/lib/systemd/system/podman.service

[Unit]
Description=Podman API Service
Requires=podman.socket
After=podman.socket
Documentation=man:podman-system-service(1)
StartLimitIntervalSec=0

[Service]
Type=exec
KillMode=process
Environment=LOGGING="--log-level=info"
ExecStart=/usr/bin/podman $LOGGING system service

[Install]
WantedBy=default.target

[root@node-1 ~]# vim /usr/lib/systemd/system/podman.socket

[Unit]
Description=Podman API Socket
Documentation=man:podman-system-service(1)

[Socket]
ListenStream=%t/podman/podman.sock
SocketMode=0660

[Install]
WantedBy=sockets.target

[root@node-1 ~]# systemctl daemon-reload
[root@node-1 ~]# systemctl start podman
[root@node-1 ~]# systemctl status podman

下载的静态程序不支持system service!!!

找到openEuler里面对应版本src.rpm过来编译安装

[root@node-1 ~]# wget https://repo.openeuler.org/openEuler-22.03-LTS-SP2/source/Packages/podman-3.4.4-1.oe2203sp2.src.rpm
--2023-11-21 11:47:30-- https://repo.openeuler.org/openEuler-22.03-LTS-SP2/source/Packages/podman-3.4.4-1.oe2203sp2.src.rpm
Resolving repo.openeuler.org (repo.openeuler.org)... 49.0.230.196
Connecting to repo.openeuler.org (repo.openeuler.org)|49.0.230.196|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 16823599 (16M) [application/x-redhat-package-manager]
Saving to: ‘podman-3.4.4-1.oe2203sp2.src.rpm’

podman-3.4.4-1.oe2203sp2.src.rpm 100%[==================================================================================>] 16.04M 3.50MB/s in 4.8s

2023-11-21 11:47:35 (3.38 MB/s) - ‘podman-3.4.4-1.oe2203sp2.src.rpm’ saved [16823599/16823599]

[root@node-1 ~]#
[root@node-1 ~]# rpm -ivUh podman-3.4.4-1.oe2203sp2.src.rpm
warning: podman-3.4.4-1.oe2203sp2.src.rpm: Header V4 RSA/SHA256 Signature, key ID fb37bc6f: NOKEY
Updating / installing...
1:podman-1:3.4.4-1.oe2203sp2 ################################# [100%]
[root@node-1 ~]# cd rpmbuild/
[root@node-1 rpmbuild]#

[root@node-1 rpmbuild]# yum-builddep -y SPECS/podman.spec

[root@node-1 rpmbuild]# rpmbuild -ba SPECS/podman.spec

[root@node-1 rpmbuild]# yum install -y RPMS/x86_64/podman-3.4.4-1.x86_64.rpm \
> RPMS/x86_64/podman-remote-3.4.4-1.x86_64.rpm \
> RPMS/x86_64/podman-plugins-3.4.4-1.x86_64.rpm \
> RPMS/x86_64/podman-help-3.4.4-1.x86_64.rpm \
> RPMS/x86_64/podman-gvproxy-3.4.4-1.x86_64.rpm \
> RPMS/noarch/podman-docker-3.4.4-1.noarch.rpm
Last metadata expiration check: 1:42:13 ago on Tue 21 Nov 2023 10:12:28 AM CST.
Error:
Problem 1: conflicting requests
- nothing provides crun needed by podman-1:3.4.4-1.x86_64
Problem 2: package podman-docker-1:3.4.4-1.noarch requires podman = 1:3.4.4-1, but none of the providers can be installed
- conflicting requests
- nothing provides crun needed by podman-1:3.4.4-1.x86_64
(try to add '--skip-broken' to skip uninstallable packages or '--nobest' to use not only best candidate packages)

[root@node-1 ~]# wget https://repo.openeuler.org/openEuler-22.03-LTS-SP2/source/Packages/crun-1.4.5-1.oe2203sp2.src.rpm
--2023-11-21 11:56:17-- https://repo.openeuler.org/openEuler-22.03-LTS-SP2/source/Packages/crun-1.4.5-1.oe2203sp2.src.rpm
Resolving repo.openeuler.org (repo.openeuler.org)... 49.0.230.196
Connecting to repo.openeuler.org (repo.openeuler.org)|49.0.230.196|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 887311 (867K) [application/x-redhat-package-manager]
Saving to: ‘crun-1.4.5-1.oe2203sp2.src.rpm’

crun-1.4.5-1.oe2203sp2.src.rpm 100%[==================================================================================>] 866.51K 1.79MB/s in 0.5s

2023-11-21 11:56:18 (1.79 MB/s) - ‘crun-1.4.5-1.oe2203sp2.src.rpm’ saved [887311/887311]

[root@node-1 ~]# rpm -ivUh crun-1.4.5-1.oe2203sp2.src.rpm
warning: crun-1.4.5-1.oe2203sp2.src.rpm: Header V4 RSA/SHA256 Signature, key ID fb37bc6f: NOKEY
Updating / installing...
1:crun-1.4.5-1.oe2203sp2 ################################# [100%]

[root@node-1 ~]# yum-builddep -y rpmbuild/SPECS/crun.spec

[root@node-1 ~]# rpmbuild -ba rpmbuild/SPECS/crun.spec

[root@node-1 ~]# cd rpmbuild/
[root@node-1 rpmbuild]# yum install -y RPMS/x86_64/podman-3.4.4-1.x86_64.rpm RPMS/x86_64/podman-remote-3.4.4-1.x86_64.rpm RPMS/x86_64/podman-plugins-3.4.4-1.x86_64.rpm RPMS/x86_64/podman-help-3.4.4-1.x86_64.rpm RPMS/x86_64/podman-gvproxy-3.4.4-1.x86_64.rpm RPMS/noarch/podman-docker-3.4.4-1.noarch.rpm RPMS/x86_64/crun-1.4.5-1.x86_64.rpm

[root@node-1 rpmbuild]# systemctl daemon-reload
[root@node-1 rpmbuild]# systemctl restart podman
[root@node-1 rpmbuild]# systemctl status podman
× podman.service - Podman API Service
Loaded: loaded (/usr/lib/systemd/system/podman.service; disabled; vendor preset: disabled)
Active: failed (Result: exit-code) since Tue 2023-11-21 12:00:51 CST; 6s ago
TriggeredBy: ● podman.socket
Docs: man:podman-system-service(1)
Process: 40106 ExecStart=/usr/bin/podman $LOGGING system service (code=exited, status=125)
Main PID: 40106 (code=exited, status=125)

Nov 21 12:00:51 node-1 systemd[1]: Starting Podman API Service...
Nov 21 12:00:51 node-1 systemd[1]: Started Podman API Service.
Nov 21 12:00:51 node-1 podman[40106]: time="2023-11-21T12:00:51+08:00" level=info msg="/usr/bin/podman filtering at log level info"
Nov 21 12:00:51 node-1 podman[40106]: time="2023-11-21T12:00:51+08:00" level=warning msg="Conmon at /usr/bin/conmon invalid: outdated conmon version"
Nov 21 12:00:51 node-1 podman[40106]: time="2023-11-21T12:00:51+08:00" level=warning msg="Conmon at /usr/bin/conmon is invalid: outdated conmon version"
Nov 21 12:00:51 node-1 podman[40106]: Error: please update to v2.0.24 or later: outdated conmon version
Nov 21 12:00:51 node-1 systemd[1]: podman.service: Main process exited, code=exited, status=125/n/a
Nov 21 12:00:51 node-1 systemd[1]: podman.service: Failed with result 'exit-code'.

[root@node-1 ~]# wget https://repo.openeuler.org/openEuler-22.03-LTS-SP2/source/Packages/conmon-2.1.0-1.oe2203sp2.src.rpm
--2023-11-21 12:03:01-- https://repo.openeuler.org/openEuler-22.03-LTS-SP2/source/Packages/conmon-2.1.0-1.oe2203sp2.src.rpm
Resolving repo.openeuler.org (repo.openeuler.org)... 49.0.230.196
Connecting to repo.openeuler.org (repo.openeuler.org)|49.0.230.196|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 211336 (206K) [application/x-redhat-package-manager]
Saving to: ‘conmon-2.1.0-1.oe2203sp2.src.rpm’

conmon-2.1.0-1.oe2203sp2.src.rpm 100%[==================================================================================>] 206.38K 1.05MB/s in 0.2s

2023-11-21 12:03:01 (1.05 MB/s) - ‘conmon-2.1.0-1.oe2203sp2.src.rpm’ saved [211336/211336]

[root@node-1 ~]# rpm -ivUh conmon-2.1.0-1.oe2203sp2.src.rpm
warning: conmon-2.1.0-1.oe2203sp2.src.rpm: Header V4 RSA/SHA256 Signature, key ID fb37bc6f: NOKEY
Updating / installing...
1:conmon-2:2.1.0-1.oe2203sp2 ################################# [100%]

[root@node-1 ~]# yum-builddep -y rpmbuild/SPECS/conmon.spec

[root@node-1 ~]# rpmbuild -ba rpmbuild/SPECS/conmon.spec

[root@node-1 ~]# rpm -ivUh /root/rpmbuild/RPMS/x86_64/conmon-2.1.0-1.x86_64.rpm
Verifying... ################################# [100%]
Preparing... ################################# [100%]
Updating / installing...
1:conmon-2:2.1.0-1 ################################# [ 50%]
Cleaning up / removing...
2:conmon-2:2.0.2-4.oe2203 ################################# [100%]

（其实一开始选择openEuler-22.03-LTS-SP2就没这些事情了，直接装podman ）

cephadm bootstrap --mon-ip

[root@node-1 ~]# cephadm bootstrap --mon-ip 10.47.76.94
Creating directory /etc/ceph for ceph.conf
Verifying podman|docker is present...
Verifying lvm2 is present...
Verifying time synchronization is in place...
Unit chronyd.service is enabled and running
Repeating the final host check...
podman (/usr/bin/podman) version 3.4.4 is present
systemctl is present
lvcreate is present
Unit chronyd.service is enabled and running
Host looks OK
Cluster fsid: 5c90c23c-8823-11ee-a5be-08002726ce7d
Verifying IP 10.47.76.94 port 3300 ...
Verifying IP 10.47.76.94 port 6789 ...
Mon IP `10.47.76.94` is in CIDR network `10.47.76.0/24`
- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network
Pulling container image quay.io/ceph/ceph:v16...
Non-zero exit code 125 from /usr/bin/podman pull quay.io/ceph/ceph:v16
/usr/bin/podman: stderr Trying to pull quay.io/ceph/ceph:v16...
/usr/bin/podman: stderr time="2023-11-21T12:06:42+08:00" level=warning msg="failed, retrying in 1s ... (1/3). Error: parsing image configuration: Get \"https://cdn02.quay.io/sha256/ae/ae96fc565b065a0c531b222ae151caca195a9a68ee516d9c9b6f4811e8fe85c5?username=None&namespace=ceph&Expires=1700643054&Signature=TcolHavApvIZG8vFeY8363vvm56ON9aH~J-O3LVRx4pZyV3NnvMuOyqNmVQWr~sE6p1iBPLc1qv4TNp-NINRvW-uIEtaFefxni5m7JXNTNuMAUOyP7vj71Pg0KS6zaVHm4IqE7bV2wts-0w53moDa9bwzEP3OchKDM72OGxzaHJpz0ZUd2Zklr7ar5MggaGCmc~at34CJvbJz8BPWWcanNdZC0valSLkQuR~bg4p6TuCdo0EKASUgjishYlv~387UTqtva5-xGWXaWi5q2oI~y5xCKCNlD2wDAW1sXfWJA5WGvkiUN7SrN3LDuf3zUvgV9VW4k4msKHFiZIumGtQ1w__&Key-Pair-Id=APKAJ67PQLWGCSP66DGA\": EOF"
/usr/bin/podman: stderr Error: parsing image configuration: Get "https://cdn02.quay.io/sha256/ae/ae96fc565b065a0c531b222ae151caca195a9a68ee516d9c9b6f4811e8fe85c5?username=None&namespace=ceph&Expires=1700643061&Signature=Fs7HMwSWy-csPD6-4zXN5MPak-m5UbW~9WOia9AJS8taPGZT6tMc71t79P7BE8id62C21MODFET1oR6uCKe9cHb1cWqxl8lrn5kMnojecjVl0rMWG2e4zeb0x4TfdgYiii7hG1Or2jcJaIeTPn7BTq3zCqLfLLddHW1HqHOR9LospJIkcSyJ-JbRNG8q0RbkFiVtSmujBJ4MfrKKFITwtW~9FTT2PBKzq0JJ8ntNhje2KKWtTd8tPIArvLthELjV6nqi~hGCXYP6yArrn4lNmnwDfHnsQs68afzLH9y402ty1Tn2UHLk5PO8gWgygT6o7F-hDCtFit7CoXDuurllMg__&Key-Pair-Id=APKAJ67PQLWGCSP66DGA": http: server gave HTTP response to HTTPS client
ERROR: Failed command: /usr/bin/podman pull quay.io/ceph/ceph:v16

坑爹的网络

离线安装

[root@node-1 ~]# podman load -i ceph-v16.tar
Getting image source signatures
Copying blob 602fe9f6cc5a done
Copying blob c9b8d8af634e done
Copying config ae96fc565b done
Writing manifest to image destination
Storing signatures
Loaded image(s): quay.io/ceph/ceph:v16

[root@node-1 ~]# cephadm bootstrap --mon-ip 10.47.76.94 --skip-pull
Verifying podman|docker is present...
Verifying lvm2 is present...
Verifying time synchronization is in place...
Unit chronyd.service is enabled and running
Repeating the final host check...
podman (/usr/bin/podman) version 3.4.4 is present
systemctl is present
lvcreate is present
Unit chronyd.service is enabled and running
Host looks OK
Cluster fsid: 18788cd6-8825-11ee-9f59-08002726ce7d
Verifying IP 10.47.76.94 port 3300 ...
Verifying IP 10.47.76.94 port 6789 ...
Mon IP `10.47.76.94` is in CIDR network `10.47.76.0/24`
- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network
Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16 -e NODE_NAME=node-1 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16 --version
ceph: stderr Error: container-init binary not found on the host: stat /usr/libexec/podman/catatonit: no such file or directory
Traceback (most recent call last):
File "/usr/sbin/cephadm", line 8571, in <module>
main()
File "/usr/sbin/cephadm", line 8559, in main
r = ctx.func(ctx)
File "/usr/sbin/cephadm", line 1787, in _default_image
return func(ctx)
File "/usr/sbin/cephadm", line 4257, in command_bootstrap
image_ver = CephContainer(ctx, ctx.image, 'ceph', ['--version']).run().strip()
File "/usr/sbin/cephadm", line 3420, in run
out, _, _ = call_throws(self.ctx, self.run_cmd(),
File "/usr/sbin/cephadm", line 1467, in call_throws
raise RuntimeError('Failed command: %s' % ' '.join(command))
RuntimeError: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16 -e NODE_NAME=node-1 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16 --version