centos7单节点部署ceph（mon/mgr/osd/mgr/rgw）

使用ceph建议采用多节点多磁盘方式部署，本文章仅作为单节点部署参考，请勿用于生产环境

1. yum 配置文件添加如下相关源
[root@testserver01 ~]# cat /etc/yum.conf
[epel]
name=CentOS7-epel
baseurl=https://mirrors.aliyun.com/epel/7/x86_64/
gpgcheck=0
enabled=1

[base]
name=CentOS7-OS
baseurl=https://mirrors.aliyun.com/centos/7/os/x86_64/
gpgcheck=0
enabled=1

[nautilus-x86_64]
name=ceph-x86_64
baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/x86_64/
gpgcheck=0
enabled=1

[nautilus-noarch]
name=ceph-noarch
baseurl=https://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/
gpgcheck=0
enabled=1

2. 安装ceph mgr依赖的python2-Werkzeug包
[root@testserver01 ~]# rpm -ivh http://rpmfind.net/linux/opensuse/distribution/leap/15.3/repo/oss/noarch/python2-Werkzeug-1.0.1-1.10.noarch.rpm

3. [root@testserver01 ~]# yum install ceph chrony gdisk ceph-radosgw ceph-deploy ceph-mgr-dashboard -y

4. 修改系统参数
[root@testserver01 ~]# sysctl -w kernel.pid_max=4194303; echo 'kernel.pid_max=4194303' >> /etc/sysctl.conf; sysctl -p

5. 添加ceph配置文件
[root@testserver01 ~]# cat /etc/ceph/ceph.conf
[global]
fsid = b080bada-7f10-11ee-8f11-666666666666
mon initial members = testserver01
mon host = 192.168.1.128
public network = 192.168.1.0/24
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd pool default pg num = 16
osd pool default pgp num = 16
osd crush update on start=false
osd_pool_default_min_size = 1
mon_osd_down_out_subtree_limit = host
osd_pool_default_size = 1
mon_warn_on_pool_no_redundancy = false
mon_warn_on_insecure_global_id_reclaim_allowed = false

[osd]
osd_scrub_begin_hour = 23
osd_scrub_end_hour = 7

6. 部署mon
[root@testserver01 ~]# ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
creating /tmp/ceph.mon.keyring
[root@testserver01 ~]# ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
creating /etc/ceph/ceph.client.admin.keyring
[root@testserver01 ~]# ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
importing contents of /etc/ceph/ceph.client.admin.keyring into /tmp/ceph.mon.keyring
[root@testserver01 ~]# monmaptool --create --add testserver01 192.168.1.128 --fsid b080bada-7f10-11ee-8f11-666666666666 /tmp/monmap
monmaptool: monmap file /tmp/monmap
monmaptool: set fsid to b080bada-7f10-11ee-8f11-666666666666
monmaptool: writing epoch 0 to /tmp/monmap (1 monitors)
[root@testserver01 ~]# ceph-mon --mkfs -i testserver01 --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring
[root@testserver01 ~]# chown ceph.ceph /var/lib/ceph/mon -R
[root@testserver01 ~]# systemctl start ceph-mon@testserver01
system[root@testserver01 ~]# systemctl enable ceph-mon@testserver01
Created symlink from /etc/systemd/system/ceph-mon.target.wants/ceph-mon@testserver01.service to /usr/lib/systemd/system/ceph-mon@.service.
[root@testserver01 ~]# ceph mon enable-msgr2
[root@testserver01 ~]# ceph -s
cluster:
id: b080bada-7f10-11ee-8f11-666666666666
health: HEALTH_OK

services:
mon: 1 daemons, quorum testserver01 (age 43s)
mgr: no daemons active
osd: 0 osds: 0 up, 0 in

data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:

7. 部署mgr
[root@testserver01 ~]# mkdir -p /var/lib/ceph/mgr/ceph-testserver01
[root@testserver01 ~]# ceph auth get-or-create mgr.testserver01 mon 'allow profile mgr' osd 'allow *' mds 'allow *' >/var/lib/ceph/mgr/ceph-testserver01/keyring
[root@testserver01 ~]# chown ceph.ceph -R /var/lib/ceph/mgr/
[root@testserver01 ~]# systemctl start ceph-mgr@testserver01
[root@testserver01 ~]# systemctl enable ceph-mgr@testserver01
Created symlink from /etc/systemd/system/ceph-mgr.target.wants/ceph-mgr@testserver01.service to /usr/lib/systemd/system/ceph-mgr@.service.
[root@testserver01 ~]# ceph -s
cluster:
id: b080bada-7f10-11ee-8f11-666666666666
health: HEALTH_WARN
OSD count 0 < osd_pool_default_size 1

services:
mon: 1 daemons, quorum testserver01 (age 4m)
mgr: testserver01(active, since 70s)
osd: 0 osds: 0 up, 0 in

data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:

8. 部署osd
[root@testserver01 ~]# ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
exported keyring for client.bootstrap-osd
[root@testserver01 ~]# sgdisk --zap-all /dev/vdb
Creating new GPT entries.
GPT data structures destroyed! You may now partition the disk using fdisk or
other utilities.
[root@testserver01 ~]# ceph-volume lvm create --bluestore --data /dev/vdb

9. 配置crushmap
[root@testserver01 ~]# ceph osd crush move osd.0 host=testserver01 root=default
moved item id 0 name 'osd.0' to location {host=testserver01,root=default} in crush map
[root@testserver01 ~]# ceph osd crush reweight osd.0 0.01
reweighted item id 0 name 'osd.0' to 0.01 in crush map

10. 创建块存储池
[root@testserver01 ~]# ceph osd pool create rbdpool01 32 32
pool 'rbdpool01' created
[root@testserver01 ~]# ceph osd pool application enable rbdpool01 rbd
enabled application 'rbd' on pool 'rbdpool01'
[root@testserver01 ~]# ceph osd pool ls detail
pool 1 'rbdpool01' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 10 flags hashpspool stripe_width 0 application rbd

11. 创建rbd存储块
[root@testserver01 ~]# rbd create rbdpool01/testimg --size 5G
[root@testserver01 ~]# rbd ls -p rbdpool01 -l
NAME SIZE PARENT FMT PROT LOCK
testimg 5 GiB 2

12. 添加mds服务
[root@testserver01 ~]# mkdir -p /var/lib/ceph/mds/ceph-testserver01
[root@testserver01 ~]# ceph-authtool --create-keyring /var/lib/ceph/mds/ceph-testserver01/keyring --gen-key -n mds.testserver01
creating /var/lib/ceph/mds/ceph-testserver01/keyring
[root@testserver01 ~]# ceph auth add mds.testserver01 osd 'allow rwx' mds 'allow *' mon 'allow profile mds' -i /var/lib/ceph/mds/ceph-testserver01/keyring
added key for mds.testserver01
[root@testserver01 ~]# chown -R ceph.ceph /var/lib/ceph/
[root@testserver01 ~]# systemctl enable ceph-mds@testserver01
Created symlink from /etc/systemd/system/ceph-mds.target.wants/ceph-mds@testserver01.service to /usr/lib/systemd/system/ceph-mds@.service.
[root@testserver01 ~]# systemctl start ceph-mds@testserver01

13. 添加cephfs存储池
[root@testserver01 ~]# ceph osd pool create cephfs-metadata 16 16
pool 'cephfs-metadata' created
[root@testserver01 ~]# ceph osd pool create cephfs-data 32 32
pool 'cephfs-data' created
[root@testserver01 ~]# ceph fs new cephfs cephfs-metadata cephfs-data
new fs with metadata pool 2 and data pool 3
[root@testserver01 ~]# ceph fs status
cephfs - 0 clients
======
+------+--------+--------------+---------------+-------+-------+
| Rank | State | MDS | Activity | dns | inos |
+------+--------+--------------+---------------+-------+-------+
| 0 | active | testserver01 | Reqs: 0 /s | 10 | 13 |
+------+--------+--------------+---------------+-------+-------+
+-----------------+----------+-------+-------+
| Pool | type | used | avail |
+-----------------+----------+-------+-------+
| cephfs-metadata | metadata | 512k | 8697M |
| cephfs-data | data | 0 | 8697M |
+-----------------+----------+-------+-------+
+-------------+
| Standby MDS |
+-------------+
+-------------+
MDS version: ceph version 14.2.22 (ca74598065096e6fcbd8433c8779a2be0c889351) nautilus (stable)
[root@testserver01 ~]#

14. 部署rgw (默认7480端口)
[root@testserver01 ~]# cp /etc/ceph/ceph.conf /tmp/
[root@testserver01 ~]# cd /tmp
[root@testserver01 tmp]# ceph auth get client.bootstrap-rgw -o ceph.bootstrap-rgw.keyring
exported keyring for client.bootstrap-rgw
[root@testserver01 tmp]# ceph-deploy --overwrite-conf rgw create testserver01
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.0.1): /usr/bin/ceph-deploy --overwrite-conf rgw create testserver01
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] rgw : [('testserver01', 'rgw.testserver01')]
[ceph_deploy.cli][INFO ] overwrite_conf : True
[ceph_deploy.cli][INFO ] subcommand : create
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7fca20b03a28>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function rgw at 0x7fca21148050>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.rgw][DEBUG ] Deploying rgw, cluster ceph hosts testserver01:rgw.testserver01
[testserver01][DEBUG ] connected to host: testserver01
[testserver01][DEBUG ] detect platform information from remote host
[testserver01][DEBUG ] detect machine type
[ceph_deploy.rgw][INFO ] Distro info: CentOS Linux 7.9.2009 Core
[ceph_deploy.rgw][DEBUG ] remote host will use systemd
[ceph_deploy.rgw][DEBUG ] deploying rgw bootstrap to testserver01
[testserver01][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[testserver01][WARNIN] rgw keyring does not exist yet, creating one
[testserver01][DEBUG ] create a keyring file
[testserver01][DEBUG ] create path recursively if it doesn't exist
[testserver01][INFO ] Running command: ceph --cluster ceph --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/ceph.keyring auth get-or-create client.rgw.testserver01 osd allow rwx mon allow rw -o /var/lib/ceph/radosgw/ceph-rgw.testserver01/keyring
[testserver01][INFO ] Running command: systemctl enable ceph-radosgw@rgw.testserver01
[testserver01][WARNIN] Created symlink from /etc/systemd/system/ceph-radosgw.target.wants/ceph-radosgw@rgw.testserver01.service to /usr/lib/systemd/system/ceph-radosgw@.service.
[testserver01][INFO ] Running command: systemctl start ceph-radosgw@rgw.testserver01
[testserver01][INFO ] Running command: systemctl enable ceph.target
[ceph_deploy.rgw][INFO ] The Ceph Object Gateway (RGW) is now running on host testserver01 and default port 7480
[root@testserver01 tmp]# radosgw-admin user create --uid=admin --display-name="admin user" --system
{
"user_id": "admin",
"display_name": "admin user",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"subusers": [],
"keys": [
{
"user": "admin",
"access_key": "7DRHRKOVTQF0EBWHL7LZ",
"secret_key": "01qnDj58DmWTjQZzCwDv9SLfC6x2bIsWgD2EgHri"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"system": "true",
"default_placement": "",
"default_storage_class": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw",
"mfa_ids": []
}

15. 开启并配置mgr dashboard
[root@testserver01 ~]# ceph mgr module enable prometheus (开启监控端口，9283端口，需要安装ceph-mgr-dashboard包)

[root@testserver01 ~]# ceph config set mgr mgr/dashboard/ssl false
[root@testserver01 ~]# ceph mgr module enable dashboard (关闭dashboard ssl，默认8080端口)
[root@testserver01 tmp]# echo 'passwordxxx' > password.txt
[root@testserver01 tmp]# ceph dashboard ac-user-create admin administrator -i password.txt
{"username": "admin", "lastUpdate": 1709353417, "name": null, "roles": ["administrator"], "password": "$2b$12$FcEB.KYWVB8BCXaXxu5VJOQbUqwO0KCUOEfi2wgSnmlFPW3RqTQp2", "email": null}

监控端口：

访问mgr dashboard http://192.168.1.128:8080

16. 配置rgw dashboard
[root@testserver01 tmp]# radosgw-admin user info --uid=admin (查看ak和sk)

[root@testserver01 tmp]# cat /tmp/a.txt
7DRHRKOVTQF0EBWHL7LZ
[root@testserver01 tmp]# cat /tmp/s.txt
01qnDj58DmWTjQZzCwDv9SLfC6x2bIsWgD2EgHri
[root@testserver01 tmp]# ceph dashboard set-rgw-api-access-key -i a.txt
Option RGW_API_ACCESS_KEY updated
[root@testserver01 tmp]# ceph dashboard set-rgw-api-secret-key -i s.txt
Option RGW_API_SECRET_KEY updated

17. 查看健康状态，osd，pool，crush rule，端口等
[root@testserver01 ~]# ceph -s
cluster:
id: b080bada-7f10-11ee-8f11-666666666666
health: HEALTH_OK

services:
mon: 1 daemons, quorum testserver01 (age 33m)
mgr: testserver01(active, since 11m)
mds: cephfs:1 {0=testserver01=up:active}
osd: 1 osds: 1 up (since 28m), 1 in (since 28m)
rgw: 1 daemon active (testserver01)

task status:

data:
pools: 7 pools, 144 pgs
objects: 216 objects, 5.7 KiB
usage: 1.0 GiB used, 9.0 GiB / 10 GiB avail
pgs: 144 active+clean

[root@testserver01 ~]# ceph osd df
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS
0 hdd 0.00999 1.00000 10 GiB 1.0 GiB 4.9 MiB 0 B 1 GiB 9.0 GiB 10.05 1.00 144 up
TOTAL 10 GiB 1.0 GiB 4.9 MiB 0 B 1 GiB 9.0 GiB 10.05
MIN/MAX VAR: 1.00/1.00 STDDEV: 0
[root@testserver01 ~]# ceph osd pool ls detail
pool 1 'rbdpool01' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 28 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd
removed_snaps [1~3]
pool 2 'cephfs-metadata' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 17 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs
pool 3 'cephfs-data' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode warn last_change 17 flags hashpspool stripe_width 0 application cephfs
pool 4 '.rgw.root' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 20 flags hashpspool stripe_width 0 application rgw
pool 5 'default.rgw.control' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 22 flags hashpspool stripe_width 0 application rgw
pool 6 'default.rgw.meta' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 24 flags hashpspool stripe_width 0 application rgw
pool 7 'default.rgw.log' replicated size 1 min_size 1 crush_rule 0 object_hash rjenkins pg_num 16 pgp_num 16 autoscale_mode warn last_change 26 flags hashpspool stripe_width 0 application rgw

[root@testserver01 ~]# ceph osd crush rule dump
[
{
"rule_id": 0,
"rule_name": "replicated_rule",
"ruleset": 0,
"type": 1,
"min_size": 1,
"max_size": 10,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "chooseleaf_firstn",
"num": 0,
"type": "host"
},
{
"op": "emit"
}
]
}
]

[root@testserver01 ~]# netstat -tnlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 192.168.1.128:6809 0.0.0.0:* LISTEN 2694/ceph-osd
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 977/master
tcp 0 0 192.168.1.128:6810 0.0.0.0:* LISTEN 3037/ceph-mds
tcp 0 0 192.168.1.128:6811 0.0.0.0:* LISTEN 3037/ceph-mds
tcp 0 0 192.168.1.128:3300 0.0.0.0:* LISTEN 1944/ceph-mon
tcp 0 0 192.168.1.128:6789 0.0.0.0:* LISTEN 1944/ceph-mon
tcp 0 0 192.168.1.128:6800 0.0.0.0:* LISTEN 4480/ceph-mgr
tcp 0 0 192.168.1.128:6801 0.0.0.0:* LISTEN 4480/ceph-mgr
tcp 0 0 192.168.1.128:6802 0.0.0.0:* LISTEN 2694/ceph-osd
tcp 0 0 192.168.1.128:6803 0.0.0.0:* LISTEN 2694/ceph-osd
tcp 0 0 192.168.1.128:6804 0.0.0.0:* LISTEN 2694/ceph-osd
tcp 0 0 192.168.1.128:6805 0.0.0.0:* LISTEN 2694/ceph-osd
tcp 0 0 192.168.1.128:6806 0.0.0.0:* LISTEN 2694/ceph-osd
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 873/sshd
tcp 0 0 192.168.1.128:6807 0.0.0.0:* LISTEN 2694/ceph-osd
tcp 0 0 0.0.0.0:7480 0.0.0.0:* LISTEN 3280/radosgw
tcp 0 0 192.168.1.128:6808 0.0.0.0:* LISTEN 2694/ceph-osd
tcp6 0 0 ::1:25 :::* LISTEN 977/master
tcp6 0 0 :::9283 :::* LISTEN 4480/ceph-mgr
tcp6 0 0 :::8080 :::* LISTEN 4480/ceph-mgr
tcp6 0 0 :::22 :::* LISTEN 873/sshd
tcp6 0 0 :::7480 :::* LISTEN 3280/radosgw