当前位置：首页 > news >正文

ceph对象存储-存储池-用户认证

news 2025/9/1 10:03:27

OSD 操作

# 查看osd硬件是否开启了 bluestore 分片
[root@test-ceph1 ~]# ceph config get osd.1 bluestore_rocksdb_cf
true# 查看集群的 osd 设备
[root@test-ceph1 ~]# ceph orch device ls# 查看ceph设备
[root@test-ceph1 ~]# ceph device ls# 准备osd设备，删除设备上的分区并清理数据
# --force，强制
ceph orch device zap test-ceph1 /dev/nvme0n4 --force# 自动配置osd
ceph orch apply osd --all-available-devices# 禁用自动管理osd
ceph orch apply osd --all-available-devices --unmanaged=true# 添加指定的主机的指定磁盘做osd
ceph orch daemon add osd test-ceph1:/dev/nvme0n4# 停止某个osd
ceph orch daemon stop osd.1
# 删除某个osd
ceph orch daemon rm osd.1
ceph osd rm 1# 通过 spec 文件部署osd
service_type: osd
service_id: osd_size_and_model
placement:host_pattern: '*' # 匹配任何主机
data_devices:size: '100G'  # 匹配100G或者以上容量的设备
db_devices:model: My-Disk  # 匹配有My-Disk标记的设备
wal_devices:size: '10G:20G' # 匹配10G-20G容量的设备
unmanaged:true
---
service_type: osd
service_id: osd_host_and_path
placement:host_pattern: 'test-ceph[7-10]'
data_devices:paths:- /dev/nvme0n4
db_devices:paths:- /dev/nvme0n3
wal_devices:paths:- /dev/nvme0n2
encrypted: true # 设备加密
# 进行部署
ceph orch apply -i service_spec.yaml# ceph-volume 支持LVM插件和裸盘设备
# 使用 nvme0n2 创建一个新的 bluestore OSD
ceph-volume lvm create --bluestore --data /dev/nvme0n2ceph-volume lvm prepare --bluestore --data /dev/nvme0n2
ceph-volume lvm list
ceph-volume lvm activate <sod-fsid>
ceph-volume lvm batch --bluestore  /dev/nvme0n2  /dev/nvme0n3  /dev/nvme0n4
ceph-volume inventory

使用逻辑卷创建 BluseStore OSD

# 查看集群状态
ceph -s # 查看集群磁盘容量
[root@test-ceph1 ~]# ceph df 
--- RAW STORAGE ---
CLASS     SIZE    AVAIL     USED  RAW USED  %RAW USED
ssd    1.5 TiB  1.5 TiB  4.3 GiB   4.3 GiB       0.28
TOTAL  1.5 TiB  1.5 TiB  4.3 GiB   4.3 GiB       0.28--- POOLS ---
POOL  ID  PGS   STORED  OBJECTS     USED  %USED  MAX AVAIL
.mgr   1    1  449 KiB        2  1.3 MiB      0    474 GiB# 查看osd布局
[root@test-ceph1 ~]# ceph osd tree
ID   CLASS  WEIGHT   TYPE NAME            STATUS  REWEIGHT  PRI-AFF-1         1.46530  root default                                  -7         0.29306      host test-ceph1                           1    ssd  0.09769          osd.1            up   1.00000  1.000007    ssd  0.09769          osd.7            up   1.00000  1.0000012    ssd  0.09769          osd.12           up   1.00000  1.00000-9         0.29306      host test-ceph2                           3    ssd  0.09769          osd.3            up   1.00000  1.000008    ssd  0.09769          osd.8            up   1.00000  1.0000013    ssd  0.09769          osd.13           up   1.00000  1.00000
-11         0.29306      host test-ceph3                           4    ssd  0.09769          osd.4            up   1.00000  1.000009    ssd  0.09769          osd.9            up   1.00000  1.0000014    ssd  0.09769          osd.14           up   1.00000  1.00000-3         0.29306      host test-ceph4                           0    ssd  0.09769          osd.0            up   1.00000  1.000005    ssd  0.09769          osd.5            up   1.00000  1.0000010    ssd  0.09769          osd.10           up   1.00000  1.00000-5         0.29306      host test-ceph6                           2    ssd  0.09769          osd.2            up   1.00000  1.000006    ssd  0.09769          osd.6            up   1.00000  1.0000011    ssd  0.09769          osd.11           up   1.00000  1.00000# 查看集群所有可用的磁盘设备
ceph orch devices ls 
ceph orch devices ls |awk test-ceph|grep Yes# 创建两个OSD daemons
ceph orch daemon add osd test-ceph1:/dev/nvme0n2
ceph orch daemon add osd test-ceph1:/dev/nvme0n3
# 确认 daemon 是否运行
ceph orch ps |grep -ie osd.1 -ie osd.7# 查看集群容量
ceph df 
# 查看osd布局
ceph osd tree# 自动管理所有设备，创建osd进程
ceph orch apply osd --all-available-devices
# 查看自动管理服务是否创建
ceph orch ls |grep all-available-devices# 查看osd 布局
ceph osd tree# 停止 test-ceph1 上的 nvme0n2 设备，并删除该设备
ceph devices ls | grep 'test-ceph1'
ceph orch daemon stop osd.ID
ceph orch daemon rm osd.ID --force
ceph osd rm ID
ceph orch osd rm status
ceph orch osd rm status # 确认删除完成
# 清理并准备该设备
ceph orch device zap --force test-ceph1 /dev/nvm0n2
# 查看并等待设备就绪
ceph orch device ls |grep /test-ceph1/ 
ceph device ls |grep 'test-ceph1:nvm0n2'
ceph orch ps |grep osd.ID
# 以yaml格式查看OSD服务
ceph orch ls --service-type osd --format yaml
# 禁止自动管理osd
ceph orch apply osd --all-available-devices --umanaged=true# 停止 test-ceph2 上面的 /dev/mvme0n4 设备的osd进程
# 确认删除进程正确完成
# 清理这个磁盘，集群不会自动使用干净的设备创建一个新的osd
ceph device ls |grep 'test-ceph2:nvm0n4'
ceph orch daemon stop osd.ID
ceph orch daemon rm osd.ID --force
# 等待删除
ceph orch osd rm status 
ceph osd rm ID
ceph orch device zap --force test-ceph2 /dev/nvm0n4
ceph orch device ls |awk /test-ceph2/
# 等待一段时间
ceph orch device ls | awk /test-ceph2/

创建并配置存储池复制池

# PG 计算方式
# osd数量 * 100 / 副本数量# 创建复制存储池
# ceph osd pool create pool-name pg-num pgp-num replicated crush-rule-name
ceph -s 
# 创建名为 replpool1 的 64 PG 复制池
[root@test-ceph1 ~]# ceph osd pool create replpool1 64 64
pool 'replpool1' created# 查看 PG 的自动扩展是否打开
[root@test-ceph1 ~]# ceph osd pool get replpool1 pg_autoscale_mode
pg_autoscale_mode: on
[root@test-ceph1 ~]# ceph config get mon osd_pool_default_pg_autoscale_mode
on# 查看存储池是否创建成功
[root@test-ceph1 ~]# ceph osd lspools
1 .mgr
2 replpool1# 查看自动扩展状态
[root@test-ceph1 ~]# ceph osd pool autoscale-status
POOL         SIZE  TARGET SIZE  RATE  RAW CAPACITY   RATIO  TARGET RATIO  EFFECTIVE RATIO  BIAS  PG_NUM  NEW PG_NUM  AUTOSCALE  BULK   
.mgr       452.0k                3.0         1499G  0.0000                                  1.0       1              on         False  
replpool1      0                 3.0         1499G  0.0000                                  1.0      64              on         False  # 设置副本数量为4
[root@test-ceph1 ~]# ceph osd pool set replpool1 size 4
set pool 2 size to 4# 最小多少个副本正常才能正常读,比副本数量要小
[root@test-ceph1 ~]# ceph osd pool set replpool1 min_size 2
set pool 2 min_size to 2# 将存储池的应用程序类型设置为 rdb
[root@test-ceph1 ~]# ceph osd pool application enable replpool1 rbd
enabled application 'rbd' on pool 'replpool1'# 查看存储池配置信息
[root@test-ceph1 ~]# ceph osd pool ls detail
pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 32 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr
pool 2 'replpool1' replicated size 4 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode on last_change 54 flags hashpspool stripe_width 0 application rbd# 查看存储池副本数量
[root@test-ceph1 ~]# ceph osd pool get replpool1 size 
size: 4# 对存储池进行重命名
[root@test-ceph1 ~]# ceph osd pool rename replpool1 newpool
pool 'replpool1' renamed to 'newpool'# 查看存储池
[root@test-ceph1 ~]# ceph osd pool ls
.mgr
newpool# 临时设置允许删除存储池
[root@test-ceph1 ~]# ceph tell mon.* config set mon_allow_pool_delete true
mon.test-ceph1: {"success": "mon_allow_pool_delete = '' "
}
mon.test-ceph3: {"success": "mon_allow_pool_delete = '' "
}
mon.test-ceph2: {"success": "mon_allow_pool_delete = '' "
}
# 永久设置允许删除存储池
[root@test-ceph1 ~]# ceph config set mon mon_allow_pool_delete true# 删除存储池
[root@test-ceph1 ~]# ceph osd pool delete newpool newpool --yes-i-really-really-mean-it
pool 'newpool' removed
# 不能正常删除
[root@test-ceph1 ~]# ceph osd pool delete newpool
Error EPERM: WARNING: this will *PERMANENTLY DESTROY* all data stored in pool newpool.  If you are *ABSOLUTELY CERTAIN* that is what you want, pass the pool name *twice*, followed by --yes-i-really-really-mean-it.

创建并配置存储池纠删码池

# 创将纠删码池
# ceph osd pool create pool-name pg-num pgp-num erasure erasure-code-profile crush-rule-name
ceph -s # 列出所有已经配置的纠删码配置文件
[root@test-ceph1 ~]# ceph osd erasure-code-profile ls 
default# 查看 default 纠删码配置文件的详细信息
[root@test-ceph1 ~]# ceph osd erasure-code-profile get default
k=2
m=2
plugin=jerasure# 创建纠删码配置文件 ecprofile-k4-m2，并设置k=4 m=2
[root@test-ceph1 ~]# ceph osd erasure-code-profile set ecprofile-k4-m2 k=4 m=2# 创建纠删码池 ecpool1 ,使用配置文件 ecprofile-k4-m2
[root@test-ceph1 ~]# ceph osd pool create ecpool1 64 64 erasure ecprofile-k4-m2
pool 'ecpool1' created# ecpool1 池启用 rgw
[root@test-ceph1 ~]# ceph osd pool application enable ecpool1 rgw 
enabled application 'rgw' on pool 'ecpool1'# 查看存储池详情
[root@test-ceph1 ~]# ceph osd pool ls detail 
pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 32 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr
pool 3 'ecpool1' erasure profile ecprofile-k4-m2 size 6 min_size 5 crush_rule 1 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode on last_change 62 flags hashpspool stripe_width 16384 application rgw# 允许 RBD 和 CephFS 也可以使用该池
[root@test-ceph1 ~]# ceph osd pool set ecpool1 allow_ec_overwrites true
set pool 3 allow_ec_overwrites to true# 删除存储池
[root@test-ceph1 ~]# ceph osd pool delete ecpool1 ecpool1 --yes-i-really-really-mean-it
pool 'ecpool1' removed

存储池命名空间

rados -p mytestpool -N system put srv /etc/services
rados -p mytestpool -N system ls 
rados -p mytestpool --all ls 
rados -p mytestpool --all ls --format=json-pretty

用户认证

# 创建一个测试存储池
[root@test-ceph1 ~]# ceph osd pool create replpool1 64 64 
pool 'replpool1' created# 创建一个读写，一个只读的用户
[root@test-ceph1 ~]# ceph osd pool create replpool1 64 64 
pool 'replpool1' created
[root@test-ceph1 ~]# ceph auth get-or-create client.docedit \
mon 'allow r' osd 'allow rw pool=replpool1 namespace=docs' \
> /etc/ceph/ceph.client.docedit.keyring
[root@test-ceph1 ~]# ceph auth get-or-create client.docget \
mon 'allow r' osd 'allow r pool=replpool1 namespace=docs' \
> /etc/ceph/ceph.client.docget.keyring# 查看创建的用户
[root@test-ceph1 ~]# ceph auth ls |grep -A 3 -ie docedit -ie docget 
client.doceditkey: AQAGHrRo1ERcBRAAcVS07Xpp00PbMZljAJL3Kw==caps: [mon] allow rcaps: [osd] allow rw pool=replpool1 namespace=docs
client.docgetkey: AQANHrRoU5+HNBAAuc44PrhlDpZj5IrLktaF6g==caps: [mon] allow rcaps: [osd] allow r pool=replpool1 namespace=docs# 同步用户认证文件到其他主机
[root@test-ceph1 ~]# rsync -v /etc/ceph/ceph.conf test-ceph6:/etc/ceph/
[root@test-ceph1 ~]# rsync -v /etc/ceph/ceph.client.docedit.keyring test-ceph6:/etc/ceph/
[root@test-ceph1 ~]# rsync -v /etc/ceph/ceph.client.docget.keyring test-ceph6:/etc/ceph/# 其他主机上进行文件传输,然后下载为 /tmp/test
[root@test-ceph6 ~]# rados --id docedit -p replpool1 -N docs put adoc /etc/hosts
[root@test-ceph6 ~]# rados --id docget -p replpool1 -N docs get adoc /tmp/test# 查看两个文件是不是同一个文件
[root@test-ceph6 ~]# diff /etc/hosts /tmp/test# 测试get权限能否上传文件
[root@test-ceph6 ~]# rados --id docget -p replpool1 -N docs put writetest /etc/hosts
error putting replpool1/writetest: (1) Operation not permitted# 进行权限覆盖
[root@test-ceph1 ~]# ceph auth caps client.docget mon 'allow r' \
osd 'allow rw pool=replpool1 namespace=docs,allow rw pool=docarchive'
updated caps for client.docget# 进行测试
[root@test-ceph6 ~]# rados --id docget -p replpool1 -N docs put writetest /etc/hosts# 删除创建的各种认证,两台主机都操作
[root@test-ceph6 ~]# rm -f /etc/ceph/ceph.client.docedit.keyring
[root@test-ceph1 ~]# rm -f /etc/ceph/ceph.client.docedit.keyring
[root@test-ceph1 ~]# ceph auth del client.docedit[root@test-ceph6 ~]# rm -f /etc/ceph/ceph.client.docget.keyring
[root@test-ceph1 ~]# rm -f /etc/ceph/ceph.client.docget.keyring
[root@test-ceph1 ~]# ceph auth del client.docget# 授权访问块存储
[root@test-ceph1 ~]# ceph auth get-or-create client.forrbd \
mon 'profile rbd' osd 'profile rbd'
[client.forrbd]key = AQAsIrRoEwY1ABAAz3Hx7wbajIUQE8ErrAMtyw==# 授权用户可以对任何存储池中的对象名称以 pref 开始的文件有读写权限
[root@test-ceph1 ~]# ceph auth get-or-create client.formyapp3 \
mon 'allow r' \
osd 'allow rw pool=myapp-pool object_prefix pref-'
[client.formyapp3]key = AQCWIrRo6R25HRAAwey6HjKp/Az3uFX2GX7KFA==# 限制仅可以执行 auth get-or-create 和 auth list 命令
[root@test-ceph1 ~]# ceph auth get-or-create client.operator1 \
mon 'allow r,allow command "auth get-or-create",allow command "auth list"'
[client.operator1]key = AQBPIrRoeVoXEBAACsAC1JM8Ygzmq6mE3quEYg==