How do i create a ceph block device (RBD) on a client?

blackrhino

Install the ceph-common package on the client.

ceph-client> $ sudo yum install -y ceph-common
ceph-client> $ sudo chown -R ceph:ceph /etc/ceph

On the management node, create a rados block device (RBD), setting permissions, the pool, and authorisation, storing the config in a file.

ceph-mgmt> $ ceph auth get-or-create client.rbd mon 'allow r' osd 'allow rwx pool=rbd' -o /etc/ceph/ceph.client.rbd.keyring

Copy the ceph config and block device config to the client node.

ceph-mgmt> $ scp /etc/ceph/ceph.conf ceph-client:/etc/ceph
ceph-mgmt> $ scp /etc/ceph/ceph.client.rbd.keyring ceph-client:/etc/ceph

From the client node, verify the cluster is available to the rbd user.

ceph-client> $ ssh ceph-client
ceph-client> $ ceph -s --id rbd
    cluster 877519e2-9b01-4982-bd51-0f88b4b591b5
     health HEALTH_OK
     monmap e1: 3 mons at {mon-node1=10.100.2.11:6789/0,mon-node2=10.100.2.12:6789/0,mon-node3=10.100.2.13:6789/0}
            election epoch 4, quorum 0,1,2 mon-node1,mon-node2,mon-node3
     osdmap e26: 9 osds: 9 up, 9 in
            flags sortbitwise
      pgmap v52: 128 pgs, 1 pools, 0 bytes data, 0 objects
            305 MB used, 863 GB / 863 GB avail
                 128 active+clean

Create a block device of 5G on the client

ceph-client> rbd create oracle --size 5000 --image-feature layering --id rbd

Verify the block device has been created.

ceph-client> $ rbd ls --id rbd
oracle
ceph-client> $ rbd info oracle --id rbd
rbd image 'oracle':
	size 5000 MB in 1250 objects
	order 22 (4096 kB objects)
	block_name_prefix: rbd_data.104d2ae8944a
	format: 2
	features: layering
	flags:

Load and verify RBD kernel module

ceph-client> $ lsmod | grep -i rbd
ceph-client> $ sudo modprobe rbd
ceph-client> $ lsmod | grep -i rbd
rbd                    73158  0 
libceph               244999  1 rbd

Map the block device to the client

ceph-client> $ sudo rbd map oracle --id rbd
/dev/rbd0

Verify

$ rbd showmapped --id rbd
id pool image  snap device    
0  rbd  oracle -    /dev/rbd0

Create a filesystem and mount.

ceph-client> $ sudo mkfs.xfs /dev/rbd0
meta-data=/dev/rbd0              isize=256    agcount=8, agsize=159744 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=0        finobt=0
data     =                       bsize=4096   blocks=1277952, imaxpct=25
         =                       sunit=1024   swidth=1024 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=0
log      =internal log           bsize=4096   blocks=2560, version=2
         =                       sectsz=512   sunit=8 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

ceph-client> $ sudo mkdir /oracle
ceph-client> $ sudo mount /dev/rbd0 /oracle
ceph-client> $ df -h /oracle
Filesystem      Size  Used Avail Use% Mounted on
/dev/rbd0       4.9G   33M  4.9G   1% /oracle