Previous: Installing Ceph Next: Service checks and CLI commands
Ensure the kernel module for rbd is loaded every time the server is restarted
echo "rbd" >> /etc/modules-load.d/modules.conf
For the current session, force
modprobe rbd
Install KVM
apt -y install qemu-kvm libvirt-daemon-system libvirt-daemon virtinst bridge-utils libguestfs-tools libosinfo-bin libvirt-daemon-driver-storage-rbd ovmf
Configure bridge interfaces
Configure bridge interfaces for 10.0.1.x, 10.0.2.x, 10.0.3.x and 10.0.4.x. 10.0.5.x is dedicated to the ceph internal network; hence, no bridge is required.
Update /etc/network/interfaces (sample below for server 1)
source /etc/network/interfaces.d/*
auto lo
iface lo inet loopback
allow-hotplug eno1
iface eno1 inet manual
mtu 9000
auto br1
iface br1 inet static
mtu 9000
address 10.0.1.1/16
gateway 10.0.0.1
bridge_ports eno1
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno2
iface eno2 inet manual
mtu 9000
auto br2
iface br2 inet static
mtu 9000
address 10.0.2.1/24
bridge_ports eno2
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno3
iface eno3 inet manual
mtu 9000
auto br3
iface br3 inet static
mtu 9000
address 10.0.3.1/24
bridge_ports eno3
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno4
iface eno4 inet manual
mtu 9000
auto br4
iface br4 inet static
mtu 9000
address 10.0.4.1/24
bridge_ports eno4
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug enp129s0
iface enp129s0 inet static
address 10.0.5.1/24
mtu 9000
Apply the network configurations
root@server1:~# systemctl restart networking
The ipv4.forward configuration in the Linux kernel controls whether the host machine is allowed to forward IP packets. By default, it’s usually disabled for security reasons. Execute the following to enable the same.
sed -i "s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g" /etc/sysctl.conf
sysctl -p
Start the default network.
modprobe vhost_net
echo vhost_net | tee -a /etc/modules
We have a DHCP service already up and running in 10.0.0.5, so we will not need a built-in dnsmasq service (libvirtd).
Edit /etc/libvirt/qemu/networks/default.xml (Remove dhcp range configuration and add dns server configuration)
<network>
<name>default</name>
<uuid>a27043ab-4bf7-4acf-826b-b601fbaf6e8c</uuid>
<forward mode='nat'/>
<bridge name='virbr0' stp='on' delay='0'/>
<mac address='52:54:00:59:9d:b9'/>
<ip address="192.168.122.1" netmask="255.255.255.0"> </ip>
<dns>
<server address="10.0.0.5" />
</dns>
</network>
Destroy default network if previously created, start with new configuration and mark it for autostart
virsh net-destroy default
virsh net-define /etc/libvirt/qemu/networks/default.xml
virsh net-start default
virsh net-autostart default
Generate a keyring for the libvirt client to connect to ceph rbd.
ceph auth get-or-create client.libvirt mon 'profile rbd' osd 'profile rbd pool=ssdpool, profile rbd pool=nvmepool'
Now, add the client.libvirt section in /etc/ceph/ceph.conf
[client.libvirt]
keyring = /etc/ceph/ceph.client.libvirt.keyring
As we have logged in as root or working in root user context, ensure the ownership is set to ceph user
chown -R ceph:ceph /etc/ceph
Copy the updated ceph.conf file to other servers
scp /etc/ceph/ceph.conf ceph2:/etc/ceph
scp /etc/ceph/ceph.conf ceph3:/etc/ceph
scp /etc/ceph/ceph.conf ceph4:/etc/ceph
Create ceph-secret.xml to define a secret using virsh (uuid is generated)
<secret>
<uuid>155d8c36-3fa6-46e2-833a-cdae1b7bf8d2</uuid>
<usage type='ceph'>
<name>libvirt</name>
</usage>
</secret>
Copy the secret file to other servers
scp ceph-secret.xml ceph2:~/
scp ceph-secret.xml ceph3:~/
scp ceph-secret.xml ceph4:~/
Define the secret (in all servers)
virsh secret-define --file ceph-secret.xml
Set the value for the secret (in all servers)
virsh secret-set-value --secret <unique-secret-id-generated above> --base64 <ceph.client.libvirt.keyring value>
Create a pool definition files nvmepool.xml and ssdpool.xml with the following contents
<pool type="rbd">
<name>nvmepool</name> <!-- name of pool in libvirt -->
<source>
<name>nvmepool</name> <!-- name of pool in ceph -->
<host name='ceph1'/>
<host name='ceph2'/> <!- host names of monitor nodes -->
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
<pool type="rbd">
<name>ssdpool</name>
<source>
<name>ssdpool</name>
<host name='ceph1'/>
<host name='ceph2'/>
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
Copy the files to other servers.
scp ssdpool.xml nvmepool.xml ceph2:~/
scp ssdpool.xml nvmepool.xml ceph3:~/
scp ssdpool.xml nvmepool.xml ceph4:~/
Define the pool, start it and also mark for autostart whenever libvirt starts (in all servers)
virsh pool-define nvmepool.xml
virsh pool-start nvmepool
virsh pool-autostart nvmepool
virsh pool-define ssdpool.xml
virsh pool-start ssdpool
virsh pool-autostart ssdpool
Observed that whenever the libvirtd service restarts, the pools are not in active mode. The workaround was to create /usr/bin/poolstart.sh with the following contents and get it executed whenever libvirtd service starts (ExecStartPost)
#!/bin/bash
export PATH=/usr/bin:$PATH
logfile=/var/log/cephcs.log
function log() {
timenow=$(date)
line_count=$(wc -l < "$logfile")
if (( line_count > 500 )); then
tail -n 500 "$logfile" > "$logfile.tmp"
mv "$logfile.tmp" "$logfile"
fi
echo "$timenow | $1" >> "$logfile"
}
function waitForRBDPool() {
maxWaitCount=20
counter=0
poolName=$1
while true; do
((counter++))
log "Waiting for CEPH pool $poolName is ready. Iteration $counter"
if [ $counter -gt $maxWaitCount ]; then
break
else
poolCount=`ceph osd pool ls | grep "$poolName" | wc -l`
if [ $poolcount -ge 1 ]; then
break
fi
sleep 6
fi
done
if [ $counter -le $maxWaitCount ]; then
log "$poolName is ready."
fi
}
secretid=$(awk '/<uuid>/,/<\/uuid>/ { if ($1 ~ /<uuid>/) { gsub(/<\/?uuid>/,""); print $1 } }' /root/kvm/ceph-secret.xml)
secretvalue=$(awk '/\[client\.libvirt\]/,/^$/ { if ($1 == "key") { print $3 } }' /etc/ceph/ceph.client.libvirt.keyring)
configuredValue=`/usr/bin/virsh secret-get-value $secretid`
if [ "$secretvalue" != "$configuredValue" ]; then
waitForRBDPool ssdpool
waitForRBDPool nvmepool
virsh secret-set-value --secret "$secretid" --base64 "$secretvalue"
else
log "Detected valid auth configuration..."
fi
log "Checking SSD Pool...."
ssdpoolcount=`/usr/bin/virsh pool-list | grep "ssdpool" | wc -l`
if [ $ssdpoolcount -lt 1 ]; then
sleep 3
ssdpoolcount=`/usr/bin/virsh pool-list | grep "ssdpool" | wc -l`
fi
log "SSD Pool count : $ssdpoolcount"
if [ $ssdpoolcount -lt 1 ]; then
log "Starting ssd pool"
/usr/bin/virsh pool-start ssdpool
/usr/bin/virsh pool-autostart ssdpool
else
log "SSD pool active."
fi
nvmepoolcount=`/usr/bin/virsh pool-list | grep "nvmepool" | wc -l`
if [ $nvmepoolcount -lt 1 ]; then
sleep 3
nvmepoolcount=`/usr/bin/virsh pool-list | grep "nvmepool" | wc -l`
fi
log "SSD Pool count : $nvmepoolcount"
if [ $nvmepoolcount -lt 1 ]; then
log "Starting nvme pool"
/usr/bin/virsh pool-start nvmepool
/usr/bin/virsh pool-autostart nvmepool
else
log "NVME pool active."
fi
log "====================================================="
exit 0
Set execute permisssion
chmod +x /usr/bin/poolstart.sh
Edit libvirtd service and add the following to be overridden in the service section.
systemctl edit libvirtd.service
### Editing /etc/systemd/system/libvirtd.service.d/override.conf
### Anything between here and the comment below will become the new contents of the file
[Service]
ExecStartPost=/usr/bin/poolstart.sh
User=root
### Lines below this comment will be discarded