Previous: Installing Ceph Next: Service checks and CLI commands
Ensure the kernel module for rbd is loaded every time the server is restarted
echo "rbd" >> /etc/modules-load.d/modules.conf
For the current session, force
modprobe rbd
Install KVM
apt -y install qemu-kvm libvirt-daemon-system libvirt-daemon virtint bridge-utils libguestfs-tools libosinfo-bin libvirt-daemon-driver-storage-rbd ovmf qemu-guest-agent
Configure bridge interfaces
Configure bridge interfaces for 10.0.1.x, 10.0.2.x, 10.0.3.x and 10.0.4.x. 10.0.5.x is dedicated to the ceph internal network; hence, no bridge is required.
Update /etc/network/interfaces (sample below for server 1)
source /etc/network/interfaces.d/*
auto lo
iface lo inet loopback
allow-hotplug eno1
iface eno1 inet manual
mtu 9000
auto br1
iface br1 inet static
mtu 9000
address 10.0.1.1/16
gateway 10.0.0.1
bridge_ports eno1
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno2
iface eno2 inet manual
mtu 9000
auto br2
iface br2 inet static
mtu 9000
address 10.0.2.1/24
bridge_ports eno2
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno3
iface eno3 inet manual
mtu 9000
auto br3
iface br3 inet static
mtu 9000
address 10.0.3.1/24
bridge_ports eno3
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno4
iface eno4 inet manual
mtu 9000
auto br4
iface br4 inet static
mtu 9000
address 10.0.4.1/24
bridge_ports eno4
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug enp129s0
iface enp129s0 inet static
address 10.0.5.1/24
mtu 9000
Apply the network configurations
root@server1:~# systemctl restart networking
The ipv4.forward configuration in the Linux kernel controls whether the host machine is allowed to forward IP packets. By default, it’s usually disabled for security reasons. Execute the following to enable the same.
sed -i "s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g" /etc/sysctl.conf
sysctl -p
Start the default network.
modprobe vhost_net
echo vhost_net | tee -a /etc/modules
By default, we had planned for three networks: one for accessing VMs, one for accessing DB, and one for accessing ceph. We have created the necessary bridges and now need to define the networks for libvirt.
Create the following XML (network definitions) br1.xml, br2.xml and br3.xml
br1.xml (Use the same template for br2 and br3)
<network>
<name>br1</name>
<bridge name="br1" />
<forward mode="bridge"/>
<interface type="bridge"/>
</network>
Destroy default network and undefine, if previously created
virsh net-destroy default
virsh net-undefine /etc/libvirt/qemu/networks/default.xml
Define 3 networks (repeat the same command for br2 and br3)
virsh net-define br1.xml
virsh net-start br1
virsh net-autostart br1
Generate a keyring for the libvirt client to connect to ceph rbd.
ceph auth get-or-create client.libvirt mon 'profile rbd' osd 'profile rbd pool=ssdpool, profile rbd pool=nvmepool'
Now, add the client.libvirt section in /etc/ceph/ceph.conf
[client.libvirt]
keyring = /etc/ceph/ceph.client.libvirt.keyring
As we have logged in as root or working in root user context, ensure the ownership is set to ceph user
chown -R ceph:ceph /etc/ceph
Copy the updated ceph.conf file to other servers
scp /etc/ceph/ceph.conf ceph2:/etc/ceph
scp /etc/ceph/ceph.conf ceph3:/etc/ceph
scp /etc/ceph/ceph.conf ceph4:/etc/ceph
Create ceph-secret.xml to define a secret using virsh (uuid is generated)
<secret>
<uuid>155d8c36-3fa6-46e2-833a-cdae1b7bf8d2</uuid>
<usage type='ceph'>
<name>libvirt</name>
</usage>
</secret>
Copy the secret file to other servers
scp ceph-secret.xml ceph2:~/
scp ceph-secret.xml ceph3:~/
scp ceph-secret.xml ceph4:~/
Define the secret (in all servers)
virsh secret-define --file ceph-secret.xml
Set the value for the secret (in all servers)
virsh secret-set-value --secret <unique-secret-id-generated above> --base64 <ceph.client.libvirt.keyring value>
Create a pool definition files nvmepool.xml and ssdpool.xml with the following contents
<pool type="rbd">
<name>nvmepool</name> <!-- name of pool in libvirt -->
<source>
<name>nvmepool</name> <!-- name of pool in ceph -->
<host name='ceph1'/>
<host name='ceph2'/> <!- host names of monitor nodes -->
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
<pool type="rbd">
<name>ssdpool</name>
<source>
<name>ssdpool</name>
<host name='ceph1'/>
<host name='ceph2'/>
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
Copy the files to other servers.
scp ssdpool.xml nvmepool.xml ceph2:~/
scp ssdpool.xml nvmepool.xml ceph3:~/
scp ssdpool.xml nvmepool.xml ceph4:~/
Define the pool, start it and also mark for autostart whenever libvirt starts (in all servers)
virsh pool-define nvmepool.xml
virsh pool-start nvmepool
virsh pool-autostart nvmepool
virsh pool-define ssdpool.xml
virsh pool-start ssdpool
virsh pool-autostart ssdpool
Observed that whenever the libvirtd service restarts, the pools are not in active mode. The workaround was to create /usr/bin/poolstart.sh with the following contents and get it executed whenever libvirtd service starts (ExecStartPost)
#!/bin/bash
export PATH=/usr/bin:$PATH
logfile=/var/log/cephcs.log
function log() {
timenow=$(date)
line_count=$(wc -l < "$logfile")
if (( line_count > 500 )); then
tail -n 500 "$logfile" > "$logfile.tmp"
mv "$logfile.tmp" "$logfile"
fi
echo "$timenow | $1" >> "$logfile"
}
function waitForRBDPool() {
maxWaitCount=20
counter=0
poolName=$1
while true; do
((counter++))
log "Waiting for CEPH pool $poolName is ready. Iteration $counter"
if [ $counter -gt $maxWaitCount ]; then
break
else
poolCount=`ceph osd pool ls | grep "$poolName" | wc -l`
if [ $poolcount -ge 1 ]; then
break
fi
sleep 6
fi
done
if [ $counter -le $maxWaitCount ]; then
log "$poolName is ready."
fi
}
secretid=$(awk '/<uuid>/,/<\/uuid>/ { if ($1 ~ /<uuid>/) { gsub(/<\/?uuid>/,""); print $1 } }' /root/kvm/ceph-secret.xml)
secretvalue=$(awk '/\[client\.libvirt\]/,/^$/ { if ($1 == "key") { print $3 } }' /etc/ceph/ceph.client.libvirt.keyring)
configuredValue=`/usr/bin/virsh secret-get-value $secretid`
if [ "$secretvalue" != "$configuredValue" ]; then
waitForRBDPool ssdpool
waitForRBDPool nvmepool
virsh secret-set-value --secret "$secretid" --base64 "$secretvalue"
else
log "Detected valid auth configuration..."
fi
log "Checking SSD Pool...."
ssdpoolcount=`/usr/bin/virsh pool-list | grep "ssdpool" | wc -l`
if [ $ssdpoolcount -lt 1 ]; then
sleep 3
ssdpoolcount=`/usr/bin/virsh pool-list | grep "ssdpool" | wc -l`
fi
log "SSD Pool count : $ssdpoolcount"
if [ $ssdpoolcount -lt 1 ]; then
log "Starting ssd pool"
/usr/bin/virsh pool-start ssdpool
/usr/bin/virsh pool-autostart ssdpool
else
log "SSD pool active."
fi
nvmepoolcount=`/usr/bin/virsh pool-list | grep "nvmepool" | wc -l`
if [ $nvmepoolcount -lt 1 ]; then
sleep 3
nvmepoolcount=`/usr/bin/virsh pool-list | grep "nvmepool" | wc -l`
fi
log "SSD Pool count : $nvmepoolcount"
if [ $nvmepoolcount -lt 1 ]; then
log "Starting nvme pool"
/usr/bin/virsh pool-start nvmepool
/usr/bin/virsh pool-autostart nvmepool
else
log "NVME pool active."
fi
log "====================================================="
exit 0
Set execute permisssion
chmod +x /usr/bin/poolstart.sh
Edit libvirtd service and add the following to be overridden in the service section.
systemctl edit libvirtd.service
### Editing /etc/systemd/system/libvirtd.service.d/override.conf
### Anything between here and the comment below will become the new contents of the file
[Service]
ExecStartPost=/usr/bin/poolstart.sh
User=root
### Lines below this comment will be discarded
Observed that when there are no VMs launched on the host, libvirtd stops due to inactivity. I could not figure out how to disable it. So, I decided to have a safeguard against the same. Have the following script run as a cron job every two minutes.
#!/bin/bash
export PATH=/usr/bin:$PATH
logfile=/var/log/cephcs.log
function log() {
timenow=$(date)
line_count=$(wc -l < "$logfile")
if (( line_count > 500 )); then
tail -n 500 "$logfile" > "$logfile.tmp"
mv "$logfile.tmp" "$logfile"
fi
echo "$timenow | $1" >> "$logfile"
}
function waitForRBDPool() {
maxWaitCount=5
counter=0
poolName=$1
poolCount=`ceph osd pool ls | grep "$poolName" | wc -l`
if [ $poolCount -le 0 ]; then
#log "CEPH : $poolName : Count is [$poolCount]"
while true; do
((counter++))
#log "Waiting for CEPH pool $poolName is ready. Iteration $counter"
if [ $counter -gt $maxWaitCount ]; then
#log "Time out waiting for CEPH pool $poolName to be ready"
break
else
sleep 3
poolCount=`ceph osd pool ls | grep "$poolName" | wc -l`
#log "CEPH : $poolName : Count is [$poolCount]"
if [ $poolCount -ge 1 ]; then
break
fi
fi
done
fi
if [ $counter -le $maxWaitCount ]; then
log "CEPH : $poolName is ready."
else
log "CEPH : $poolName is not ready."
fi
}
function checkVirshPool() {
#log "Virsh : $1 : Checking if active...."
poolcount=`virsh pool-list | grep "$1" | wc -l`
if [ $poolcount -lt 1 ]; then
log "Virsh : $1 : Starting pool"
virsh pool-start $1
virsh pool-autostart $1
sleep 3
poolcount=`virsh pool-list | grep "$1" | wc -l`
fi
if [ $poolcount -eq 1 ]; then
log "Virsh : $1 : active."
else
log "Virsh : $1 : not active."
fi
}
function checkCephServices() {
systemctl list-dependencies ceph.target | grep -oE 'ceph-[a-z]+@[a-z0-9.]+|ceph-[a-z]+.service' | sort -u > /usr/bin/ceph.services
for service in $(cat /usr/bin/ceph.services); do
if ! systemctl is-active "$service" > /dev/null; then
log "Starting service $service"
systemctl start $service
sleep 3
else
log "$service is active."
fi
done
}
function checkLibVirtdServices() {
if ! systemctl is-active libvirtd.service > /dev/null; then
log "Libvirt is not active .... checking ceph services..."
for service in $(cat /usr/bin/ceph.services); do
if ! systemctl is-active "$service" > /dev/null; then
log "Service $service is not active!"
all_services_active=false
break
fi
done
if $all_services_active; then
log "Starting libvirtd..."
systemctl start libvirtd.service
else
log "Not all ceph services are up. Not starting libvirtd..."
fi
else
log "Libvirtd service is active."
fi
}
function checkActivePools() {
if systemctl is-active libvirtd.service > /dev/null; then
log "Libvitd is active... Checking for active pools...."
secretid=$(awk '/<uuid>/,/<\/uuid>/ { if ($1 ~ /<uuid>/) { gsub(/<\/?uuid>/,""); print $1 } }' /root/kvm/ceph-secret.xml)
secretvalue=$(awk '/\[client\.libvirt\]/,/^$/ { if ($1 == "key") { print $3 } }' /etc/ceph/ceph.client.libvirt.keyring)
configuredValue=`virsh secret-get-value $secretid`
if [ "$secretvalue" != "$configuredValue" ]; then
log "Setting secret value..."
virsh secret-set-value --secret "$secretid" --base64 "$secretvalue"
fi
waitForRBDPool ssdpool
waitForRBDPool nvmepool
checkVirshPool ssdpool
checkVirshPool nvmepool
else
log "Libvirtd is not active, not checking for active pools."
fi
}
log "-------- Periodic check start ---------------"
checkCephServices
checkLibVirtdServices
checkActivePools
log "-------- Periodic check end -----------------"
exit 0