Previous: Installing Ceph Next: Orchestrator RBD backed VMs on KVM
Ensure the kernel module for RBD is loaded every time the server is restarted
echo "rbd" >> /etc/modules-load.d/modules.conf
For the current session, force
modprobe rbd
Install KVM
apt -y install qemu-system-x86 libvirt-daemon-system libvirt-daemon virtinst bridge-utils libguestfs-tools libosinfo-bin libvirt-daemon-driver-storage-rbd ovmf qemu-guest-agent
Configure bridge interfaces
Configure bridge interfaces for 10.0.1.x, 10.0.2.x, 10.0.3.x and 10.0.4.x. 10.0.5.x is dedicated to the ceph internal network; hence, no bridge is required.
Update /etc/network/interfaces (sample below for server 1)
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
allow-hotplug eno1
iface eno1 inet manual
mtu 9000
# VM Access / Management Traffic
auto br1
iface br1 inet static
mtu 9000
address 10.0.1.1/16
gateway 10.0.0.1
bridge_ports eno1
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno2
iface eno2 inet manual
mtu 9000
# Message Queue Traffic
auto br2
iface br2 inet static
mtu 9000
address 10.0.2.1/24
bridge_ports eno2
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno3
iface eno3 inet manual
mtu 9000
# DB Access Traffic
auto br3
iface br3 inet static
mtu 9000
address 10.0.3.1/24
bridge_ports eno3
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno4
iface eno4 inet manual
mtu 9000
# Ceph Access Traffic
auto br4
iface br4 inet static
mtu 9000
address 10.0.4.1/24
bridge_ports eno4
bridge_stp off
bridge_waitport 0
bridge_fd 0
# 40G - Ceph Cluster Traffic
allow-hotplug enp3s0
iface enp3s0 inet static
address 10.0.5.1/24
mtu 9000
#40G - DB Cluster Traffic
allow-hotplug enp3s0d1
iface enp3s0d1 inet static
address 10.0.6.1/24
mtu 9000
Apply the network configurations
root@server1:~# systemctl restart networking
The ipv4.forward Configuration in the Linux kernel controls whether the host machine is allowed to forward IP packets. By default, it’s usually disabled for security reasons. Execute the following to enable the same.
sed -i "s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g" /etc/sysctl.conf
sysctl -p
Start the default network.
modprobe vhost_net
echo vhost_net | tee -a /etc/modules
Create bridges for VM Management, Message Queue Traffic, DB Access Traffic, Ceph Access Traffic. Any VM Orchestrated will have 4 Virtual NICs attached.
Create the following XML (network definitions): br1.xml, br2.xml and br3.xml
br1.xml (Use the same template for br2,br3 and br4)
<network>
<name>br1</name>
<bridge name=”br1″ />
<forward mode=”bridge”/>
<interface type=”bridge”/>
</network>
Destroy the default network and undefine it, if previously created.
virsh net-destroy default
virsh net-undefine /etc/libvirt/qemu/networks/default.xml
Define four networks (repeat the same command for br2, br3 and br4)
virsh net-define br1.xml
virsh net-start br1
virsh net-autostart br1
Generate a keyring for the libvirt client to connect to Ceph RBD.
ceph-authtool --create-keyring /etc/ceph/ceph.client.libvirt.keyring --gen-key -n client.libvirt --cap mon 'profile rbd' --cap osd 'profile rbd pool=ssdpool, profile rbd pool=nvmepool'
ceph auth add client.libvirt -i /etc/ceph/ceph.client.libvirt.keyring
Now, add the client.libvirt section in /etc/ceph/ceph.conf
[client.libvirt]
keyring = /etc/ceph/ceph.client.libvirt.keyring
As we have logged in as root or working in root user context, ensure the ownership is set to the ceph user
chown -R ceph:ceph /etc/ceph
Copy the updated ceph.conf file to other servers
scp /etc/ceph/ceph.client.libvirt.keyring /etc/ceph/ceph.conf ceph2:/etc/ceph
scp /etc/ceph/ceph.client.libvirt.keyring /etc/ceph/ceph.conf ceph3:/etc/ceph
scp /etc/ceph/ceph.client.libvirt.keyring /etc/ceph/ceph.conf ceph4:/etc/ceph
In all the servers, set permissions for libvirt to access
chown root:libvirt /etc/ceph/ceph.client.libvirt.keyring
chmod 640 /etc/ceph/ceph.client.libvirt.keyring
In the first server, create ceph-secret.xml to define a secret using virsh (the uuid is generated)
<secret>
<uuid>155d8c36-3fa6-46e2-833a-cdae1b7bf8d2</uuid>
<usage type='ceph'>
<name>libvirt</name>
</usage>
</secret>
Copy the secret file to other servers
scp ceph-secret.xml ceph2:~/
scp ceph-secret.xml ceph3:~/
scp ceph-secret.xml ceph4:~/
Define the secret (in all servers)
virsh secret-define --file ceph-secret.xml
Set the value for the secret (in all servers)
virsh secret-set-value --secret <unique-secret-id-generated above> --base64 <ceph.client.libvirt.keyring value>
Create a pool definition files nvmepool.xml and ssdpool.xml with the following contents
<pool type="rbd">
<name>nvmepool</name> <!-- name of pool in libvirt -->
<source>
<name>nvmepool</name> <!-- name of pool in ceph -->
<host name='ceph1'/>
<host name='ceph2'/> <!- host names of monitor nodes -->
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
<pool type="rbd">
<name>ssdpool</name>
<source>
<name>ssdpool</name>
<host name='ceph1'/>
<host name='ceph2'/>
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
Copy the files to other servers.
scp ssdpool.xml nvmepool.xml ceph2:~/
scp ssdpool.xml nvmepool.xml ceph3:~/
scp ssdpool.xml nvmepool.xml ceph4:~/
Define the pool, start it and also mark for autostart whenever libvirt starts (in all servers)
virsh pool-define nvmepool.xml
virsh pool-start nvmepool
virsh pool-autostart nvmepool
virsh pool-define ssdpool.xml
virsh pool-start ssdpool
virsh pool-autostart ssdpool
Observed that whenever the libvirtd service restarts, the pools are not in active mode. The workaround was to create /usr/bin/poolstart.sh with the following contents and get it executed whenever the libvirtd service starts (ExecStartPost)
#!/bin/bash
export PATH=/usr/bin:$PATH
logfile=/var/log/cephcs.log
function log() {
timenow=$(date)
line_count=$(wc -l < "$logfile")
if (( line_count > 500 )); then
tail -n 500 "$logfile" > "$logfile.tmp"
mv "$logfile.tmp" "$logfile"
fi
echo "$timenow | $1" >> "$logfile"
}
function waitForRBDPool() {
maxWaitCount=20
counter=0
poolName=$1
while true; do
((counter++))
log "Waiting for CEPH pool $poolName is ready. Iteration $counter"
if [ $counter -gt $maxWaitCount ]; then
break
else
poolCount=`ceph osd pool ls | grep "$poolName" | wc -l`
if [ $poolcount -ge 1 ]; then
break
fi
sleep 6
fi
done
if [ $counter -le $maxWaitCount ]; then
log "$poolName is ready."
fi
}
secretid=$(awk '/<uuid>/,/<\/uuid>/ { if ($1 ~ /<uuid>/) { gsub(/<\/?uuid>/,""); print $1 } }' /root/kvm/ceph-secret.xml)
secretvalue=$(awk '/\[client\.libvirt\]/,/^$/ { if ($1 == "key") { print $3 } }' /etc/ceph/ceph.client.libvirt.keyring)
configuredValue=`/usr/bin/virsh secret-get-value $secretid`
if [ "$secretvalue" != "$configuredValue" ]; then
waitForRBDPool ssdpool
waitForRBDPool nvmepool
virsh secret-set-value --secret "$secretid" --base64 "$secretvalue"
else
log "Detected valid auth configuration..."
fi
log "Checking SSD Pool...."
ssdpoolcount=`/usr/bin/virsh pool-list | grep "ssdpool" | wc -l`
if [ $ssdpoolcount -lt 1 ]; then
sleep 3
ssdpoolcount=`/usr/bin/virsh pool-list | grep "ssdpool" | wc -l`
fi
log "SSD Pool count : $ssdpoolcount"
if [ $ssdpoolcount -lt 1 ]; then
log "Starting ssd pool"
/usr/bin/virsh pool-start ssdpool
/usr/bin/virsh pool-autostart ssdpool
else
log "SSD pool active."
fi
nvmepoolcount=`/usr/bin/virsh pool-list | grep "nvmepool" | wc -l`
if [ $nvmepoolcount -lt 1 ]; then
sleep 3
nvmepoolcount=`/usr/bin/virsh pool-list | grep "nvmepool" | wc -l`
fi
log "SSD Pool count : $nvmepoolcount"
if [ $nvmepoolcount -lt 1 ]; then
log "Starting nvme pool"
/usr/bin/virsh pool-start nvmepool
/usr/bin/virsh pool-autostart nvmepool
else
log "NVME pool active."
fi
log "====================================================="
exit 0
Set execute permission
chmod +x /usr/bin/poolstart.sh
Edit the libvirtd service and add the following to be overridden in the service section. [ Note: ExecStart is being overridden to avoid the default timeout of 2 minutes ]
systemctl edit libvirtd.service
### Editing /etc/systemd/system/libvirtd.service.d/override.conf
### Anything between here and the comment below will become the new contents of the file
[Service]
ExecStart=
ExecStart=/usr/sbin/libvirtd
ExecStartPost=/usr/bin/poolstart.sh
User=root
### Lines below this comment will be discarded
Auto-start vms on host reboot
We need the guest vms to autostart on host startup – Even if we configure virsh autostart <vmname> it will not start unless it is enabled in libvirt-guests config
Configure ON BOOT=true in /etc/default/libvirt-guests