Previous: Installing Ceph Next: Kubekey + Kubesphere
Ensure the kernel module for rbd is loaded every time the server is restarted
echo "rbd" >> /etc/modules-load.d/modules.conf
For the current session, force
modprobe rbd
Install KVM
apt -y install qemu-kvm libvirt-daemon-system libvirt-daemon virtinst bridge-utils libguestfs-tools libosinfo-bin libvirt-daemon-driver-storage-rbd
Configure bridge interfaces
Configure bridge interfaces for 10.0.1.x, 10.0.2.x, 10.0.3.x and 10.0.4.x. 10.0.5.x is dedicated to the ceph internal network; hence, no bridge is required.
Update /etc/network/interfaces (sample below for server 1)
source /etc/network/interfaces.d/*
auto lo
iface lo inet loopback
allow-hotplug eno1
iface eno1 inet manual
mtu 9000
auto br1
iface br1 inet static
mtu 9000
address 10.0.1.1/16
gateway 10.0.0.1
bridge_ports eno1
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno2
iface eno2 inet manual
mtu 9000
auto br2
iface br2 inet static
mtu 9000
address 10.0.2.1/24
bridge_ports eno2
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno3
iface eno3 inet manual
mtu 9000
auto br3
iface br3 inet static
mtu 9000
address 10.0.3.1/24
bridge_ports eno3
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug eno4
iface eno4 inet manual
mtu 9000
auto br4
iface br4 inet static
mtu 9000
address 10.0.4.1/24
bridge_ports eno4
bridge_stp off
bridge_waitport 0
bridge_fd 0
allow-hotplug enp129s0
iface enp129s0 inet static
address 10.0.5.1/24
mtu 9000
Apply the network configurations
root@server1:~# systemctl restart networking
The ipv4.forward configuration in the Linux kernel controls whether the host machine is allowed to forward IP packets. By default, it’s usually disabled for security reasons. Execute the following to enable the same.
sed -i "s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g" /etc/sysctl.conf
sysctl -p
Start the default network
modprobe vhost_net
echo vhost_net | tee -a /etc/modules
virsh net-start default
virsh net-autostart default
Generate a keyring for the libvirt client to connect to ceph rbd.
ceph auth get-or-create client.libvirt mon 'profile rbd' osd 'profile rbd pool=ssdpool, profile rbd pool=nvmepool'
Now, add the client.libvirt section in /etc/ceph/ceph.conf
[client.libvirt]
keyring = /etc/ceph/ceph.client.libvirt.keyring
As we have logged in as root or working in root user context, ensure the ownership is set to ceph user
chown -R ceph:ceph /etc/ceph
Copy the updated ceph.conf file to other servers
scp /etc/ceph/ceph.conf ceph2:/etc/ceph
scp /etc/ceph/ceph.conf ceph3:/etc/ceph
scp /etc/ceph/ceph.conf ceph4:/etc/ceph
Create ceph-secret.xml to define a secret using virsh (uuid is generated)
<secret>
<uuid>155d8c36-3fa6-46e2-833a-cdae1b7bf8d2</uuid>
<usage type='ceph'>
<name>libvirt</name>
</usage>
</secret>
Copy the secret file to other servers
scp ceph-secret.xml ceph2:~/
scp ceph-secret.xml ceph3:~/
scp ceph-secret.xml ceph4:~/
Define the secret (in all servers)
virsh secret-define --file ceph-secret.xml
Set the value for the secret (in all servers)
virsh secret-set-value --secret <unique-secret-id-generated above> --base64 <ceph.client.libvirt.keyring value>
Create a pool definition files nvmepool.xml and ssdpool.xml with the following contents
<pool type="rbd">
<name>nvmepool</name> <!-- name of pool in libvirt -->
<source>
<name>nvmepool</name> <!-- name of pool in ceph -->
<host name='ceph1'/>
<host name='ceph2'/> <!- host names of monitor nodes -->
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
<pool type="rbd">
<name>ssdpool</name>
<source>
<name>ssdpool</name>
<host name='ceph1'/>
<host name='ceph2'/>
<host name='ceph3'/>
<auth username='libvirt' type='ceph'>
<secret uuid='<unique-secret-id-generated above>'/>
</auth>
</source>
</pool>
Copy the files to other servers.
scp ssdpool.xml nvmepool.xml ceph2:~/
scp ssdpool.xml nvmepool.xml ceph3:~/
scp ssdpool.xml nvmepool.xml ceph4:~/
Define the pool, start it and also mark for autostart whenever libvirt starts (in all servers)
virsh pool-define nvmepool.xml
virsh pool-start nvmepool
virsh pool-autostart nvmepool
virsh pool-define ssdpool.xml
virsh pool-start ssdpool
virsh pool-autostart ssdpool
Though we have configured the autostart of pools, it is possible that the underlying ceph rbd pools may not be ready when KVM starts. Until we identify the dependencies and solutions, I decided to go with a simple script that will do the job. This script will be executed every 2 mins (crontab) [ Create /usr/bin/start-libvirt-pools.sh ]
#!/bin/bash
logfile=/var/log/pooltrack.log
function log() {
timenow=`date`
line_count=$(wc -l < "$logfile")
if (( line_count > 500 )); then
tail -n 500 "$logfile" > "$logfile.tmp"
mv "$logfile.tmp" "$logfile"
rm "$logfile.tmp"
fi
echo "$timenow | $1" >> $logfile
}
while ! /usr/bin/ceph osd pool ls | grep -q ssdpool; do sleep 1; log "Waiting for active rbd pools."; done; while ! /usr/bin/ceph osd pool ls | grep -q nvmepool; do sleep 1; done
secretvalue=`awk '/\[client\.libvirt\]/,/^$/ { if ($1 == "key") { print $3 } }' /etc/ceph/ceph.conf`
secretid=`awk '/<uuid>/,/<\/uuid>/ { if ($1 ~ /<uuid>/) { gsub(/<\/?uuid>/,""); print $1 } }' /root/ceph-secret.xml`
/usr/bin/virsh secret-set-value --secret $secretid --base64 $secretvalue
ssdpoolcount=`/usr/bin/virsh pool-list | grep 'ssdpool' | wc -l`
if [ $ssdpoolcount -lt 1 ]; then
log "Starting ssd pool "
/usr/bin/virsh pool-start ssdpool
/usr/bin/virsh pool-autostart ssdpool
else
log "SSD pool active."
fi
nvmepoolcount=`/usr/bin/virsh pool-list | grep 'nvmepool' | wc -l`
if [ $nvmepoolcount -lt 1 ]; then
log "Starting nvme ppol"
/usr/bin/virsh pool-start nvmepool
/usr/bin/virsh pool-autostart nvmepool
else
log "NVME pool active."
fi
if ! /usr/bin/systemctl is-active --quiet libvirtd.service; then
log "Starting libvirtd...."
/usr/bin/systemctl start libvirtd.service
else
log "libvirtd active...."
fi
Copy the script to all servers
scp /usr/bin/start-libvirt-pools.sh ceph2:/usr/bin/
scp /usr/bin/start-libvirt-pools.sh ceph3:/usr/bin/
scp /usr/bin/start-libvirt-pools.sh ceph4:/usr/bin/
Set executable permission to the script (in all servers)
chmod +x /usr/bin/start-libvirt-pools.sh
Add the following crontab entry (crontab -e) (in all servers)
*/2 * * * * /usr/bin/start-libvirt-pools.sh