I run a Centos 7 virtualized in Ubuntu 20.04. There is a huge I/O performance drop in the VM compared to the host. The host has plenty of RAM.
Host:
* MB/s = 1,000,000 bytes/s
* KB = 1000 bytes, KiB = 1024 bytes
Sequential Read (Q= 32,T= 1) : 4080MB/s [ 124 IOPS]
Sequential Write (Q= 32,T= 1) : 2799MB/s [ 85 IOPS]
Random Read 4KiB (Q= 8,T= 8) : 2932MB/s [ 733039 IOPS]
Random Write 4KiB (Q= 8,T= 8) : 1631MB/s [ 407961 IOPS]
Random Read 4KiB (Q= 32,T= 1) : 600MB/s [ 150036 IOPS]
Random Write 4KiB (Q= 32,T= 1) : 344MB/s [ 86050 IOPS]
Random Read 4KiB (Q= 1,T= 1) : 587MB/s [ 146810 IOPS]
Random Write 4KiB (Q= 1,T= 1) : 339MB/s [ 84979 IOPS]
VM:
* MB/s = 1,000,000 bytes/s
* KB = 1000 bytes, KiB = 1024 bytes
Sequential Read (Q= 32,T= 1) : 633MB/s [ 19 IOPS]
Sequential Write (Q= 32,T= 1) : 4660MB/s [ 142 IOPS]
Random Read 4KiB (Q= 8,T= 8) : 447MB/s [ 111872 IOPS]
Random Write 4KiB (Q= 8,T= 8) : 358MB/s [ 89671 IOPS]
Random Read 4KiB (Q= 32,T= 1) : 282MB/s [ 70620 IOPS]
Random Write 4KiB (Q= 32,T= 1) : 287MB/s [ 71859 IOPS]
Random Read 4KiB (Q= 1,T= 1) : 27MB/s [ 6805 IOPS]
Random Write 4KiB (Q= 1,T= 1) : 33MB/s [ 8415 IOPS]
The VM disk is a ZVOL (I also tested with a .qcow2 in a ZFS dataset which is as bad) :
# zfs get all ssd1/vm/isp-master
NAME PROPERTY VALUE SOURCE
ssd1/vm/isp-master type volume -
ssd1/vm/isp-master creation Tue Feb 22 21:44 2022 -
ssd1/vm/isp-master used 703G -
ssd1/vm/isp-master available 164G -
ssd1/vm/isp-master referenced 393G -
ssd1/vm/isp-master compressratio 1.09x -
ssd1/vm/isp-master reservation none default
ssd1/vm/isp-master volsize 900G local
ssd1/vm/isp-master volblocksize 8K default
ssd1/vm/isp-master checksum on default
ssd1/vm/isp-master compression lz4 inherited from ssd1
ssd1/vm/isp-master readonly off default
ssd1/vm/isp-master createtxg 9025875 -
ssd1/vm/isp-master copies 1 default
ssd1/vm/isp-master refreservation none default
ssd1/vm/isp-master guid 14573562710197222671 -
ssd1/vm/isp-master primarycache metadata local
ssd1/vm/isp-master secondarycache all default
ssd1/vm/isp-master usedbysnapshots 310G -
ssd1/vm/isp-master usedbydataset 393G -
ssd1/vm/isp-master usedbychildren 0B -
ssd1/vm/isp-master usedbyrefreservation 0B -
ssd1/vm/isp-master logbias latency default
ssd1/vm/isp-master objsetid 660 -
ssd1/vm/isp-master dedup off default
ssd1/vm/isp-master mlslabel none default
ssd1/vm/isp-master sync standard default
ssd1/vm/isp-master refcompressratio 1.11x -
ssd1/vm/isp-master written 76.6M -
ssd1/vm/isp-master logicalused 763G -
ssd1/vm/isp-master logicalreferenced 436G -
ssd1/vm/isp-master volmode default default
ssd1/vm/isp-master snapshot_limit none default
ssd1/vm/isp-master snapshot_count none default
ssd1/vm/isp-master snapdev hidden default
ssd1/vm/isp-master context none default
ssd1/vm/isp-master fscontext none default
ssd1/vm/isp-master defcontext none default
ssd1/vm/isp-master rootcontext none default
ssd1/vm/isp-master redundant_metadata all default
ssd1/vm/isp-master encryption off default
ssd1/vm/isp-master keylocation none default
ssd1/vm/isp-master keyformat none default
ssd1/vm/isp-master pbkdf2iters 0 default
I use libvirt for virtualization, Qemu guest agent is installed in Centos7 which runs a recent kernel (5.4.231-1.el7.elrepo.x86_64):
<domain type='kvm'>
<name>isp-master</name>
<uuid>[redacted]</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://centos.org/centos/7.0"/>
</libosinfo:libosinfo>
</metadata>
<memory unit='KiB'>33554432</memory>
<currentMemory unit='KiB'>33554432</currentMemory>
<vcpu placement='static'>12</vcpu>
<os>
<type arch='x86_64' machine='pc-i440fx-bionic'>hvm</type>
</os>
<features>
<acpi/>
<apic/>
<vmport state='off'/>
</features>
<cpu mode='host-passthrough' check='none'/>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/zvol/ssd1/vm/isp-master'/>
<target dev='vda' bus='virtio'/>
<boot order='1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/zvol/pool1/vm/isp-master-backups'/>
<target dev='vdb' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images-ssd1/vm-isp-master.qcow2'/>
<target dev='vdc' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<target dev='hda' bus='ide'/>
<readonly/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<controller type='usb' index='0' model='ich9-ehci1'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci1'>
<master startport='0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci2'>
<master startport='2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci3'>
<master startport='4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'/>
<controller type='ide' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</controller>
<controller type='scsi' index='0' model='virtio-scsi'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</controller>
<interface type='bridge'>
<mac address='[redacted]'/>
<source bridge='br0'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0'/>
<address type='virtio-serial' controller='0' bus='0' port='2'/>
</channel>
<input type='tablet' bus='usb'>
<address type='usb' bus='0' port='1'/>
</input>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice' autoport='yes'>
<listen type='address'/>
</graphics>
<video>
<model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='2'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='3'/>
</redirdev>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</memballoon>
</devices>
</domain>
I have no idea why the I/O performance is so bad. Any idea ?
