cloudopenstacklibvirtnumahuge-pages

openstack using hugepages for instance


I am tring to config openstack and libvirt to use hugepages for instance, here is my config:

  1. GRUB
vim /etc/default/grub

GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos_mmb-x86/root rd.lvm.lv=centos_mmb-x86/swap rhgb quiet default_hugepagesz=1G hugepagesz=1G hugepages=64 hugepagesz=2M hugepages=4096 transparent_hugepage=never"
GRUB_DISABLE_RECOVERY="true"
  1. disable transparent hugepage
echo never > /sys/kernel/mm/transparent_hugepage/enabled

disabled forver

vim /etc/default/grub
transparent_hugepage=never
grub2-mkconfig -o /boot/grub2/grub.cfg; reboot
  1. check numa cpu
numactl --hardware

available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3 4 5 6 7 8 9 20 21 22 23 24 25 26 27 28 29
node 0 size: 63909 MB
node 0 free: 409 MB
node 1 cpus: 10 11 12 13 14 15 16 17 18 19 30 31 32 33 34 35 36 37 38 39
node 1 size: 64476 MB
node 1 free: 399 MB
node distances:
node   0   1 
  0:  10  21 
  1:  21  10
  1. set hugepage and check
echo 4096 > /proc/sys/vm/nr_hugepages
cat /proc/meminfo | grep Huge
AnonHugePages:         0 kB
HugePages_Total:    4096
HugePages_Free:     3072
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB


  1. set mem per node and check
echo 32 > /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages
echo 16384 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages
cat /sys/devices/system/node/node*/meminfo | fgrep Huge
Node 0 AnonHugePages:         0 kB
Node 0 HugePages_Total:    52
Node 0 HugePages_Free:     52
Node 0 HugePages_Surp:      0
Node 1 AnonHugePages:         0 kB
Node 1 HugePages_Total:  4044
Node 1 HugePages_Free:   3020
Node 1 HugePages_Surp:      0
  1. mount hugetlbfs
mount -t hugetlbfs hugetlbfs /dev/hugepages2M/
  1. qemu.conf on compute node
vim /etc/libvirt/qemu.conf
hugetlbfs_mount=["/dev/hugepages2M"]
systemctl restart libvirtd
  1. openstack controller nova.conf
vim /etc/nova/nova.conf
[filter_scheduler]
enabled_filters=ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,NUMATopologyFilter
systemctl restart openstack-nova-scheduler
  1. openstack compute nova.conf
vim /etc/nova/nova.conf
[filter_scheduler]
enabled_filters=NUMATopologyFilter
systemctl restart openstack-nova-compute
  1. openstack flavor
openstack flavor create --vcpus 2 --ram 2048 --disk 5 --property hw:mem_page_size=2MB --property hw:numa_nodes=2  --property hw:cpu_policy=dedicated --property  mmb.hp

openstack flavor set mmb.hp --property hw:numa_nodes=1
  1. create server
openstack server create --image 4d234158-fb8c-4d61-ab54-556fb40703e0 --flavor mmb.hp --nic net-id=357dfd3d-2279-4226-8189-75c9483a45db --availability-zone nova:intel-compute hugepage2M

or

nova boot --flavor mmb.hp --image 4d234158-fb8c-4d61-ab54-556fb40703e0 --nic net-id=357dfd3d-2279-4226-8189-75c9483a45db --availability-zone nova:intel-compute --hint scheduler_hints={"hugepages_requested":2} hugepage2M
  1. error
Requested instance NUMA topology cannot fit the given host NUMA topology.

Did i miss something here or what i did wrong, please help, thanks in advance.


Solution

  • libvirt wasn't compiled with numa lib

    [root@intel-compute ~]# ldd /usr/sbin/libvirtd | grep numa
            libnuma.so.1 => /lib64/libnuma.so.1 (0x00007fa41b2d7000)
    

    should come with this result, if not recompile libvirt with

    --with-numactl          with libnuma support [default=check]
    --with-numad            use numad to manage CPU placement dynamically
    

    also qemu should be compiled with numa:

    1. download qemu
     git clone git://git.qemu-project.org/qemu.git
    
    1. compile
    ./configure --enable-numa
    
    1. install
    make && make install
    

    now qemu has numa

    [root@intel-compute ~]# ldd /usr/bin/qemu-system-x86* | grep numa
            libnuma.so.1 => /lib64/libnuma.so.1 (0x00007f4db8dc1000)