公司用于开通虚拟机集群的CPU类型有很多，hygon,Intel，kunpeng, 这些cpu类型的机器统一被纳管到k8s集群当中，当然也可能存在操作系统kernel的差别，比如公司自研的操作系统TOS和原操作系统CentOs之间的都被纳管到统一集群当中。

      于是，就会存在K8S和kubevirt可能会出现的不适配问题，需要做兼容或者适配。

       kubevirt虚拟机借助于底层的qemu和libvirt的相关能力，实现了在k8s上自由的虚拟化，开通资源，但是在混合集群上还没有先例。我们在测试热迁移的时候发现了如下问题，这些问题可以被大家借鉴的。

        热迁移测试结果：

	TOS-->CentOs	CentOs--->TOs
Intel	热迁移偶现失败	热迁移偶现失败
Hygon	成功	成功
Kunpeng	失败	失败

         问题：虚拟机热迁移失败，这个报错不影响，忽略。

cannot check guest CPU feature for aarch64 architecture的error不影响热迁移

        于是查看virt-launcher报错

       出现报错：acpi表大小不足：Size too large

       于是修改qemu代码：

修改max_size，max_size=0x10000后重新编译后解决。

       然后，在libvirt中添加debug，发现读取设备内存报错，虚拟机没有开通GPU卡，但是qemu启动参数中带有GPU设备-device virtio-gpu-pci,id=video0,max_outputs=1,bus=pci.6,addr-0x0，在VirtualMachine中添加配置autoattachGraphicsDevice: false，依然迁移失败。

      Operation not Permerted

/usr/libexec/qemu-kvm -name guest=evm-1024_evm-cvehg6jjm91ond2mruvg,debug-threads=on -S -object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-evm-1024_evm-cvehg6j/master-key.aes"} -blockdev {"driver":"file","filename":"/usr/share/AAVMF/AAVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"} -blockdev {"driver":"file","filename":"/tmp/evm-1024_evm-cvehg6jjm91ond2mruvg","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"} -machine virt-rhel8.5.0,accel=kvm,usb=off,dump-guest-core=off,gic-version=3,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=mach-virt.ram -cpu host -m 8192 -object {"qom-type":"memory-backend-ram","id":"mach-virt.ram","size":8589934592} -overcommit mem-lock=off -smp 4,sockets=1,dies=1,cores=4,threads=1 -object {"qom-type":"iothread","id":"iothread1"} -uuid 41a4057d-ba29-5bde-8b15-bb3327725cc8 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=19,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot strict=on -device pcie-root-port,port=0x8,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x1 -device pcie-root-port,port=0x9,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1 -device pcie-root-port,port=0xa,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2 -device pcie-root-port,port=0xb,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3 -device pcie-root-port,port=0xc,chassis=5,id=pci.5,bus=pcie.0,addr=0x1.0x4 -device pcie-root-port,port=0xd,chassis=6,id=pci.6,bus=pcie.0,addr=0x1.0x5 -device pcie-root-port,port=0xe,chassis=7,id=pci.7,bus=pcie.0,addr=0x1.0x6 -device pcie-root-port,port=0xf,chassis=8,id=pci.8,bus=pcie.0,addr=0x1.0x7 -device pcie-root-port,port=0x10,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=0x11,chassis=10,id=pci.10,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=0x12,chassis=11,id=pci.11,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=0x13,chassis=12,id=pci.12,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=0x14,chassis=13,id=pci.13,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=0x15,chassis=14,id=pci.14,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=0x16,chassis=15,id=pci.15,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=0x17,chassis=16,id=pci.16,bus=pcie.0,addr=0x2.0x7 -device pcie-root-port,port=0x18,chassis=17,id=pci.17,bus=pcie.0,multifunction=on,addr=0x3 -device pcie-root-port,port=0x19,chassis=18,id=pci.18,bus=pcie.0,addr=0x3.0x1 -device pcie-root-port,port=0x1a,chassis=19,id=pci.19,bus=pcie.0,addr=0x3.0x2 -device pcie-root-port,port=0x1b,chassis=20,id=pci.20,bus=pcie.0,addr=0x3.0x3 -device pcie-root-port,port=0x1c,chassis=21,id=pci.21,bus=pcie.0,addr=0x3.0x4 -device pcie-root-port,port=0x1d,chassis=22,id=pci.22,bus=pcie.0,addr=0x3.0x5 -device pcie-root-port,port=0x1e,chassis=23,id=pci.23,bus=pcie.0,addr=0x3.0x6 -device pcie-root-port,port=0x1f,chassis=24,id=pci.24,bus=pcie.0,addr=0x3.0x7 -device pcie-root-port,port=0x20,chassis=25,id=pci.25,bus=pcie.0,multifunction=on,addr=0x4 -device pcie-root-port,port=0x21,chassis=26,id=pci.26,bus=pcie.0,addr=0x4.0x1 -device pcie-root-port,port=0x22,chassis=27,id=pci.27,bus=pcie.0,addr=0x4.0x2 -device qemu-xhci,id=usb,bus=pci.2,addr=0x0 -device virtio-scsi-pci-non-transitional,id=scsi0,bus=pci.3,addr=0x0 -device virtio-serial-pci-non-transitional,id=virtio-serial0,bus=pci.4,addr=0x0 -blockdev {"driver":"host_device","filename":"/dev/cd-cvehg7e61c4qr45ls1kg","aio":"native","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"} -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,device_id=hdd-dv-cvehg7e61c4qr45ls1k0,drive=libvirt-2-format,id=ua-cd-cvehg7e61c4qr45ls1kg,bootindex=1,write-cache=on,serial=hdd-dv-cvehg7e61c4qr45ls1k0,werror=report,rerror=report -blockdev {"driver":"file","filename":"/var/run/kubevirt-ephemeral-disks/cloud-init-data/evm-1024/evm-cvehg6jjm91ond2mruvg/noCloud.iso","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"} -device scsi-cd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,device_id=drive-ua-cloud-init-volume,drive=libvirt-1-format,id=ua-cloud-init-volume,write-cache=on,werror=report,rerror=report -netdev tap,fds=21:22:23:24,id=hostua-nic-cvehg7ejga4b4efablm0,vhost=on,vhostfds=25:26:27:28 -device virtio-net-pci-non-transitional,mq=on,vectors=10,rx_queue_size=1024,tx_queue_size=1024,host_mtu=1400,netdev=hostua-nic-cvehg7ejga4b4efablm0,id=ua-nic-cvehg7ejga4b4efablm0,mac=00:00:00:14:bd:ae,bus=pci.1,addr=0x0,romfile= -chardev socket,id=charserial0,fd=29,server=on,wait=off -serial chardev:charserial0 -chardev socket,id=charchannel0,fd=30,server=on,wait=off -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=ua-tablet1,bus=usb.0,port=1 -device usb-kbd,id=input1,bus=usb.0,port=2 -audiodev id=audio1,driver=none -vnc vnc=unix:/var/run/kubevirt-private/5c3e0afa-fd84-4765-8376-e1818a2edc10/virt-vnc,audiodev=audio1 -device virtio-gpu-pci,id=video0,max_outputs=1,bus=pci.6,addr=0x0 -device virtio-balloon-pci-non-transitional,id=balloon0,bus=pci.5,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

依然热迁移失败后，

主要报错为：

"msg":"internal error: qemu unexpectedly closed the monitor: 2024-10-29T12:20:32.771894Z qemu-kvm: Unknown ramblock \"pvtime\", cannot accept migration","pos":"qemuProcessReportLogError:2046","subcomponent":"libvirt","thread":"95","timestamp":"2024-10-29T12:20:32.773000Z"}

centos+Tos环境虚拟机热迁移由于操作系统存在差异，在x86/arm上热迁移会出现cpu-feature，以及未知ram问题，具体分析日志如下：

{"component":"virt-launcher","level":"error","msg":"operation failed: guest CPU doesn't match specification: missing features: pdcm,fsrm,tsx-ctrl","pos":"virCPUx86UpdateLive:3131","subcomponent":"libvirt","thread":"52","timestamp":"2024-10-21T07:17:22.040000Z"}

cpu-feature.node.kubevirt.io/tsx-ctrl: "true"
cpu-feature.node.kubevirt.io/pdcm: "true"
cpu-feature.node.kubevirt.io/fsrm: "true"

"msg":"internal error: qemu unexpectedly closed the monitor: 2024-10-29T12:20:32.771894Z qemu-kvm: Unknown ramblock \"pvtime\", cannot accept migration","pos":"qemuProcessReportLogError:2046","subcomponent":"libvirt","thread":"95","timestamp":"2024-10-29T12:20:32.773000Z"}
{"component":"virt-launcher","level":"info","msg":"2024-10-29T12:20:32.771920Z qemu-kvm: error while loading state for instance 0x0 of device 'ram'","subcomponent":"libvirt","timestamp":"2024-10-29T12:20:32.773777Z"} 

尝试在宿主机创建虚拟机debug热迁移，适配版本如下：

aarch64 + centos：libvirtd  4.5.0，qemu 2.12.0，gdb 7.6.1-120.el7

aarch64 + Tos：libvirtd  6.2.0，qemu 6.2.0，gdb 11.1-9

版本差异过大，无法debug

于是重新梳理环境的差异，

        1. OS（CentOS7.7 <-> TOS23.01.2）

        2. Kernel（CentOS: 4.18.0-3.3.el7 <->  TOS23: 5.10.0-136.12.0.88.4）

        3. PAGESIZE（CentOS: 65536 <-> TOS: 4096）

        4. 默认内存页支持（CentOS: 64K <->  TOS: 4K）

        5. 两端内核均未开启pvtime特性（kvmclock）

       两端PAGE SIZE不一致，虚拟机默认内存页大小与宿主机一致，导致热迁移过程中脏页位图不一致，定位失败

        a. kubevirt没有提供修改默认内存页大小的属性配置

        b. qemu版本不支持配置alignment来修改默认内存页大小

        c. 修改宿主机的PAGE SIZE来修复迁移问题，需要重新编译内核代码

      于是选择方案c来解决问题，内核团队编译内核输出临时版本5.10.0-136.12.0.88.4.64k.temp.ctl3.aarch64，两端PAGE_SIZE一致，解决了error while loading state for instance 0x0 of device 'ram'问题。可以利用在操作系统中使用：getconf PAGESIZE 来查看page_size大小。

       但是测试又出现新的问题：error while loading state for instance 0x0 of device 'cpu'，OS和Kernel差异导致两端CPU指令集不匹配，导致热迁移失败，可能是

• 微码（固件补丁）
• 内核态配置
• 用户态权限配置

迁移配置为unsafe（全局允许不安全迁移，绕过 NUMA、CPU 兼容性等检查）

        VirtualMachine对象配置CPUFeature为可选（optional），在host-model的模式下不强制要求迁移对端拥有相同的CPUFeature因为CPUFeature不一致会影响kubevirt热迁移。该配置对Running状态的虚拟机不生效，配置后需要重启才能生效，影响到qemu的启动参数。