I'm building a Virtual Machine Manager on top of Hypervisor Framework and having a problem with hv_vcpu_run
never return somehow. It is suppose to return because the code it is going run will execute MMIO. I have tested the same code on Asahi Linux using KVM and everything are working correctly, the KVM_RUN
return with KVM_EXIT_MMIO
as expected.
How I setup a vCPU:
// Set PSTATE.
states.set_pstate(
Pstate::new()
.with_m(0b0101) // EL1 with SP_EL1 (EL1h).
.with_f(true)
.with_i(true)
.with_a(true)
.with_d(true),
);
// Enable MMU to enable virtual address and set TCR_EL1.
states.set_sctlr(
Sctlr::new()
.with_m(true)
.with_c(true)
.with_itd(true)
.with_i(true)
.with_tscxt(true)
.with_span(true)
.with_ntlsmd(true)
.with_lsmaoe(true),
);
states.set_mair_el1(map.memory_attrs);
states.set_tcr(
Tcr::new()
.with_ips(feats.mmfr0.pa_range())
.with_tg1(match map.page_size.get() {
0x4000 => 0b01, // 16K page for TTBR1_EL1.
_ => todo!(),
})
.with_sh1(0b11)
.with_orgn1(0b01)
.with_irgn1(0b01)
.with_t1sz(16)
.with_tg0(match map.page_size.get() {
0x4000 => 0b10, // 16K page for TTBR0_EL1.
_ => todo!(),
})
.with_sh0(0b11)
.with_orgn0(0b01)
.with_irgn0(0b01)
.with_t0sz(16),
);
// Set page table. We need both lower and higher VA here because the virtual devices mapped with
// identity mapping.
states.set_ttbr0_el1(map.page_table);
states.set_ttbr1_el1(map.page_table);
// Set entry point, its argument and stack pointer.
states.set_x0(map.env_vaddr);
states.set_x1(map.conf_vaddr);
states.set_sp_el1(map.stack_vaddr + map.stack_len); // Top-down.
states.set_pc(map.kern_vaddr + entry);
states
.commit()
.map_err(|e| MainCpuError::CommitCpuStatesFailed(Box::new(e)))
The commit
implementation:
fn commit(self) -> Result<(), Self::Err> {
use hv_sys::{
hv_reg_t_HV_REG_CPSR as HV_REG_CPSR, hv_reg_t_HV_REG_PC as HV_REG_PC,
hv_reg_t_HV_REG_X0 as HV_REG_X0, hv_reg_t_HV_REG_X1 as HV_REG_X1,
hv_sys_reg_t_HV_SYS_REG_MAIR_EL1 as HV_SYS_REG_MAIR_EL1,
hv_sys_reg_t_HV_SYS_REG_SCTLR_EL1 as HV_SYS_REG_SCTLR_EL1,
hv_sys_reg_t_HV_SYS_REG_SP_EL1 as HV_SYS_REG_SP_EL1,
hv_sys_reg_t_HV_SYS_REG_TCR_EL1 as HV_SYS_REG_TCR_EL1,
hv_sys_reg_t_HV_SYS_REG_TTBR0_EL1 as HV_SYS_REG_TTBR0_EL1,
hv_sys_reg_t_HV_SYS_REG_TTBR1_EL1 as HV_SYS_REG_TTBR1_EL1, hv_vcpu_set_reg,
hv_vcpu_set_sys_reg,
};
// Set PSTATE. Hypervisor Framework use CPSR to represent PSTATE.
let cpu = self.cpu.instance;
let set_reg = |reg, val| match NonZero::new(unsafe { hv_vcpu_set_reg(cpu, reg, val) }) {
Some(v) => Err(v),
None => Ok(()),
};
if let State::Dirty(v) = self.pstate {
set_reg(HV_REG_CPSR, v).map_err(StatesError::SetPstateFailed)?;
}
// Set system registers.
let set_sys = |reg, val| match NonZero::new(unsafe { hv_vcpu_set_sys_reg(cpu, reg, val) }) {
Some(v) => Err(v),
None => Ok(()),
};
if let State::Dirty(v) = self.mair_el1 {
set_sys(HV_SYS_REG_MAIR_EL1, v).map_err(StatesError::SetMairEl1Failed)?;
}
if let State::Dirty(v) = self.ttbr0_el1 {
set_sys(HV_SYS_REG_TTBR0_EL1, v).map_err(StatesError::SetTtbr0El1Failed)?;
}
if let State::Dirty(v) = self.ttbr1_el1 {
set_sys(HV_SYS_REG_TTBR1_EL1, v).map_err(StatesError::SetTtbr1El1Failed)?;
}
if let State::Dirty(v) = self.tcr {
set_sys(HV_SYS_REG_TCR_EL1, v).map_err(StatesError::SetTcrFailed)?;
}
if let State::Dirty(v) = self.sctlr {
set_sys(HV_SYS_REG_SCTLR_EL1, v).map_err(StatesError::SetSctlrFailed)?;
}
if let State::Dirty(v) = self.sp_el1 {
set_sys(HV_SYS_REG_SP_EL1, v).map_err(StatesError::SetSpEl1Failed)?;
}
// Set general registers.
if let State::Dirty(v) = self.pc {
set_reg(HV_REG_PC, v).map_err(StatesError::SetPcFailed)?;
}
if let State::Dirty(v) = self.x0 {
set_reg(HV_REG_X0, v).map_err(StatesError::SetX0Failed)?;
}
if let State::Dirty(v) = self.x1 {
set_reg(HV_REG_X1, v).map_err(StatesError::SetX1Failed)?;
}
Ok(())
}
Found the problem. The problem is hv_vm_map
ignore all memory that returned from mmap
with PROT_NONE
. So the fix is just invoke hv_vm_map
each time the memory protection is changed from PROT_NONE
.