macosrustarm64hypervisor-framework

hv_vcpu_run on macOS with M1 suppose to return but it is not


I'm building a Virtual Machine Manager on top of Hypervisor Framework and having a problem with hv_vcpu_run never return somehow. It is suppose to return because the code it is going run will execute MMIO. I have tested the same code on Asahi Linux using KVM and everything are working correctly, the KVM_RUN return with KVM_EXIT_MMIO as expected.

How I setup a vCPU:

    // Set PSTATE.
    states.set_pstate(
        Pstate::new()
            .with_m(0b0101) // EL1 with SP_EL1 (EL1h).
            .with_f(true)
            .with_i(true)
            .with_a(true)
            .with_d(true),
    );

    // Enable MMU to enable virtual address and set TCR_EL1.
    states.set_sctlr(
        Sctlr::new()
            .with_m(true)
            .with_c(true)
            .with_itd(true)
            .with_i(true)
            .with_tscxt(true)
            .with_span(true)
            .with_ntlsmd(true)
            .with_lsmaoe(true),
    );
    states.set_mair_el1(map.memory_attrs);
    states.set_tcr(
        Tcr::new()
            .with_ips(feats.mmfr0.pa_range())
            .with_tg1(match map.page_size.get() {
                0x4000 => 0b01, // 16K page for TTBR1_EL1.
                _ => todo!(),
            })
            .with_sh1(0b11)
            .with_orgn1(0b01)
            .with_irgn1(0b01)
            .with_t1sz(16)
            .with_tg0(match map.page_size.get() {
                0x4000 => 0b10, // 16K page for TTBR0_EL1.
                _ => todo!(),
            })
            .with_sh0(0b11)
            .with_orgn0(0b01)
            .with_irgn0(0b01)
            .with_t0sz(16),
    );

    // Set page table. We need both lower and higher VA here because the virtual devices mapped with
    // identity mapping.
    states.set_ttbr0_el1(map.page_table);
    states.set_ttbr1_el1(map.page_table);

    // Set entry point, its argument and stack pointer.
    states.set_x0(map.env_vaddr);
    states.set_x1(map.conf_vaddr);
    states.set_sp_el1(map.stack_vaddr + map.stack_len); // Top-down.
    states.set_pc(map.kern_vaddr + entry);

    states
        .commit()
        .map_err(|e| MainCpuError::CommitCpuStatesFailed(Box::new(e)))

The commit implementation:

    fn commit(self) -> Result<(), Self::Err> {
        use hv_sys::{
            hv_reg_t_HV_REG_CPSR as HV_REG_CPSR, hv_reg_t_HV_REG_PC as HV_REG_PC,
            hv_reg_t_HV_REG_X0 as HV_REG_X0, hv_reg_t_HV_REG_X1 as HV_REG_X1,
            hv_sys_reg_t_HV_SYS_REG_MAIR_EL1 as HV_SYS_REG_MAIR_EL1,
            hv_sys_reg_t_HV_SYS_REG_SCTLR_EL1 as HV_SYS_REG_SCTLR_EL1,
            hv_sys_reg_t_HV_SYS_REG_SP_EL1 as HV_SYS_REG_SP_EL1,
            hv_sys_reg_t_HV_SYS_REG_TCR_EL1 as HV_SYS_REG_TCR_EL1,
            hv_sys_reg_t_HV_SYS_REG_TTBR0_EL1 as HV_SYS_REG_TTBR0_EL1,
            hv_sys_reg_t_HV_SYS_REG_TTBR1_EL1 as HV_SYS_REG_TTBR1_EL1, hv_vcpu_set_reg,
            hv_vcpu_set_sys_reg,
        };

        // Set PSTATE. Hypervisor Framework use CPSR to represent PSTATE.
        let cpu = self.cpu.instance;
        let set_reg = |reg, val| match NonZero::new(unsafe { hv_vcpu_set_reg(cpu, reg, val) }) {
            Some(v) => Err(v),
            None => Ok(()),
        };

        if let State::Dirty(v) = self.pstate {
            set_reg(HV_REG_CPSR, v).map_err(StatesError::SetPstateFailed)?;
        }

        // Set system registers.
        let set_sys = |reg, val| match NonZero::new(unsafe { hv_vcpu_set_sys_reg(cpu, reg, val) }) {
            Some(v) => Err(v),
            None => Ok(()),
        };

        if let State::Dirty(v) = self.mair_el1 {
            set_sys(HV_SYS_REG_MAIR_EL1, v).map_err(StatesError::SetMairEl1Failed)?;
        }

        if let State::Dirty(v) = self.ttbr0_el1 {
            set_sys(HV_SYS_REG_TTBR0_EL1, v).map_err(StatesError::SetTtbr0El1Failed)?;
        }

        if let State::Dirty(v) = self.ttbr1_el1 {
            set_sys(HV_SYS_REG_TTBR1_EL1, v).map_err(StatesError::SetTtbr1El1Failed)?;
        }

        if let State::Dirty(v) = self.tcr {
            set_sys(HV_SYS_REG_TCR_EL1, v).map_err(StatesError::SetTcrFailed)?;
        }

        if let State::Dirty(v) = self.sctlr {
            set_sys(HV_SYS_REG_SCTLR_EL1, v).map_err(StatesError::SetSctlrFailed)?;
        }

        if let State::Dirty(v) = self.sp_el1 {
            set_sys(HV_SYS_REG_SP_EL1, v).map_err(StatesError::SetSpEl1Failed)?;
        }

        // Set general registers.
        if let State::Dirty(v) = self.pc {
            set_reg(HV_REG_PC, v).map_err(StatesError::SetPcFailed)?;
        }

        if let State::Dirty(v) = self.x0 {
            set_reg(HV_REG_X0, v).map_err(StatesError::SetX0Failed)?;
        }

        if let State::Dirty(v) = self.x1 {
            set_reg(HV_REG_X1, v).map_err(StatesError::SetX1Failed)?;
        }

        Ok(())
    }

Solution

  • Found the problem. The problem is hv_vm_map ignore all memory that returned from mmap with PROT_NONE. So the fix is just invoke hv_vm_map each time the memory protection is changed from PROT_NONE.