assemblyx86dosmasmprotected-mode

Entering protected mode from a DOS program


I think i've read a dozen or so questions that are basically a duplicate of this one, but I still haven't found a solution.

The desired result is to enter protected mode and halt with no faults. The problem i'm experiencing is a triple fault after executing the intersegment jmp with 6 byte immediate value.

here's my code that produces the fault in DOSBox and on a Pentium II PC running MS-DOS 7. Assembler is MASM 5.10

single segment stack                                                      
assume cs:single,ds:single 

gdt        dq 0

c_limit_lo dw 0ffffh     
c_base_lo  dw 0
c_base_mid db 0
c_priv     db 10011110b ;present set, highest priv, type set, conforming, read      
c_limit_hi db 11001111b ;granularity set, operand size 32  
c_base_hi  db 0

d_limit_lo dw 0ffffh
d_base_lo  dw 0
d_base_mid db 0
d_priv     db 10010010b ;present set, highest priv, type clr, expand dn, write  
d_limit_hi db 11001111b ;granularity set, big set 
d_base_hi  db 0
gdt_end:

gdt_limit  dw gdt_end-offset gdt-1
gdt_addr   dd ?

start:
     mov ax, cs
     mov ds, ax
                                   ;calc phys address of current code segment and
                                   ;insert it into code and data descriptors 
     .386p                         
     xor eax, eax
     mov ax, cs                    
     mov cl, 4              
     shl eax, cl                   ;multiply cs by 16 to get phys address of seg
     mov edx, eax
     mov c_base_lo, ax
     mov d_base_lo, ax             ;low word
     mov cl, 16
     shr eax, cl
     mov c_base_mid, al
     mov d_base_mid, al            ;middle byte
     mov c_base_hi, ah
     mov d_base_hi, ah             ;high byte
     add edx, offset gdt           ;add offset of gdt 
     mov gdt_addr, edx             ;gdt address set 

                                   ;attempt to enter protected mode
     cli                           ;disable interrupts
     in al, 70h
     or al, 80h
     out 70h, al                   ;turn off nonmasked interrupts
     in al, 92h
     or al, 2
     out 92h, al                   ;enable A20 line
     lgdt [gdt_limit]
     mov eax, cr0         
     or eax, 1
     mov cr0, eax                  ;enter protected mode
     db 66h                        ;specify 32-bit operand
     jmp_op  db 0eah               ;manually encoded "jmp 8h:enter_32" TRIPLE FAULT
     jmp_loc_lo dw offset enter_32
     jmp_loc_hi dw 0
     jmp_sel dw 8
enter_32:  
     mov eax, 0ffffffffh           ;sometimes doesn't triple fault on infinite jump or hlt instruction
back:jmp back                      ;but always triple faults on mov



the_stack db 64 dup (0ffh)          ;64 byte stack

single ends                                                                  
end start       

The triple fault seems to be "luck" based to some extent. Certain configurations of 0x67 prefixes and nops after the far jump cause the cpu to behave as if it's halted. I don't really understand it.

I think i'm generating the wrong jump target.

Update: It's not faulting with single byte instructions (instructions with a single encoding regardless of cpu mode). I think I'll try jumping into a USE32 defined segment.

This code does not fault:

     jmp_op  db 0eah               
     jmp_loc_lo dw offset enter_32
     jmp_loc_hi dw 0
     jmp_sel dw 8
enter_32:  
     aaa
     daa
     cmc
     cld
     cli
     stc
     nop
     aaa
     daa
     cmc
     cld
     cli
     stc
     nop
     hlt

Solution

  • Answered by fuz, I just needed to jump into code that was assembled for 32-bit mode. You tell the assembler to make 32-bit code by defining a segment with the USE32 keyword.

    Complete protected mode program with a VGA demo and a mode switch back to real mode:

    single segment stack                                                      
    assume cs:single,ds:single 
    
    gdt        dq 0                             ;global descriptor table
    p_code     dq 00cf9e000000ffffh             ;protected mode code descriptor
    p_data     dq 00cf92000000ffffh             ;protected mode data descriptor
    r_code     dq 008f9a000000ffffh             ;real mode code descriptor
    r_data     dq 008f92000000ffffh             ;real mode data descriptor
    v_buff     dq 00cf920a0000ffffh             ;vga buffer descriptor
    
    gdt_limit  dw offset gdt_limit-offset gdt-1 ;gdt_limit <- gdt byte size -1
    gdt_addr   dd offset gdt                    ;gdt_addr <- offset of gdt, phys address of
                                                ;code segment will be added
    start:
        mov ax, cs 
        mov ds, ax                   ;ds = cs, single segment
        mov ax, 13h
        int 10h                      ;enter vga 320x200x256
    
        .386p                        ;enable 32-bit extensions
        xor eax, eax                 ;clear high word of eax
        mov ax, cs                   ;eax <- cs 
        shl eax, 4                   ;eax <- physical address of cs 
        add [gdt_addr], eax          ;gdt_addr <- physical address of gdt
        mov word ptr [r_code+2], ax  
        mov word ptr [r_data+2], ax  ;insert low word of cs phys address
        shr eax, 16
        mov byte ptr [r_code+4], al
        mov byte ptr [r_data+4], al  ;insert middle byte of cs address
        mov byte ptr [r_code+7], ah
        mov byte ptr [r_data+7], ah  ;insert high byte of cs address
    
        xor eax, eax                 ;clear high word of eax
        mov ax, seg32                ;eax <- seg32 segment address
        shl eax, 4                   ;eax <- physical address of seg32 
        mov word ptr [p_code+2], ax   
        mov word ptr [p_data+2], ax  ;insert low word of seg32 phys address
        shr eax, 16
        mov byte ptr [p_code+4], al
        mov byte ptr [p_data+4], al  ;insert middle byte of seg32 address
        mov byte ptr [p_code+7], ah
        mov byte ptr [p_data+7], ah  ;insert high byte of seg32 address
    
        cli                          ;disable interrupts
        in al, 70h                   ;al <- cmos ram index register port
        or al, 80h                   ;set bit 7 to disable nmi 
        out 70h, al                  ;nmi disabled
        in al, 92h                   ;al <- ps/2 system control port
        or al, 2                     ;set bit 1 to enable a20
        out 92h, al                  ;a20 enabled
    
        lgdt [gdt_limit]             ;load gdt 
        mov eax, cr0                  
        or eax, 1                    ;set pe bit
        mov cr0, eax                 ;enter protected mode
        db 66h                       ;specify 32-bit operand
        db 0eah                      ;manually encoded jmp 8h:0, jump to offset 0 of seg32
        dd offset enter_32                  
        dw 8 
    
    ret_real:
        mov eax, cr0
        and al, 11111110b            ;clear pe bit
        mov cr0, eax                 ;real mode enabled     
        db 0eah                      ;jmp single:real_cs to load cs:ip
        dw offset real_cs
        dw seg single  
    
    real_cs:
        mov ax, cs                      
        mov ds, ax                   ;ds = cs
        mov ss, ax                   ;ss = cs
        mov sp, offset s16_end       ;top of stack is end of stack
        in al, 70h                   ;al <- cmos ram index register port
        and al, 01111111b            ;clear bit 7 to enable nmi 
        out 70h, al                  ;nmi enabled
        sti                          ;enable interrupts
    
        mov ax, 40h
        mov es, ax                   ;access kbd data area via segment 40h
        mov word ptr es:[1ah], 1eh   ;set the kbd buff head to start of buff
        mov word ptr es:[1ch], 1eh   ;set kbd buff tail to same as buff head
                                     ;now the keyboard buffer is cleared.
        xor ah, ah                   ;select video mode function
        mov al, 3                    ;select 80x25 16 colors
        int 10h                      ;restore vga compatible text mode
        mov ax, 4c00h                ;Terminate process function selected
        int 21h                      ;return to ms-dos
    
    s16 db 256 dup (0ffh)            ;needed 256 bytes to call int 10h on fx5200 vga bios
    s16_end: 
    single ends   
    
    
    seg32 segment use32
    assume cs:seg32,ds:seg32
    
    enter_32:
        mov ax, 10h                  ;protected mode data segment selector
        mov ds, ax                   ;ds references main data segment
        mov ss, ax                   ;stack is in main data segment 
        mov esp, offset s32_end      ;initial top of stack is end of stack
        mov ax, 28h                  ;vga buffer selector
        mov es, ax                   ;es references vga buffer  
        mov eax, 0ffffffffh          ;initialize eax
    write_scr:
        inc al
        inc ah
        rol eax, 16
        inc al
        inc ah                       ;increment each byte of eax
        xor edi, edi                 ;init index
        mov ecx, 320*200/4           ;vga buffer length in bytes
    
        push eax
        mov dx, 3dah                 ;dx <- vga status register
    vrb_set:
        in al, dx                    ;al <- status byte
        test al, 8                   ;is bit vertical retrace bit set
        jnz vrb_set                  ;if so, wait for it to clear
    vrb_clr:                         ;when clear, wait for it to be set
        in al, dx
        test al, 8
        jz vrb_clr                   ;loop back until vertical retrace bit has been set
        pop eax
        rep stosd                    ;fill vga buffer
    
        push eax
        in al, 60h                   ;al <- keyboard data port
        mov ebx, eax                 
        pop eax                     
        cmp bl, 1                    ;escape key scancode? 
        jne write_scr                ;if not, update screen
        mov ax, 20h                  ;real mode data selector
        mov ds, ax                   
        mov es, ax                   ;setup ds and es for real mode
        db 0eah                      ;jmp 18h:ret_real to load real mode code descriptor 
        dd offset ret_real
        dw 18h
    
    s32 db 128 dup (0ffh)            ;128 byte stack
    s32_end:
    seg32 ends
    
    end start
    

    Assembling code for 16-bit real mode and running it in 32-bit protected mode may produce unexpected behaviour and crashes. This 16-bit code:

        mov eax, 0ffffffffh
    back:jmp back
    

    Is encoded as:

    66B8FFFFFFFF      mov eax,0xffffffff
    EBFE              jmp short 0x6
    

    However, if this sequence of bytes is decoded as 32-bit protected mode instructions they would be interpreted as:

    66B8FFFF          mov ax,0xffff
    FF                db 0xff
    FF                db 0xff
    EBFE              jmp short 0x6
    

    After moving 0xffff to register AX the processor would raise a general protection fault (#GP) when it found the invalid instruction (byte 0xff). In the absence of a proper Interrupt Descriptor Table (IDT) and exception handler for #GP a triple fault occurs.