vgabios: Don't use extra stack if it appears a modern OS is in use

If the last mode set (while not in vm86 mode) was done from a VBE mode
set call then disable the extra stack.  This works under the premise
that only a modern OS would invoke the VBE mode changing facilities
and a modern OS would always call the vgabios with sufficient stack
space.

This is an ugly hack to work around a problem Windows Vista (and
possibly later Windows releases) has with the VGA BIOS using a stack
in the e-segment.

Reported-by: Richard Laager <rlaager@wiktel.com>
Signed-off-by: Kevin O'Connor <kevin@koconnor.net>

diff --git a/src/x86.h b/src/x86.h
index 7798b1c17c3d3d0fe91310cf0ba2d843a91a506a..14ebb7de877a277497844ff741d7e9aa30692546 100644 (file)
--- a/src/x86.h
+++ b/src/x86.h
@@ -83,6 +83,11 @@ static inline u32 getcr0(void) {
 static inline void setcr0(u32 cr0) {
     asm("movl %0, %%cr0" : : "r"(cr0));
 }
+static inline u16 getcr0_vm86(void) {
+    u16 cr0;
+    asm("smsww %0" : "=r"(cr0));
+    return cr0;
+}
 
 static inline u64 rdmsr(u32 index)
 {

diff --git a/vgasrc/vgabios.c b/vgasrc/vgabios.c
index 4aa50e1c19bd2bf561f3d5c4dc364cb6b6b82a67..7c1f0b81aee29041c503ee10476bc3381bd33168 100644 (file)
--- a/vgasrc/vgabios.c
+++ b/vgasrc/vgabios.c
@@ -304,6 +304,12 @@ vga_set_mode(int mode, int flags)
         SET_BDA(video_mode, 0xff);
     SET_BDA_EXT(vbe_mode, mode | (flags & MF_VBEFLAGS));
     SET_BDA_EXT(vgamode_offset, (u32)vmode_g);
+    if (CONFIG_VGA_ALLOCATE_EXTRA_STACK && !(getcr0_vm86() & CR0_PE))
+        // Disable extra stack if it appears a modern OS is in use.
+        // This works around bugs in some versions of Windows (Vista
+        // and possibly later) when the stack is in the e-segment.
+        MASK_BDA_EXT(flags, BF_EXTRA_STACK
+                     , (flags & MF_LEGACY) ? BF_EXTRA_STACK : 0);
     if (memmodel == MM_TEXT) {
         SET_BDA(video_cols, width);
         SET_BDA(video_rows, height-1);

diff --git a/vgasrc/vgabios.h b/vgasrc/vgabios.h
index fd796f2e6b680acd9ab4318ae39916849f969ecc..831f69407ee51c0c7a3b695cd0ff7092f52d811c 100644 (file)
--- a/vgasrc/vgabios.h
+++ b/vgasrc/vgabios.h
@@ -62,7 +62,8 @@ struct gfx_op {
 #define GO_MEMSET  3
 #define GO_MEMMOVE 4
 
-// Custom internal storage in BDA
+// Custom internal storage in BDA (don't change here without also
+// updating vgaentry.S)
 #define VGA_CUSTOM_BDA 0xb9
 
 struct vga_bda_s {
@@ -74,6 +75,7 @@ struct vga_bda_s {
 #define BF_PM_MASK      0x0f
 #define BF_EMULATE_TEXT 0x10
 #define BF_SWCURSOR     0x20
+#define BF_EXTRA_STACK  0x40
 
 #define GET_BDA_EXT(var) \
     GET_FARVAR(SEG_BDA, ((struct vga_bda_s *)VGA_CUSTOM_BDA)->var)

diff --git a/vgasrc/vgaentry.S b/vgasrc/vgaentry.S
index f9cf6568988566b883c2c3b6e5db6ab5cda48f38..e0ab9547c471aefee1b17bd0edb688a371291ce0 100644 (file)
--- a/vgasrc/vgaentry.S
+++ b/vgasrc/vgaentry.S
@@ -104,14 +104,23 @@ entry_10:
         ENTRY_ARG_VGA handle_10
         iretw
 
+#define VGA_CUSTOM_BDA_FLAGS 0xb9
+#define BF_EXTRA_STACK 0x40
+
         // Entry point using extra stack
         DECLFUNC entry_10_extrastack
 entry_10_extrastack:
         cli
         cld
-        pushw %ds               // Set %ds:%eax to space on ExtraStack
+        pushw %ds
         pushl %eax
-        movw %cs:ExtraStackSeg, %ds
+
+        movw $SEG_BDA, %ax      // Check if extra stack is enabled
+        movw %ax, %ds
+        testb $BF_EXTRA_STACK, VGA_CUSTOM_BDA_FLAGS
+        jz 1f
+
+        movw %cs:ExtraStackSeg, %ds // Set %ds:%eax to space on ExtraStack
         movl $(CONFIG_VGA_EXTRA_STACK_SIZE-PUSHBREGS_size-16), %eax
         SAVEBREGS_POP_DSEAX     // Save registers on extra stack
         movl %esp, PUSHBREGS_size+8(%eax)
@@ -134,6 +143,11 @@ entry_10_extrastack:
         RESTOREBREGS_DSEAX
         iretw
 
+1:      // Use regular entry point if the extra stack is disabled
+        popl %eax
+        popw %ds
+        jmp entry_10
+
         // Timer irq handling
         DECLFUNC entry_timer_hook
 entry_timer_hook: