vmd(8): provide a detailed e820 memory map.

When booting guests with SeaBIOS, vmd(8) supplied details about the
available guest memory via CMOS registers. Consequently, we've been
carrying some patches in the ports tree to SeaBIOS to fetch this
information like it's the 1990s.

When a vm initializes memory ranges, we now track what each range
represents. This information can be used to supply the e820 memory
map to SeaBIOS via the fw_cfg interface allowing it to properly
communicate memory ranges to a guest operating system. (This will
also allow us to drop some patches from the port.)

Given the ranges can now be marked with a purpose, this also allows
vmm(4) to switch from hard-coded mmio ranges and instead let the
information on the memory range dictate if vmm should be handling
a page fault or sending to vmd for a memory assist.

Tested by Mischa Peters and others. OK mlarkin@.

diff --git a/sys/arch/amd64/amd64/vmm.c b/sys/arch/amd64/amd64/vmm.c
index d7659a2..13cf764 100644 (file)
--- a/sys/arch/amd64/amd64/vmm.c
+++ b/sys/arch/amd64/amd64/vmm.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: vmm.c,v 1.333 2022/12/19 04:48:07 dlg Exp $   */
+/*     $OpenBSD: vmm.c,v 1.334 2022/12/26 23:50:20 dv Exp $    */
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -1626,8 +1626,8 @@ vmx_remote_vmclear(struct cpu_info *ci, struct vcpu *vcpu)
  * The last physical address may not exceed VMM_MAX_VM_MEM_SIZE.
  *
  * Return Values:
- *   The total memory size in MB if the checks were successful
- *   0: One of the memory ranges was invalid, or VMM_MAX_VM_MEM_SIZE was
+ *   The total memory size in bytes if the checks were successful
+ *   0: One of the memory ranges was invalid or VMM_MAX_VM_MEM_SIZE was
  *   exceeded
  */
 size_t
@@ -1638,21 +1638,27 @@ vm_create_check_mem_ranges(struct vm_create_params *vcp)
        const paddr_t maxgpa = VMM_MAX_VM_MEM_SIZE;
 
        if (vcp->vcp_nmemranges == 0 ||
-           vcp->vcp_nmemranges > VMM_MAX_MEM_RANGES)
+           vcp->vcp_nmemranges > VMM_MAX_MEM_RANGES) {
+               DPRINTF("invalid number of guest memory ranges\n");
                return (0);
+       }
 
        for (i = 0; i < vcp->vcp_nmemranges; i++) {
                vmr = &vcp->vcp_memranges[i];
 
                /* Only page-aligned addresses and sizes are permitted */
                if ((vmr->vmr_gpa & PAGE_MASK) || (vmr->vmr_va & PAGE_MASK) ||
-                   (vmr->vmr_size & PAGE_MASK) || vmr->vmr_size == 0)
+                   (vmr->vmr_size & PAGE_MASK) || vmr->vmr_size == 0) {
+                       DPRINTF("memory range %zu is not page aligned\n", i);
                        return (0);
+               }
 
                /* Make sure that VMM_MAX_VM_MEM_SIZE is not exceeded */
                if (vmr->vmr_gpa >= maxgpa ||
-                   vmr->vmr_size > maxgpa - vmr->vmr_gpa)
+                   vmr->vmr_size > maxgpa - vmr->vmr_gpa) {
+                       DPRINTF("exceeded max memory size\n");
                        return (0);
+               }
 
                /*
                 * Make sure that all virtual addresses are within the address
@@ -1662,39 +1668,29 @@ vm_create_check_mem_ranges(struct vm_create_params *vcp)
                 */
                if (vmr->vmr_va < VM_MIN_ADDRESS ||
                    vmr->vmr_va >= VM_MAXUSER_ADDRESS ||
-                   vmr->vmr_size >= VM_MAXUSER_ADDRESS - vmr->vmr_va)
-                       return (0);
-
-               /*
-                * Specifying ranges within the PCI MMIO space is forbidden.
-                * Disallow ranges that start inside the MMIO space:
-                * [VMM_PCI_MMIO_BAR_BASE .. VMM_PCI_MMIO_BAR_END]
-                */
-               if (vmr->vmr_gpa >= VMM_PCI_MMIO_BAR_BASE &&
-                   vmr->vmr_gpa <= VMM_PCI_MMIO_BAR_END)
-                       return (0);
-
-               /*
-                * ... and disallow ranges that end inside the MMIO space:
-                * (VMM_PCI_MMIO_BAR_BASE .. VMM_PCI_MMIO_BAR_END]
-                */
-               if (vmr->vmr_gpa + vmr->vmr_size > VMM_PCI_MMIO_BAR_BASE &&
-                   vmr->vmr_gpa + vmr->vmr_size <= VMM_PCI_MMIO_BAR_END)
+                   vmr->vmr_size >= VM_MAXUSER_ADDRESS - vmr->vmr_va) {
+                       DPRINTF("guest va not within range or wraps\n");
                        return (0);
+               }
 
                /*
                 * Make sure that guest physical memory ranges do not overlap
                 * and that they are ascending.
                 */
-               if (i > 0 && pvmr->vmr_gpa + pvmr->vmr_size > vmr->vmr_gpa)
+               if (i > 0 && pvmr->vmr_gpa + pvmr->vmr_size > vmr->vmr_gpa) {
+                       DPRINTF("guest range %zu overlaps or !ascending\n", i);
                        return (0);
+               }
 
-               memsize += vmr->vmr_size;
+               /*
+                * No memory is mappable in MMIO ranges, so don't count towards
+                * the total guest memory size.
+                */
+               if (vmr->vmr_type != VM_MEM_MMIO)
+                       memsize += vmr->vmr_size;
                pvmr = vmr;
        }
 
-       if (memsize % (1024 * 1024) != 0)
-               return (0);
        return (memsize);
 }
 
@@ -5628,11 +5624,6 @@ vmm_get_guest_memtype(struct vm *vm, paddr_t gpa)
        int i;
        struct vm_mem_range *vmr;
 
-       if (gpa >= VMM_PCI_MMIO_BAR_BASE && gpa <= VMM_PCI_MMIO_BAR_END) {
-               DPRINTF("guest mmio access @ 0x%llx\n", (uint64_t)gpa);
-               return (VMM_MEM_TYPE_MMIO);
-       }
-
        /* XXX Use binary search? */
        for (i = 0; i < vm->vm_nmemranges; i++) {
                vmr = &vm->vm_memranges[i];
@@ -5644,8 +5635,11 @@ vmm_get_guest_memtype(struct vm *vm, paddr_t gpa)
                if (gpa < vmr->vmr_gpa)
                        break;
 
-               if (gpa < vmr->vmr_gpa + vmr->vmr_size)
+               if (gpa < vmr->vmr_gpa + vmr->vmr_size) {
+                       if (vmr->vmr_type == VM_MEM_MMIO)
+                               return (VMM_MEM_TYPE_MMIO);
                        return (VMM_MEM_TYPE_REGULAR);
+               }
        }
 
        DPRINTF("guest memtype @ 0x%llx unknown\n", (uint64_t)gpa);

diff --git a/sys/arch/amd64/include/vmmvar.h b/sys/arch/amd64/include/vmmvar.h
index 94feca1..6b4802a 100644 (file)
--- a/sys/arch/amd64/include/vmmvar.h
+++ b/sys/arch/amd64/include/vmmvar.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: vmmvar.h,v 1.84 2022/11/10 11:46:39 dv Exp $  */
+/*     $OpenBSD: vmmvar.h,v 1.85 2022/12/26 23:50:20 dv Exp $  */
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -451,6 +451,10 @@ struct vm_mem_range {
        paddr_t vmr_gpa;
        vaddr_t vmr_va;
        size_t  vmr_size;
+       int     vmr_type;
+#define VM_MEM_RAM             0       /* Presented as usable system memory. */
+#define VM_MEM_RESERVED                1       /* Reserved for BIOS, etc. */
+#define VM_MEM_MMIO            2       /* Special region for device mmio. */
 };
 
 /*

diff --git a/usr.sbin/vmd/fw_cfg.c b/usr.sbin/vmd/fw_cfg.c
index 4ec0369..ed4386b 100644 (file)
--- a/usr.sbin/vmd/fw_cfg.c
+++ b/usr.sbin/vmd/fw_cfg.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: fw_cfg.c,v 1.5 2021/11/05 10:18:50 jan Exp $  */
+/*     $OpenBSD: fw_cfg.c,v 1.6 2022/12/26 23:50:20 dv Exp $   */
 /*
  * Copyright (c) 2018 Claudio Jeker <claudio@openbsd.org>
  *
@@ -16,6 +16,7 @@
  */
 #include <sys/types.h>
 #include <sys/uio.h>
+#include <machine/biosvar.h>   /* bios_memmap_t */
 #include <machine/vmmvar.h>
 
 #include <stdlib.h>
@@ -63,6 +64,8 @@ static struct fw_cfg_state {
 
 static uint64_t        fw_cfg_dma_addr;
 
+static bios_memmap_t e820[VMM_MAX_MEM_RANGES];
+
 static int     fw_cfg_select_file(uint16_t);
 static void    fw_cfg_file_dir(void);
 
@@ -71,6 +74,22 @@ fw_cfg_init(struct vmop_create_params *vmc)
 {
        const char *bootorder = NULL;
        unsigned int sd = 0;
+       size_t i, e820_len = 0;
+
+       /* Define e820 memory ranges. */
+       memset(&e820, 0, sizeof(e820));
+       for (i = 0; i < vmc->vmc_params.vcp_nmemranges; i++) {
+               struct vm_mem_range *range = &vmc->vmc_params.vcp_memranges[i];
+               bios_memmap_t *entry = &e820[i];
+               entry->addr = range->vmr_gpa;
+               entry->size = range->vmr_size;
+               if (range->vmr_type == VM_MEM_RAM)
+                       entry->type = BIOS_MAP_FREE;
+               else
+                       entry->type = BIOS_MAP_RES;
+               e820_len += sizeof(bios_memmap_t);
+       }
+       fw_cfg_add_file("etc/e820", &e820, e820_len);
 
        /* do not double print chars on serial port */
        fw_cfg_add_file("etc/screen-and-debug", &sd, sizeof(sd));

diff --git a/usr.sbin/vmd/loadfile_elf.c b/usr.sbin/vmd/loadfile_elf.c
index 6517195..95e5e96 100644 (file)
--- a/usr.sbin/vmd/loadfile_elf.c
+++ b/usr.sbin/vmd/loadfile_elf.c
@@ -1,5 +1,5 @@
 /* $NetBSD: loadfile.c,v 1.10 2000/12/03 02:53:04 tsutsui Exp $ */
-/* $OpenBSD: loadfile_elf.c,v 1.43 2022/11/28 18:24:52 dv Exp $ */
+/* $OpenBSD: loadfile_elf.c,v 1.44 2022/12/26 23:50:20 dv Exp $ */
 
 /*-
  * Copyright (c) 1997 The NetBSD Foundation, Inc.
@@ -334,38 +334,23 @@ loadfile_elf(gzFile fp, struct vm_create_params *vcp,
 static size_t
 create_bios_memmap(struct vm_create_params *vcp, bios_memmap_t *memmap)
 {
-       size_t i, n = 0, sz;
-       paddr_t gpa;
+       size_t i, n = 0;
        struct vm_mem_range *vmr;
 
-       for (i = 0; i < vcp->vcp_nmemranges; i++) {
+       for (i = 0; i < vcp->vcp_nmemranges; i++, n++) {
                vmr = &vcp->vcp_memranges[i];
-               gpa = vmr->vmr_gpa;
-               sz = vmr->vmr_size;
-
-               /*
-                * Make sure that we do not mark the ROM/video RAM area in the
-                * low memory as physcal memory available to the kernel.
-                */
-               if (gpa < 0x100000 && gpa + sz > LOWMEM_KB * 1024) {
-                       if (gpa >= LOWMEM_KB * 1024)
-                               sz = 0;
-                       else
-                               sz = LOWMEM_KB * 1024 - gpa;
-               }
-
-               if (sz != 0) {
-                       memmap[n].addr = gpa;
-                       memmap[n].size = sz;
-                       memmap[n].type = 0x1;   /* Type 1 : Normal memory */
-                       n++;
-               }
+               memmap[n].addr = vmr->vmr_gpa;
+               memmap[n].size = vmr->vmr_size;
+               if (vmr->vmr_type == VM_MEM_RAM)
+                       memmap[n].type = BIOS_MAP_FREE;
+               else
+                       memmap[n].type = BIOS_MAP_RES;
        }
 
        /* Null mem map entry to denote the end of the ranges */
        memmap[n].addr = 0x0;
        memmap[n].size = 0x0;
-       memmap[n].type = 0x0;
+       memmap[n].type = BIOS_MAP_END;
        n++;
 
        return (n);

diff --git a/usr.sbin/vmd/vm.c b/usr.sbin/vmd/vm.c
index 8f3eed7..befee2e 100644 (file)
--- a/usr.sbin/vmd/vm.c
+++ b/usr.sbin/vmd/vm.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: vm.c,v 1.77 2022/12/23 19:25:22 dv Exp $      */
+/*     $OpenBSD: vm.c,v 1.78 2022/12/26 23:50:20 dv Exp $      */
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -899,6 +899,7 @@ create_memory_map(struct vm_create_params *vcp)
        len = LOWMEM_KB * 1024;
        vcp->vcp_memranges[0].vmr_gpa = 0x0;
        vcp->vcp_memranges[0].vmr_size = len;
+       vcp->vcp_memranges[0].vmr_type = VM_MEM_RAM;
        mem_bytes -= len;
 
        /*
@@ -913,12 +914,14 @@ create_memory_map(struct vm_create_params *vcp)
        len = MB(1) - (LOWMEM_KB * 1024);
        vcp->vcp_memranges[1].vmr_gpa = LOWMEM_KB * 1024;
        vcp->vcp_memranges[1].vmr_size = len;
+       vcp->vcp_memranges[1].vmr_type = VM_MEM_RESERVED;
        mem_bytes -= len;
 
        /* If we have less than 2MB remaining, still create a 2nd BIOS area. */
        if (mem_bytes <= MB(2)) {
                vcp->vcp_memranges[2].vmr_gpa = VMM_PCI_MMIO_BAR_END;
                vcp->vcp_memranges[2].vmr_size = MB(2);
+               vcp->vcp_memranges[2].vmr_type = VM_MEM_RESERVED;
                vcp->vcp_nmemranges = 3;
                return;
        }
@@ -939,18 +942,27 @@ create_memory_map(struct vm_create_params *vcp)
        /* Third memory region: area above 1MB to MMIO region */
        vcp->vcp_memranges[2].vmr_gpa = MB(1);
        vcp->vcp_memranges[2].vmr_size = above_1m;
+       vcp->vcp_memranges[2].vmr_type = VM_MEM_RAM;
 
-       /* Fourth region: 2nd copy of BIOS above MMIO ending at 4GB */
-       vcp->vcp_memranges[3].vmr_gpa = VMM_PCI_MMIO_BAR_END + 1;
-       vcp->vcp_memranges[3].vmr_size = MB(2);
+       /* Fourth region: PCI MMIO range */
+       vcp->vcp_memranges[3].vmr_gpa = VMM_PCI_MMIO_BAR_BASE;
+       vcp->vcp_memranges[3].vmr_size = VMM_PCI_MMIO_BAR_END -
+           VMM_PCI_MMIO_BAR_BASE + 1;
+       vcp->vcp_memranges[3].vmr_type = VM_MEM_MMIO;
 
-       /* Fifth region: any remainder above 4GB */
+       /* Fifth region: 2nd copy of BIOS above MMIO ending at 4GB */
+       vcp->vcp_memranges[4].vmr_gpa = VMM_PCI_MMIO_BAR_END + 1;
+       vcp->vcp_memranges[4].vmr_size = MB(2);
+       vcp->vcp_memranges[4].vmr_type = VM_MEM_RESERVED;
+
+       /* Sixth region: any remainder above 4GB */
        if (above_4g > 0) {
-               vcp->vcp_memranges[4].vmr_gpa = GB(4);
-               vcp->vcp_memranges[4].vmr_size = above_4g;
-               vcp->vcp_nmemranges = 5;
+               vcp->vcp_memranges[5].vmr_gpa = GB(4);
+               vcp->vcp_memranges[5].vmr_size = above_4g;
+               vcp->vcp_memranges[5].vmr_type = VM_MEM_RAM;
+               vcp->vcp_nmemranges = 6;
        } else
-               vcp->vcp_nmemranges = 4;
+               vcp->vcp_nmemranges = 5;
 }
 
 /*