int ukplat_pt_walk(struct uk_pagetable *pt, __vaddr_t vaddr,
unsigned int *level, __vaddr_t *pt_vaddr, __pte_t *pte);
+/* Forward declaration */
+struct ukplat_page_mapx;
+
+/**
+ * Page mapper function that allows controlling the mappings that are created
+ * in a call to ukplat_page_mapx().
+ *
+ * @param pt
+ * The page table that the mapping is done in
+ * @param vaddr
+ * The virtual address for which a mapping is established
+ * @param pt_vaddr
+ * The virtual address of the actual hardware page table that is modified.
+ * Use this to retrieve the current PTE, if needed.
+ * @param level
+ * The page table level at which the mapping will be created
+ * @param[in,out] pte
+ * The new PTE that will be set. The handler may freely modify the PTE to
+ * control the mapping.
+ * @param ctx
+ * An optional user-supplied context
+ *
+ * @return
+ * - 0 on success (i.e., the PTE should be applied)
+ * - a negative error code to indicate a fatal error
+ * - UKPLAT_PAGE_MAPX_ESKIP to skip the current PTE (do not apply the changes)
+ * - UKPLAT_PAGE_MAPX_ETOOBIG to indicate that the mapping should be retried
+ * using a smaller page size
+ */
+typedef int (*ukplat_page_mapx_func_t)(struct uk_pagetable *pt,
+ __vaddr_t vaddr, __vaddr_t pt_vaddr,
+ unsigned int level, __pte_t *pte,
+ void *ctx);
+
+/* Page mapper function return codes */
+#define UKPLAT_PAGE_MAPX_ESKIP 1 /* Skip the current PTE */
+#define UKPLAT_PAGE_MAPX_ETOOBIG 2 /* Retry with smaller page */
+
+struct ukplat_page_mapx {
+ /** Handler called before updating the PTE in the page table */
+ ukplat_page_mapx_func_t map;
+ /** Optional user context */
+ void *ctx;
+};
+
/**
- * Creates a mapping from a range of continuous virtual addresses to a range of
+ * Creates a mapping from a range of contiguous virtual addresses to a range of
* physical addresses using the specified attributes.
*
- * @param pt the page table instance on which to operate.
- * @param vaddr the virtual address of the first page in the new mapping.
- * @param paddr the physical address of the memory which the virtual region is
- * mapped to. This parameter can be __PADDR_ANY to dynamically allocate
- * physical memory as needed.
- * @param pages the number of pages in requested page size to map.
- * @param attr page attributes to set for the new mapping (PAGE_ATTR_* flags).
- * @param flags page flags (PAGE_FLAG_* flags). The page size can be specified
- * with PAGE_FLAG_SIZE(). If PAGE_FLAG_FORCE_SIZE is not specified, the
- * function tries to map the given range (i.e., pages * requested page size)
- * using the largest possible pages. The actual mapping thus may use larger
- * or smaller pages than requested depending on address alignment, supported
- * page sizes, and available continuous physical memory (if paddr is
- * __PADDR_ANY).
+ * @param pt
+ * The page table instance on which to operate
+ * @param vaddr
+ * The virtual address of the first page in the new mapping
+ * @param paddr
+ * The physical address of the memory which the virtual region is mapped to.
+ * This parameter can be __PADDR_ANY to dynamically allocate physical memory
+ * as needed. Note that, the physical memory might not be contiguous.
*
- * @return 0 on success, a non-zero value otherwise. May fail if:
- * - the physical or virtual address is not aligned to the page size;
- * - a page in the region is already mapped;
- * - a page table could not be set up;
- * - if __PADDR_ANY flag is set and there are no more free frames;
+ * paddr should be 0 if physical addresses should be handled by the mapx page
+ * mapper.
+ * @param pages
+ * The number of pages in requested page size to map
+ * @param attr
+ * Page attributes to set for the new mapping (see PAGE_ATTR_* flags)
+ * @param flags
+ * Page flags (see PAGE_FLAG_* flags)
+ *
+ * The page size can be specified with PAGE_FLAG_SIZE(). If
+ * PAGE_FLAG_FORCE_SIZE is not specified, the function tries to map the given
+ * range (i.e., pages * requested page size) using the largest possible pages.
+ * The actual mapping thus may use larger or smaller pages than requested
+ * depending on address alignment, supported page sizes, and, if paddr is
+ * __PADDR_ANY, the available contiguous physical memory. If
+ * PAGE_FLAG_FORCE_SIZE is specified, only mappings of the given page size
+ * are created.
+ *
+ * If PAGE_FLAG_KEEP_PTES is specified, the new mapping will incorporate the
+ * PTEs currently present in the page table. The physical address and
+ * permission flags will be updated according to the new mapping.
+ * @param mapx
+ * Optional page mapper object. If the page mapper is supplied, it is called
+ * right before applying a new PTE, giving the mapper a chance to affect the
+ * mapping. Depending on the return code of the mapper it is also possible to
+ * skip the current PTE or force a smaller page size (if PAGE_FLAG_FORCE_SIZE
+ * is not set). Note that the page mapper is not called for PTEs referencing
+ * other page tables.
+ *
+ * If paddr is __PADDR_ANY, the PTE supplied to the mapper will point to newly
+ * allocated physical memory that can be initialized before becoming visible.
+ * Use PT_Lx_PTE_PADDR() to retrieve the physical address from the PTE and
+ * ukplat_page_kmap() to temporarily map the physical memory. Note that, if
+ * the PTE currently present in the page table already points to a valid
+ * mapping (i.e., PT_Lx_PTE_PRESENT() returns a non-zero value), no new
+ * physical memory will be allocated. Instead, the physical address will
+ * remain unchanged. It is the mapper's responsibilty to properly free the
+ * referenced physical memory, if the physical address is changed.
+ *
+ * Before calling the page mapper, existing large pages may be split up if a
+ * smaller page size is enforced.
+ *
+ * @return
+ * 0 on success, a non-zero value otherwise. May fail if:
+ * - the physical or virtual address is not aligned to the page size
+ * - a page in the region is already mapped and no mapx is supplied
+ * - a page table could not be set up
+ * - if __PADDR_ANY flag is set and there are no more free frames
* - the platform rejected the operation
+ * - the mapx page mapper returned a fatal error
*/
-int ukplat_page_map(struct uk_pagetable *pt, __vaddr_t vaddr,
- __paddr_t paddr, unsigned long pages,
- unsigned long attr, unsigned long flags);
+int ukplat_page_mapx(struct uk_pagetable *pt, __vaddr_t vaddr,
+ __paddr_t paddr, unsigned long pages,
+ unsigned long attr, unsigned long flags,
+ struct ukplat_page_mapx *mapx);
+
+#define ukplat_page_map(pt, va, pa, pages, attr, flags) \
+ ukplat_page_mapx(pt, va, pa, pages, attr, flags, __NULL)
/**
* Removes the mappings from a range of continuous virtual addresses and frees
static inline void pg_pt_free(struct uk_pagetable *pt, __vaddr_t pt_vaddr,
unsigned int level);
+static int pg_page_split(struct uk_pagetable *pt, __vaddr_t pt_vaddr,
+ __vaddr_t vaddr, unsigned int level);
+
static int pg_page_unmap(struct uk_pagetable *pt, __vaddr_t pt_vaddr,
unsigned int level, __vaddr_t vaddr, __sz len,
unsigned long flags);
return lvl;
}
-static int pg_page_map(struct uk_pagetable *pt, __vaddr_t pt_vaddr,
- unsigned int level, __vaddr_t vaddr, __paddr_t paddr,
- __sz len, unsigned long attr, unsigned long flags,
- __pte_t template, unsigned int template_level)
+static int pg_page_mapx(struct uk_pagetable *pt, __vaddr_t pt_vaddr,
+ unsigned int level, __vaddr_t vaddr, __paddr_t paddr,
+ __sz len, unsigned long attr, unsigned long flags,
+ __pte_t template, unsigned int template_level,
+ struct ukplat_page_mapx *mapx)
{
unsigned int to_lvl = PAGE_FLAG_SIZE_TO_LEVEL(flags);
unsigned int max_lvl = pg_page_largest_level;
unsigned int tmp_lvl;
__vaddr_t pt_vaddr_cache[PT_LEVELS];
__paddr_t pt_paddr;
- __pte_t pte;
+ __pte_t pte, orig_pte;
__sz page_size;
unsigned int pte_idx;
int rc, alloc_pmem;
if (PT_Lx_PTE_PRESENT(pte, lvl)) {
/* If there is already a larger page mapped
- * at this address return an error.
+ * at this address and we have a mapx, we
+ * split the page until we reach the target
+ * level and let the mapx decide what to do.
+ * Otherwise, we bail out.
*/
- if (PAGE_Lx_IS(pte, lvl))
- return -EEXIST;
+ if (PAGE_Lx_IS(pte, lvl)) {
+ if (!mapx)
+ return -EEXIST;
+
+ rc = pg_page_split(pt, pt_vaddr, vaddr,
+ lvl);
+ if (unlikely(rc))
+ return rc;
+
+ continue;
+ }
pt_vaddr = pgarch_pt_pte_to_vaddr(pt, pte, lvl);
} else {
if (unlikely(rc))
return rc;
+ orig_pte = pte;
+
if (PT_Lx_PTE_PRESENT(pte, lvl)) {
/* It could be that there is a page table linked at
* this PTE. In this case, we descent further down to
continue;
}
- return -EEXIST;
- }
+ /* This is not a page table. However, we are on the
+ * correct level. If we have a mapping function we let
+ * the mapping function decide what we should do with
+ * the existing mapping. Otherwise, bail out.
+ */
+ if (!mapx)
+ return -EEXIST;
- UK_ASSERT(!PT_Lx_PTE_PRESENT(pte, lvl));
+ paddr = PT_Lx_PTE_PADDR(pte, lvl);
+ } else if (alloc_pmem) {
+ UK_ASSERT(!PT_Lx_PTE_PRESENT(pte, lvl));
- /* This is a non-present PTE so we can map the new page here */
- if (alloc_pmem) {
paddr = __PADDR_ANY;
-
rc = pg_falloc(pt, &paddr, lvl);
- if (rc) {
+ if (unlikely(rc)) {
+TOO_BIG:
/* We could not allocate a contiguous,
* self-aligned block of physical memory with
* the requested size. If we should map largest
continue;
}
+
+ /* If something goes wrong in the following, we must
+ * free the physical memory!
+ */
}
UK_ASSERT(PAGE_Lx_ALIGNED(vaddr, lvl));
template_level = lvl;
pte = pgarch_pte_create(paddr, attr, lvl, pte, template_level);
+
+ if (mapx) {
+ /* pte will always be prepared with the PTE that we
+ * intend to write if the mapx returns success.
+ * Accessing the current PTE can be done by reading it
+ * from the page table.
+ */
+ UK_ASSERT(mapx->map);
+ rc = mapx->map(pt, vaddr, pt_vaddr, lvl, &pte,
+ mapx->ctx);
+ if (unlikely(rc)) {
+ if (alloc_pmem &&
+ !PT_Lx_PTE_PRESENT(orig_pte, lvl))
+ pg_ffree(pt, paddr, lvl);
+
+ if (rc == UKPLAT_PAGE_MAPX_ESKIP)
+ goto NEXT_PTE;
+
+ if (rc == UKPLAT_PAGE_MAPX_ETOOBIG) {
+ rc = -ENOMEM;
+ goto TOO_BIG;
+ }
+
+ UK_ASSERT(rc < 0);
+ return rc;
+ }
+ }
+
+ UK_ASSERT(PAGE_Lx_ALIGNED(PT_Lx_PTE_PADDR(pte, lvl), lvl));
+
rc = ukarch_pte_write(pt_vaddr, lvl, pte_idx, pte);
if (unlikely(rc)) {
- if (alloc_pmem)
+ if (alloc_pmem &&
+ !PT_Lx_PTE_PRESENT(orig_pte, lvl))
pg_ffree(pt, paddr, lvl);
return rc;
pt->nr_lx_pages[lvl]++;
#endif /* CONFIG_PAGING_STATS */
+ if (PT_Lx_PTE_PRESENT(orig_pte, lvl))
+ ukarch_tlb_flush_entry(vaddr);
+
+NEXT_PTE:
UK_ASSERT(len >= page_size);
len -= page_size;
return 0;
}
-int ukplat_page_map(struct uk_pagetable *pt, __vaddr_t vaddr,
- __paddr_t paddr, unsigned long pages,
- unsigned long attr, unsigned long flags)
+int ukplat_page_mapx(struct uk_pagetable *pt, __vaddr_t vaddr,
+ __paddr_t paddr, unsigned long pages,
+ unsigned long attr, unsigned long flags,
+ struct ukplat_page_mapx *mapx)
{
unsigned int level = PAGE_FLAG_SIZE_TO_LEVEL(flags);
__sz len;
UK_ASSERT(pt->pt_vbase != __VADDR_INV);
UK_ASSERT(pt->pt_pbase != __PADDR_INV);
- return pg_page_map(pt, pt->pt_vbase, PT_LEVELS - 1, vaddr, paddr, len,
- attr, flags, PT_Lx_PTE_INVALID(PAGE_LEVEL),
- PAGE_LEVEL);
+ return pg_page_mapx(pt, pt->pt_vbase, PT_LEVELS - 1, vaddr, paddr, len,
+ attr, flags, PT_Lx_PTE_INVALID(PAGE_LEVEL),
+ PAGE_LEVEL, mapx);
}
static int pg_page_split(struct uk_pagetable *pt, __vaddr_t pt_vaddr,
* contiguous range of physical memory than the input page
*/
paddr = PT_Lx_PTE_PADDR(pte, level);
- rc = pg_page_map(pt, new_pt_vaddr, level - 1, vaddr, paddr,
- PAGE_Lx_SIZE(level), attr, flags, pte, level);
+ rc = pg_page_mapx(pt, new_pt_vaddr, level - 1, vaddr, paddr,
+ PAGE_Lx_SIZE(level), attr, flags, pte, level, NULL);
if (unlikely(rc))
goto EXIT_FREE;
projects / unikraft / unikraft.git / commitdiff