+++ /dev/null
-/* Copyright (c) 2007,2008 Sandia National Laboratories */
-
-#include <lwk/kernel.h>
-#include <lwk/aspace.h>
-#include <lwk/task.h>
-#include <lwk/init_task.h>
-#include <arch/page.h> /* TODO: remove */
-#include <arch/pgtable.h> /* TODO: remove */
-#include <arch/page_table.h>
-
-
-/**
- * Architecture specific address space initialization. This allocates a new
- * page table root for the aspace and copies the kernel page tables into it.
- */
-int
-arch_aspace_create(
- struct aspace * aspace
-)
-{
- unsigned int i;
-
- /* Allocate a root page table for the address space */
- if ((aspace->arch.pgd = kmem_get_pages(0)) == NULL)
- return -ENOMEM;
-
- /* Copy the current kernel page tables into the address space */
- for (i = pgd_index(PAGE_OFFSET); i < PTRS_PER_PGD; i++)
- aspace->arch.pgd[i] = bootstrap_task.aspace->arch.pgd[i];
-
- return 0;
-}
-
-
-/**
- * Architecture specific address space destruction. This frees all page table
- * memory that the aspace was using.
- */
-void
-arch_aspace_destroy(
- struct aspace * aspace
-)
-{
- unsigned int i, j, k;
-
- xpte_t *pgd; /* Page Global Directory: level 0 (root of tree) */
- xpte_t *pud; /* Page Upper Directory: level 1 */
- xpte_t *pmd; /* Page Middle Directory: level 2 */
- xpte_t *ptd; /* Page Table Directory: level 3 */
-
- /* Walk and then free the Page Global Directory */
- pgd = aspace->arch.pgd;
- for (i = 0; i < pgd_index(PAGE_OFFSET); i++) {
- if (!pgd[i].present)
- continue;
-
- /* Walk and then free the Page Upper Directory */
- pud = __va(pgd[i].base_paddr << 12);
- for (j = 0; j < 512; j++) {
- if (!pud[j].present || pud[j].pagesize)
- continue;
-
- /* Walk and then free the Page Middle Directory */
- pmd = __va(pud[j].base_paddr << 12);
- for (k = 0; k < 512; k++) {
- if (!pmd[k].present || pmd[k].pagesize)
- continue;
-
- /* Free the last level Page Table Directory */
- ptd = __va(pmd[k].base_paddr << 12);
- kmem_free_pages(ptd, 0);
- }
- kmem_free_pages(pmd, 0);
- }
- kmem_free_pages(pud, 0);
- }
- kmem_free_pages(pgd, 0);
-}
-
-
-/**
- * Loads the address space object's root page table pointer into the calling
- * CPU's CR3 register, causing the aspace to become active.
- */
-void
-arch_aspace_activate(
- struct aspace * aspace
-)
-{
- asm volatile(
- "movq %0,%%cr3" :: "r" (__pa(aspace->arch.pgd)) : "memory"
- );
-}
-
-
-/**
- * Allocates a new page table and links it to a parent page table entry.
- */
-static xpte_t *
-alloc_page_table(
- xpte_t * parent_pte
-)
-{
- xpte_t *new_table;
-
- new_table = kmem_get_pages(0);
- if (!new_table)
- return NULL;
-
- if (parent_pte) {
- xpte_t _pte;
-
- memset(&_pte, 0, sizeof(_pte));
- _pte.present = 1;
- _pte.write = 1;
- _pte.user = 1;
- _pte.base_paddr = __pa(new_table) >> 12;
-
- *parent_pte = _pte;
- }
-
- return new_table;
-}
-
-
-/**
- * Locates an existing page table entry or creates a new one if none exists.
- * Returns a pointer to the page table entry.
- */
-static xpte_t *
-find_or_create_pte(
- struct aspace * aspace,
- vaddr_t vaddr,
- vmpagesize_t pagesz
-)
-{
- xpte_t *pgd; /* Page Global Directory: level 0 (root of tree) */
- xpte_t *pud; /* Page Upper Directory: level 1 */
- xpte_t *pmd; /* Page Middle Directory: level 2 */
- xpte_t *ptd; /* Page Table Directory: level 3 */
-
- xpte_t *pge; /* Page Global Directory Entry */
- xpte_t *pue; /* Page Upper Directory Entry */
- xpte_t *pme; /* Page Middle Directory Entry */
- xpte_t *pte; /* Page Table Directory Entry */
-
- /* Calculate indices into above directories based on vaddr specified */
- const unsigned int pgd_index = (vaddr >> 39) & 0x1FF;
- const unsigned int pud_index = (vaddr >> 30) & 0x1FF;
- const unsigned int pmd_index = (vaddr >> 21) & 0x1FF;
- const unsigned int ptd_index = (vaddr >> 12) & 0x1FF;
-
- /* Traverse the Page Global Directory */
- pgd = aspace->arch.pgd;
- pge = &pgd[pgd_index];
- if (!pge->present && !alloc_page_table(pge))
- return NULL;
-
- /* Traverse the Page Upper Directory */
- pud = __va(pge->base_paddr << 12);
- pue = &pud[pud_index];
- if (pagesz == VM_PAGE_1GB)
- return pue;
- else if (!pue->present && !alloc_page_table(pue))
- return NULL;
- else if (pue->pagesize)
- panic("BUG: Can't follow PUD entry, pagesize bit set.");
-
- /* Traverse the Page Middle Directory */
- pmd = __va(pue->base_paddr << 12);
- pme = &pmd[pmd_index];
- if (pagesz == VM_PAGE_2MB)
- return pme;
- else if (!pme->present && !alloc_page_table(pme))
- return NULL;
- else if (pme->pagesize)
- panic("BUG: Can't follow PMD entry, pagesize bit set.");
-
- /* Traverse the Page Table Entry Directory */
- ptd = __va(pme->base_paddr << 12);
- pte = &ptd[ptd_index];
- return pte;
-}
-
-
-/**
- * Examines a page table to determine if it has any active entries. If not,
- * the page table is freed.
- */
-static int
-try_to_free_table(
- xpte_t * table,
- xpte_t * parent_pte
-)
-{
- int i;
-
- /* Determine if the table can be freed */
- for (i = 0; i < 512; i++) {
- if (table[i].present)
- return -1; /* Nope */
- }
-
- /* Yup, free the page table */
- kmem_free_pages(table, 0);
- memset(parent_pte, 0, sizeof(xpte_t));
- return 0;
-}
-
-
-/**
- * Zeros a page table entry. If the page table that the PTE was in becomes
- * empty (contains no active mappings), it is freed. Page table freeing
- * continues up to the top of the page table tree (e.g., a single call may
- * result in a PTD, PMD, and PUD being freed; the PGD is never freed by this
- * function).
- */
-static void
-find_and_delete_pte(
- struct aspace * aspace,
- vaddr_t vaddr,
- vmpagesize_t pagesz
-)
-{
- xpte_t *pgd; /* Page Global Directory: level 0 (root of tree) */
- xpte_t *pud; /* Page Upper Directory: level 1 */
- xpte_t *pmd; /* Page Middle Directory: level 2 */
- xpte_t *ptd; /* Page Table Directory: level 3 */
-
- xpte_t *pge; /* Page Global Directory Entry */
- xpte_t *pue; /* Page Upper Directory Entry */
- xpte_t *pme; /* Page Middle Directory Entry */
- xpte_t *pte; /* Page Table Directory Entry */
-
- /* Calculate indices into above directories based on vaddr specified */
- const unsigned int pgd_index = (vaddr >> 39) & 0x1FF;
- const unsigned int pud_index = (vaddr >> 30) & 0x1FF;
- const unsigned int pmd_index = (vaddr >> 21) & 0x1FF;
- const unsigned int ptd_index = (vaddr >> 12) & 0x1FF;
-
- /* Traverse the Page Global Directory */
- pgd = aspace->arch.pgd;
- pge = &pgd[pgd_index];
- if (!pge->present)
- return;
-
- /* Traverse the Page Upper Directory */
- pud = __va(pge->base_paddr << 12);
- pue = &pud[pud_index];
- if (!pue->present) {
- return;
- } else if (pagesz == VM_PAGE_1GB) {
- if (!pue->pagesize)
- panic("BUG: 1GB PTE has child page table attached.\n");
-
- /* Unmap the 1GB page that this PTE was mapping */
- memset(pue, 0, sizeof(xpte_t));
-
- /* Try to free PUD that the PTE was in */
- try_to_free_table(pud, pge);
- return;
- }
-
- /* Traverse the Page Middle Directory */
- pmd = __va(pue->base_paddr << 12);
- pme = &pmd[pmd_index];
- if (!pme->present) {
- return;
- } else if (pagesz == VM_PAGE_2MB) {
- if (!pme->pagesize)
- panic("BUG: 2MB PTE has child page table attached.\n");
-
- /* Unmap the 2MB page that this PTE was mapping */
- memset(pme, 0, sizeof(xpte_t));
-
- /* Try to free the PMD that the PTE was in */
- if (try_to_free_table(pmd, pue))
- return; /* nope, couldn't free it */
-
- /* Try to free the PUD that contained the PMD just freed */
- try_to_free_table(pud, pge);
- return;
- }
-
- /* Traverse the Page Table Entry Directory */
- ptd = __va(pme->base_paddr << 12);
- pte = &ptd[ptd_index];
- if (pte->present) {
- return;
- } else {
- /* Unmap the 4KB page that this PTE was mapping */
- memset(pme, 0, sizeof(xpte_t));
-
- /* Try to free the PTD that the PTE was in */
- if (try_to_free_table(ptd, pme))
- return; /* nope, couldn't free it */
-
- /* Try to free the PMD that contained the PTD just freed */
- if (try_to_free_table(pmd, pue))
- return; /* nope, couldn't free it */
-
- /* Try to free the PUD that contained the PMD just freed */
- try_to_free_table(pud, pge);
- return;
- }
-}
-
-
-/**
- * Writes a new value to a PTE.
- * TODO: Determine if this is atomic enough.
- */
-static void
-write_pte(
- xpte_t * pte,
- paddr_t paddr,
- vmflags_t flags,
- vmpagesize_t pagesz
-)
-{
- xpte_t _pte;
- memset(&_pte, 0, sizeof(_pte));
-
- _pte.present = 1;
- if (flags & VM_WRITE)
- _pte.write = 1;
- if (flags & VM_USER)
- _pte.user = 1;
- if (flags & VM_GLOBAL)
- _pte.global = 1;
- if ((flags & VM_EXEC) == 0)
- _pte.no_exec = 1;
-
- if (pagesz == VM_PAGE_4KB) {
- _pte.base_paddr = paddr >> 12;
- } else if (pagesz == VM_PAGE_2MB) {
- _pte.pagesize = 1;
- _pte.base_paddr = paddr >> 21;
- } else if (pagesz == VM_PAGE_1GB) {
- _pte.pagesize = 1;
- _pte.base_paddr = paddr >> 30;
- } else {
- panic("Invalid page size 0x%lx.", pagesz);
- }
-
- *pte = _pte;
-}
-
-
-/**
- * Maps a page into an address space.
- *
- * Arguments:
- * [IN] aspace: Address space to map page into.
- * [IN] start: Address in aspace to map page to.
- * [IN] paddr: Physical address of the page to map.
- * [IN] flags: Protection and memory type flags.
- * [IN] pagesz: Size of the page being mapped, in bytes.
- *
- * Returns:
- * Success: 0
- * Failure: Error Code, the page was not mapped.
- */
-int
-arch_aspace_map_page(
- struct aspace * aspace,
- vaddr_t start,
- paddr_t paddr,
- vmflags_t flags,
- vmpagesize_t pagesz
-)
-{
- xpte_t *pte;
-
- /* Locate page table entry that needs to be updated to map the page */
- pte = find_or_create_pte(aspace, start, pagesz);
- if (!pte)
- return -ENOMEM;
-
- /* Update the page table entry */
- write_pte(pte, paddr, flags, pagesz);
-
- return 0;
-}
-
-
-/**
- * Unmaps a page from an address space.
- *
- * Arguments:
- * [IN] aspace: Address space to unmap page from.
- * [IN] start: Address in aspace to unmap page from.
- * [IN] pagesz: Size of the page to unmap.
- */
-void
-arch_aspace_unmap_page(
- struct aspace * aspace,
- vaddr_t start,
- vmpagesize_t pagesz
-)
-{
- find_and_delete_pte(aspace, start, pagesz);
-}
-
-int
-arch_aspace_smartmap(struct aspace *src, struct aspace *dst,
- vaddr_t start, size_t extent)
-{
- size_t n = extent / SMARTMAP_ALIGN;
- size_t i;
- xpte_t *src_pgd = src->arch.pgd;
- xpte_t *dst_pgd = dst->arch.pgd;
- xpte_t *src_pge, *dst_pge;
-
- /* Make sure all of the source PGD entries are present */
- for (i = 0; i < n; i++) {
- src_pge = &src_pgd[i];
- if (!src_pge->present && !alloc_page_table(src_pge))
- return -ENOMEM;
- }
-
- /* Perform the SMARTMAP... just copy src PGEs to the dst PGD */
- for (i = 0; i < n; i++) {
- src_pge = &src_pgd[i];
- dst_pge = &dst_pgd[(start >> 39) & 0x1FF];
- BUG_ON(dst_pge->present);
- dst_pge = src_pge;
- }
-
- return 0;
-}
-
-int
-arch_aspace_unsmartmap(struct aspace *src, struct aspace *dst,
- vaddr_t start, size_t extent)
-{
- size_t n = extent / SMARTMAP_ALIGN;
- size_t i;
- xpte_t *dst_pgd = dst->arch.pgd;
- xpte_t *dst_pge;
-
- /* Unmap the SMARTMAP PGEs */
- for (i = 0; i < n; i++) {
- dst_pge = &dst_pgd[(start >> 39) & 0x1FF];
- dst_pge->present = 0;
- }
-
- return 0;
-}