--- /dev/null
+#include <lwk/kernel.h>
+#include <lwk/bootmem.h>
+#include <lwk/string.h>
+#include <lwk/pmem.h>
+#include <arch/page.h>
+#include <arch/pgtable.h>
+#include <arch/e820.h>
+#include <arch/tlbflush.h>
+#include <arch/proto.h>
+
+/**
+ * Start and end page frame numbers of the kernel page tables.
+ */
+unsigned long __initdata table_start, table_end; /* page frame numbers */
+
+static __init void *early_ioremap(unsigned long addr, unsigned long size)
+{
+ unsigned long vaddr;
+ pmd_t *pmd, *last_pmd;
+ int i, pmds;
+
+ pmds = ((addr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
+ vaddr = __START_KERNEL_map;
+ pmd = level2_kernel_pgt;
+ last_pmd = level2_kernel_pgt + PTRS_PER_PMD - 1;
+ for (; pmd <= last_pmd; pmd++, vaddr += PMD_SIZE) {
+ for (i = 0; i < pmds; i++) {
+ if (pmd_present(pmd[i]))
+ goto next;
+ }
+ vaddr += addr & ~PMD_MASK;
+ addr &= PMD_MASK;
+ for (i = 0; i < pmds; i++, addr += PMD_SIZE)
+ set_pmd(pmd + i,__pmd(addr | _KERNPG_TABLE | _PAGE_PSE));
+ __flush_tlb();
+ return (void *)vaddr;
+ next:
+ ;
+ }
+ printk("early_ioremap(0x%lx, %lu) failed\n", addr, size);
+ return NULL;
+}
+
+/* To avoid virtual aliases later */
+static __init void early_iounmap(void *addr, unsigned long size)
+{
+ unsigned long vaddr;
+ pmd_t *pmd;
+ int i, pmds;
+
+ vaddr = (unsigned long)addr;
+ pmds = ((vaddr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
+ pmd = level2_kernel_pgt + pmd_index(vaddr);
+ for (i = 0; i < pmds; i++)
+ pmd_clear(pmd + i);
+ __flush_tlb();
+}
+
+static __init void *alloc_low_page(unsigned long *phys)
+{
+ unsigned long pfn = table_end++;
+ void *adr;
+
+ if (pfn >= end_pfn)
+ panic("alloc_low_page: ran out of memory");
+
+ adr = early_ioremap(pfn * PAGE_SIZE, PAGE_SIZE);
+ memset(adr, 0, PAGE_SIZE);
+ *phys = pfn * PAGE_SIZE;
+ return adr;
+}
+
+/**
+ * Destroys a temporary mapping that was setup by alloc_low_page().
+ */
+static void __init unmap_low_page(void *adr)
+{
+ early_iounmap(adr, PAGE_SIZE);
+}
+
+/**
+ * Initializes a fixmap entry to point to a given physical page.
+ */
+void __init
+__set_fixmap(
+ enum fixed_addresses fixmap_index, /* fixmap entry index to setup */
+ unsigned long phys_addr, /* map fixmap entry to this addr */
+ pgprot_t prot /* page protection bits */
+)
+{
+ unsigned long virt_addr;
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte, new_pte;
+
+ if (fixmap_index >= __end_of_fixed_addresses)
+ panic("Invalid FIXMAP index");
+
+ /* Calculate the virtual address of the fixmap entry */
+ virt_addr = __fix_to_virt(fixmap_index);
+
+ /* Look up PGD entry covering the fixmap entry */
+ pgd = pgd_offset_k(virt_addr);
+ if (pgd_none(*pgd))
+ panic("PGD FIXMAP MISSING, it should be setup in head.S!\n");
+
+ /* Look up the PMD entry covering the fixmap entry */
+ pud = pud_offset(pgd, virt_addr);
+ if (pud_none(*pud)) {
+ /* PUD entry is empty... allocate a new PMD directory for it */
+ pmd = (pmd_t *) alloc_bootmem_aligned(PAGE_SIZE, PAGE_SIZE);
+ set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
+ BUG_ON(pmd != pmd_offset(pud, 0));
+ }
+
+ /* Look up the PMD entry covering the fixmap entry */
+ pmd = pmd_offset(pud, virt_addr);
+ if (pmd_none(*pmd)) {
+ /* PMD entry is empty... allocate a new PTE directory for it */
+ pte = (pte_t *) alloc_bootmem_aligned(PAGE_SIZE, PAGE_SIZE);
+ set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE | _PAGE_USER));
+ BUG_ON(pte != pte_offset_kernel(pmd, 0));
+ }
+
+ /*
+ * Construct and install a new PTE that maps the fixmap entry
+ * to the requested physical address.
+ */
+ pte = pte_offset_kernel(pmd, virt_addr);
+ new_pte = pfn_pte(phys_addr >> PAGE_SHIFT, prot);
+ set_pte(pte, new_pte);
+ __flush_tlb_one(virt_addr);
+}
+
+/**
+ * Finds enough space for the kernel page tables.
+ */
+static void __init
+find_early_table_space(unsigned long end)
+{
+ unsigned long puds; /* # of pud page tables needed */
+ unsigned long pmds; /* # of pmd page tables needed */
+ unsigned long tables; /* page table memory needed, in bytes */
+ unsigned long start; /* start address for kernel page tables */
+
+ /*
+ * The kernel page tables map memory using 2 MB pages.
+ * This means only puds and pmds are needed.
+ */
+ puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
+ pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
+ tables = round_up(puds * sizeof(pud_t), PAGE_SIZE) +
+ round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
+
+ /*
+ * Consult the memory map to find a region of suitable size.
+ */
+ start = 0x8000;
+ table_start = find_e820_area(start, end, tables);
+ if (table_start == -1UL)
+ panic("Cannot find space for the kernel page tables");
+
+ /*
+ * Store table_start and table_end as page frame numbers.
+ * table_end starts out as the same as table_start.
+ * It will be incremented for each page table allocated.
+ */
+ table_start >>= PAGE_SHIFT;
+ table_end = table_start;
+}
+
+
+/**
+ * Configures the input Page Middle Directory to map physical addresses
+ * [address, end). PMD entries outside of this range are zeroed.
+ *
+ * Each PMD table maps 1 GB of memory (512 entries, each mapping 2 MB).
+ */
+static void __init
+phys_pmd_init(pmd_t *pmd, unsigned long address, unsigned long end)
+{
+ int i;
+
+ for (i = 0; i < PTRS_PER_PMD; pmd++, i++, address += PMD_SIZE) {
+ unsigned long entry;
+
+ if (address > end) {
+ for (; i < PTRS_PER_PMD; i++, pmd++)
+ set_pmd(pmd, __pmd(0));
+ break;
+ }
+ entry = _PAGE_NX|_PAGE_PSE|_KERNPG_TABLE|_PAGE_GLOBAL|address;
+ entry &= __supported_pte_mask;
+ set_pmd(pmd, __pmd(entry));
+ }
+}
+
+/**
+ * Configures the input Page Upper Directory to map physical addresses
+ * [address, end). PUD entries outside of this range are zeroed.
+ *
+ * Each PUD table maps 512 GB of memory (512 entries, each pointing to a PMD).
+ */
+static void __init
+phys_pud_init(pud_t *pud, unsigned long address, unsigned long end)
+{
+ long i = pud_index(address);
+
+ pud = pud + i;
+
+ for (; i < PTRS_PER_PUD; pud++, i++) {
+ unsigned long paddr, pmd_phys;
+ pmd_t *pmd;
+
+ paddr = (address & PGDIR_MASK) + i*PUD_SIZE;
+ if (paddr >= end)
+ break;
+
+ if (!e820_any_mapped(paddr, paddr+PUD_SIZE, 0)) {
+ set_pud(pud, __pud(0));
+ continue;
+ }
+
+ pmd = alloc_low_page(&pmd_phys);
+ set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE));
+ phys_pmd_init(pmd, paddr, end);
+ unmap_low_page(pmd);
+ }
+ __flush_tlb();
+}
+
+/**
+ * Setup the initial kernel page tables, These map all of physical memory
+ * (0 to the top of physical memory) starting at virtual address PAGE_OFFSET.
+ * This runs before bootmem is initialized and therefore has to get pages
+ * directly from physical memory.
+ */
+void __init
+init_kernel_pgtables(unsigned long start, unsigned long end)
+{
+ unsigned long next;
+
+ /*
+ * Find a contiguous region of memory large enough to hold the
+ * kernel page tables.
+ */
+ find_early_table_space(end);
+
+ /*
+ * Calculate the start and end kernel virtual addresses
+ * corresponding to the input physical address range.
+ */
+ start = (unsigned long)__va(start);
+ end = (unsigned long)__va(end);
+
+ for (; start < end; start = next) {
+ unsigned long pud_phys;
+ pud_t *pud;
+
+ /*
+ * Allocate a new page for the Page Upper Directory.
+ *
+ * pud = kernel virtual address where the new
+ * page can be accessed.
+ * pud_phys = physical address of the new page.
+ * map = cookie needed to free the temporary mapping.
+ */
+ pud = alloc_low_page(&pud_phys);
+
+ /*
+ * Calculate the upper bound address for the PUD.
+ * The PUD maps [start, next).
+ */
+ next = start + PGDIR_SIZE;
+ if (next > end)
+ next = end;
+
+ /*
+ * Initialize the new PUD.
+ * phys_pud_init internally calls phys_pmd_init for
+ * each entry in the PUD.
+ */
+ phys_pud_init(pud, __pa(start), __pa(next));
+
+ /*
+ * Point the Page Global Directory at the new PUD.
+ * The PGD is the root of the page tables.
+ */
+ set_pgd(pgd_offset_k(start), mk_kernel_pgd(pud_phys));
+
+ /* Destroy the temporary kernel mapping of the new PUD */
+ unmap_low_page(pud);
+ }
+
+ asm volatile("movq %%cr4,%0" : "=r" (mmu_cr4_features));
+ __flush_tlb_all();
+
+ printk(KERN_DEBUG
+ "Allocated %lu KB for kernel page tables [0x%lx - 0x%lx)\n",
+ ((table_end - table_start) << PAGE_SHIFT) / 1024,
+ table_start << PAGE_SHIFT,
+ table_end << PAGE_SHIFT);
+}
+
+/**
+ * This performs architecture-specific memory subsystem initialization. It is
+ * called from the platform-independent memsys_init(). For x86_64, the only
+ * thing that needs to be done is to relocate the initrd image to user memory.
+ */
+void __init
+arch_memsys_init(size_t kmem_size)
+{
+ struct pmem_region query, result;
+ size_t initrd_size, umem_size;
+
+ if (!initrd_start)
+ return;
+
+ initrd_size = initrd_end - initrd_start;
+ umem_size = round_up(initrd_size, PAGE_SIZE);
+
+ /* Relocate the initrd image to an unallocated chunk of umem */
+ if (pmem_alloc_umem(umem_size, PAGE_SIZE, &result))
+ panic("Failed to allocate umem for initrd image.");
+ result.type = PMEM_TYPE_INITRD;
+ pmem_update(&result);
+ memmove(__va(result.start), __va(initrd_start), initrd_size);
+
+ /* Free the memory used by the old initrd location */
+ pmem_region_unset_all(&query);
+ query.start = round_down( initrd_start, PAGE_SIZE );
+ query.end = round_up( initrd_end, PAGE_SIZE );
+ while (pmem_query(&query, &result) == 0) {
+ result.type = (result.start < kmem_size) ? PMEM_TYPE_KMEM
+ : PMEM_TYPE_UMEM;
+ result.allocated = false;
+ BUG_ON(pmem_update(&result));
+ query.start = result.end;
+ }
+
+ /* Update initrd_start and initrd_end to their new values */
+ initrd_start = result.start;
+ initrd_end = initrd_start + initrd_size;
+}
+
