1 #include <lwk/kernel.h>
2 #include <lwk/bootmem.h>
3 #include <lwk/string.h>
6 #include <arch/pgtable.h>
8 #include <arch/tlbflush.h>
9 #include <arch/proto.h>
12 * Start and end page frame numbers of the kernel page tables.
14 unsigned long __initdata table_start, table_end; /* page frame numbers */
16 static __init void *early_ioremap(unsigned long addr, unsigned long size)
19 pmd_t *pmd, *last_pmd;
22 pmds = ((addr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
23 vaddr = __START_KERNEL_map;
24 pmd = level2_kernel_pgt;
25 last_pmd = level2_kernel_pgt + PTRS_PER_PMD - 1;
26 for (; pmd <= last_pmd; pmd++, vaddr += PMD_SIZE) {
27 for (i = 0; i < pmds; i++) {
28 if (pmd_present(pmd[i]))
31 vaddr += addr & ~PMD_MASK;
33 for (i = 0; i < pmds; i++, addr += PMD_SIZE)
34 set_pmd(pmd + i,__pmd(addr | _KERNPG_TABLE | _PAGE_PSE));
40 printk("early_ioremap(0x%lx, %lu) failed\n", addr, size);
44 /* To avoid virtual aliases later */
45 static __init void early_iounmap(void *addr, unsigned long size)
51 vaddr = (unsigned long)addr;
52 pmds = ((vaddr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
53 pmd = level2_kernel_pgt + pmd_index(vaddr);
54 for (i = 0; i < pmds; i++)
59 static __init void *alloc_low_page(unsigned long *phys)
61 unsigned long pfn = table_end++;
65 panic("alloc_low_page: ran out of memory");
67 adr = early_ioremap(pfn * PAGE_SIZE, PAGE_SIZE);
68 memset(adr, 0, PAGE_SIZE);
69 *phys = pfn * PAGE_SIZE;
74 * Destroys a temporary mapping that was setup by alloc_low_page().
76 static void __init unmap_low_page(void *adr)
78 early_iounmap(adr, PAGE_SIZE);
82 * Initializes a fixmap entry to point to a given physical page.
86 enum fixed_addresses fixmap_index, /* fixmap entry index to setup */
87 unsigned long phys_addr, /* map fixmap entry to this addr */
88 pgprot_t prot /* page protection bits */
91 unsigned long virt_addr;
97 if (fixmap_index >= __end_of_fixed_addresses)
98 panic("Invalid FIXMAP index");
100 /* Calculate the virtual address of the fixmap entry */
101 virt_addr = __fix_to_virt(fixmap_index);
103 /* Look up PGD entry covering the fixmap entry */
104 pgd = pgd_offset_k(virt_addr);
106 panic("PGD FIXMAP MISSING, it should be setup in head.S!\n");
108 /* Look up the PMD entry covering the fixmap entry */
109 pud = pud_offset(pgd, virt_addr);
110 if (pud_none(*pud)) {
111 /* PUD entry is empty... allocate a new PMD directory for it */
112 pmd = (pmd_t *) alloc_bootmem_aligned(PAGE_SIZE, PAGE_SIZE);
113 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
114 BUG_ON(pmd != pmd_offset(pud, 0));
117 /* Look up the PMD entry covering the fixmap entry */
118 pmd = pmd_offset(pud, virt_addr);
119 if (pmd_none(*pmd)) {
120 /* PMD entry is empty... allocate a new PTE directory for it */
121 pte = (pte_t *) alloc_bootmem_aligned(PAGE_SIZE, PAGE_SIZE);
122 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE | _PAGE_USER));
123 BUG_ON(pte != pte_offset_kernel(pmd, 0));
127 * Construct and install a new PTE that maps the fixmap entry
128 * to the requested physical address.
130 pte = pte_offset_kernel(pmd, virt_addr);
131 new_pte = pfn_pte(phys_addr >> PAGE_SHIFT, prot);
132 set_pte(pte, new_pte);
133 __flush_tlb_one(virt_addr);
137 * Finds enough space for the kernel page tables.
140 find_early_table_space(unsigned long end)
142 unsigned long puds; /* # of pud page tables needed */
143 unsigned long pmds; /* # of pmd page tables needed */
144 unsigned long tables; /* page table memory needed, in bytes */
145 unsigned long start; /* start address for kernel page tables */
148 * The kernel page tables map memory using 2 MB pages.
149 * This means only puds and pmds are needed.
151 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
152 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
153 tables = round_up(puds * sizeof(pud_t), PAGE_SIZE) +
154 round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
157 * Consult the memory map to find a region of suitable size.
160 table_start = find_e820_area(start, end, tables);
161 if (table_start == -1UL)
162 panic("Cannot find space for the kernel page tables");
165 * Store table_start and table_end as page frame numbers.
166 * table_end starts out as the same as table_start.
167 * It will be incremented for each page table allocated.
169 table_start >>= PAGE_SHIFT;
170 table_end = table_start;
175 * Configures the input Page Middle Directory to map physical addresses
176 * [address, end). PMD entries outside of this range are zeroed.
178 * Each PMD table maps 1 GB of memory (512 entries, each mapping 2 MB).
181 phys_pmd_init(pmd_t *pmd, unsigned long address, unsigned long end)
185 for (i = 0; i < PTRS_PER_PMD; pmd++, i++, address += PMD_SIZE) {
189 for (; i < PTRS_PER_PMD; i++, pmd++)
190 set_pmd(pmd, __pmd(0));
193 entry = _PAGE_NX|_PAGE_PSE|_KERNPG_TABLE|_PAGE_GLOBAL|address;
194 entry &= __supported_pte_mask;
195 set_pmd(pmd, __pmd(entry));
200 * Configures the input Page Upper Directory to map physical addresses
201 * [address, end). PUD entries outside of this range are zeroed.
203 * Each PUD table maps 512 GB of memory (512 entries, each pointing to a PMD).
206 phys_pud_init(pud_t *pud, unsigned long address, unsigned long end)
208 long i = pud_index(address);
212 for (; i < PTRS_PER_PUD; pud++, i++) {
213 unsigned long paddr, pmd_phys;
216 paddr = (address & PGDIR_MASK) + i*PUD_SIZE;
220 if (!e820_any_mapped(paddr, paddr+PUD_SIZE, 0)) {
221 set_pud(pud, __pud(0));
225 pmd = alloc_low_page(&pmd_phys);
226 set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE));
227 phys_pmd_init(pmd, paddr, end);
234 * Setup the initial kernel page tables, These map all of physical memory
235 * (0 to the top of physical memory) starting at virtual address PAGE_OFFSET.
236 * This runs before bootmem is initialized and therefore has to get pages
237 * directly from physical memory.
240 init_kernel_pgtables(unsigned long start, unsigned long end)
245 * Find a contiguous region of memory large enough to hold the
246 * kernel page tables.
248 find_early_table_space(end);
251 * Calculate the start and end kernel virtual addresses
252 * corresponding to the input physical address range.
254 start = (unsigned long)__va(start);
255 end = (unsigned long)__va(end);
257 for (; start < end; start = next) {
258 unsigned long pud_phys;
262 * Allocate a new page for the Page Upper Directory.
264 * pud = kernel virtual address where the new
265 * page can be accessed.
266 * pud_phys = physical address of the new page.
267 * map = cookie needed to free the temporary mapping.
269 pud = alloc_low_page(&pud_phys);
272 * Calculate the upper bound address for the PUD.
273 * The PUD maps [start, next).
275 next = start + PGDIR_SIZE;
280 * Initialize the new PUD.
281 * phys_pud_init internally calls phys_pmd_init for
282 * each entry in the PUD.
284 phys_pud_init(pud, __pa(start), __pa(next));
287 * Point the Page Global Directory at the new PUD.
288 * The PGD is the root of the page tables.
290 set_pgd(pgd_offset_k(start), mk_kernel_pgd(pud_phys));
292 /* Destroy the temporary kernel mapping of the new PUD */
296 asm volatile("movq %%cr4,%0" : "=r" (mmu_cr4_features));
300 "Allocated %lu KB for kernel page tables [0x%lx - 0x%lx)\n",
301 ((table_end - table_start) << PAGE_SHIFT) / 1024,
302 table_start << PAGE_SHIFT,
303 table_end << PAGE_SHIFT);
307 * This performs architecture-specific memory subsystem initialization. It is
308 * called from the platform-independent memsys_init(). For x86_64, the only
309 * thing that needs to be done is to relocate the initrd image to user memory.
312 arch_memsys_init(size_t kmem_size)
314 struct pmem_region query, result;
315 size_t initrd_size, umem_size;
320 initrd_size = initrd_end - initrd_start;
321 umem_size = round_up(initrd_size, PAGE_SIZE);
323 /* Relocate the initrd image to an unallocated chunk of umem */
324 if (pmem_alloc_umem(umem_size, PAGE_SIZE, &result))
325 panic("Failed to allocate umem for initrd image.");
326 result.type = PMEM_TYPE_INITRD;
327 pmem_update(&result);
328 memmove(__va(result.start), __va(initrd_start), initrd_size);
330 /* Free the memory used by the old initrd location */
331 pmem_region_unset_all(&query);
332 query.start = round_down( initrd_start, PAGE_SIZE );
333 query.end = round_up( initrd_end, PAGE_SIZE );
334 while (pmem_query(&query, &result) == 0) {
335 result.type = (result.start < kmem_size) ? PMEM_TYPE_KMEM
337 result.allocated = false;
338 BUG_ON(pmem_update(&result));
339 query.start = result.end;
342 /* Update initrd_start and initrd_end to their new values */
343 initrd_start = result.start;
344 initrd_end = initrd_start + initrd_size;