palacios/src/palacios/mmu/vmm_shdw_pg_tlb_64.h

   1 /*
   2  * This file is part of the Palacios Virtual Machine Monitor developed
   3  * by the V3VEE Project with funding from the United States National
   4  * Science Foundation and the Department of Energy.
   5  *
   6  * The V3VEE Project is a joint project between Northwestern University
   7  * and the University of New Mexico.  You can find out more at
   8  * http://www.v3vee.org
   9  *
  10  * Copyright (c) 2008, Jack Lange <jarusl@cs.northwestern.edu>
  11  * Copyright (c) 2008, The V3VEE Project <http://www.v3vee.org>
  12  * All rights reserved.
  13  *
  14  * Author: Jack Lange <jarusl@cs.northwestern.edu>
  15  *
  16  * This is free software.  You are permitted to use,
  17  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
  18  */
  19
  20 #ifdef V3_CONFIG_TM_FUNC
  21 #include <extensions/trans_mem.h>
  22 #endif
  23
  24 static inline int activate_shadow_pt_64(struct guest_info * info) {
  25     struct cr3_64 * shadow_cr3 = (struct cr3_64 *)&(info->ctrl_regs.cr3);
  26     struct cr3_64 * guest_cr3 = (struct cr3_64 *)&(info->shdw_pg_state.guest_cr3);
  27     struct shadow_page_data * shadow_pt = create_new_shadow_pt(info);
  28     addr_t shadow_pt_addr = shadow_pt->page_pa;
  29
  30     // Because this is a new CR3 load the allocated page is the new CR3 value
  31     shadow_pt->cr3 = shadow_pt->page_pa;
  32
  33     PrintDebug(info->vm_info, info, "Top level Shadow page pa=%p\n", (void *)shadow_pt_addr);
  34
  35     shadow_cr3->pml4t_base_addr = PAGE_BASE_ADDR_4KB(shadow_pt_addr);
  36     PrintDebug(info->vm_info, info, "Creating new 64 bit shadow page table %p\n", (void *)BASE_TO_PAGE_ADDR(shadow_cr3->pml4t_base_addr));
  37
  38
  39     shadow_cr3->pwt = guest_cr3->pwt;
  40     shadow_cr3->pcd = guest_cr3->pcd;
  41
  42     return 0;
  43 }
  44
  45
  46
  47
  48
  49
  50 /*
  51  * *
  52  * *
  53  * * 64 bit Page table fault handlers
  54  * *
  55  * *
  56  */
  57
  58 static int handle_2MB_shadow_pagefault_pde_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
  59                                               pt_access_status_t shadow_pde_access, pde64_2MB_t * shadow_pt,
  60                                               pde64_2MB_t * large_guest_pde);
  61 static int handle_2MB_shadow_pagefault_pte_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
  62                                               pte64_t * shadow_pt, pde64_2MB_t * large_guest_pde);
  63
  64 static int handle_pte_shadow_pagefault_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
  65                                           pte64_t * shadow_pt, pte64_t * guest_pt);
  66
  67 static int handle_pde_shadow_pagefault_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
  68                                           pde64_t * shadow_pd, pde64_t * guest_pd);
  69
  70 static int handle_pdpe_shadow_pagefault_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
  71                                            pdpe64_t * shadow_pdp, pdpe64_t * guest_pdp);
  72
  73
  74 static inline int handle_shadow_pagefault_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code) {
  75     pml4e64_t * guest_pml = NULL;
  76     pml4e64_t * shadow_pml = CR3_TO_PML4E64_VA(info->ctrl_regs.cr3);
  77     addr_t guest_cr3 = CR3_TO_PML4E64_PA(info->shdw_pg_state.guest_cr3);
  78     pt_access_status_t guest_pml4e_access;
  79     pt_access_status_t shadow_pml4e_access;
  80     pml4e64_t * guest_pml4e = NULL;
  81     pml4e64_t * shadow_pml4e = (pml4e64_t *)&(shadow_pml[PML4E64_INDEX(fault_addr)]);
  82
  83     PrintDebug(info->vm_info, info, "64 bit Shadow page fault handler: %p\n", (void *)fault_addr);
  84     PrintDebug(info->vm_info, info, "Handling PML fault\n");
  85
  86     if (v3_gpa_to_hva(info, guest_cr3, (addr_t*)&guest_pml) == -1) {
  87         PrintError(info->vm_info, info, "Invalid Guest PML4E Address: 0x%p\n",  (void *)guest_cr3);
  88         return -1;
  89     }
  90
  91     guest_pml4e = (pml4e64_t *)&(guest_pml[PML4E64_INDEX(fault_addr)]);
  92
  93     PrintDebug(info->vm_info, info, "Checking Guest %p\n", (void *)guest_pml);
  94     // Check the guest page permissions
  95     guest_pml4e_access = v3_can_access_pml4e64(guest_pml, fault_addr, error_code);
  96
  97     PrintDebug(info->vm_info, info, "Checking shadow %p\n", (void *)shadow_pml);
  98     // Check the shadow page permissions
  99     shadow_pml4e_access = v3_can_access_pml4e64(shadow_pml, fault_addr, error_code);
 100
 101     /* Was the page fault caused by the Guest's page tables? */
 102     if (v3_is_guest_pf(guest_pml4e_access, shadow_pml4e_access) == 1) {
 103         PrintDebug(info->vm_info, info, "Injecting PML4E pf to guest: (guest access error=%d) (pf error code=%d)\n",
 104                    *(uint_t *)&guest_pml4e_access, *(uint_t *)&error_code);
 105         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 106             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 107             return -1;
 108         }
 109         return 0;
 110     }
 111
 112     if (shadow_pml4e_access == PT_ACCESS_USER_ERROR) {
 113         //
 114         // PML4 Entry marked non-user
 115         //
 116         PrintDebug(info->vm_info, info, "Shadow Paging User access error (shadow_pml4e_access=0x%x, guest_pml4e_access=0x%x)\n",
 117                    shadow_pml4e_access, guest_pml4e_access);
 118         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 119             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 120             return -1;
 121         }
 122         return 0;
 123     } else if ((shadow_pml4e_access != PT_ACCESS_NOT_PRESENT) &&
 124                (shadow_pml4e_access != PT_ACCESS_OK)) {
 125         // inject page fault in guest
 126         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 127             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 128             return -1;
 129         }
 130         PrintDebug(info->vm_info, info, "Unknown Error occurred (shadow_pde_access=%d)\n", shadow_pml4e_access);
 131         PrintDebug(info->vm_info, info, "Manual Says to inject page fault into guest\n");
 132         return 0;
 133     }
 134
 135
 136     pdpe64_t * shadow_pdp = NULL;
 137     pdpe64_t * guest_pdp = NULL;
 138
 139     // Get the next shadow page level, allocate if not present
 140
 141     if (shadow_pml4e_access == PT_ACCESS_NOT_PRESENT) {
 142         struct shadow_page_data * shdw_page =  create_new_shadow_pt(info);
 143         shadow_pdp = (pdpe64_t *)V3_VAddr((void *)shdw_page->page_pa);
 144
 145
 146         shadow_pml4e->present = 1;
 147         shadow_pml4e->user_page = guest_pml4e->user_page;
 148         shadow_pml4e->writable = guest_pml4e->writable;
 149         shadow_pml4e->cache_disable = guest_pml4e->cache_disable;
 150         shadow_pml4e->write_through = guest_pml4e->write_through;
 151
 152         guest_pml4e->accessed = 1;
 153
 154         shadow_pml4e->pdp_base_addr = PAGE_BASE_ADDR(shdw_page->page_pa);
 155     } else {
 156         shadow_pdp = (pdpe64_t *)V3_VAddr((void *)(addr_t)BASE_TO_PAGE_ADDR(shadow_pml4e->pdp_base_addr));
 157     }
 158
 159     // Continue processing at the next level
 160
 161     if (v3_gpa_to_hva(info, BASE_TO_PAGE_ADDR(guest_pml4e->pdp_base_addr), (addr_t *)&guest_pdp) == -1) {
 162         // Machine check the guest
 163         PrintError(info->vm_info, info, "Invalid Guest PDP Address: 0x%p\n", (void *)BASE_TO_PAGE_ADDR(guest_pml4e->pdp_base_addr));
 164         v3_raise_exception(info, MC_EXCEPTION);
 165         return 0;
 166     }
 167
 168     if (handle_pdpe_shadow_pagefault_64(info, fault_addr, error_code, shadow_pdp, guest_pdp) == -1) {
 169         PrintError(info->vm_info, info, "Error handling Page fault caused by PDPE\n");
 170         return -1;
 171     }
 172
 173     return 0;
 174 }
 175
 176
 177
 178 // For now we are not going to handle 1 Gigabyte pages
 179 static int handle_pdpe_shadow_pagefault_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
 180                                            pdpe64_t * shadow_pdp, pdpe64_t * guest_pdp) {
 181     pt_access_status_t guest_pdpe_access;
 182     pt_access_status_t shadow_pdpe_access;
 183     pdpe64_t * guest_pdpe = (pdpe64_t *)&(guest_pdp[PDPE64_INDEX(fault_addr)]);
 184     pdpe64_t * shadow_pdpe = (pdpe64_t *)&(shadow_pdp[PDPE64_INDEX(fault_addr)]);
 185
 186     PrintDebug(info->vm_info, info, "Handling PDP fault\n");
 187
 188     if (fault_addr==0) {
 189         PrintDebug(info->vm_info, info, "Guest Page Tree for guest virtual address zero fault\n");
 190         PrintGuestPageTree(info,fault_addr,(addr_t)(info->shdw_pg_state.guest_cr3));
 191         PrintDebug(info->vm_info, info, "Host Page Tree for guest virtual address zero fault\n");
 192         PrintHostPageTree(info,fault_addr,(addr_t)(info->ctrl_regs.cr3));
 193     }
 194
 195     // Check the guest page permissions
 196     guest_pdpe_access = v3_can_access_pdpe64(guest_pdp, fault_addr, error_code);
 197
 198     // Check the shadow page permissions
 199     shadow_pdpe_access = v3_can_access_pdpe64(shadow_pdp, fault_addr, error_code);
 200
 201     /* Was the page fault caused by the Guest's page tables? */
 202     if (v3_is_guest_pf(guest_pdpe_access, shadow_pdpe_access) == 1) {
 203         PrintDebug(info->vm_info, info, "Injecting PDPE pf to guest: (guest access error=%d) (pf error code=%d)\n",
 204                    *(uint_t *)&guest_pdpe_access, *(uint_t *)&error_code);
 205         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 206             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 207             return -1;
 208         }
 209         return 0;
 210     }
 211
 212     if (shadow_pdpe_access == PT_ACCESS_USER_ERROR) {
 213         //
 214         // PML4 Entry marked non-user
 215         //
 216         PrintDebug(info->vm_info, info, "Shadow Paging User access error (shadow_pdpe_access=0x%x, guest_pdpe_access=0x%x)\n",
 217                    shadow_pdpe_access, guest_pdpe_access);
 218         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 219             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 220             return -1;
 221         }
 222         return 0;
 223     } else if ((shadow_pdpe_access != PT_ACCESS_NOT_PRESENT) &&
 224                (shadow_pdpe_access != PT_ACCESS_OK)) {
 225         // inject page fault in guest
 226         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 227             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 228             return -1;
 229         }
 230         PrintDebug(info->vm_info, info, "Unknown Error occurred (shadow_pde_access=%d)\n", shadow_pdpe_access);
 231         PrintDebug(info->vm_info, info, "Manual Says to inject page fault into guest\n");
 232         return 0;
 233     }
 234
 235
 236     pde64_t * shadow_pd = NULL;
 237     pde64_t * guest_pd = NULL;
 238
 239     // Get the next shadow page level, allocate if not present
 240
 241     if (shadow_pdpe_access == PT_ACCESS_NOT_PRESENT) {
 242         struct shadow_page_data * shdw_page = create_new_shadow_pt(info);
 243         shadow_pd = (pde64_t *)V3_VAddr((void *)shdw_page->page_pa);
 244
 245
 246         shadow_pdpe->present = 1;
 247         shadow_pdpe->user_page = guest_pdpe->user_page;
 248         shadow_pdpe->writable = guest_pdpe->writable;
 249         shadow_pdpe->write_through = guest_pdpe->write_through;
 250         shadow_pdpe->cache_disable = guest_pdpe->cache_disable;
 251
 252
 253         guest_pdpe->accessed = 1;
 254
 255         shadow_pdpe->pd_base_addr = PAGE_BASE_ADDR(shdw_page->page_pa);
 256     } else {
 257         shadow_pd = (pde64_t *)V3_VAddr((void *)(addr_t)BASE_TO_PAGE_ADDR(shadow_pdpe->pd_base_addr));
 258     }
 259
 260     // Continue processing at the next level
 261
 262     if (v3_gpa_to_hva(info, BASE_TO_PAGE_ADDR(guest_pdpe->pd_base_addr), (addr_t *)&guest_pd) == -1) {
 263         // Machine check the guest
 264         PrintError(info->vm_info, info, "Invalid Guest PTE Address: 0x%p\n", (void *)BASE_TO_PAGE_ADDR(guest_pdpe->pd_base_addr));
 265         v3_raise_exception(info, MC_EXCEPTION);
 266         return 0;
 267     }
 268
 269     if (handle_pde_shadow_pagefault_64(info, fault_addr, error_code, shadow_pd, guest_pd) == -1) {
 270         PrintError(info->vm_info, info, "Error handling Page fault caused by PDE\n");
 271         return -1;
 272     }
 273
 274     return 0;
 275 }
 276
 277 static int handle_pde_shadow_pagefault_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
 278                                           pde64_t * shadow_pd, pde64_t * guest_pd) {
 279     pt_access_status_t guest_pde_access;
 280     pt_access_status_t shadow_pde_access;
 281     pde64_t * guest_pde = (pde64_t *)&(guest_pd[PDE64_INDEX(fault_addr)]);
 282     pde64_t * shadow_pde = (pde64_t *)&(shadow_pd[PDE64_INDEX(fault_addr)]);
 283
 284     PrintDebug(info->vm_info, info, "Handling PDE fault\n");
 285
 286     // Check the guest page permissions
 287     guest_pde_access = v3_can_access_pde64(guest_pd, fault_addr, error_code);
 288
 289     // Check the shadow page permissions
 290     shadow_pde_access = v3_can_access_pde64(shadow_pd, fault_addr, error_code);
 291
 292     /* Was the page fault caused by the Guest's page tables? */
 293     if (v3_is_guest_pf(guest_pde_access, shadow_pde_access) == 1) {
 294         PrintDebug(info->vm_info, info, "Injecting PDE pf to guest: (guest access error=%d) (pf error code=%d)\n",
 295                    *(uint_t *)&guest_pde_access, *(uint_t *)&error_code);
 296         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 297             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 298             return -1;
 299         }
 300         return 0;
 301     }
 302
 303     if (shadow_pde_access == PT_ACCESS_USER_ERROR) {
 304         //
 305         // PDE Entry marked non-user
 306         //
 307         PrintDebug(info->vm_info, info, "Shadow Paging User access error (shadow_pdpe_access=0x%x, guest_pdpe_access=0x%x)\n",
 308                    shadow_pde_access, guest_pde_access);
 309         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 310             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 311             return -1;
 312         }
 313         return 0;
 314
 315     } else if ((shadow_pde_access == PT_ACCESS_WRITE_ERROR) &&
 316                (guest_pde->large_page == 1)) {
 317
 318         ((pde64_2MB_t *)guest_pde)->dirty = 1;
 319         shadow_pde->writable = guest_pde->writable;
 320
 321         //PrintDebug(info->vm_info, info, "Returning due to large page Write Error\n");
 322         //PrintHostPageTree(info, fault_addr, info->ctrl_regs.cr3);
 323
 324         return 0;
 325     } else if ((shadow_pde_access != PT_ACCESS_NOT_PRESENT) &&
 326                (shadow_pde_access != PT_ACCESS_OK)) {
 327         // inject page fault in guest
 328         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 329             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 330             return -1;
 331         }
 332         PrintDebug(info->vm_info, info, "Unknown Error occurred (shadow_pde_access=%d)\n", shadow_pde_access);
 333         PrintDebug(info->vm_info, info, "Manual Says to inject page fault into guest\n");
 334         return 0;
 335     }
 336
 337     pte64_t * shadow_pt = NULL;
 338     pte64_t * guest_pt = NULL;
 339
 340     // get the next shadow page level, allocate if not present
 341     if (shadow_pde_access == PT_ACCESS_NOT_PRESENT) {
 342         // Check if  we can use large pages and the guest memory is properly aligned
 343         // to potentially use a large page
 344
 345         if ((info->use_large_pages == 1) && (guest_pde->large_page == 1)) {
 346             addr_t guest_pa = BASE_TO_PAGE_ADDR_2MB(((pde64_2MB_t *)guest_pde)->page_base_addr);
 347             uint32_t page_size = v3_get_max_page_size(info, guest_pa, LONG);
 348
 349             if (page_size == PAGE_SIZE_2MB) {
 350                 if (handle_2MB_shadow_pagefault_pde_64(info, fault_addr, error_code, shadow_pde_access,
 351                                                        (pde64_2MB_t *)shadow_pde, (pde64_2MB_t *)guest_pde) == -1) {
 352                     PrintError(info->vm_info, info, "Error handling large pagefault with large page\n");
 353                     return -1;
 354                 }
 355
 356                 return 0;
 357             }
 358             // Fallthrough to handle the region with small pages
 359         }
 360
 361         struct shadow_page_data * shdw_page = create_new_shadow_pt(info);
 362         shadow_pt = (pte64_t *)V3_VAddr((void *)shdw_page->page_pa);
 363
 364         PrintDebug(info->vm_info, info, "Creating new shadow PT: %p\n", shadow_pt);
 365
 366         shadow_pde->present = 1;
 367         shadow_pde->user_page = guest_pde->user_page;
 368
 369         if (guest_pde->large_page == 0) {
 370             shadow_pde->writable = guest_pde->writable;
 371         } else {
 372             // This large page flag is temporary until we can get a working cache....
 373             ((pde64_2MB_t *)guest_pde)->vmm_info = V3_LARGE_PG;
 374
 375             if (error_code.write) {
 376                 shadow_pde->writable = guest_pde->writable;
 377                 ((pde64_2MB_t *)guest_pde)->dirty = 1;
 378             } else {
 379                 shadow_pde->writable = 0;
 380                 ((pde64_2MB_t *)guest_pde)->dirty = 0;
 381             }
 382         }
 383
 384         // VMM Specific options
 385         shadow_pde->write_through = guest_pde->write_through;
 386         shadow_pde->cache_disable = guest_pde->cache_disable;
 387         shadow_pde->global_page = guest_pde->global_page;
 388         //
 389
 390         guest_pde->accessed = 1;
 391
 392         shadow_pde->pt_base_addr = PAGE_BASE_ADDR(shdw_page->page_pa);
 393     } else { // when PT_ACCESS_OK is true
 394         if ((info->use_large_pages == 1) && (guest_pde->large_page == 1) ) {
 395             addr_t guest_pa = BASE_TO_PAGE_ADDR_4MB(((pde32_4MB_t *)guest_pde)->page_base_addr);
 396             uint32_t page_size = v3_get_max_page_size(info, guest_pa, LONG);
 397             if (page_size == PAGE_SIZE_2MB)   {
 398                 if (shadow_pde_access == PT_ACCESS_OK) {
 399                     // Inconsistent state...
 400                     // Guest Re-Entry will flush tables and everything should now workd
 401                     PrintDebug(info->vm_info, info, "Inconsistent state PDE... Guest re-entry should flush tlb\n");
 402                     //PrintDebug(info->vm_info, info, "Bug here: shadow_pde_access is %d page_size is %d\n",
 403                     //     (uint_t)shadow_pde_access,(uint_t)page_size);
 404                     return 0;
 405                 }
 406             }
 407         }
 408         shadow_pt = (pte64_t *)V3_VAddr((void *)BASE_TO_PAGE_ADDR(shadow_pde->pt_base_addr));
 409     }
 410
 411     // Continue processing at the next level
 412     if (guest_pde->large_page == 0) {
 413         if (v3_gpa_to_hva(info, BASE_TO_PAGE_ADDR(guest_pde->pt_base_addr), (addr_t *)&guest_pt) == -1) {
 414             // Machine check the guest
 415             PrintError(info->vm_info, info, "Invalid Guest PTE Address: 0x%p\n", (void *)BASE_TO_PAGE_ADDR(guest_pde->pt_base_addr));
 416             v3_raise_exception(info, MC_EXCEPTION);
 417             return 0;
 418         }
 419
 420         if (handle_pte_shadow_pagefault_64(info, fault_addr, error_code, shadow_pt, guest_pt) == -1) {
 421             PrintError(info->vm_info, info, "Error handling Page fault caused by PDE\n");
 422             return -1;
 423         }
 424     } else {
 425         if (handle_2MB_shadow_pagefault_pte_64(info, fault_addr, error_code, shadow_pt, (pde64_2MB_t *)guest_pde) == -1) {
 426             PrintError(info->vm_info, info, "Error handling large pagefault with small page\n");
 427             return -1;
 428         }
 429     }
 430
 431     return 0;
 432 }
 433
 434
 435 static int handle_pte_shadow_pagefault_64(struct guest_info * info, addr_t fault_addr, pf_error_t error_code,
 436                                           pte64_t * shadow_pt, pte64_t * guest_pt) {
 437     pt_access_status_t guest_pte_access;
 438     pt_access_status_t shadow_pte_access;
 439     pte64_t * guest_pte = (pte64_t *)&(guest_pt[PTE64_INDEX(fault_addr)]);;
 440     pte64_t * shadow_pte = (pte64_t *)&(shadow_pt[PTE64_INDEX(fault_addr)]);
 441     addr_t guest_pa = BASE_TO_PAGE_ADDR((addr_t)(guest_pte->page_base_addr)) +  PAGE_OFFSET(fault_addr);
 442     //  struct shadow_page_state * state = &(info->shdw_pg_state);
 443
 444     PrintDebug(info->vm_info, info, "Handling PTE fault\n");
 445
 446     struct v3_mem_region * shdw_reg =  v3_get_mem_region(info->vm_info, info->vcpu_id, guest_pa);
 447
 448
 449
 450     if (shdw_reg == NULL) {
 451         // Inject a machine check in the guest
 452         PrintError(info->vm_info, info, "Invalid Guest Address in page table (0x%p)\n", (void *)guest_pa);
 453         v3_raise_exception(info, MC_EXCEPTION);
 454         return 0;
 455     }
 456
 457     // Check the guest page permissions
 458     guest_pte_access = v3_can_access_pte64(guest_pt, fault_addr, error_code);
 459
 460     // Check the shadow page permissions
 461     shadow_pte_access = v3_can_access_pte64(shadow_pt, fault_addr, error_code);
 462
 463     /* Was the page fault caused by the Guest's page tables? */
 464     if (v3_is_guest_pf(guest_pte_access, shadow_pte_access) == 1) {
 465
 466         PrintDebug(info->vm_info, info, "Access error injecting pf to guest (guest access error=%d) (pf error code=%d)\n",
 467                    guest_pte_access, *(uint_t*)&error_code);
 468
 469         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 470             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 471             return -1;
 472         }
 473
 474         return 0;
 475     }
 476
 477
 478     if (shadow_pte_access == PT_ACCESS_OK) {
 479         // Inconsistent state...
 480         // Guest Re-Entry will flush page tables and everything should now work
 481         PrintDebug(info->vm_info, info, "Inconsistent state... Guest re-entry should flush tlb\n");
 482         return 0;
 483     }
 484
 485     if (shadow_pte_access == PT_ACCESS_NOT_PRESENT) {
 486         // Page Table Entry Not Present
 487         PrintDebug(info->vm_info, info, "guest_pa =%p\n", (void *)guest_pa);
 488
 489 #ifdef V3_CONFIG_TM_FUNC
 490         addr_t page_to_use;
 491         if (v3_tm_handle_pf_64(info, error_code, fault_addr, &page_to_use) == -1) {
 492             return -1;
 493         }
 494 #endif
 495
 496         if ((shdw_reg->flags.alloced == 1) ||
 497             (shdw_reg->flags.read == 1)) {
 498             addr_t shadow_pa = 0;
 499
 500             if (v3_gpa_to_hpa(info, guest_pa, &shadow_pa) == -1) {
 501                 PrintError(info->vm_info, info, "could not translate page fault address (%p)\n", (void *)guest_pa);
 502                 return -1;
 503             }
 504
 505 #ifdef V3_CONFIG_TM_FUNC
 506             v3_tm_handle_usr_tlb_miss(info, error_code, page_to_use, &shadow_pa);
 507 #endif
 508
 509             shadow_pte->page_base_addr = PAGE_BASE_ADDR(shadow_pa);
 510
 511             shadow_pte->present = guest_pte->present;
 512             shadow_pte->user_page = guest_pte->user_page;
 513
 514             //set according to VMM policy
 515             shadow_pte->write_through = guest_pte->write_through;
 516             shadow_pte->cache_disable = guest_pte->cache_disable;
 517             shadow_pte->global_page = guest_pte->global_page;
 518             //
 519
 520             guest_pte->accessed = 1;
 521
 522             if (guest_pte->dirty == 1) {
 523                 shadow_pte->writable = guest_pte->writable;
 524             } else if ((guest_pte->dirty == 0) && (error_code.write == 1)) {
 525                 shadow_pte->writable = guest_pte->writable;
 526                 guest_pte->dirty = 1;
 527             } else if ((guest_pte->dirty == 0) && (error_code.write == 0)) {
 528                 shadow_pte->writable = 0;
 529             }
 530
 531
 532             // Write hooks trump all, and are set Read Only
 533             if (shdw_reg->flags.write == 0) {
 534                 shadow_pte->writable = 0;
 535             }
 536
 537 #ifdef V3_CONFIG_TM_FUNC
 538         v3_tm_handle_read_fault(info, error_code, shadow_pte);
 539 #endif
 540         } else {
 541             // Pass to unhandled call back
 542             if (shdw_reg->unhandled(info, fault_addr, guest_pa, shdw_reg, error_code) == -1) {
 543                 PrintError(info->vm_info, info, "Special Page fault handler returned error for address: %p\n",  (void *)fault_addr);
 544                 return -1;
 545             }
 546         }
 547     } else if (shadow_pte_access == PT_ACCESS_WRITE_ERROR) {
 548         guest_pte->dirty = 1;
 549
 550
 551         if (shdw_reg->flags.write == 1) {
 552             PrintDebug(info->vm_info, info, "Shadow PTE Write Error\n");
 553             shadow_pte->writable = guest_pte->writable;
 554         } else {
 555             if (shdw_reg->unhandled(info, fault_addr, guest_pa, shdw_reg, error_code) == -1) {
 556                 PrintError(info->vm_info, info, "Special Page fault handler returned error for address: %p\n",  (void *)fault_addr);
 557                 return -1;
 558             }
 559         }
 560
 561
 562
 563         return 0;
 564
 565     } else {
 566         // Inject page fault into the guest
 567         if (v3_inject_guest_pf(info, fault_addr, error_code) == -1) {
 568             PrintError(info->vm_info, info, "Could not inject guest page fault\n");
 569             return -1;
 570         }
 571         PrintError(info->vm_info, info, "PTE Page fault fell through... Not sure if this should ever happen\n");
 572         PrintError(info->vm_info, info, "Manual Says to inject page fault into guest\n");
 573         return -1;
 574     }
 575
 576     return 0;
 577 }
 578
 579
 580 static int handle_2MB_shadow_pagefault_pde_64(struct guest_info * info,
 581                                               addr_t fault_addr, pf_error_t error_code,
 582                                               pt_access_status_t shadow_pde_access,
 583                                               pde64_2MB_t * large_shadow_pde, pde64_2MB_t * large_guest_pde)
 584 {
 585     addr_t guest_fault_pa = BASE_TO_PAGE_ADDR_2MB(large_guest_pde->page_base_addr) + PAGE_OFFSET_2MB(fault_addr);
 586     //  struct shadow_page_state * state = &(info->shdw_pg_state);
 587
 588     PrintDebug(info->vm_info, info, "Handling 2MB fault with large page (guest_fault_pa=%p) (error_code=%x)\n", (void *)guest_fault_pa, *(uint_t*)&error_code);
 589     PrintDebug(info->vm_info, info, "LargeShadowPDE=%p, LargeGuestPDE=%p\n", large_shadow_pde, large_guest_pde);
 590
 591     struct v3_mem_region * shdw_reg = v3_get_mem_region(info->vm_info, info->vcpu_id, guest_fault_pa);
 592
 593     if (shdw_reg == NULL) {
 594         // Inject a machine check in the guest
 595         PrintError(info->vm_info, info, "Invalid Guest Address in page table (0x%p)\n", (void *)guest_fault_pa);
 596         v3_raise_exception(info, MC_EXCEPTION);
 597         return 0;
 598     }
 599
 600     /* Note that this handler is currently invoked only when PT_ACCESS_OK is PT_NOT_PRESENT. */
 601     if (shadow_pde_access == PT_ACCESS_OK) {
 602         // Inconsistent state...
 603         // Guest Re-Entry will flush tables and everything should now workd
 604         PrintDebug(info->vm_info, info, "Inconsistent state... Guest re-entry should flush tlb\n");
 605         PrintError(info->vm_info, info, "Hmm... did not expect call to this handler with PT_ACCESS_OK to happen...\n");
 606         //PrintHostPageTree(info, fault_addr, info->ctrl_regs.cr3);
 607         return 0;
 608     }
 609
 610
 611
 612     if (shadow_pde_access == PT_ACCESS_NOT_PRESENT) {
 613         // Get the guest physical address of the fault
 614
 615         if ((shdw_reg->flags.alloced == 1) ||
 616             (shdw_reg->flags.read == 1)) {
 617             addr_t shadow_pa = 0;
 618
 619             if (v3_gpa_to_hpa(info, guest_fault_pa, &shadow_pa) == -1) {
 620                 PrintError(info->vm_info, info, "could not translate page fault address (%p)\n", (void *)guest_fault_pa);
 621                 return -1;
 622             }
 623
 624             large_guest_pde->vmm_info = V3_LARGE_PG; /* For invalidations */
 625             large_shadow_pde->page_base_addr = PAGE_BASE_ADDR_2MB(shadow_pa);
 626             large_shadow_pde->large_page = 1;
 627             large_shadow_pde->present = 1;
 628             large_shadow_pde->user_page = 1;
 629
 630             if (shdw_reg->flags.write == 0) {
 631                 large_shadow_pde->writable = 0;
 632             } else {
 633                 large_shadow_pde->writable = 1;
 634             }
 635
 636             //set according to VMM policy
 637             large_shadow_pde->write_through = large_guest_pde->write_through;
 638             large_shadow_pde->cache_disable = large_guest_pde->cache_disable;
 639             large_shadow_pde->global_page = large_guest_pde->global_page;
 640             //
 641
 642         } else {
 643             if (shdw_reg->unhandled(info, fault_addr, guest_fault_pa, shdw_reg, error_code) == -1) {
 644                 PrintError(info->vm_info, info, "Special Page Fault handler returned error for address: %p\n", (void *)fault_addr);
 645                 return -1;
 646             }
 647         }
 648     } else if (shadow_pde_access == PT_ACCESS_WRITE_ERROR) {
 649         if (shdw_reg->unhandled(info, fault_addr, guest_fault_pa, shdw_reg, error_code) == -1) {
 650             PrintError(info->vm_info, info, "Special Page Fault handler returned error for address: %p\n", (void *)fault_addr);
 651             return -1;
 652         }
 653     } else {
 654         PrintError(info->vm_info, info, "Error in large page fault handler...\n");
 655         PrintError(info->vm_info, info, "This case should have been handled at the top level handler\n");
 656         return -1;
 657     }
 658
 659     //  PrintHostPageTree(info, fault_addr, info->ctrl_regs.cr3);
 660     PrintDebug(info->vm_info, info, "Returning from large page->large page fault handler\n");
 661     return 0;
 662 }
 663
 664 static int handle_2MB_shadow_pagefault_pte_64(struct guest_info * info,
 665                                               addr_t fault_addr, pf_error_t error_code,
 666                                               pte64_t * shadow_pt, pde64_2MB_t * large_guest_pde)
 667 {
 668     pt_access_status_t shadow_pte_access = v3_can_access_pte64(shadow_pt, fault_addr, error_code);
 669     pte64_t * shadow_pte = (pte64_t *)&(shadow_pt[PTE64_INDEX(fault_addr)]);
 670     addr_t guest_fault_pa = BASE_TO_PAGE_ADDR_2MB(large_guest_pde->page_base_addr) + PAGE_OFFSET_2MB(fault_addr);
 671     //  struct shadow_page_state * state = &(info->shdw_pg_state);
 672
 673     PrintDebug(info->vm_info, info, "Handling 2MB fault (guest_fault_pa=%p) (error_code=%x)\n", (void *)guest_fault_pa, *(uint_t*)&error_code);
 674     PrintDebug(info->vm_info, info, "ShadowPT=%p, LargeGuestPDE=%p\n", shadow_pt, large_guest_pde);
 675
 676     struct v3_mem_region * shdw_reg = v3_get_mem_region(info->vm_info, info->vcpu_id, guest_fault_pa);
 677
 678
 679     if (shdw_reg == NULL) {
 680         // Inject a machine check in the guest
 681         PrintError(info->vm_info, info, "Invalid Guest Address in page table (0x%p)\n", (void *)guest_fault_pa);
 682         v3_raise_exception(info, MC_EXCEPTION);
 683         return 0;
 684     }
 685
 686     if (shadow_pte_access == PT_ACCESS_OK) {
 687         // Inconsistent state...
 688         // Guest Re-Entry will flush tables and everything should now workd
 689         PrintDebug(info->vm_info, info, "Inconsistent state... Guest re-entry should flush tlb\n");
 690         //PrintHostPageTree(info, fault_addr, info->ctrl_regs.cr3);
 691         return 0;
 692     }
 693
 694
 695     if (shadow_pte_access == PT_ACCESS_NOT_PRESENT) {
 696         // Get the guest physical address of the fault
 697
 698         if ((shdw_reg->flags.alloced == 1) ||
 699             (shdw_reg->flags.read == 1)) {
 700             addr_t shadow_pa = 0;
 701
 702             if (v3_gpa_to_hpa(info, guest_fault_pa, &shadow_pa) == -1) {
 703                 PrintError(info->vm_info, info, "could not translate page fault address (%p)\n", (void *)guest_fault_pa);
 704                 return -1;
 705             }
 706
 707             shadow_pte->page_base_addr = PAGE_BASE_ADDR(shadow_pa);
 708
 709             shadow_pte->present = 1;
 710
 711             /* We are assuming that the PDE entry has precedence
 712              * so the Shadow PDE will mirror the guest PDE settings,
 713              * and we don't have to worry about them here
 714              * Allow everything
 715              */
 716             shadow_pte->user_page = 1;
 717
 718             if (shdw_reg->flags.write == 0) {
 719                 shadow_pte->writable = 0;
 720             } else {
 721                 shadow_pte->writable = 1;
 722             }
 723
 724             //set according to VMM policy
 725             shadow_pte->write_through = large_guest_pde->write_through;
 726             shadow_pte->cache_disable = large_guest_pde->cache_disable;
 727             shadow_pte->global_page = large_guest_pde->global_page;
 728             //
 729
 730         } else {
 731             if (shdw_reg->unhandled(info, fault_addr, guest_fault_pa, shdw_reg, error_code) == -1) {
 732                 PrintError(info->vm_info, info, "Special Page Fault handler returned error for address: %p\n", (void *)fault_addr);
 733                 return -1;
 734             }
 735         }
 736     } else if (shadow_pte_access == PT_ACCESS_WRITE_ERROR) {
 737         if (shdw_reg->unhandled(info, fault_addr, guest_fault_pa, shdw_reg, error_code) == -1) {
 738             PrintError(info->vm_info, info, "Special Page Fault handler returned error for address: %p\n", (void *)fault_addr);
 739             return -1;
 740         }
 741     } else {
 742         PrintError(info->vm_info, info, "Error in large page fault handler...\n");
 743         PrintError(info->vm_info, info, "This case should have been handled at the top level handler\n");
 744         return -1;
 745     }
 746
 747     //  PrintHostPageTree(info, fault_addr, info->ctrl_regs.cr3);
 748     PrintDebug(info->vm_info, info, "Returning from large page->small page fault handler\n");
 749     return 0;
 750 }
 751
 752
 753
 754
 755 static int invalidation_cb_64(struct guest_info * info, page_type_t type,
 756                               addr_t vaddr, addr_t page_ptr, addr_t page_pa,
 757                               void * private_data) {
 758
 759     switch (type) {
 760         case PAGE_PML464:
 761             {
 762                 pml4e64_t * pml = (pml4e64_t *)page_ptr;
 763
 764                 if (pml[PML4E64_INDEX(vaddr)].present == 0) {
 765                     return 1;
 766                 }
 767                 return 0;
 768             }
 769         case PAGE_PDP64:
 770             {
 771                 pdpe64_t * pdp = (pdpe64_t *)page_ptr;
 772                 pdpe64_t * pdpe = &(pdp[PDPE64_INDEX(vaddr)]);
 773
 774                 if (pdpe->present == 0) {
 775                     return 1;
 776                 }
 777
 778                 if (pdpe->vmm_info == V3_LARGE_PG) {
 779                     PrintError(info->vm_info, info, "1 Gigabyte pages not supported\n");
 780                     return -1;
 781                 }
 782
 783                 return 0;
 784             }
 785         case PAGE_PD64:
 786             {
 787                 pde64_t * pd = (pde64_t *)page_ptr;
 788                 pde64_t * pde = &(pd[PDE64_INDEX(vaddr)]);
 789
 790                 if (pde->present == 0) {
 791                     return 1;
 792                 }
 793
 794                 if (pde->vmm_info == V3_LARGE_PG) {
 795                     pde->present = 0;
 796                     return 1;
 797                 }
 798
 799                 return 0;
 800             }
 801         case PAGE_PT64:
 802             {
 803                 pte64_t * pt = (pte64_t *)page_ptr;
 804
 805                 pt[PTE64_INDEX(vaddr)].present = 0;
 806
 807                 return 1;
 808             }
 809         default:
 810             PrintError(info->vm_info, info, "Invalid Page Type\n");
 811             return -1;
 812
 813     }
 814
 815     // should not get here
 816     PrintError(info->vm_info, info, "Should not get here....\n");
 817     return -1;
 818 }
 819
 820
 821 static inline int handle_shadow_invlpg_64(struct guest_info * info, addr_t vaddr) {
 822     PrintDebug(info->vm_info, info, "INVLPG64 - %p\n",(void*)vaddr);
 823
 824     int ret =  v3_drill_host_pt_64(info, info->ctrl_regs.cr3, vaddr, invalidation_cb_64, NULL);
 825     if (ret == -1) {
 826         PrintError(info->vm_info, info, "Page table drill returned error.... \n");
 827         PrintHostPageTree(info, vaddr, info->ctrl_regs.cr3);
 828     }
 829
 830     return (ret == -1) ? -1 : 0;
 831 }