2 * This file is part of the Palacios Virtual Machine Monitor developed
3 * by the V3VEE Project with funding from the United States National
4 * Science Foundation and the Department of Energy.
6 * The V3VEE Project is a joint project between Northwestern University
7 * and the University of New Mexico. You can find out more at
10 * Copyright (c) 2008, Jack Lange <jarusl@cs.northwestern.edu>
11 * Copyright (c) 2008, The V3VEE Project <http://www.v3vee.org>
12 * All rights reserved.
14 * Author: Jack Lange <jarusl@cs.northwestern.edu>
16 * This is free software. You are permitted to use,
17 * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
20 #include <palacios/vmm_shadow_paging.h>
21 #include <palacios/vmm_swapbypass.h>
22 #include <palacios/vmm_ctrl_regs.h>
24 #include <palacios/vm_guest.h>
25 #include <palacios/vm_guest_mem.h>
26 #include <palacios/vmm_paging.h>
27 #include <palacios/vmm_hashtable.h>
28 #include <palacios/vmm_list.h>
30 #define DEFAULT_CACHE_SIZE ((32 * 1024 * 1024) / 4096)
32 #define V3_CACHED_PG 0x1
34 #ifndef V3_CONFIG_DEBUG_SHDW_PG_CACHE
36 #define PrintDebug(fmt, ...)
40 struct shdw_back_ptr {
42 struct shdw_pg_data * pg_data;
43 struct list_head back_ptr_node;
46 struct guest_pg_tuple {
49 } __attribute__((packed));
57 struct list_head rmap_node;
61 struct guest_pg_tuple tuple;
66 struct list_head back_ptrs;
67 struct list_head pg_queue_node;
73 struct cache_core_state {
79 struct cache_vm_state {
83 struct hashtable * page_htable; // GPA to shdw_pg_data
84 struct hashtable * reverse_map;
90 struct list_head pg_queue;
93 struct list_head free_list;
98 static inline int evict_pt(void * pt, addr_t va, page_type_t pt_type) {
103 pde[PDE32_INDEX(va)].present = 0;
107 pde32_4MB_t * pde = pt;
108 pde[PDE32_INDEX(va)].present = 0;
113 pte[PTE32_INDEX(va)].present = 0;
117 pml4e64_t * pml = pt;
118 pml[PML4E64_INDEX(va)].present = 0;
123 pdp[PDPE64_INDEX(va)].present = 0;
128 pde[PDE64_INDEX(va)].present = 0;
133 pte[PTE64_INDEX(va)].present = 0;
137 PrintError(VM_NONE, VCORE_NONE, "Invalid page type: %d\n", pt_type);
146 static inline int grab_pt(void * pt, addr_t va, page_type_t pt_type) {
151 pde[PDE32_INDEX(va)].writable = 0;
155 pde32_4MB_t * pde = pt;
156 pde[PDE32_INDEX(va)].writable = 0;
161 pte[PTE32_INDEX(va)].writable = 0;
165 pml4e64_t * pml = pt;
166 pml[PML4E64_INDEX(va)].writable = 0;
171 pdp[PDPE64_INDEX(va)].writable = 0;
176 pde[PDE64_INDEX(va)].writable = 0;
181 pte[PTE64_INDEX(va)].writable = 0;
185 PrintError(VM_NONE, VCORE_NONE, "Invalid page type: %d\n", pt_type);
193 static int unlink_shdw_pg(struct shdw_pg_data * pg_data) {
194 struct shdw_back_ptr * back_ptr = NULL;
195 struct shdw_back_ptr * tmp_ptr = NULL;
197 PrintError(VM_NONE, VCORE_NONE, "Unlinking gpa=%p, type=%d\n", (void *)pg_data->tuple.gpa, pg_data->tuple.pt_type);
199 list_for_each_entry_safe(back_ptr, tmp_ptr, &(pg_data->back_ptrs), back_ptr_node) {
200 struct shdw_pg_data * parent = back_ptr->pg_data;
202 evict_pt(parent->hva, back_ptr->gva, parent->tuple.pt_type);
203 list_del(&(back_ptr->back_ptr_node));
213 static int add_rmap(struct v3_vm_info * vm, struct shdw_pg_data * pg_data, addr_t gpa, addr_t gva) {
214 struct cache_vm_state * cache_state = vm->shdw_impl.impl_data;
215 struct list_head * rmap_list = NULL;
216 struct rmap_entry * entry = NULL;
219 rmap_list = (struct list_head *)v3_htable_search(cache_state->reverse_map, gpa);
221 if (rmap_list == NULL) {
222 rmap_list = V3_Malloc(sizeof(struct list_head));
225 PrintError(vm, VCORE_NONE, "Cannot allocate\n");
229 INIT_LIST_HEAD(rmap_list);
231 v3_htable_insert(cache_state->reverse_map, gpa, (addr_t)rmap_list);
234 entry = V3_Malloc(sizeof(struct rmap_entry));
237 PrintError(vm, VCORE_NONE, "Cannot allocate\n");
242 entry->gpa = pg_data->tuple.gpa;
243 entry->pt_type = pg_data->tuple.pt_type;
245 list_add(&(entry->rmap_node), rmap_list);
252 static int update_rmap_entries(struct v3_vm_info * vm, addr_t gpa) {
253 struct cache_vm_state * cache_state = vm->shdw_impl.impl_data;
254 struct list_head * rmap_list = NULL;
255 struct rmap_entry * entry = NULL;
258 rmap_list = (struct list_head *)v3_htable_search(cache_state->reverse_map, gpa);
260 if (rmap_list == NULL) {
264 PrintError(vm, VCORE_NONE, "Updating rmap entries\n\t");
266 list_for_each_entry(entry, rmap_list, rmap_node) {
267 struct shdw_pg_data * pg_data = NULL;
268 struct guest_pg_tuple tuple = {entry->gpa, entry->pt_type};
270 V3_Print(vm, VCORE_NONE, "%d \n", i);
272 pg_data = (struct shdw_pg_data *)v3_htable_search(cache_state->page_htable, (addr_t)&tuple);
275 PrintError(vm, VCORE_NONE, "Invalid PTE reference... Should Delete rmap entry\n");
279 if (grab_pt(pg_data->hva, entry->gva, entry->pt_type) == -1) {
280 PrintError(vm, VCORE_NONE, "Could not invalidate reverse map entry\n");
294 static int link_shdw_pg(struct shdw_pg_data * child_pg, struct shdw_pg_data * parent_pg, addr_t gva) {
295 struct shdw_back_ptr * back_ptr = V3_Malloc(sizeof(struct shdw_back_ptr));
298 PrintError(VM_NONE, VCORE_NONE, "Cannot allocate\n");
302 memset(back_ptr, 0, sizeof(struct shdw_back_ptr));
304 back_ptr->pg_data = parent_pg;
307 list_add(&(back_ptr->back_ptr_node), &(child_pg->back_ptrs));
314 static struct shdw_pg_data * find_shdw_pt(struct v3_vm_info * vm, addr_t gpa, page_type_t pt_type) {
315 struct cache_vm_state * cache_state = vm->shdw_impl.impl_data;
316 struct shdw_pg_data * pg_data = NULL;
317 struct guest_pg_tuple tuple = {gpa, pt_type};
319 pg_data = (struct shdw_pg_data *)v3_htable_search(cache_state->page_htable, (addr_t)&tuple);
321 if (pg_data != NULL) {
322 // move pg_data to head of queue, for LRU policy
323 list_move(&(pg_data->pg_queue_node), &(cache_state->pg_queue));
330 static int evict_shdw_pg(struct v3_vm_info * vm, addr_t gpa, page_type_t pt_type) {
331 struct cache_vm_state * cache_state = vm->shdw_impl.impl_data;
332 struct shdw_pg_data * pg_data = NULL;
334 pg_data = find_shdw_pt(vm, gpa, pt_type);
336 PrintError(vm, VCORE_NONE, "Evicting GPA: %p, type=%d\n", (void *)gpa, pt_type);
338 if (pg_data != NULL) {
339 if (unlink_shdw_pg(pg_data) == -1) {
340 PrintError(vm, VCORE_NONE, "Error unlinking page...\n");
344 v3_htable_remove(cache_state->page_htable, (addr_t)&(pg_data->tuple), 0);
347 // Move Page to free list
348 list_move(&(pg_data->pg_queue_node), &(cache_state->free_list));
349 cache_state->pgs_in_free_list++;
350 cache_state->pgs_in_cache--;
357 static struct shdw_pg_data * pop_queue_pg(struct v3_vm_info * vm,
358 struct cache_vm_state * cache_state) {
359 struct shdw_pg_data * pg_data = NULL;
361 PrintError(vm, VCORE_NONE, "popping page from queue\n");
363 pg_data = list_tail_entry(&(cache_state->pg_queue), struct shdw_pg_data, pg_queue_node);
366 if (unlink_shdw_pg(pg_data) == -1) {
367 PrintError(vm, VCORE_NONE, "Error unlinking cached page\n");
371 v3_htable_remove(cache_state->page_htable, (addr_t)&(pg_data->tuple), 0);
372 list_del(&(pg_data->pg_queue_node));
374 cache_state->pgs_in_cache--;
379 static struct shdw_pg_data * create_shdw_pt(struct v3_vm_info * vm, addr_t gpa, page_type_t pt_type) {
380 struct cache_vm_state * cache_state = vm->shdw_impl.impl_data;
381 struct shdw_pg_data * pg_data = NULL;
384 PrintError(vm, VCORE_NONE, "Creating shdw page: gpa=%p, type=%d\n", (void *)gpa, pt_type);
386 if (cache_state->pgs_in_cache < cache_state->max_cache_pgs) {
387 pg_data = V3_Malloc(sizeof(struct shdw_pg_data));
390 PrintError(vm, VCORE_NONE, "Cannot allocate\n");
394 pg_data->hpa = (addr_t)V3_AllocPages(1);
397 PrintError(vm, VCORE_NONE, "Cannot allocate page for shadow page table\n");
401 pg_data->hva = (void *)V3_VAddr((void *)pg_data->hpa);
403 } else if (cache_state->pgs_in_free_list) {
405 PrintError(vm, VCORE_NONE, "pulling page from free list\n");
406 // pull from free list
407 pg_data = list_tail_entry(&(cache_state->free_list), struct shdw_pg_data, pg_queue_node);
409 list_del(&(pg_data->pg_queue_node));
410 cache_state->pgs_in_free_list--;
414 pg_data = pop_queue_pg(vm, cache_state);
418 if (pg_data == NULL) {
419 PrintError(vm, VCORE_NONE, "Error creating Shadow Page table page\n");
423 memset(pg_data->hva, 0, PAGE_SIZE_4KB);
425 pg_data->tuple.gpa = gpa;
426 pg_data->tuple.pt_type = pt_type;
428 INIT_LIST_HEAD(&(pg_data->back_ptrs));
430 v3_htable_insert(cache_state->page_htable, (addr_t)&(pg_data->tuple), (addr_t)pg_data);
432 list_add(&(pg_data->pg_queue_node), &(cache_state->pg_queue));
433 cache_state->pgs_in_cache++;
440 #include "vmm_shdw_pg_cache_32.h"
441 //#include "vmm_shdw_pg_cache_32pae.h"
442 //#include "vmm_shdw_pg_cache_64.h"
445 static uint_t cache_hash_fn(addr_t key) {
446 struct guest_pg_tuple * tuple = (struct guest_pg_tuple *)key;
448 return v3_hash_buffer((uint8_t *)tuple, sizeof(struct guest_pg_tuple));
451 static int cache_eq_fn(addr_t key1, addr_t key2) {
452 struct guest_pg_tuple * tuple1 = (struct guest_pg_tuple *)key1;
453 struct guest_pg_tuple * tuple2 = (struct guest_pg_tuple *)key2;
455 return ((tuple1->gpa == tuple2->gpa) && (tuple1->pt_type == tuple2->pt_type));
458 static uint_t rmap_hash_fn(addr_t key) {
459 return v3_hash_long(key, sizeof(addr_t) * 8);
462 static int rmap_eq_fn(addr_t key1, addr_t key2) {
463 return (key1 == key2);
467 static int cache_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
468 struct v3_shdw_impl_state * vm_state = &(vm->shdw_impl);
469 struct cache_vm_state * cache_state = NULL;
470 int cache_size = DEFAULT_CACHE_SIZE;
471 char * cache_sz_str = v3_cfg_val(cfg, "cache_size");
473 if (cache_sz_str != NULL) {
474 cache_size = ((atoi(cache_sz_str) * 1024 * 1024) / 4096);
477 V3_Print(vm, VCORE_NONE, "Shadow Page Cache initialization\n");
479 cache_state = V3_Malloc(sizeof(struct cache_vm_state));
482 PrintError(vm, VCORE_NONE, "Cannot allocate\n");
486 memset(cache_state, 0, sizeof(struct cache_vm_state));
488 cache_state->page_htable = v3_create_htable(0, cache_hash_fn, cache_eq_fn);
489 cache_state->reverse_map = v3_create_htable(0, rmap_hash_fn, rmap_eq_fn);
490 v3_lock_init(&(cache_state->cache_lock));
491 INIT_LIST_HEAD(&(cache_state->pg_queue));
492 INIT_LIST_HEAD(&(cache_state->free_list));
493 cache_state->max_cache_pgs = cache_size;
495 vm_state->impl_data = cache_state;
501 static int cache_deinit(struct v3_vm_info * vm) {
506 static int cache_local_init(struct guest_info * core) {
507 // struct v3_shdw_pg_state * core_state = &(vm->shdw_pg_state);
513 static int cache_activate_shdw_pt(struct guest_info * core) {
514 switch (v3_get_vm_cpu_mode(core)) {
517 PrintError(core->vm_info, core, "Calling 32 bit cache activation\n");
518 return activate_shadow_pt_32(core);
520 // return activate_shadow_pt_32pae(core);
524 // return activate_shadow_pt_64(core);
526 PrintError(core->vm_info, core, "Invalid CPU mode: %s\n", v3_cpu_mode_to_str(v3_get_vm_cpu_mode(core)));
533 static int cache_invalidate_shdw_pt(struct guest_info * core) {
534 // wipe everything...
535 V3_Print(core->vm_info, core, "Cache invalidation called\n");
537 return cache_activate_shdw_pt(core);
542 static int cache_handle_pf(struct guest_info * core, addr_t fault_addr, pf_error_t error_code) {
544 switch (v3_get_vm_cpu_mode(core)) {
546 return handle_shadow_pagefault_32(core, fault_addr, error_code);
549 // return handle_shadow_pagefault_32pae(core, fault_addr, error_code);
553 // return handle_shadow_pagefault_64(core, fault_addr, error_code);
555 PrintError(core->vm_info, core, "Unhandled CPU Mode: %s\n", v3_cpu_mode_to_str(v3_get_vm_cpu_mode(core)));
561 static int cache_handle_invlpg(struct guest_info * core, addr_t vaddr) {
562 PrintError(core->vm_info, core, "INVLPG called for %p\n", (void *)vaddr);
564 switch (v3_get_vm_cpu_mode(core)) {
566 return handle_shadow_invlpg_32(core, vaddr);
568 // return handle_shadow_invlpg_32pae(core, vaddr);
572 // return handle_shadow_invlpg_64(core, vaddr);
574 PrintError(core->vm_info, core, "Invalid CPU mode: %s\n", v3_cpu_mode_to_str(v3_get_vm_cpu_mode(core)));
584 static struct v3_shdw_pg_impl cache_impl = {
585 .name = "SHADOW_CACHE",
587 .deinit = cache_deinit,
588 .local_init = cache_local_init,
589 .handle_pagefault = cache_handle_pf,
590 .handle_invlpg = cache_handle_invlpg,
591 .activate_shdw_pt = cache_activate_shdw_pt,
592 .invalidate_shdw_pt = cache_invalidate_shdw_pt
597 register_shdw_pg_impl(&cache_impl);