From: Peter Dinda Date: Mon, 31 Aug 2015 20:34:00 +0000 (-0500) Subject: Cache partitioning support X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=f1c8d924817188c4f08a97205e97392ff304913f Cache partitioning support This is an experimental feature that allows you to map all of a VM's page allocations such that they map to a partition of the last level cache. The syntax is: The BLOCK_SIZE needs to be one page, and your base region size for the VM needs to be one page, and your allocations of physical memory to Palacios needs to support enough one page allocations to satisfy the constraints and the amount of memory you are asking for. NUM_COLORS is your view of how many colors (distinct pages) the abstract cache has while MIN_COLOR to MAX_COLOR are the range of cache pages of the abstract cache you want to use. The system will conform this requested color range on your abstract cache to a real color range on the actual cache, provided it is possible. Note that currently this is implemented (on Linux) by having the allocator (the buddy allocator) filter prospective free blocks by their color (the color of their first page). As a consequence, allocating the VM is slow: you need to allocate a lot of separate pages (one page per base region), and each page needs to be the right color. --- diff --git a/Kconfig b/Kconfig index 8826535..6b2cf2a 100644 --- a/Kconfig +++ b/Kconfig @@ -31,6 +31,9 @@ config LINUX select BUILT_IN_STDLIB select BUILT_IN_STRDUP select BUILT_IN_ATOI + select BUILT_IN_STRTOX + select BUILT_IN_STRTOI + select BUILT_IN_ATOX select FILE select V3_DECODER help @@ -186,6 +189,26 @@ config LAZY_FP_SWITCH the floating point state is explicitly saved on each exit and restored on each entry---this save/restore is entirely done in Palacios. + +config CACHEPART + bool "Support last-level cache partitioning" + depends on CACHE_INFO && EXPERIMENTAL + default y + help + When true, can be used to select which page colors + are allowed to be used by the VM, thus limiting it to a portion + of the last level shared cache. + This is an experimental option and requires a lot of careful + configuration to work. In particular, memory base regions must be + a page size, and initial allocation of the VM may take a long + time depending on the host's page allocator. + +config DEBUG_CACHEPART + bool "Enable debugging of cache partitioning" + depends on CACHEPART + default n + help + Generate output from the debugging statements in cache partitioning endmenu diff --git a/Makefile b/Makefile index f483b05..bb0860b 100644 --- a/Makefile +++ b/Makefile @@ -521,7 +521,7 @@ NOSTDINC_FLAGS += # disable pointer signedness warnings in gcc 4.0 CFLAGS += $(call cc-option,-Wno-pointer-sign,) -CFLAGS += -O +CFLAGS += -O2 # Default kernel image to build when no specific target is given. # KBUILD_IMAGE may be overruled on the commandline or diff --git a/palacios/include/palacios/vmm_cachepart.h b/palacios/include/palacios/vmm_cachepart.h new file mode 100644 index 0000000..f355720 --- /dev/null +++ b/palacios/include/palacios/vmm_cachepart.h @@ -0,0 +1,55 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2015, The V3VEE Project + * All rights reserved. + * + * Author: Peter Dinda + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + + +#ifndef __VMM_CACHEPART_H +#define __VMM_CACHEPART_H + + +#ifdef __V3VEE__ + +#include + +typedef struct v3_cachepart { + uint64_t mem_block_size; + uint64_t expected_num_colors; + uint64_t actual_num_colors; + uint64_t min_color; // with respect to actual colors + uint64_t max_color; + uint64_t color_shift; + uint64_t color_mask; +} v3_cachepart_t ; + + +struct v3_xml; + +int v3_init_cachepart(); +int v3_deinit_cachepart(); + +int v3_init_cachepart_vm(struct v3_vm_info *vm, struct v3_xml *config); +int v3_deinit_cachepart_vm(struct v3_vm_info *vm); + +int v3_init_cachepart_core(struct guest_info *core); +int v3_deinit_cachepart_core(struct guest_info *core); + +int v3_cachepart_filter(void *paddr, struct v3_cachepart *p); + +#endif /* ! __V3VEE__ */ + + +#endif diff --git a/palacios/src/palacios/Makefile b/palacios/src/palacios/Makefile index 733bfe7..4a9ab90 100644 --- a/palacios/src/palacios/Makefile +++ b/palacios/src/palacios/Makefile @@ -92,9 +92,13 @@ obj-$(V3_CONFIG_SYMCALL) += vmm_symcall.o obj-$(V3_CONFIG_SYMMOD) += vmm_symmod.o +obj-$(V3_CONFIG_CACHEPART) += vmm_cachepart.o + obj-$(V3_CONFIG_MEM_TRACK) += vmm_mem_track.o obj-$(V3_CONFIG_MULTIBOOT) += vmm_multiboot.o obj-$(V3_CONFIG_HVM) += vmm_hvm.o vmm_hvm_lowlevel.o + + obj-y += mmu/ diff --git a/palacios/src/palacios/vm_guest.c b/palacios/src/palacios/vm_guest.c index 41158e7..7736e38 100644 --- a/palacios/src/palacios/vm_guest.c +++ b/palacios/src/palacios/vm_guest.c @@ -36,6 +36,9 @@ #ifdef V3_CONFIG_MEM_TRACK #include #endif +#ifdef V3_CONFIG_CACHEPART +#include +#endif v3_cpu_mode_t v3_get_vm_cpu_mode(struct guest_info * info) { @@ -394,6 +397,9 @@ int v3_free_vm_internal(struct v3_vm_info * vm) { v3_fw_cfg_deinit(vm); +#ifdef V3_CONFIG_CACHEPART + v3_deinit_cachepart_vm(vm); +#endif return 0; } @@ -408,6 +414,12 @@ int v3_init_core(struct guest_info * core) { /* * Initialize the subsystem data strutures */ + + +#ifdef V3_CONFIG_CACHEPART + v3_init_cachepart_core(core); +#endif + #ifdef V3_CONFIG_TELEMETRY v3_init_core_telemetry(core); #endif @@ -502,6 +514,7 @@ int v3_free_core(struct guest_info * core) { #endif + switch (v3_mach_type) { #ifdef V3_CONFIG_SVM case V3_SVM_CPU: @@ -527,6 +540,10 @@ int v3_free_core(struct guest_info * core) { return -1; } +#ifdef V3_CONFIG_CACHEPART + v3_deinit_cachepart_core(core); +#endif + return 0; } diff --git a/palacios/src/palacios/vmm_cachepart.c b/palacios/src/palacios/vmm_cachepart.c new file mode 100644 index 0000000..ceab364 --- /dev/null +++ b/palacios/src/palacios/vmm_cachepart.c @@ -0,0 +1,298 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2015, The V3VEE Project + * All rights reserved. + * + * Author: Peter Dinda + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#include +#include +#include + +#ifndef V3_CONFIG_DEBUG_CACHEPART +#undef PrintDebug +#define PrintDebug(fmt, args...) +#endif + +/* + + + + Cache partitioning support + + We are partioning the last level shared cache here. + + Cache partitioning is applied to the guest physical memory allocation + and page allocations for nested and shadow paging. To the extent + possible, we also try to apply this paritioning to other allocations + and as early as possible during the creation of the VM + + BLOCK_SIZE = required v3_mem_block_size needed for partitioning in bytes + this is typically one page (4096 bytes) + NUM_COLORS = the number of page colors the cache is expected to hold + this is used to make the min/max color fields sensible + [MIN_COLOR,MAX_COLOR] + = the range of allowed page colors allowed for this VM + with respect to the range [0,NUMCOLORS) +*/ + +static int inited=0; +static struct v3_cache_info cache_info; + +#define CEIL_DIV(x,y) (((x)/(y)) + !!((x)%(y))) +#define FLOOR_DIV(x,y) (((x)/(y))) +#define DIVIDES(x,y) (!((x)%(y))) + + +static inline int getcacheinfo() +{ + if (inited) { + return 0; + } else { + if (v3_get_cache_info(V3_CACHE_COMBINED,0xffffffff,&cache_info)) { + PrintError(VM_NONE,VCORE_NONE,"Unable to get information about cache\n"); + return -1; + } + V3_Print(VM_NONE,VCORE_NONE,"cachepart: last level combined cache: 0x%x bytes, 0x%x blocksize, 0x%x associativity\n",cache_info.size,cache_info.blocksize,cache_info.associativity); + inited=1; + if (!(DIVIDES(cache_info.size,cache_info.blocksize) && + DIVIDES(cache_info.size,cache_info.associativity) && + DIVIDES(cache_info.size/cache_info.blocksize,cache_info.associativity))) { + PrintError(VM_NONE,VCORE_NONE,"cachepart: CACHE INFO IS NONSENSICAL\n"); + return -1; + } + return 0; + } +} + +int v3_init_cachepart() +{ + PrintDebug(VM_NONE,VCORE_NONE,"cachepart: init\n"); + return 0; +} + +int v3_deinit_cachepart() +{ + PrintDebug(VM_NONE,VCORE_NONE,"cachepart: deinit\n"); + return 0; +} + +/* +static int bitcount(uint64_t x) +{ + int c=0; + + while (x) { + c+=x&0x1; + x>>=1; + } + return c; +} + +static int is_pow2(x) +{ + int c = bitcount(x); + + return c==0 || c==1; +} +*/ + +static uint64_t log2(uint64_t x) +{ + uint64_t i=-1; + + while (x) { + i++; + x>>=1; + } + + return i; +} + +static uint64_t pow2(uint64_t n) +{ + uint64_t x = 1; + while (n) { + x*=2; + n--; + } + return x; +} + +static uint64_t num_lines() +{ + return FLOOR_DIV(cache_info.size,cache_info.blocksize); +} + +static uint64_t num_sets() +{ + return FLOOR_DIV(num_lines(),cache_info.associativity); +} + + +static void build_colors(v3_cachepart_t *c) +{ + uint64_t bo_bits, set_bits, bs_bits; + + // number of bits in block offset + bo_bits = log2(cache_info.blocksize); + set_bits = log2(num_sets()); + bs_bits = log2(c->mem_block_size); + + if (bs_bits= (bo_bits+set_bits)) { + c->actual_num_colors = 1; + c->color_mask=0; + c->color_shift=0; + c->min_color=0; + c->max_color=0; + } else { + c->actual_num_colors = pow2(bo_bits+set_bits-bs_bits); + c->color_mask = (c->actual_num_colors-1); + c->color_shift = bs_bits; + c->min_color = FLOOR_DIV(c->min_color,CEIL_DIV(c->expected_num_colors,c->actual_num_colors)); + c->max_color = FLOOR_DIV(c->max_color,CEIL_DIV(c->expected_num_colors,c->actual_num_colors)); + } +} + +int v3_init_cachepart_vm(struct v3_vm_info *vm, struct v3_xml *config) +{ + extern uint64_t v3_mem_block_size; + v3_cfg_tree_t *cp; + char *block_size_s; + char *num_colors_s; + char *min_color_s; + char *max_color_s; + uint64_t req_block_size, req_num_colors, req_min_color, req_max_color; + v3_cachepart_t *c = &(vm->cachepart_state); + + if (getcacheinfo()) { + PrintError(VM_NONE,VCORE_NONE,"cachepart: getcacheinfo failed!\n"); + return -1; + } + + if (!config || !(cp=v3_cfg_subtree(config,"cachepart"))) { + PrintDebug(vm,VCORE_NONE,"cachepart: no cachepart configuration found\n"); + return 0; + } + + if (!(block_size_s=v3_cfg_val(cp,"block_size"))) { + PrintError(vm,VCORE_NONE,"cachepart: missing block_size parameter\n"); + return 0; + } + + req_block_size = atoi(block_size_s); + + if (!(num_colors_s=v3_cfg_val(cp,"num_colors"))) { + PrintError(vm,VCORE_NONE,"cachepart: missing num_colors parameter\n"); + return -1; + } + + req_num_colors=atoi(num_colors_s); + + if (!(min_color_s=v3_cfg_val(cp,"min_color"))) { + PrintError(vm,VCORE_NONE,"cachepart: missing min_color parameter\n"); + return -1; + } + + req_min_color=atoi(min_color_s); + + if (!(max_color_s=v3_cfg_val(cp,"max_color"))) { + PrintError(vm,VCORE_NONE,"cachepart: missing max_color parameter\n"); + return -1; + } + + req_max_color=atoi(max_color_s); + + PrintDebug(vm,VCORE_NONE,"cachepart: request block_size=0x%llx, num_colors=0x%llx, min_color=0x%llx, max_color=0x%llx\n", req_block_size, req_num_colors, req_min_color, req_max_color); + + if (req_block_size!=v3_mem_block_size) { + PrintError(vm,VCORE_NONE,"cachepart: requested block size is %llu, but palacios is configured with v3_mem_block_size=%llu\n",req_block_size,v3_mem_block_size); + return -1; + } + + if (req_max_color>=req_num_colors || req_min_color>req_max_color) { + PrintError(vm,VCORE_NONE,"cachepart: min/max color request is nonsensical\n"); + return -1; + } + + memset(c,0,sizeof(*c)); + + c->mem_block_size=req_block_size; + c->expected_num_colors=req_num_colors; + c->min_color=req_min_color; + c->max_color=req_max_color; + + // Now update the color structure to reflect the cache constraints + build_colors(c); + + + PrintDebug(vm,VCORE_NONE,"cachepart: cache has 0x%llx colors so finalized to block_size=0x%llx, num_colors=0x%llx, min_color=0x%llx, max_color=0x%llx, color_shift=0x%llx, color_mask=0x%llx\n", c->actual_num_colors, c->mem_block_size, c->actual_num_colors, c->min_color, c->max_color, c->color_shift, c->color_mask); + + if (vm->resource_control.pg_filter_func || vm->resource_control.pg_filter_state) { + PrintError(vm,VCORE_NONE, "cachepart: paging filter functions and state are already installed...\n"); + return -1; + } + + vm->resource_control.pg_filter_func = (int (*)(void *,void*)) v3_cachepart_filter; + vm->resource_control.pg_filter_state = &vm->cachepart_state; + + + // V3_Sleep(50000000); + + return 0; + +} + +int v3_deinit_cachepart_vm(struct v3_vm_info *vm) +{ + // nothing to do + return 0; +} + +int v3_init_cachepart_core(struct guest_info *core) +{ + // nothing to do yet + return 0; +} + +int v3_deinit_cachepart_core(struct guest_info *core) +{ + // nothing to do yet + return 0; +} + +static unsigned count=0; + +int v3_cachepart_filter(void *paddr, v3_cachepart_t *c) +{ + uint64_t a = (uint64_t)paddr; + uint64_t color = (a >> c->color_shift) & c->color_mask; + + PrintDebug(VM_NONE,VCORE_NONE,"cachepart: %p is color 0x%llx required colors: [0x%llx,0x%llx] %s\n",paddr,color,c->min_color,c->max_color, color>=c->min_color && color<=c->max_color ? "ACCEPT" : "REJECT"); + + /* + if (count<10) { + V3_Sleep(5000000); + count++; + } + */ + + return color>=c->min_color && color<=c->max_color; + +} +