From: Peter Dinda Date: Mon, 31 Aug 2015 20:20:23 +0000 (-0500) Subject: Implementation of resource control host os interface for Linux X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=6234775894cac514f495b751a046db245ecb124a Implementation of resource control host os interface for Linux This associates a pointer to a resource control structure with each thread created by Palacios. If this structure exists, its contents are then used to control resource allocations. Currently, the controls are on page allocation: alignment, numa node, and page allocation filtering. The latter is what cache partitioning builds on. This also changes the /proc/v3vee/v3-guest-info-details output slightly. For a guest with too many cores or memory regions to display, the output is now truncated in a graceful way. --- diff --git a/linux_module/main.c b/linux_module/main.c index 723e5ea..b4c4637 100644 --- a/linux_module/main.c +++ b/linux_module/main.c @@ -34,6 +34,8 @@ #include "linux-exts.h" +#include "util-hashtable.h" + MODULE_LICENSE("GPL"); // Module parameter @@ -63,6 +65,10 @@ static struct proc_dir_entry * palacios_proc_dir = NULL; struct class * v3_class = NULL; static struct cdev ctrl_dev; + +// mapping from thread ids to their resource control blocks +struct hashtable *v3_thread_resource_map=0; + static int register_vm(struct v3_guest * guest) { int i = 0; @@ -265,41 +271,47 @@ struct proc_dir_entry *palacios_get_procdir(void) } -#define MAX_VCORES 256 +#define MAX_CORES 1024 #define MAX_REGIONS 1024 - - +#define MIN(x,y) ((x)<(y) ? (x) : (y)) static int read_guests_details(struct seq_file *s, void *v) { unsigned int i = 0; unsigned int j = 0; uint64_t num_vcores, num_regions; + uint64_t alloc_num_vcores, alloc_num_regions; struct v3_vm_base_state *base=0; struct v3_vm_core_state *core=0; struct v3_vm_mem_state *mem=0; - base = palacios_alloc(sizeof(struct v3_vm_base_state)); + + base = palacios_valloc(sizeof(struct v3_vm_base_state)); + if (!base) { ERROR("No space for base state structure\n"); goto out; } + for(i = 0; i < MAX_VMS; i++) { if (guest_map[i] != NULL) { v3_get_state_sizes_vm(guest_map[i]->v3_ctx,&num_vcores,&num_regions); - core = palacios_alloc(sizeof(struct v3_vm_core_state) + num_vcores*sizeof(struct v3_vm_vcore_state)); + alloc_num_vcores = MIN(num_vcores,MAX_CORES); + alloc_num_regions = MIN(num_regions,MAX_REGIONS); + + core = palacios_valloc(sizeof(struct v3_vm_core_state) + alloc_num_vcores*sizeof(struct v3_vm_vcore_state)); if (!core) { ERROR("No space for core state structure\n"); goto out; } - mem = palacios_alloc(sizeof(struct v3_vm_mem_state) + num_regions*sizeof(struct v3_vm_mem_region)); + mem = palacios_valloc(sizeof(struct v3_vm_mem_state) + alloc_num_regions*sizeof(struct v3_vm_mem_region)); if (!mem) { ERROR("No space for memory state structure\n"); @@ -315,8 +327,8 @@ static int read_guests_details(struct seq_file *s, void *v) i,guest_map[i]->name, i); // Get extended data - core->num_vcores=num_vcores; - mem->num_regions=num_regions; + core->num_vcores=alloc_num_vcores; + mem->num_regions=alloc_num_regions; if (v3_get_state_vm(guest_map[i]->v3_ctx, base, core, mem)) { ERROR("Cannot get VM info\n"); @@ -325,8 +337,8 @@ static int read_guests_details(struct seq_file *s, void *v) seq_printf(s, "Type: %s\n" "State: %s\n" - "Cores: %llu\n" - "Regions: %llu\n" + "Cores: %llu (%llu shown)\n" + "Regions: %llu (%llu shown)\n" "Memsize: %llu (%llu ROS)\n\n", base->vm_type==V3_VM_GENERAL ? "general" : base->vm_type==V3_VM_HVM ? "HVM" : "UNKNOWN", @@ -337,7 +349,9 @@ static int read_guests_details(struct seq_file *s, void *v) base->state==V3_VM_ERROR ? "ERROR" : base->state==V3_VM_SIMULATING ? "simulating" : base->state==V3_VM_RESETTING ? "resetting" : "UNKNOWN", + num_vcores, core->num_vcores, + num_regions, mem->num_regions, mem->mem_size, mem->ros_mem_size); @@ -370,6 +384,7 @@ static int read_guests_details(struct seq_file *s, void *v) core->vcore[j].vcore_type==V3_VCORE_HRT ? "hrt" : "UNKNOWN"); } + seq_printf(s, "\nMemory Regions\n"); for (j=0;jnum_regions;j++) { seq_printf(s," region %u has HPAs 0x%016llx-0x%016llx (node %d) GPA 0x%016llx %s %s\n", @@ -379,12 +394,13 @@ static int read_guests_details(struct seq_file *s, void *v) mem->region[j].swapped ? "swapped" : "", mem->region[j].pinned ? "pinned" : ""); } + } seq_printf(s, "---------------------------------------------------------------------------------------\n"); - palacios_free(mem); mem=0; - palacios_free(core); core=0; + palacios_vfree(mem); mem=0; + palacios_vfree(core); core=0; } @@ -392,9 +408,9 @@ static int read_guests_details(struct seq_file *s, void *v) out: - if (mem) { palacios_free(mem); } - if (core) { palacios_free(core); } - if (base) { palacios_free(base); } + if (mem) { palacios_vfree(mem); } + if (core) { palacios_vfree(core); } + if (base) { palacios_vfree(base); } return 0; } @@ -405,34 +421,43 @@ static int read_guests(struct seq_file *s, void *v) struct v3_vm_base_state *base=0; struct v3_vm_core_state *core=0; struct v3_vm_mem_state *mem=0; + uint64_t num_vcores, num_regions; + + + INFO("READ GUEST\n"); - base = palacios_alloc(sizeof(struct v3_vm_base_state)); + base = palacios_valloc(sizeof(struct v3_vm_base_state)); if (!base) { ERROR("No space for base state structure\n"); goto out; } - core = palacios_alloc(sizeof(struct v3_vm_core_state) + MAX_VCORES*sizeof(struct v3_vm_vcore_state)); + core = palacios_valloc(sizeof(struct v3_vm_core_state)); if (!core) { ERROR("No space for core state structure\n"); goto out; } - mem = palacios_alloc(sizeof(struct v3_vm_mem_state) + MAX_REGIONS*sizeof(struct v3_vm_mem_region)); + mem = palacios_valloc(sizeof(struct v3_vm_mem_state)); if (!mem) { ERROR("No space for memory state structure\n"); goto out; } + for(i = 0; i < MAX_VMS; i++) { if (guest_map[i] != NULL) { + + v3_get_state_sizes_vm(guest_map[i]->v3_ctx,&num_vcores,&num_regions); + seq_printf(s,"%s\t/dev/v3-vm%d", guest_map[i]->name, i); - // Get extended data - core->num_vcores=MAX_VCORES; // max we can handle - mem->num_regions=MAX_REGIONS; // max we can handle + + // Skip getting per core and per-region + core->num_vcores=0; + mem->num_regions=0; if (v3_get_state_vm(guest_map[i]->v3_ctx, base, core, mem)) { ERROR("Cannot get VM info\n"); @@ -445,8 +470,8 @@ static int read_guests(struct seq_file *s, void *v) base->state==V3_VM_PAUSED ? "paused" : base->state==V3_VM_ERROR ? "ERROR" : base->state==V3_VM_SIMULATING ? "simulating" : "UNKNOWN", - core->num_vcores, - mem->num_regions, + num_vcores, + num_regions, mem->mem_size, base->vm_type == V3_VM_GENERAL ? "general" : base->vm_type == V3_VM_HVM ? "hvm" : "UNKNOWN"); @@ -456,9 +481,9 @@ static int read_guests(struct seq_file *s, void *v) out: - if (mem) { palacios_free(mem); } - if (core) { palacios_free(core); } - if (base) { palacios_free(base); } + if (mem) { palacios_vfree(mem); } + if (core) { palacios_vfree(core); } + if (base) { palacios_vfree(base); } return 0; } @@ -568,6 +593,16 @@ static struct file_operations info_proc_ops = { }; +static inline uint_t thr_hash_func(addr_t key) +{ + return palacios_hash_long((long)key,64); +} + +static inline int thr_hash_comp(addr_t k1, addr_t k2) +{ + return k1==k2; +} + static int __init v3_init(void) { dev_t dev = MKDEV(0, 0); // We dynamicallly assign the major number @@ -576,6 +611,13 @@ static int __init v3_init(void) { LOCKCHECK_INIT(); MEMCHECK_INIT(); + + if (!(v3_thread_resource_map = palacios_create_htable(MAX_THREADS,thr_hash_func,thr_hash_comp))) { + ERROR("Could not create thread/resource map\n"); + ret = -1; + goto failure0; + } + palacios_proc_dir = proc_mkdir("v3vee", NULL); if (!palacios_proc_dir) { ERROR("Could not create proc entry\n"); @@ -689,6 +731,8 @@ static int __init v3_init(void) { failure2: remove_proc_entry("v3vee", NULL); failure1: + palacios_free_htable(v3_thread_resource_map,0,0); + failure0: MEMCHECK_DEINIT(); LOCKCHECK_DEINIT(); @@ -762,6 +806,8 @@ static void __exit v3_exit(void) { DEBUG("Palacios Module Mallocs = %d, Frees = %d\n", mod_allocs, mod_frees); + palacios_free_htable(v3_thread_resource_map,0,0); + MEMCHECK_DEINIT(); LOCKCHECK_DEINIT(); } diff --git a/linux_module/palacios-stubs.c b/linux_module/palacios-stubs.c index 10d2fde..7814987 100644 --- a/linux_module/palacios-stubs.c +++ b/linux_module/palacios-stubs.c @@ -28,6 +28,8 @@ #include "palacios.h" +#include "util-hashtable.h" + #include "mm.h" #include "memcheck.h" @@ -58,6 +60,9 @@ extern int cpu_list[NR_CPUS]; extern int cpu_list_len; +extern struct hashtable *v3_thread_resource_map; + + static char *print_buffer[NR_CPUS]; static void deinit_print_buffers(void) @@ -177,12 +182,30 @@ void palacios_print_scoped(void * vm, int vcore, const char *fmt, ...) { */ void *palacios_allocate_pages(int num_pages, unsigned int alignment, int node_id, int (*filter_func)(void *paddr, void *filter_state), void *filter_state) { void * pg_addr = NULL; + v3_resource_control_t *r; if (num_pages<=0) { ERROR("ALERT ALERT Attempt to allocate zero or fewer pages (%d pages, alignment %d, node %d, filter_func %p, filter_state %p)\n",num_pages, alignment, node_id, filter_func, filter_state); return NULL; } + if ((r=(v3_resource_control_t *)palacios_htable_search(v3_thread_resource_map,(addr_t)current))) { + // thread has a registered resource control structure + // these override any default values + // INFO("Overridden page search: (pre) alignment=%x, node_id=%x, filter_func=%p, filter_state=%p\n",alignment,node_id,filter_func,filter_state); + if (alignment==4096) { + alignment = r->pg_alignment; + } + if (node_id==-1) { + node_id = r->pg_node_id; + } + if (!filter_func) { + filter_func = r->pg_filter_func; + filter_state = r->pg_filter_state; + } + //INFO("Overridden page search: (post) alignment=%x, node_id=%x, filter_func=%p, filter_state=%p\n",alignment,node_id,filter_func,filter_state); + } + pg_addr = (void *)alloc_palacios_pgs(num_pages, alignment, node_id, filter_func, filter_state); if (!pg_addr) { @@ -379,6 +402,7 @@ palacios_xcall( struct lnx_thread_arg { int (*fn)(void * arg); void * arg; + v3_resource_control_t *resource_control; char name[MAX_THREAD_NAME]; }; @@ -399,10 +423,14 @@ static int lnx_thread_target(void * arg) { fpu_alloc(&(current->thread.fpu)); #endif + palacios_htable_insert(v3_thread_resource_map,(addr_t)current,(addr_t)thread_info->resource_control); + ret = thread_info->fn(thread_info->arg); INFO("Palacios Thread (%s) EXITING\n", thread_info->name); + palacios_htable_remove(v3_thread_resource_map,(addr_t)current,0); + palacios_free(thread_info); // handle cleanup @@ -421,7 +449,8 @@ void * palacios_create_and_start_kernel_thread( int (*fn) (void * arg), void * arg, - char * thread_name) { + char * thread_name, + v3_resource_control_t *resource_control) { struct lnx_thread_arg * thread_info = palacios_alloc(sizeof(struct lnx_thread_arg)); @@ -434,6 +463,7 @@ palacios_create_and_start_kernel_thread( thread_info->arg = arg; strncpy(thread_info->name,thread_name,MAX_THREAD_NAME); thread_info->name[MAX_THREAD_NAME-1] =0; + thread_info->resource_control = resource_control; return kthread_run( lnx_thread_target, thread_info, thread_info->name ); } @@ -444,9 +474,10 @@ palacios_create_and_start_kernel_thread( */ void * palacios_create_thread_on_cpu(int cpu_id, - int (*fn)(void * arg), - void * arg, - char * thread_name ) { + int (*fn)(void * arg), + void * arg, + char * thread_name, + v3_resource_control_t *resource_control) { struct task_struct * thread = NULL; struct lnx_thread_arg * thread_info = palacios_alloc(sizeof(struct lnx_thread_arg)); @@ -459,6 +490,7 @@ palacios_create_thread_on_cpu(int cpu_id, thread_info->arg = arg; strncpy(thread_info->name,thread_name,MAX_THREAD_NAME); thread_info->name[MAX_THREAD_NAME-1] =0; + thread_info->resource_control=resource_control; thread = kthread_create( lnx_thread_target, thread_info, thread_info->name ); @@ -493,9 +525,10 @@ void * palacios_create_and_start_thread_on_cpu(int cpu_id, int (*fn)(void * arg), void * arg, - char * thread_name ) { + char * thread_name, + v3_resource_control_t *resource_control) { - void *t = palacios_create_thread_on_cpu(cpu_id, fn, arg, thread_name); + void *t = palacios_create_thread_on_cpu(cpu_id, fn, arg, thread_name, resource_control); if (t) { palacios_start_thread(t); diff --git a/linux_module/palacios-vnet.c b/linux_module/palacios-vnet.c index d4f09b0..446bd73 100644 --- a/linux_module/palacios-vnet.c +++ b/linux_module/palacios-vnet.c @@ -140,10 +140,10 @@ host_del_timer(void * vnet_timer){ static void * -host_allocate_pages(int num_pages, unsigned int alignment, int node_id, int constraint) +host_allocate_pages(int num_pages, unsigned int alignment, int node_id) { // allocates pages preferentially on the caller's node - return palacios_allocate_pages(num_pages, alignment, node_id, constraint); + return palacios_allocate_pages(num_pages, alignment, node_id, 0, 0); } diff --git a/linux_module/palacios.h b/linux_module/palacios.h index 63d948f..a546cb4 100644 --- a/linux_module/palacios.h +++ b/linux_module/palacios.h @@ -152,7 +152,7 @@ struct v3_guest { // For now MAX_VMS must be a multiple of 8 // This is due to the minor number bitmap #define MAX_VMS 32 - +#define MAX_THREADS (MAX_VMS*64) int palacios_vmm_init( char *options ); @@ -162,6 +162,8 @@ int palacios_vmm_exit( void ); // This is how a component finds the proc dir we are using for global state struct proc_dir_entry *palacios_get_procdir(void); +struct v3_resource_control; + // Selected exported stubs, for use in other palacios components, like vnet // The idea is that everything uses the same stubs void palacios_print_scoped(void *vm, int vcore, const char *fmt, ...); @@ -178,10 +180,10 @@ void palacios_vfree(void *); // use instead of vfree void *palacios_vaddr_to_paddr(void *vaddr); void *palacios_paddr_to_vaddr(void *paddr); void palacios_xcall(int cpu_id, void (*fn)(void *arg), void *arg); -void *palacios_create_and_start_kernel_thread(int (*fn)(void * arg), void *arg, char *thread_name); -void *palacios_create_thread_on_cpu(int cpu_id, int (*fn)(void * arg), void *arg, char *thread_name); +void *palacios_create_and_start_kernel_thread(int (*fn)(void * arg), void *arg, char *thread_name, struct v3_resource_control *rc); +void *palacios_create_thread_on_cpu(int cpu_id, int (*fn)(void * arg), void *arg, char *thread_name, struct v3_resource_control *rc); void palacios_start_thread(void *thread_ptr); -void *palacios_creeate_and_start_thread_on_cpu(int cpu_id, int (*fn)(void * arg), void *arg, char *thread_name); +void *palacios_creeate_and_start_thread_on_cpu(int cpu_id, int (*fn)(void * arg), void *arg, char *thread_name, struct v3_resource_control *rc); int palacios_move_thread_to_cpu(int new_cpu_id, void *thread_ptr); void palacios_yield_cpu(void); void palacios_sleep_cpu(unsigned int us);