From: Jack Lange Date: Tue, 13 Sep 2011 19:44:25 +0000 (-0400) Subject: Added barrier locking to the core migration functionality X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?a=commitdiff_plain;h=c58da27ff29c9cd0511c7c3f64eff2644fa85961;p=palacios.releases.git Added barrier locking to the core migration functionality --- diff --git a/linux_module/palacios-stubs.c b/linux_module/palacios-stubs.c index 36f7625..26ada0b 100644 --- a/linux_module/palacios-stubs.c +++ b/linux_module/palacios-stubs.c @@ -234,10 +234,12 @@ palacios_start_thread_on_cpu(int cpu_id, */ static int palacios_move_thread_to_cpu(int new_cpu_id, - void * thread_ptr) { + void * thread_ptr) { struct task_struct * thread = (struct task_struct *)thread_ptr; - if(thread == NULL){ + printk("Moving thread (%p) to cpu %d\n", thread, new_cpu_id); + + if (thread == NULL) { thread = current; } @@ -245,7 +247,7 @@ palacios_move_thread_to_cpu(int new_cpu_id, * Bind to the specified CPU. When this call returns, * the thread should be running on the target CPU. */ - return set_cpus_allowed(thread, cpumask_of_cpu(new_cpu_id)); + return set_cpus_allowed_ptr(thread, cpumask_of(new_cpu_id)); } diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index eae4bd7..1ff5899 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -128,9 +128,6 @@ struct guest_info { v3_core_operating_mode_t core_run_state; - v3_core_moving_state_t core_move_state; /* if 1, then it is to be migrated to current pcpu_id */ - uint32_t target_pcpu_id; /* the target physical CPU id for core moving */ - void * core_thread; /* thread struct for virtual core */ /* the logical cpu on which this core runs */ diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 4243d02..8a75498 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -207,12 +207,12 @@ struct guest_info; #define V3_CREATE_THREAD_ON_CPU(cpu, fn, arg, name) ({ \ void * thread = NULL; \ - extern struct v3_os_hooks * os_hooks; \ - if ((os_hooks) && (os_hooks)->start_thread_on_cpu) { \ - thread = (os_hooks)->start_thread_on_cpu(cpu, fn, arg, name); \ - } \ - thread; \ - }) + extern struct v3_os_hooks * os_hooks; \ + if ((os_hooks) && (os_hooks)->start_thread_on_cpu) { \ + thread = (os_hooks)->start_thread_on_cpu(cpu, fn, arg, name); \ + } \ + thread; \ + }) #define V3_MOVE_THREAD_TO_CPU(pcpu, thread) ({ \ int ret = -1; \ diff --git a/palacios/include/palacios/vmm_types.h b/palacios/include/palacios/vmm_types.h index aced5d8..8d45bcd 100644 --- a/palacios/include/palacios/vmm_types.h +++ b/palacios/include/palacios/vmm_types.h @@ -29,7 +29,6 @@ typedef enum {SHADOW_PAGING, NESTED_PAGING} v3_paging_mode_t; typedef enum {VM_RUNNING, VM_STOPPED, VM_PAUSED, VM_ERROR} v3_vm_operating_mode_t; typedef enum {CORE_RUNNING, CORE_STOPPED} v3_core_operating_mode_t; -typedef enum {CORE_MOVE_DONE, CORE_MOVE_PENDING} v3_core_moving_state_t; typedef enum {REAL, /*UNREAL,*/ PROTECTED, PROTECTED_PAE, LONG, LONG_32_COMPAT, LONG_16_COMPAT} v3_cpu_mode_t; typedef enum {PHYSICAL_MEM, VIRTUAL_MEM} v3_mem_mode_t; diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index d9b44f0..1bf16cc 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -547,29 +547,6 @@ int v3_svm_enter(struct guest_info * info) { v3_time_enter_vm(info); guest_ctrl->TSC_OFFSET = v3_tsc_host_offset(&info->time_state); - if(info->core_move_state == CORE_MOVE_PENDING) { - v3_stgi(); - - if(V3_MOVE_THREAD_TO_CPU(info->target_pcpu_id, info->core_thread) != 0){ - PrintError("Failed to move Vcore %d to CPU %d\n", - info->vcpu_id, - info->target_pcpu_id); - } else { - info->pcpu_id = info->target_pcpu_id; - V3_Print("Core move done, vcore %d is running on CPU %d now\n", - info->vcpu_id, - V3_Get_CPU()); - } - - info->core_move_state = CORE_MOVE_DONE; - - /* disable global interrupts, - * NOTE now it is being running on a different CPU - */ - v3_clgi(); - } - - //V3_Print("Calling v3_svm_launch\n"); diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index 34b6c2c..961bbca 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -377,24 +377,55 @@ int v3_reset_vm_core(struct guest_info * core, addr_t rip) { int v3_move_vm_core(struct v3_vm_info * vm, int vcore_id, int target_cpu) { struct guest_info * core = NULL; - if(vcore_id < 0 || vcore_id > vm->num_cores) { + if ((vcore_id < 0) || (vcore_id >= vm->num_cores)) { + PrintError("Attempted to migrate invalid virtual core (%d)\n", vcore_id); return -1; } core = &(vm->cores[vcore_id]); - if(target_cpu != core->pcpu_id && - core->core_move_state != CORE_MOVE_PENDING){ - core->core_move_state = CORE_MOVE_PENDING; - core->target_pcpu_id = target_cpu; - v3_interrupt_cpu(vm, core->pcpu_id, 0); + if (target_cpu == core->pcpu_id) { + PrintError("Attempted to migrate to local core (%d)\n", target_cpu); + // well that was pointless + return 0; + } - while(core->core_move_state != CORE_MOVE_DONE){ - v3_yield(NULL); - } + if (core->core_thread == NULL) { + PrintError("Attempted to migrate a core without a valid thread context\n"); + return -1; } + while (v3_raise_barrier(vm, NULL) == -1); + + V3_Print("Performing Migration from %d to %d\n", core->pcpu_id, target_cpu); + + // Double check that we weren't preemptively migrated + if (target_cpu != core->pcpu_id) { + + V3_Print("Moving Core\n"); + + if (V3_MOVE_THREAD_TO_CPU(target_cpu, core->core_thread) != 0) { + PrintError("Failed to move Vcore %d to CPU %d\n", + core->vcpu_id, target_cpu); + v3_lower_barrier(vm); + return -1; + } + /* There will be a benign race window here: + core->pcpu_id will be set to the target core before its fully "migrated" + However the core will NEVER run on the old core again, its just in flight to the new core + */ + core->pcpu_id = target_cpu; + + V3_Print("core now at %d\n", core->pcpu_id); + + } + + + + + v3_lower_barrier(vm); + return 0; } diff --git a/palacios/src/palacios/vmm_halt.c b/palacios/src/palacios/vmm_halt.c index 58a0374..bfe2e5a 100644 --- a/palacios/src/palacios/vmm_halt.c +++ b/palacios/src/palacios/vmm_halt.c @@ -59,12 +59,6 @@ int v3_handle_halt(struct guest_info * info) { /* asm("hlt"); */ } - /* check if there is a core move request pending - * if there is, resume the guest with RIP on halt instruction again - */ - if (info->core_move_state == CORE_MOVE_PENDING){ - return 0; - } } /* V3_Print("palacios: done with halt\n"); */ diff --git a/palacios/src/palacios/vmx.c b/palacios/src/palacios/vmx.c index 03dab64..54ccd3c 100644 --- a/palacios/src/palacios/vmx.c +++ b/palacios/src/palacios/vmx.c @@ -758,7 +758,9 @@ int v3_vmx_enter(struct guest_info * info) { v3_update_timers(info); if (vmcs_store() != vmx_info->vmcs_ptr_phys) { + vmcs_clear(vmx_info->vmcs_ptr_phys); vmcs_load(vmx_info->vmcs_ptr_phys); + vmx_info->state = VMX_UNLAUNCHED; } v3_vmx_restore_vmcs(info); @@ -787,30 +789,6 @@ int v3_vmx_enter(struct guest_info * info) { check_vmcs_write(VMCS_TSC_OFFSET, tsc_offset_low); - /* determine if we need to move to a different physical core */ - if(info->core_move_state == CORE_MOVE_PENDING) { - vmcs_clear(vmx_info->vmcs_ptr_phys); - - v3_enable_ints(); - - if(V3_MOVE_THREAD_TO_CPU(info->target_pcpu_id, info->core_thread) != 0){ - PrintError("Failed to move vcore %d to CPU %d\n", info->vcpu_id, info->target_pcpu_id); - } else { - info->pcpu_id = info->target_pcpu_id; - PrintDebug("Core move done, vcore %d is running on CPU %d now\n", info->vcpu_id, V3_Get_CPU()); - } - - /* disable global interrupts, - * NOTE now it is being running on a different CPU - */ - v3_disable_ints(); - - vmcs_load(vmx_info->vmcs_ptr_phys); - vmx_info->state = VMX_UNLAUNCHED; - info->core_move_state= CORE_MOVE_DONE; - } - - if (v3_update_vmcs_host_state(info)) { v3_enable_ints(); PrintError("Could not write host state\n"); @@ -820,6 +798,7 @@ int v3_vmx_enter(struct guest_info * info) { if (vmx_info->state == VMX_UNLAUNCHED) { vmx_info->state = VMX_LAUNCHED; + info->vm_info->run_state = VM_RUNNING; ret = v3_vmx_launch(&(info->vm_regs), info, &(info->ctrl_regs)); } else {