From: Jack Lange Date: Wed, 14 Sep 2011 16:33:31 +0000 (-0400) Subject: Merge branch 'devel' of newskysaw.cs.northwestern.edu:/home/palacios/palacios into... X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?a=commitdiff_plain;h=b1aec98739d57c1bc4d1834414a0206eb37aab92;hp=6b32ab71071174408dba942dab82adc2c8324643;p=palacios.releases.git Merge branch 'devel' of newskysaw.cs.northwestern.edu:/home/palacios/palacios into devel --- diff --git a/linux_module/palacios-stubs.c b/linux_module/palacios-stubs.c index 05789fb..26ada0b 100644 --- a/linux_module/palacios-stubs.c +++ b/linux_module/palacios-stubs.c @@ -226,6 +226,31 @@ palacios_start_thread_on_cpu(int cpu_id, return thread; } + +/** + * Rebind a kernel thread to the specified CPU + * The thread will be running on target CPU on return + * non-zero return means failure + */ +static int +palacios_move_thread_to_cpu(int new_cpu_id, + void * thread_ptr) { + struct task_struct * thread = (struct task_struct *)thread_ptr; + + printk("Moving thread (%p) to cpu %d\n", thread, new_cpu_id); + + if (thread == NULL) { + thread = current; + } + + /* + * Bind to the specified CPU. When this call returns, + * the thread should be running on the target CPU. + */ + return set_cpus_allowed_ptr(thread, cpumask_of(new_cpu_id)); +} + + /** * Returns the CPU ID that the caller is running on. */ @@ -461,6 +486,7 @@ static struct v3_os_hooks palacios_os_hooks = { .interrupt_cpu = palacios_interrupt_cpu, .call_on_cpu = palacios_xcall, .start_thread_on_cpu = palacios_start_thread_on_cpu, + .move_thread_to_cpu = palacios_move_thread_to_cpu, }; diff --git a/linux_module/palacios.h b/linux_module/palacios.h index 633ede1..3a253ee 100644 --- a/linux_module/palacios.h +++ b/linux_module/palacios.h @@ -18,9 +18,10 @@ #define V3_VM_PAUSE 23 #define V3_VM_CONTINUE 24 - #define V3_VM_INSPECT 30 +#define V3_VM_MOVE_CORE 33 + #define V3_VM_FB_INPUT (256+1) #define V3_VM_FB_QUERY (256+2) @@ -40,6 +41,11 @@ struct v3_mem_region { unsigned long long num_pages; }; +struct v3_core_move_cmd{ + uint16_t vcore_id; + uint16_t pcore_id; +}; + void * trace_malloc(size_t size, gfp_t flags); void trace_free(const void * objp); diff --git a/linux_module/vm.c b/linux_module/vm.c index 890cb69..f53a598 100644 --- a/linux_module/vm.c +++ b/linux_module/vm.c @@ -143,6 +143,22 @@ static long v3_vm_ioctl(struct file * filp, v3_continue_vm(guest->v3_ctx); break; } + case V3_VM_MOVE_CORE: { + struct v3_core_move_cmd cmd; + void __user * argp = (void __user *)arg; + + memset(&cmd, 0, sizeof(struct v3_core_move_cmd)); + + if (copy_from_user(&cmd, argp, sizeof(struct v3_core_move_cmd))) { + printk("copy from user error getting migrate command...\n"); + return -EFAULT; + } + + printk("moving guest %s vcore %d to CPU %d\n", guest->name, cmd.vcore_id, cmd.pcore_id); + + v3_move_vm_core(guest->v3_ctx, cmd.vcore_id, cmd.pcore_id); + } + break; default: { struct vm_ctrl * ctrl = get_ctrl(guest, ioctl); diff --git a/linux_usr/Makefile b/linux_usr/Makefile index b8a1014..3aa822c 100644 --- a/linux_usr/Makefile +++ b/linux_usr/Makefile @@ -1,4 +1,4 @@ -all: v3_ctrl v3_stop v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file +all: v3_ctrl v3_stop v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file v3_core_move @@ -35,8 +35,11 @@ v3_user_keyed_stream_example: v3_user_keyed_stream_example.c v3_user_keyed_strea v3_user_keyed_stream_file: v3_user_keyed_stream_file.c v3_user_keyed_stream.h v3_user_keyed_stream.c gcc -static -I../linux_module v3_user_keyed_stream_file.c v3_user_keyed_stream.c -o v3_user_keyed_stream_file +v3_core_move : v3_core_move.c v3_ctrl.h + gcc -static v3_core_move.c -o v3_core_move + v3_inject_ecc_scrubber_mce: v3_inject_ecc_scrubber_mce.c gcc -static -I../linux_module v3_inject_ecc_scrubber_mce.c -o v3_inject_ecc_scrubber_mce clean: - rm -f v3_ctrl v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file + rm -f v3_ctrl v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file v3_core_migrate diff --git a/linux_usr/v3_core_move.c b/linux_usr/v3_core_move.c new file mode 100644 index 0000000..d7439d8 --- /dev/null +++ b/linux_usr/v3_core_move.c @@ -0,0 +1,59 @@ +/* + * V3 Virtual Core Migrate Control + * (c) Lei Xia, 2011 + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v3_ctrl.h" + +struct v3_core_move_cmd { + unsigned short vcore_id; + unsigned short pcore_id; +}; + + +int main(int argc, char* argv[]) { + int vm_fd; + char * vm_dev = NULL; + struct v3_core_move_cmd cmd; + + if (argc < 4) { + printf("Usage: v3_core_migrate \n"); + return -1; + } + + vm_dev = argv[1]; + cmd.vcore_id = atoi(argv[2]); + cmd.pcore_id = atoi(argv[3]); + + printf("Migrate vcore %d to physical CPU %d\n", cmd.vcore_id, cmd.pcore_id); + + vm_fd = open(vm_dev, O_RDONLY); + + if (vm_fd == -1) { + printf("Error opening VM device: %s\n", vm_dev); + return -1; + } + + int err = ioctl(vm_fd, V3_VM_MOVE_CORE, &cmd); + + if (err < 0) { + printf("Error write core migrating command to vm\n"); + return -1; + } + + close(vm_fd); + + return 0; +} + + diff --git a/linux_usr/v3_ctrl.h b/linux_usr/v3_ctrl.h index eb92d7d..126330d 100644 --- a/linux_usr/v3_ctrl.h +++ b/linux_usr/v3_ctrl.h @@ -14,6 +14,8 @@ #define V3_VM_SERIAL_CONNECT 21 #define V3_VM_STOP 22 +#define V3_VM_MOVE_CORE 33 + static const char * v3_dev = "/dev/v3vee"; struct v3_guest_img { diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index 51ca2de..1ff5899 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -128,6 +128,8 @@ struct guest_info { v3_core_operating_mode_t core_run_state; + void * core_thread; /* thread struct for virtual core */ + /* the logical cpu on which this core runs */ uint32_t pcpu_id; diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 4c95f2c..8a75498 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -214,6 +214,15 @@ struct guest_info; thread; \ }) +#define V3_MOVE_THREAD_TO_CPU(pcpu, thread) ({ \ + int ret = -1; \ + extern struct v3_os_hooks * os_hooks; \ + if((os_hooks) && (os_hooks)->move_thread_to_cpu) { \ + ret = (os_hooks)->move_thread_to_cpu(pcpu, thread); \ + } \ + ret; \ + }) + #endif /* ** */ @@ -305,7 +314,7 @@ struct v3_os_hooks { void (*interrupt_cpu)(struct v3_vm_info * vm, int logical_cpu, int vector); void (*call_on_cpu)(int logical_cpu, void (*fn)(void * arg), void * arg); void * (*start_thread_on_cpu)(int cpu_id, int (*fn)(void * arg), void * arg, char * thread_name); - + int (*move_thread_to_cpu)(int cpu_id, void * thread); }; @@ -334,6 +343,7 @@ int v3_stop_vm(struct v3_vm_info * vm); int v3_pause_vm(struct v3_vm_info * vm); int v3_continue_vm(struct v3_vm_info * vm); +int v3_move_vm_core(struct v3_vm_info * vm, int vcore_id, int target_cpu); int v3_free_vm(struct v3_vm_info * vm); diff --git a/palacios/src/devices/serial.c b/palacios/src/devices/serial.c index 52c3202..d6bdc2d 100644 --- a/palacios/src/devices/serial.c +++ b/palacios/src/devices/serial.c @@ -258,7 +258,7 @@ struct dll_register { struct dlm_register { uint8_t data; }; -#define SERIAL_BUF_LEN 16 +#define SERIAL_BUF_LEN 128 struct serial_buffer { int head; // most recent data diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index c3bce64..1bf16cc 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -547,6 +547,7 @@ int v3_svm_enter(struct guest_info * info) { v3_time_enter_vm(info); guest_ctrl->TSC_OFFSET = v3_tsc_host_offset(&info->time_state); + //V3_Print("Calling v3_svm_launch\n"); v3_svm_launch((vmcb_t *)V3_PAddr(info->vmm_data), &(info->vm_regs), (vmcb_t *)host_vmcbs[V3_Get_CPU()]); diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index 078ae4a..961bbca 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -269,7 +269,6 @@ int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { for (i = 0, vcore_id = 1; (i < MAX_CORES) && (vcore_id < vm->num_cores); i++) { int major = 0; int minor = 0; - void * core_thread = NULL; struct guest_info * core = &(vm->cores[vcore_id]); char * specified_cpu = v3_cfg_val(core->core_cfg_data, "target_cpu"); uint32_t core_idx = 0; @@ -319,9 +318,9 @@ int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { // TODO: actually manage these threads instead of just launching them core->pcpu_id = core_idx; - core_thread = V3_CREATE_THREAD_ON_CPU(core_idx, start_core, core, core->exec_name); + core->core_thread = V3_CREATE_THREAD_ON_CPU(core_idx, start_core, core, core->exec_name); - if (core_thread == NULL) { + if (core->core_thread == NULL) { PrintError("Thread launch failed\n"); v3_stop_vm(vm); return -1; @@ -374,6 +373,62 @@ int v3_reset_vm_core(struct guest_info * core, addr_t rip) { +/* move a virtual core to different physical core */ +int v3_move_vm_core(struct v3_vm_info * vm, int vcore_id, int target_cpu) { + struct guest_info * core = NULL; + + if ((vcore_id < 0) || (vcore_id >= vm->num_cores)) { + PrintError("Attempted to migrate invalid virtual core (%d)\n", vcore_id); + return -1; + } + + core = &(vm->cores[vcore_id]); + + if (target_cpu == core->pcpu_id) { + PrintError("Attempted to migrate to local core (%d)\n", target_cpu); + // well that was pointless + return 0; + } + + if (core->core_thread == NULL) { + PrintError("Attempted to migrate a core without a valid thread context\n"); + return -1; + } + + while (v3_raise_barrier(vm, NULL) == -1); + + V3_Print("Performing Migration from %d to %d\n", core->pcpu_id, target_cpu); + + // Double check that we weren't preemptively migrated + if (target_cpu != core->pcpu_id) { + + V3_Print("Moving Core\n"); + + if (V3_MOVE_THREAD_TO_CPU(target_cpu, core->core_thread) != 0) { + PrintError("Failed to move Vcore %d to CPU %d\n", + core->vcpu_id, target_cpu); + v3_lower_barrier(vm); + return -1; + } + + /* There will be a benign race window here: + core->pcpu_id will be set to the target core before its fully "migrated" + However the core will NEVER run on the old core again, its just in flight to the new core + */ + core->pcpu_id = target_cpu; + + V3_Print("core now at %d\n", core->pcpu_id); + + } + + + + + v3_lower_barrier(vm); + + return 0; +} + int v3_stop_vm(struct v3_vm_info * vm) { diff --git a/palacios/src/palacios/vmm_halt.c b/palacios/src/palacios/vmm_halt.c index bb4a46a..bfe2e5a 100644 --- a/palacios/src/palacios/vmm_halt.c +++ b/palacios/src/palacios/vmm_halt.c @@ -58,6 +58,7 @@ int v3_handle_halt(struct guest_info * info) { /* if no interrupt, then we do halt */ /* asm("hlt"); */ } + } /* V3_Print("palacios: done with halt\n"); */ diff --git a/palacios/src/palacios/vmx.c b/palacios/src/palacios/vmx.c index e1b427a..18f7ee7 100644 --- a/palacios/src/palacios/vmx.c +++ b/palacios/src/palacios/vmx.c @@ -759,7 +759,9 @@ int v3_vmx_enter(struct guest_info * info) { v3_update_timers(info); if (vmcs_store() != vmx_info->vmcs_ptr_phys) { + vmcs_clear(vmx_info->vmcs_ptr_phys); vmcs_load(vmx_info->vmcs_ptr_phys); + vmx_info->state = VMX_UNLAUNCHED; } v3_vmx_restore_vmcs(info); @@ -787,6 +789,7 @@ int v3_vmx_enter(struct guest_info * info) { check_vmcs_write(VMCS_TSC_OFFSET_HIGH, tsc_offset_high); check_vmcs_write(VMCS_TSC_OFFSET, tsc_offset_low); + if (v3_update_vmcs_host_state(info)) { v3_enable_ints(); PrintError("Could not write host state\n"); @@ -796,6 +799,7 @@ int v3_vmx_enter(struct guest_info * info) { if (vmx_info->state == VMX_UNLAUNCHED) { vmx_info->state = VMX_LAUNCHED; + info->vm_info->run_state = VM_RUNNING; ret = v3_vmx_launch(&(info->vm_regs), info, &(info->ctrl_regs)); } else {