From: Peter Dinda Date: Sat, 10 Jul 2010 00:37:43 +0000 (-0500) Subject: Implemented SMP, still not functional X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?a=commitdiff_plain;h=6a686aa0cf80126e77d4f0f3a0eb882a7d302ae0;p=palacios.git Implemented SMP, still not functional --- diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 1ef6cb1..bf13c3f 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -266,7 +266,7 @@ struct v3_os_hooks { unsigned int (*get_cpu)(void); void (*interrupt_cpu)(struct v3_vm_info * vm, int logical_cpu, int vector); void (*call_on_cpu)(int logical_cpu, void (*fn)(void * arg), void * arg); - void (*start_thread_on_cpu)(int logical_cpu, int (*fn)(void * arg), void * arg, char * thread_name); + void * (*start_thread_on_cpu)(int logical_cpu, int (*fn)(void * arg), void * arg, char * thread_name); }; diff --git a/palacios/include/palacios/vmm_types.h b/palacios/include/palacios/vmm_types.h index 9a3fe61..0c95d0d 100644 --- a/palacios/include/palacios/vmm_types.h +++ b/palacios/include/palacios/vmm_types.h @@ -30,7 +30,7 @@ typedef enum {SHADOW_PAGING, NESTED_PAGING} v3_paging_mode_t; typedef enum {VM_RUNNING, VM_STOPPED, VM_SUSPENDED, VM_ERROR, VM_EMULATING} v3_vm_operating_mode_t; -typedef enum {REAL, /*UNREAL,*/ PROTECTED, PROTECTED_PAE, LONG, LONG_32_COMPAT, LONG_16_COMPAT} v3_cpu_mode_t; +typedef enum {INIT, SIPI, REAL, /*UNREAL,*/ PROTECTED, PROTECTED_PAE, LONG, LONG_32_COMPAT, LONG_16_COMPAT} v3_cpu_mode_t; typedef enum {PHYSICAL_MEM, VIRTUAL_MEM} v3_mem_mode_t; diff --git a/palacios/src/devices/icc_bus.c b/palacios/src/devices/icc_bus.c index e75734d..be94618 100644 --- a/palacios/src/devices/icc_bus.c +++ b/palacios/src/devices/icc_bus.c @@ -32,7 +32,9 @@ #endif -void v3_force_exit() { +void v3_force_exit(void *p) { + struct guest_info *core=(struct guest_info *)p; + PrintDebug("core %u: Forced to exit!\n",core->cpu_id); } struct ipi_thunk_data { @@ -91,10 +93,16 @@ static int deliver(uint32_t src_apic, struct apic_data *dest_apic, struct int_cm case 0: //fixed case 1: // lowest priority - PrintDebug("icc_bus: delivering to core %u\n",dest_apic->core->cpu_id); + PrintDebug("icc_bus: delivering IRQ to core %u\n",dest_apic->core->cpu_id); dest_apic->ops->raise_intr(dest_apic->core, icr->vec, dest_apic->priv_data); if (src_apic!=state->ioapic_id && dest_apic->core->cpu_id != src_apic) { + // Assume core # is same as logical processor for now + // TODO FIX THIS FIX THIS + // THERE SHOULD BE: guestapicid->virtualapicid map, + // cpu_id->logical processor map + // host maitains logical proc->phsysical proc PrintDebug("icc_bus: non-local core, forcing it to exit\n"); + V3_Call_On_CPU(dest_apic->core->cpu_id,v3_force_exit,(void*)(dest_apic->core)); // TODO: do what the print says } break; @@ -115,14 +123,64 @@ static int deliver(uint32_t src_apic, struct apic_data *dest_apic, struct int_cm return -1; break; - case 5: //INIT - PrintError("icc_bus: INIT delivery is unsupported\n"); - return -1; + case 5: { //INIT + struct guest_info *core = dest_apic->core; + + PrintDebug("icc_bus: INIT delivery to core %u\n",core->cpu_id); + + // TODO: any APIC reset on dest core (shouldn't be needed, but not sure...) + + // Sanity check + if (core->cpu_mode!=INIT) { + PrintError("icc_bus: Warning: core %u is not in INIT state, ignored\n",core->cpu_id); + // Only a warning, since INIT INIT SIPI is common + break; + } + + // We transition the target core to SIPI state + core->cpu_mode=SIPI; // note: locking should not be needed here + + // That should be it since the target core should be + // waiting in host on this transition + // either it's on another core or on a different preemptive thread + // in both cases, it will quickly notice this transition + // in particular, we should not need to force an exit here + + PrintDebug("icc_bus: INIT delivery done\n"); + + } break; - case 6: //Start Up - PrintError("icc_bus: Startup Delivery is unsupported\n"); - return -1; + case 6: { //SIPI + struct guest_info *core = dest_apic->core; + uint64_t rip = icr->vec << 12; // vector encodes target address; + + PrintDebug("icc_bus: SIPI delivery (0x%x -> rip=0x%p) to core %u\n", + icr->vec, (void*)rip, core->cpu_id); + + // Sanity check + if (core->cpu_mode!=SIPI) { + PrintError("icc_bus: core %u is not in SIPI state, ignored!\n",core->cpu_id); + break; + } + + // Write the RIP, CS, and descriptor + // assume the rest is already good to go + core->rip=rip & 0xffff; + core->segments.cs.selector = (rip >> 4) & 0xf000; + core->segments.cs.limit= 0xffff; + core->segments.cs.base = rip & 0xf0000; + + // Maybe need to adjust the APIC? + + // We transition the target core to SIPI state + core->cpu_mode=REAL; // note: locking should not be needed here + + // As with INIT, we should not need to do anything else + + PrintDebug("icc_bus: SIPI delivery done\n"); + + } break; } diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index 752e249..35eef9c 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -555,8 +555,36 @@ int v3_start_svm_guest(struct guest_info *info) { // vmcb_ctrl_t * guest_ctrl = GET_VMCB_CTRL_AREA((vmcb_t*)(info->vmm_data)); + PrintDebug("Starting SVM core %u\n",info->cpu_id); + if (info->cpu_mode==INIT) { + PrintDebug("SVM core %u: I am an AP in INIT mode, waiting for that to change\n",info->cpu_id); + while (info->cpu_mode==INIT) { + v3_yield(info); + //PrintDebug("SVM core %u: still waiting for INIT\n",info->cpu_id); + } + PrintDebug("SVM core %u: I am out of INIT\n",info->cpu_id); + if (info->cpu_mode==SIPI) { + PrintDebug("SVM core %u: I am waiting on a SIPI to set my starting address\n",info->cpu_id); + while (info->cpu_mode==SIPI) { + v3_yield(info); + //PrintDebug("SVM core %u: still waiting for SIPI\n",info->cpu_id); + } + } + PrintDebug("SVM core %u: I have my SIPI\n", info->cpu_id); + } + + if (info->cpu_mode!=REAL) { + PrintError("SVM core %u: I am not in REAL mode at launch! Huh?!\n", info->cpu_id); + return -1; + } + + PrintDebug("SVM core %u: I am starting at CS=0x%x (base=0x%p, limit=0x%x), RIP=0x%p\n", + info->cpu_id, info->segments.cs.selector, (void*)(info->segments.cs.base), + info->segments.cs.limit,(void*)(info->rip)); + - PrintDebug("Launching SVM VM (vmcb=%p)\n", (void *)info->vmm_data); + + PrintDebug("SVM core %u: Launching SVM VM (vmcb=%p)\n", info->cpu_id, (void *)info->vmm_data); //PrintDebugVMCB((vmcb_t*)(info->vmm_data)); info->vm_info->run_state = VM_RUNNING; @@ -571,17 +599,17 @@ int v3_start_svm_guest(struct guest_info *info) { info->vm_info->run_state = VM_ERROR; - V3_Print("SVM ERROR!!\n"); + V3_Print("SVM core %u: SVM ERROR!!\n", info->cpu_id); v3_print_guest_state(info); - V3_Print("SVM Exit Code: %p\n", (void *)(addr_t)guest_ctrl->exit_code); + V3_Print("SVM core %u: SVM Exit Code: %p\n", info->cpu_id, (void *)(addr_t)guest_ctrl->exit_code); - V3_Print("exit_info1 low = 0x%.8x\n", *(uint_t*)&(guest_ctrl->exit_info1)); - V3_Print("exit_info1 high = 0x%.8x\n", *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info1)) + 4)); + V3_Print("SVM core %u: exit_info1 low = 0x%.8x\n", info->cpu_id, *(uint_t*)&(guest_ctrl->exit_info1)); + V3_Print("SVM core %u: exit_info1 high = 0x%.8x\n", info->cpu_id, *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info1)) + 4)); - V3_Print("exit_info2 low = 0x%.8x\n", *(uint_t*)&(guest_ctrl->exit_info2)); - V3_Print("exit_info2 high = 0x%.8x\n", *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info2)) + 4)); + V3_Print("SVM core %u: exit_info2 low = 0x%.8x\n", info->cpu_id, *(uint_t*)&(guest_ctrl->exit_info2)); + V3_Print("SVM core %u: exit_info2 high = 0x%.8x\n", info->cpu_id, *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info2)) + 4)); linear_addr = get_addr_linear(info, info->rip, &(info->segments.cs)); @@ -591,9 +619,9 @@ int v3_start_svm_guest(struct guest_info *info) { v3_gva_to_hva(info, linear_addr, &host_addr); } - V3_Print("Host Address of rip = 0x%p\n", (void *)host_addr); + V3_Print("SVM core %u: Host Address of rip = 0x%p\n", info->cpu_id, (void *)host_addr); - V3_Print("Instr (15 bytes) at %p:\n", (void *)host_addr); + V3_Print("SVM core %u: Instr (15 bytes) at %p:\n", info->cpu_id, (void *)host_addr); v3_dump_mem((uint8_t *)host_addr, 15); v3_print_stack(info); @@ -608,6 +636,9 @@ int v3_start_svm_guest(struct guest_info *info) { */ } + + // Need to take down the other cores on error... + return 0; } diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index 5811cba..dae7d34 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -129,37 +129,103 @@ struct v3_vm_info * v3_create_vm(void * cfg) { } -int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { - int i = 0; - V3_Print("V3 -- Starting VM\n"); +static int start_core(void *p) +{ + struct guest_info * info = (struct guest_info*)p; - for (i = 0; i < vm->num_cores; i++) { - struct guest_info * info = &(vm->cores[i]); + PrintDebug("core %u: in start_core\n",info->cpu_id); + + // we assume here that the APs are in INIT mode + // and only the BSP is in REAL + // the per-architecture code will rely on this + // assumption - /* GRUESOM HACK... */ - // vm->cpu_id = v3_get_cpu_id(); - switch (v3_cpu_types[info->cpu_id]) { + switch (v3_cpu_types[info->cpu_id]) { #ifdef CONFIG_SVM - case V3_SVM_CPU: - case V3_SVM_REV3_CPU: - return v3_start_svm_guest(info); - break; + case V3_SVM_CPU: + case V3_SVM_REV3_CPU: + return v3_start_svm_guest(info); + break; #endif #if CONFIG_VMX - case V3_VMX_CPU: - case V3_VMX_EPT_CPU: - return v3_start_vmx_guest(info); - break; + case V3_VMX_CPU: + case V3_VMX_EPT_CPU: + return v3_start_vmx_guest(info); + break; #endif - default: - PrintError("Attemping to enter a guest on an invalid CPU\n"); - return -1; + default: + PrintError("Attemping to enter a guest on an invalid CPU\n"); + return -1; + } + // should not happen + return 0; +} + + +static uint32_t get_next_core(unsigned int cpu_mask, uint32_t last_proc) +{ + uint32_t proc_to_use; + + PrintDebug("In get_next_core cpu_mask=0x%x last_proc=%u\n",cpu_mask,last_proc); + + proc_to_use=(last_proc+1) % 32; // only 32 procs + // This will wrap around, and eventually we can use proc 0, + // since that's clearly available + while (!((cpu_mask >> proc_to_use)&0x1)) { + proc_to_use=(proc_to_use+1)%32; + } + return proc_to_use; +} + +int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { + uint32_t i; + uint32_t last_proc; + uint32_t proc_to_use; + char tname[16]; + + V3_Print("V3 -- Starting VM (%u cores)\n",vm->num_cores); + + // We assume that we are running on CPU 0 of the underlying system + last_proc=0; + + // We will fork off cores 1..n first, then boot core zero + + // for the AP, we need to create threads + + for (i = 1; i < vm->num_cores; i++) { + if (!os_hooks->start_thread_on_cpu) { + PrintError("Host OS does not support start_thread_on_cpu - FAILING\n"); + return -1; } + + proc_to_use=get_next_core(cpu_mask,last_proc); + last_proc=proc_to_use; + + PrintDebug("Starting virtual core %u on logical core %u\n",i,proc_to_use); + + sprintf(tname,"core%u",i); + + PrintDebug("run: core=%u, func=0x%p, arg=0x%p, name=%s\n", + proc_to_use, start_core, &(vm->cores[i]), tname); + + // TODO: actually manage these threads instead of just launching them + if (!(os_hooks->start_thread_on_cpu(proc_to_use,start_core,&(vm->cores[i]),tname))) { + PrintError("Thread launch failed\n"); + return -1; + } + } + + // Finally launch the BSP on core 0 + sprintf(tname,"core%u",0); + if (!os_hooks->start_thread_on_cpu(0,start_core,&(vm->cores[0]),tname)) { + PrintError("Thread launch failed\n"); + return -1; } return 0; + } diff --git a/palacios/src/palacios/vmm_config.c b/palacios/src/palacios/vmm_config.c index 2c59f83..251fe05 100644 --- a/palacios/src/palacios/vmm_config.c +++ b/palacios/src/palacios/vmm_config.c @@ -413,7 +413,6 @@ struct v3_vm_info * v3_config_guest(void * cfg_blob) { for (i = 0; i < vm->num_cores; i++) { struct guest_info * info = &(vm->cores[i]); - info->cpu_id = i; info->vm_info = vm; diff --git a/palacios/src/palacios/vmm_config_class.h b/palacios/src/palacios/vmm_config_class.h index 1965059..1770a6f 100644 --- a/palacios/src/palacios/vmm_config_class.h +++ b/palacios/src/palacios/vmm_config_class.h @@ -22,8 +22,17 @@ static int pre_config_pc_core(struct guest_info * info, v3_cfg_tree_t * cfg) { + if (info->cpu_id!=0) { + // I am an AP, so I will start in INIT mode, + // not in real mode. This means I will wait for + // an INIT and then for a SIPI. The SIPI will + // tell me where to start executing in real mode + info->cpu_mode = INIT; + } else { + // I am the MP, so I will start as normal + info->cpu_mode = REAL; + } - info->cpu_mode = REAL; info->mem_mode = PHYSICAL_MEM;