From: Jack Lange Date: Tue, 26 Oct 2010 23:24:51 +0000 (-0500) Subject: Removed a lot of the highly specific multicore code from the general core VMM code X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?a=commitdiff_plain;h=921772c23185aa48406a390c58546ee463f49dc4;p=palacios.releases.git Removed a lot of the highly specific multicore code from the general core VMM code The ICC BUS is broken because it still hasn't been merged into the APIC --- diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index 04a5632..9fc40df 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -119,6 +119,8 @@ struct guest_info { struct v3_vm_info * vm_info; + v3_core_operating_mode_t core_run_state; + /* the logical cpu on which this core runs */ uint32_t cpu_id; }; diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index b3b56cb..a5b47dc 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -160,6 +160,9 @@ struct guest_info; } while (0) + + + #define V3_Hook_Interrupt(vm, irq) ({ \ int ret = 0; \ extern struct v3_os_hooks * os_hooks; \ @@ -188,6 +191,17 @@ struct guest_info; } while (0) + +#define V3_CREATE_THREAD_ON_CPU(cpu, fn, arg, name) ({ \ + void * thread = NULL; \ + extern struct v3_os_hooks * os_hooks; \ + if ((os_hooks) && (os_hooks)->start_thread_on_cpu) { \ + thread = (os_hooks)->start_thread_on_cpu(cpu, fn, arg, name); \ + } \ + thread; \ + }) + + #define V3_ACK_IRQ(irq) \ do { \ extern struct v3_os_hooks * os_hooks; \ diff --git a/palacios/include/palacios/vmm_file.h b/palacios/include/palacios/vmm_file.h index bf4a1d0..b523475 100644 --- a/palacios/include/palacios/vmm_file.h +++ b/palacios/include/palacios/vmm_file.h @@ -51,14 +51,14 @@ ({ \ extern struct v3_file_hooks * file_hooks; \ ((file_hooks) && (file_hooks)->file_read) ? \ - (file_hooks)->file_read((fd), (start), (buf), (len)) : -1; \ + (file_hooks)->file_read((fd), (buf), (len), (start)) : -1; \ }) #define V3_FileWrite(fd,start,buf,len) \ ({ \ extern struct v3_file_hooks * file_hooks; \ ((file_hooks) && (file_hooks)->file_write) ? \ - (file_hooks)->file_write((fd), (start), (buf), (len)) : -1; \ + (file_hooks)->file_write((fd), (buf), (len), (start)) : -1; \ }) @@ -75,8 +75,8 @@ struct v3_file_hooks { long long (*file_size)(int fd); // blocking reads and writes - long long (*file_read)(int fd, long long start, void * buffer, long long length); - long long (*file_write)(int fd, long long start, void * buffer, long long length); + long long (*file_read)(int fd, void * buffer, long long length, long long offset); + long long (*file_write)(int fd, void * buffer, long long length, long long offset); }; diff --git a/palacios/include/palacios/vmm_types.h b/palacios/include/palacios/vmm_types.h index fc4fd5f..021d0e8 100644 --- a/palacios/include/palacios/vmm_types.h +++ b/palacios/include/palacios/vmm_types.h @@ -28,10 +28,11 @@ typedef enum {SHADOW_PAGING, NESTED_PAGING} v3_paging_mode_t; typedef enum {VM_RUNNING, VM_STOPPED, VM_SUSPENDED, VM_ERROR, VM_EMULATING} v3_vm_operating_mode_t; +typedef enum {CORE_RUNNING, CORE_STOPPED} v3_core_operating_mode_t; typedef enum {PAGING_4KB, PAGING_2MB} v3_paging_size_t; -typedef enum {INIT, SIPI, REAL, /*UNREAL,*/ PROTECTED, PROTECTED_PAE, LONG, LONG_32_COMPAT, LONG_16_COMPAT} v3_cpu_mode_t; +typedef enum {REAL, /*UNREAL,*/ PROTECTED, PROTECTED_PAE, LONG, LONG_32_COMPAT, LONG_16_COMPAT} v3_cpu_mode_t; typedef enum {PHYSICAL_MEM, VIRTUAL_MEM} v3_mem_mode_t; diff --git a/palacios/src/devices/apic.c b/palacios/src/devices/apic.c index 40b90a1..0254d67 100644 --- a/palacios/src/devices/apic.c +++ b/palacios/src/devices/apic.c @@ -135,6 +135,9 @@ struct apic_msr { } __attribute__((packed)); + +typedef enum {INIT, SIPI, STARTED} ipi_state_t; + struct apic_state { addr_t base_addr; @@ -176,6 +179,8 @@ struct apic_state { uint32_t rem_rd_data; + ipi_state_t ipi_state; + uint8_t int_req_reg[32]; uint8_t int_svc_reg[32]; uint8_t int_en_reg[32]; @@ -228,6 +233,8 @@ static void init_apic_state(struct apic_state * apic, uint32_t id, struct vm_dev apic->icc_bus = icc; + apic->ipi_state = INIT; + // The P6 has 6 LVT entries, so we set the value to (6-1)... apic->apic_ver.val = 0x80050010; @@ -906,7 +913,7 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u PrintDebug("apic %u: core %u: sending cmd 0x%llx to apic %u\n", apic->lapic_id.val, core->cpu_id, apic->int_cmd.val, apic->int_cmd.dst); - if (v3_icc_send_ipi(apic->icc_bus, apic->lapic_id.val, apic->int_cmd.val,0)==-1) { + if (v3_icc_send_ipi(apic->icc_bus, apic->lapic_id.val, apic->int_cmd.val, 0)==-1) { return -1; } break; diff --git a/palacios/src/devices/icc_bus.c b/palacios/src/devices/icc_bus.c index e9fe95c..f1cb982 100644 --- a/palacios/src/devices/icc_bus.c +++ b/palacios/src/devices/icc_bus.c @@ -96,17 +96,20 @@ static int deliver(uint32_t src_apic, struct apic_data *dest_apic, struct int_cm case 1: // lowest priority case 7: // ExtInt PrintDebug("icc_bus: delivering IRQ to core %u\n",dest_apic->core->cpu_id); + dest_apic->ops->raise_intr(dest_apic->core, - icr->del_mode!=7 ? icr->vec : extirq, + (icr->del_mode != 7) ? icr->vec : extirq, dest_apic->priv_data); - if (src_apic!=state->ioapic_id && dest_apic->core->cpu_id != src_apic) { + + if ((src_apic != state->ioapic_id) && (dest_apic->core->cpu_id != src_apic)) { // Assume core # is same as logical processor for now // TODO FIX THIS FIX THIS // THERE SHOULD BE: guestapicid->virtualapicid map, // cpu_id->logical processor map // host maitains logical proc->phsysical proc PrintDebug("icc_bus: non-local core, forcing it to exit\n"); - V3_Call_On_CPU(dest_apic->core->cpu_id,v3_force_exit,(void*)(dest_apic->core)); + + V3_Call_On_CPU(dest_apic->core->cpu_id, v3_force_exit, (void *)(dest_apic->core)); // TODO: do what the print says } break; @@ -129,19 +132,20 @@ static int deliver(uint32_t src_apic, struct apic_data *dest_apic, struct int_cm case 5: { //INIT struct guest_info *core = dest_apic->core; - PrintDebug("icc_bus: INIT delivery to core %u\n",core->cpu_id); + PrintDebug("icc_bus: INIT delivery to core %u\n", core->cpu_id); // TODO: any APIC reset on dest core (shouldn't be needed, but not sure...) // Sanity check - if (core->cpu_mode != INIT) { - PrintError("icc_bus: Warning: core %u is not in INIT state (mode = %d), ignored\n",core->cpu_id, core->cpu_mode); + if (dest_apic->ipi_state != INIT) { + PrintError("icc_bus: Warning: core %u is not in INIT state (mode = %d), ignored\n", + core->cpu_id, core->cpu_mode); // Only a warning, since INIT INIT SIPI is common break; } // We transition the target core to SIPI state - core->cpu_mode = SIPI; // note: locking should not be needed here + dest_apic->ipi_state = SIPI; // note: locking should not be needed here // That should be it since the target core should be // waiting in host on this transition @@ -158,7 +162,7 @@ static int deliver(uint32_t src_apic, struct apic_data *dest_apic, struct int_cm struct guest_info *core = dest_apic->core; // Sanity check - if (core->cpu_mode!=SIPI) { + if (dest_apic->ipi_state != SIPI) { PrintError("icc_bus: core %u is not in SIPI state (mode = %d), ignored!\n",core->cpu_id, core->cpu_mode); break; } @@ -183,7 +187,7 @@ static int deliver(uint32_t src_apic, struct apic_data *dest_apic, struct int_cm // Maybe need to adjust the APIC? // We transition the target core to SIPI state - core->cpu_mode = REAL; // note: locking should not be needed here + core->core_run_state = CORE_RUNNING; // note: locking should not be needed here // As with INIT, we should not need to do anything else diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index 35f2047..793f2d0 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -555,44 +555,35 @@ int v3_svm_enter(struct guest_info * info) { } -int v3_start_svm_guest(struct guest_info *info) { +int v3_start_svm_guest(struct guest_info * info) { // vmcb_saved_state_t * guest_state = GET_VMCB_SAVE_STATE_AREA((vmcb_t*)(info->vmm_data)); // vmcb_ctrl_t * guest_ctrl = GET_VMCB_CTRL_AREA((vmcb_t*)(info->vmm_data)); + PrintDebug("Starting SVM core %u\n", info->cpu_id); - PrintDebug("Starting SVM core %u\n",info->cpu_id); - if (info->cpu_mode==INIT) { - PrintDebug("SVM core %u: I am an AP in INIT mode, waiting for that to change\n",info->cpu_id); - while (info->cpu_mode==INIT) { + if (info->cpu_id == 0) { + info->core_run_state = CORE_RUNNING; + info->vm_info->run_state = VM_RUNNING; + } else { + PrintDebug("SVM core %u: Waiting for core initialization\n", info->cpu_id); + + while (info->core_run_state == CORE_STOPPED) { v3_yield(info); //PrintDebug("SVM core %u: still waiting for INIT\n",info->cpu_id); } - PrintDebug("SVM core %u: I am out of INIT\n",info->cpu_id); - if (info->cpu_mode==SIPI) { - PrintDebug("SVM core %u: I am waiting on a SIPI to set my starting address\n",info->cpu_id); - while (info->cpu_mode==SIPI) { - v3_yield(info); - //PrintDebug("SVM core %u: still waiting for SIPI\n",info->cpu_id); - } - } - PrintDebug("SVM core %u: I have my SIPI\n", info->cpu_id); - } - if (info->cpu_mode!=REAL) { - PrintError("SVM core %u: I am not in REAL mode at launch! Huh?!\n", info->cpu_id); - return -1; - } + PrintDebug("SVM core %u initialized\n", info->cpu_id); + } PrintDebug("SVM core %u: I am starting at CS=0x%x (base=0x%p, limit=0x%x), RIP=0x%p\n", - info->cpu_id, info->segments.cs.selector, (void*)(info->segments.cs.base), - info->segments.cs.limit,(void*)(info->rip)); + info->cpu_id, info->segments.cs.selector, (void *)(info->segments.cs.base), + info->segments.cs.limit,(void *)(info->rip)); PrintDebug("SVM core %u: Launching SVM VM (vmcb=%p)\n", info->cpu_id, (void *)info->vmm_data); //PrintDebugVMCB((vmcb_t*)(info->vmm_data)); - info->vm_info->run_state = VM_RUNNING; v3_start_time(info); while (1) { diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index e129d29..0eaa216 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -131,12 +131,12 @@ struct v3_vm_info * v3_create_vm(void * cfg, void * priv_data) { } -static int start_core(void *p) +static int start_core(void * p) { struct guest_info * info = (struct guest_info *)p; - PrintDebug("core %u: in start_core\n",info->cpu_id); + PrintDebug("core %u: in start_core\n", info->cpu_id); // we assume here that the APs are in INIT mode // and only the BSP is in REAL @@ -166,84 +166,68 @@ static int start_core(void *p) } -static uint32_t get_next_core(unsigned int cpu_mask, uint32_t last_proc) -{ - uint32_t proc_to_use; - - PrintDebug("In get_next_core cpu_mask=0x%x last_proc=%u\n",cpu_mask,last_proc); - - proc_to_use=(last_proc+1) % 32; // only 32 procs - // This will wrap around, and eventually we can use proc 0, - // since that's clearly available - while (!((cpu_mask >> proc_to_use)&0x1)) { - proc_to_use=(proc_to_use+1)%32; - } - return proc_to_use; -} +// For the moment very ugly. Eventually we will shift the cpu_mask to an arbitrary sized type... +#define MAX_CORES 32 int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { uint32_t i; - uint32_t last_proc; - uint32_t proc_to_use; char tname[16]; + int vcore_id = 0; + uint8_t * core_mask = (uint8_t *)&cpu_mask; // This is to make future expansion easier + uint32_t avail_cores = 0; - V3_Print("V3 -- Starting VM (%u cores)\n",vm->num_cores); - // We assume that we are running on CPU 0 of the underlying system - last_proc=0; - // We will fork off cores 1..n first, then boot core zero - - // for the AP, we need to create threads - - for (i = 1; i < vm->num_cores; i++) { - if (!os_hooks->start_thread_on_cpu) { - PrintError("Host OS does not support start_thread_on_cpu - FAILING\n"); - return -1; + /// CHECK IF WE ARE MULTICORE ENABLED.... + + V3_Print("V3 -- Starting VM (%u cores)\n", vm->num_cores); + + // Check that enough cores are present in the mask to handle vcores + for (i = 0; i < MAX_CORES; i++) { + int major = i / 8; + int minor = i % 8; + + if (core_mask[major] & (0x1 << minor)) { + avail_cores++; } + + } + + if (vm->num_cores > avail_cores) { + PrintError("Attempted to start a VM with too many cores (MAX=%d)\n", MAX_CORES); + return -1; + } + - proc_to_use=get_next_core(cpu_mask,last_proc); - last_proc=proc_to_use; + for (i = 0; (i < MAX_CORES) && (vcore_id < vm->num_cores); i++) { + int major = i / 8; + int minor = i % 8; + void * core_thread = NULL; - // vm->cores[i].cpu_id=i; - // vm->cores[i].physical_cpu_id=proc_to_use; + if ((core_mask[major] & (0x1 << minor)) == 0) { + // cpuid not set in cpu_mask + continue; + } - PrintDebug("Starting virtual core %u on logical core %u\n",i,proc_to_use); + PrintDebug("Starting virtual core %u on logical core %u\n", + vcore_id, i); - sprintf(tname,"core%u",i); + sprintf(tname, "core%u", vcore_id); PrintDebug("run: core=%u, func=0x%p, arg=0x%p, name=%s\n", - proc_to_use, start_core, &(vm->cores[i]), tname); + i, start_core, &(vm->cores[vcore_id]), tname); // TODO: actually manage these threads instead of just launching them - if (!(os_hooks->start_thread_on_cpu(proc_to_use,start_core,&(vm->cores[i]),tname))) { - PrintError("Thread launch failed\n"); - return -1; - } - } - - // vm->cores[0].cpu_id=0; - // vm->cores[0].physical_cpu_id=0; + core_thread = V3_CREATE_THREAD_ON_CPU(i, start_core, + &(vm->cores[vcore_id]), tname); - // Finally launch the BSP on core 0 - sprintf(tname,"core%u",0); - -#if CONFIG_LINUX - if (vm->num_cores==1) { - start_core(&(vm->cores[0])); - return -1; - } else { - if (!os_hooks->start_thread_on_cpu(0,start_core,&(vm->cores[0]),tname)) { + if (core_thread == NULL) { PrintError("Thread launch failed\n"); return -1; } + + vcore_id++; } -#else - if (!os_hooks->start_thread_on_cpu(0,start_core,&(vm->cores[0]),tname)) { - PrintError("Thread launch failed\n"); - return -1; - } -#endif return 0; diff --git a/palacios/src/palacios/vmm_config.c b/palacios/src/palacios/vmm_config.c index ebad8b7..8cdc907 100644 --- a/palacios/src/palacios/vmm_config.c +++ b/palacios/src/palacios/vmm_config.c @@ -282,10 +282,10 @@ static int determine_paging_mode(struct guest_info *info, v3_cfg_tree_t * core_c { extern v3_cpu_arch_t v3_cpu_types[]; - v3_cfg_tree_t *vm_tree = info->vm_info->cfg_data->cfg; - v3_cfg_tree_t *pg_tree = v3_cfg_subtree(vm_tree, "paging"); - char *pg_mode = v3_cfg_val(pg_tree, "mode"); - char *page_size = v3_cfg_val(pg_tree, "page_size"); + v3_cfg_tree_t * vm_tree = info->vm_info->cfg_data->cfg; + v3_cfg_tree_t * pg_tree = v3_cfg_subtree(vm_tree, "paging"); + char * pg_mode = v3_cfg_val(pg_tree, "mode"); + char * page_size = v3_cfg_val(pg_tree, "page_size"); PrintDebug("Paging mode specified as %s\n", pg_mode); @@ -403,7 +403,7 @@ static int post_config_vm(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) { static int post_config_core(struct guest_info * info, v3_cfg_tree_t * cfg) { - + info->core_run_state = CORE_STOPPED; if (info->vm_info->vm_class == V3_PC_VM) { if (post_config_pc_core(info, cfg) == -1) { diff --git a/palacios/src/palacios/vmm_config_class.h b/palacios/src/palacios/vmm_config_class.h index 1770a6f..8f74a44 100644 --- a/palacios/src/palacios/vmm_config_class.h +++ b/palacios/src/palacios/vmm_config_class.h @@ -22,17 +22,6 @@ static int pre_config_pc_core(struct guest_info * info, v3_cfg_tree_t * cfg) { - if (info->cpu_id!=0) { - // I am an AP, so I will start in INIT mode, - // not in real mode. This means I will wait for - // an INIT and then for a SIPI. The SIPI will - // tell me where to start executing in real mode - info->cpu_mode = INIT; - } else { - // I am the MP, so I will start as normal - info->cpu_mode = REAL; - } - info->mem_mode = PHYSICAL_MEM; diff --git a/palacios/src/palacios/vmx.c b/palacios/src/palacios/vmx.c index 7cea8a1..d565d1f 100644 --- a/palacios/src/palacios/vmx.c +++ b/palacios/src/palacios/vmx.c @@ -744,35 +744,29 @@ int v3_vmx_enter(struct guest_info * info) { } -int v3_start_vmx_guest(struct guest_info* info) { +int v3_start_vmx_guest(struct guest_info * info) { - PrintDebug("Starting VMX core %u\n",info->cpu_id); - if (info->cpu_mode==INIT) { - PrintDebug("VMX core %u: I am an AP in INIT mode, waiting for that to change\n",info->cpu_id); - while (info->cpu_mode==INIT) { + PrintDebug("Starting VMX core %u\n", info->cpu_id); + + if (info->cpu_id == 0) { + info->core_run_state = CORE_RUNNING; + info->vm_info->run_state = VM_RUNNING; + } else { + + PrintDebug("VMX core %u: Waiting for core initialization\n", info->cpu_id); + + while (info->core_run_state == CORE_STOPPED) { v3_yield(info); //PrintDebug("VMX core %u: still waiting for INIT\n",info->cpu_id); } - PrintDebug("VMX core %u: I am out of INIT\n",info->cpu_id); - if (info->cpu_mode==SIPI) { - PrintDebug("VMX core %u: I am waiting on a SIPI to set my starting address\n",info->cpu_id); - while (info->cpu_mode==SIPI) { - v3_yield(info); - //PrintDebug("VMX core %u: still waiting for SIPI\n",info->cpu_id); - } - } - PrintDebug("VMX core %u: I have my SIPI\n", info->cpu_id); + + PrintDebug("VMX core %u initialized\n", info->cpu_id); } - if (info->cpu_mode!=REAL) { - PrintError("VMX core %u: I am not in REAL mode at launch! Huh?!\n", info->cpu_id); - return -1; - } PrintDebug("VMX core %u: I am starting at CS=0x%x (base=0x%p, limit=0x%x), RIP=0x%p\n", - info->cpu_id, info->segments.cs.selector, (void*)(info->segments.cs.base), - info->segments.cs.limit,(void*)(info->rip)); - + info->cpu_id, info->segments.cs.selector, (void *)(info->segments.cs.base), + info->segments.cs.limit, (void *)(info->rip)); PrintDebug("VMX core %u: Launching VMX VM\n", info->cpu_id); diff --git a/test/geekos_test_vm/build/Makefile b/test/geekos_test_vm/build/Makefile index a7669bd..ff3dbb6 100644 --- a/test/geekos_test_vm/build/Makefile +++ b/test/geekos_test_vm/build/Makefile @@ -156,7 +156,7 @@ TARGET_OBJCOPY := $(TARGET_CC_PREFIX)objcopy # Nasm (http://nasm.sourceforge.net) #NASM := $(PROJECT_ROOT)/../../devtools/bin/nasm -NASM := nasm +NASM := nasm -f elf AS = as --32