From: Jack Lange Date: Fri, 4 Sep 2009 02:41:10 +0000 (-0500) Subject: hardware multicore support X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=068d7e66db537f981fde19a098a12e4390fa3de4 hardware multicore support --- diff --git a/Kconfig b/Kconfig index 0b55661..024170b 100644 --- a/Kconfig +++ b/Kconfig @@ -2,6 +2,7 @@ mainmenu "Palacios VMM Configuration" menu "Target Configuration" + config CRAY_XT bool "Red Storm (Cray XT3/XT4)" help @@ -20,6 +21,23 @@ config VMX Compile with support for Intel VMX +config MULTITHREAD_OS + bool "Compile for a multi threaded OS" + default y + help + This allows Palacios to use OS thread mechanisms + + +config MAX_CPUS + int "Maximum number of cpus" + range 1 255 + default "16" + help + Specifies the maximum number of hardware cpus Palacios can support + + For uniprocessor environments, set this to 1 + + config SOCKET bool "Include Network Socket Support" default y @@ -28,6 +46,7 @@ config SOCKET + config TELEMETRY bool "Enable VMM telemetry support" default y diff --git a/palacios/include/palacios/svm.h b/palacios/include/palacios/svm.h index 9c3d2c9..333d38e 100644 --- a/palacios/include/palacios/svm.h +++ b/palacios/include/palacios/svm.h @@ -77,8 +77,8 @@ #define SVM_HANDLER_HALT 0x2 - -void v3_init_SVM(struct v3_ctrl_ops * vmm_ops); +void v3_init_svm_cpu(int cpu_id); +void v3_init_svm_hooks(struct v3_ctrl_ops * vmm_ops); int v3_is_svm_capable(); #endif diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index 241ec8a..68504b4 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -191,6 +191,9 @@ struct guest_info { #endif void * decoder_state; + + // the logical cpu this guest context is executing on + int cpu_id; }; diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 2da7ee2..81bad2c 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -221,8 +221,6 @@ struct v3_os_hooks { void *(*paddr_to_vaddr)(void *addr); void *(*vaddr_to_paddr)(void *addr); - void (*interrupt_cpu)(struct guest_info * vm, int logical_cpu); - int (*hook_interrupt)(struct guest_info * vm, unsigned int irq); int (*ack_irq)(int irq); @@ -237,6 +235,10 @@ struct v3_os_hooks { void (*mutex_free)(void * mutex); void (*mutex_lock)(void * mutex, int must_spin); void (*mutex_unlock)(void * mutex); + + void (*interrupt_cpu)(struct guest_info * vm, int logical_cpu); + void (*call_on_cpu)(int logical_cpu, void (*fn)(void * arg), void * arg); + void (*start_thread_on_cpu)(int logical_cpu, int (*fn)(void * arg), void * arg, char * thread_name); }; @@ -270,6 +272,8 @@ struct v3_vm_config { int enable_swap; + int guest_cpu; + unsigned long schedule_freq; // in HZ v3_disk_type_t pri_disk_type; @@ -314,7 +318,7 @@ struct v3_interrupt { -void Init_V3(struct v3_os_hooks * hooks, struct v3_ctrl_ops * vmm_ops); +void Init_V3(struct v3_os_hooks * hooks, struct v3_ctrl_ops * vmm_ops, int num_cpus); int v3_deliver_irq(struct guest_info * vm, struct v3_interrupt * intr); diff --git a/palacios/include/palacios/vmx.h b/palacios/include/palacios/vmx.h index f009a24..67e546b 100644 --- a/palacios/include/palacios/vmx.h +++ b/palacios/include/palacios/vmx.h @@ -230,8 +230,8 @@ struct vmx_data { }; int v3_is_vmx_capable(); -void v3_init_vmx(struct v3_ctrl_ops * vm_ops); - +void v3_init_vmx_hooks(struct v3_ctrl_ops * vm_ops); +void v3_init_vmx_cpu(int cpu_id); #endif // ! __V3VEE__ diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index 490e342..3c33292 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -45,7 +45,9 @@ // This is a global pointer to the host's VMCB -static void * host_vmcb = NULL; +static addr_t host_vmcbs[CONFIG_MAX_CPUS] = {0}; + + extern void v3_stgi(); extern void v3_clgi(); @@ -299,7 +301,7 @@ static int start_svm_guest(struct guest_info *info) { rdtscll(info->time_state.cached_host_tsc); // guest_ctrl->TSC_OFFSET = info->time_state.guest_tsc - info->time_state.cached_host_tsc; - v3_svm_launch((vmcb_t*)V3_PAddr(info->vmm_data), &(info->vm_regs), (vmcb_t *)host_vmcb); + v3_svm_launch((vmcb_t *)V3_PAddr(info->vmm_data), &(info->vm_regs), (vmcb_t *)host_vmcbs[info->cpu_id]); rdtscll(tmp_tsc); @@ -434,10 +436,9 @@ static int has_svm_nested_paging() { } - -void v3_init_SVM(struct v3_ctrl_ops * vmm_ops) { +void v3_init_svm_cpu(int cpu_id) { reg_ex_t msr; - extern v3_cpu_arch_t v3_cpu_type; + extern v3_cpu_arch_t v3_cpu_types[]; // Enable SVM on the CPU v3_get_msr(EFER_MSR, &(msr.e_reg.high), &(msr.e_reg.low)); @@ -447,24 +448,26 @@ void v3_init_SVM(struct v3_ctrl_ops * vmm_ops) { PrintDebug("SVM Enabled\n"); // Setup the host state save area - host_vmcb = V3_AllocPages(4); + host_vmcbs[cpu_id] = (addr_t)V3_AllocPages(4); /* 64-BIT-ISSUE */ // msr.e_reg.high = 0; //msr.e_reg.low = (uint_t)host_vmcb; - msr.r_reg = (addr_t)host_vmcb; + msr.r_reg = host_vmcbs[cpu_id]; - PrintDebug("Host State being saved at %p\n", (void *)(addr_t)host_vmcb); + PrintDebug("Host State being saved at %p\n", (void *)host_vmcbs[cpu_id]); v3_set_msr(SVM_VM_HSAVE_PA_MSR, msr.e_reg.high, msr.e_reg.low); - - if (has_svm_nested_paging() == 1) { - v3_cpu_type = V3_SVM_REV3_CPU; + v3_cpu_types[cpu_id] = V3_SVM_REV3_CPU; } else { - v3_cpu_type = V3_SVM_CPU; + v3_cpu_types[cpu_id] = V3_SVM_CPU; } +} + + +void v3_init_svm_hooks(struct v3_ctrl_ops * vmm_ops) { // Setup the SVM specific vmm operations vmm_ops->init_guest = &init_svm_guest; @@ -544,7 +547,7 @@ void v3_init_SVM(struct v3_ctrl_ops * vmm_ops) { vmsave "rdtsc ; " : "=D"(start_hi), "=S"(start_lo), "=a"(end_lo),"=d"(end_hi) - : "c"(host_vmcb), "0"(0), "1"(0), "2"(0), "3"(0) + : "c"(host_vmcb[cpu_id]), "0"(0), "1"(0), "2"(0), "3"(0) ); start = start_hi; @@ -565,7 +568,7 @@ void v3_init_SVM(struct v3_ctrl_ops * vmm_ops) { vmload "rdtsc ; " : "=D"(start_hi), "=S"(start_lo), "=a"(end_lo),"=d"(end_hi) - : "c"(host_vmcb), "0"(0), "1"(0), "2"(0), "3"(0) + : "c"(host_vmcb[cpu_id]), "0"(0), "1"(0), "2"(0), "3"(0) ); start = start_hi; diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index 2af99d2..fcc6ea5 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -33,7 +33,7 @@ #endif -v3_cpu_arch_t v3_cpu_type; +v3_cpu_arch_t v3_cpu_types[CONFIG_MAX_CPUS]; struct v3_os_hooks * os_hooks = NULL; @@ -45,12 +45,54 @@ static struct guest_info * allocate_guest() { } +struct vmm_init_arg { + int cpu_id; + struct v3_ctrl_ops * vmm_ops; +}; + +static void init_cpu(void * arg) { + struct vmm_init_arg * vmm_arg = (struct vmm_init_arg *)arg; + int cpu_id = vmm_arg->cpu_id; + struct v3_ctrl_ops * vmm_ops = vmm_arg->vmm_ops; + +#ifdef CONFIG_SVM + if (v3_is_svm_capable()) { + PrintDebug("Machine is SVM Capable\n"); + v3_init_svm_cpu(cpu_id); + + if (cpu_id == 0) { + v3_init_svm_hooks(vmm_ops); + } + } else +#endif +#ifdef CONFIG_VMX + if (v3_is_vmx_capable()) { + PrintDebug("Machine is VMX Capable\n"); + v3_init_vmx_cpu(cpu_id); + + if (cpu_id == 0) { + v3_init_vmx_hooks(vmm_ops); + } + } else +#endif + { + PrintError("CPU has no virtualization Extensions\n"); + } +} + + + +void Init_V3(struct v3_os_hooks * hooks, struct v3_ctrl_ops * vmm_ops, int num_cpus) { + int i; + struct vmm_init_arg arg; + arg.vmm_ops = vmm_ops; -void Init_V3(struct v3_os_hooks * hooks, struct v3_ctrl_ops * vmm_ops) { - // Set global variables. os_hooks = hooks; - v3_cpu_type = V3_INVALID_CPU; + + for (i = 0; i < CONFIG_MAX_CPUS; i++) { + v3_cpu_types[i] = V3_INVALID_CPU; + } // Register all the possible device types v3_init_devices(); @@ -62,23 +104,18 @@ void Init_V3(struct v3_os_hooks * hooks, struct v3_ctrl_ops * vmm_ops) { vmm_ops->allocate_guest = &allocate_guest; -#ifdef CONFIG_SVM - if (v3_is_svm_capable()) { - PrintDebug("Machine is SVM Capable\n"); - v3_init_SVM(vmm_ops); - } else -#endif -#ifdef CONFIG_VMX - if (v3_is_vmx_capable()) { - PrintDebug("Machine is VMX Capable\n"); - v3_init_vmx(vmm_ops); - - } else -#endif - { - PrintError("CPU has no virtualization Extensions\n"); + if ((hooks) && (hooks->call_on_cpu)) { + + for (i = 0; i < num_cpus; i++) { + arg.cpu_id = i; + + V3_Print("Initializing VMM extensions on cpu %d\n", i); + hooks->call_on_cpu(i, &init_cpu, &arg); + } } + + } diff --git a/palacios/src/palacios/vmm_config.c b/palacios/src/palacios/vmm_config.c index 1cadad7..c728223 100644 --- a/palacios/src/palacios/vmm_config.c +++ b/palacios/src/palacios/vmm_config.c @@ -70,11 +70,13 @@ static int passthrough_mem_write(addr_t guest_addr, void * src, uint_t length, v #endif int v3_pre_config_guest(struct guest_info * info, struct v3_vm_config * config_ptr) { - extern v3_cpu_arch_t v3_cpu_type; + extern v3_cpu_arch_t v3_cpu_types[]; // Amount of ram the Guest will have, rounded to a 4K page boundary info->mem_size = config_ptr->mem_size & ~(addr_t)0xfff; + info->cpu_id = config_ptr->guest_cpu; + /* * Initialize the subsystem data strutures */ @@ -110,7 +112,7 @@ int v3_pre_config_guest(struct guest_info * info, struct v3_vm_config * config_p // Initialize the memory map v3_init_shadow_map(info); - if ((v3_cpu_type == V3_SVM_REV3_CPU) && + if ((v3_cpu_types[info->cpu_id] == V3_SVM_REV3_CPU) && (config_ptr->enable_nested_paging == 1)) { PrintDebug("Guest Page Mode: NESTED_PAGING\n"); info->shdw_pg_mode = NESTED_PAGING; diff --git a/palacios/src/palacios/vmx.c b/palacios/src/palacios/vmx.c index 4f8abec..7789b70 100644 --- a/palacios/src/palacios/vmx.c +++ b/palacios/src/palacios/vmx.c @@ -32,7 +32,9 @@ #include #include -static addr_t vmxon_ptr_phys; +static addr_t host_vmcs_ptrs[CONFIG_MAX_CPUS] = {0}; + + extern int v3_vmx_exit_handler(); extern int v3_vmx_vmlaunch(struct v3_gprs * vm_regs, struct guest_info * info, struct v3_ctrl_regs * ctrl_regs); @@ -484,8 +486,8 @@ static int has_vmx_nested_paging() { -void v3_init_vmx(struct v3_ctrl_ops * vm_ops) { - extern v3_cpu_arch_t v3_cpu_type; +void v3_init_vmx_cpu(int cpu_id) { + extern v3_cpu_arch_t v3_cpu_types[]; struct v3_msr tmp_msr; uint64_t ret = 0; @@ -524,11 +526,11 @@ void v3_init_vmx(struct v3_ctrl_ops * vm_ops) { // Setup VMXON Region - vmxon_ptr_phys = allocate_vmcs(); + host_vmcs_ptrs[cpu_id] = allocate_vmcs(); - PrintDebug("VMXON pointer: 0x%p\n", (void *)vmxon_ptr_phys); + PrintDebug("VMXON pointer: 0x%p\n", (void *)host_vmcs_ptrs[cpu_id]); - if (v3_enable_vmx(vmxon_ptr_phys) == VMX_SUCCESS) { + if (v3_enable_vmx(host_vmcs_ptrs[cpu_id]) == VMX_SUCCESS) { PrintDebug("VMX Enabled\n"); } else { PrintError("VMX initialization failure\n"); @@ -537,11 +539,16 @@ void v3_init_vmx(struct v3_ctrl_ops * vm_ops) { if (has_vmx_nested_paging() == 1) { - v3_cpu_type = V3_VMX_EPT_CPU; + v3_cpu_types[cpu_id] = V3_VMX_EPT_CPU; } else { - v3_cpu_type = V3_VMX_CPU; + v3_cpu_types[cpu_id] = V3_VMX_CPU; } +} + + +void v3_init_vmx_hooks(struct v3_ctrl_ops * vm_ops) { + // Setup the VMX specific vmm operations vm_ops->init_guest = &init_vmx_guest; vm_ops->start_guest = &start_vmx_guest;