From: Kevin Pedretti Date: Fri, 15 Jan 2010 16:54:41 +0000 (-0700) Subject: Added ICC bus/apic stuff. X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=fa3ef2ff5003b7cd27bfe11e970760c940525966 Added ICC bus/apic stuff. --- diff --git a/palacios/include/devices/icc_bus.h b/palacios/include/devices/icc_bus.h new file mode 100644 index 0000000..d75aa57 --- /dev/null +++ b/palacios/include/devices/icc_bus.h @@ -0,0 +1,37 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2008, Jack Lange + * Copyright (c) 2008, The V3VEE Project + * All rights reserved. + * + * Author: Jack Lange + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#ifndef ICC_BUS_H_ +#define ICC_BUS_H_ + +/** + * + */ +int v3_icc_register_apic(struct v3_vm_info *info, struct vm_device *icc_bus, struct vm_device *apic, uint32_t apic_num); + +/** + * Send an inter-processor interrupt (IPI) from this local APIC to another local APIC. + * + * @param icc_bus The ICC bus that facilitates the communication. + * @param apic_num The remote APIC number. + * @param intr_num The interrupt number. + */ +int v3_icc_send_ipi(struct vm_device * icc_bus, uint32_t apic_num, uint32_t intr_num); + +#endif /* ICC_BUS_H_ */ diff --git a/palacios/src/devices/Kconfig b/palacios/src/devices/Kconfig index 182ce47..262dbed 100644 --- a/palacios/src/devices/Kconfig +++ b/palacios/src/devices/Kconfig @@ -62,7 +62,7 @@ config DEBUG_IDE config IO_APIC bool "IOAPIC" - depends on APIC + depends on ICC_BUS default y help Includes the Virtual IO APIC diff --git a/palacios/src/devices/apic.c b/palacios/src/devices/apic.c index c8ec4c7..f45e7b8 100644 --- a/palacios/src/devices/apic.c +++ b/palacios/src/devices/apic.c @@ -181,11 +181,12 @@ struct apic_state { uint32_t eoi; + uint32_t my_apic_id; + struct vm_device *icc_bus; - struct guest_info * core; + v3_lock_t lock; }; -static void apic_incoming_ipi(void *val); static int apic_read(addr_t guest_addr, void * dst, uint_t length, void * priv_data); static int apic_write(addr_t guest_addr, void * src, uint_t length, void * priv_data); @@ -233,6 +234,8 @@ static void init_apic_state(struct apic_state * apic) { apic->ext_apic_feature.val = 0x00040007; apic->ext_apic_ctrl.val = 0x00000000; apic->spec_eoi.val = 0x00000000; + + v3_lock_init(&(apic->lock)); } @@ -241,7 +244,9 @@ static void init_apic_state(struct apic_state * apic) { static int read_apic_msr(uint_t msr, v3_msr_t * dst, void * priv_data) { struct vm_device * dev = (struct vm_device *)priv_data; struct apic_state * apic = (struct apic_state *)dev->private_data; + v3_lock(apic->lock); dst->value = apic->base_addr; + v3_unlock(apic->lock); return 0; } @@ -251,21 +256,26 @@ static int write_apic_msr(uint_t msr, v3_msr_t src, void * priv_data) { struct apic_state * apic = (struct apic_state *)dev->private_data; struct v3_shadow_region * old_reg = v3_get_shadow_region(dev->vm, apic->base_addr); + if (old_reg == NULL) { // uh oh... PrintError("APIC Base address region does not exit...\n"); return -1; } + v3_lock(apic->lock); + v3_delete_shadow_region(dev->vm, old_reg); apic->base_addr = src.value; if (v3_hook_full_mem(dev->vm, apic->base_addr, apic->base_addr + PAGE_SIZE_4KB, apic_read, apic_write, dev) == -1) { PrintError("Could not hook new APIC Base address\n"); + v3_unlock(apic->lock); return -1; } + v3_unlock(apic->lock); return 0; } @@ -692,6 +702,9 @@ static int apic_read(addr_t guest_addr, void * dst, uint_t length, void * priv_d } +/** + * + */ static int apic_write(addr_t guest_addr, void * src, uint_t length, void * priv_data) { struct vm_device * dev = (struct vm_device *)priv_data; struct apic_state * apic = (struct apic_state *)dev->private_data; @@ -848,8 +861,7 @@ static int apic_write(addr_t guest_addr, void * src, uint_t length, void * priv_ case INT_CMD_LO_OFFSET: apic->int_cmd.lo = op_val; - V3_Call_On_CPU(apic->int_cmd.dst, apic_incoming_ipi, (void *)apic->int_cmd.val); - + v3_icc_send_ipi(apic->icc_bus, apic->int_cmd.dst, apic->int_cmd.val); break; case INT_CMD_HI_OFFSET: apic->int_cmd.hi = op_val; @@ -902,6 +914,7 @@ static int apic_get_intr_number(struct guest_info * info, void * private_data) { return -1; } +#if 0 static int apic_raise_intr(struct guest_info * info, void * private_data, int irq) { #ifdef CONFIG_CRAY_XT // The Seastar is connected directly to the LAPIC via LINT0 on the ICC bus @@ -920,6 +933,7 @@ static int apic_raise_intr(struct guest_info * info, void * private_data, int ir static int apic_lower_intr(struct guest_info * info, void * private_data, int irq) { return 0; } +#endif static int apic_begin_irq(struct guest_info * info, void * private_data, int irq) { struct vm_device * dev = (struct vm_device *)private_data; @@ -947,12 +961,17 @@ static int apic_begin_irq(struct guest_info * info, void * private_data, int irq int v3_apic_raise_intr(struct guest_info * info, struct vm_device * apic_dev, int intr_num) { struct apic_state * apic = (struct apic_state *)apic_dev->private_data; + // Special cases go here + // startup, etc + if (activate_apic_irq(apic, intr_num) == -1) { PrintError("Error: Could not activate apic_irq\n"); return -1; } - v3_interrupt_cpu(info, 0); + // Don't need this since we'll have the IPI force an exit if + // This is called on a different core + /* v3_interrupt_cpu(info, 0); */ return 0; } @@ -1029,7 +1048,7 @@ static void apic_update_time(struct guest_info * info, ullong_t cpu_cycles, ullo apic->tmr_vec_tbl.tmr_mode, apic->tmr_init_cnt, shift_num); if (apic_intr_pending(info, priv_data)) { - PrintDebug("Overriding pending IRQ %d\n", apic_get_intr_number(dev->vm, priv_data)); + PrintDebug("Overriding pending IRQ %d\n", apic_get_intr_number(info, priv_data)); } if (activate_internal_irq(apic, APIC_TMR_INT) == -1) { @@ -1045,57 +1064,11 @@ static void apic_update_time(struct guest_info * info, ullong_t cpu_cycles, ullo } -static void apic_incoming_ipi(void *val) -{ -PrintError("In apic_incoming_ipi, val=%p\n", val); - struct int_cmd_reg int_cmd; - char *type = NULL, *dest; - char foo[8]; - int_cmd.val = (uint64_t)val; - switch (int_cmd.dst_shorthand) - { - case 0x0: - sprintf(foo, "%d", int_cmd.dst); - dest = foo; - break; - case 0x1: - dest = "(self)"; - break; - case 0x2: - dest = "(broadcast inclusive)"; - break; - case 0x3: - dest = "(broadcast)"; - break; - } - switch (int_cmd.msg_type) - { - case 0x0: - type = ""; - break; - case 0x4: - type = "(NMI)"; - break; - case 0x5: - type = "(INIT)"; - break; - case 0x6: - type = "(Startup)"; - break; - } - PrintError("Receieved IPI on CPU %d type=%s dest=%s\n", - V3_Get_CPU(), type, dest); -//%p %s to CPU %d on CPU %d.\n", val, foo, type, dest, (int)V3_Get_CPU()); - return; -} - static struct intr_ctrl_ops intr_ops = { .intr_pending = apic_intr_pending, .get_intr_number = apic_get_intr_number, - .raise_intr = apic_raise_intr, .begin_irq = apic_begin_irq, - .lower_intr = apic_lower_intr, }; @@ -1127,6 +1100,14 @@ static struct v3_device_ops dev_ops = { static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) { PrintDebug("Creating APIC\n"); char * name = v3_cfg_val(cfg, "name"); + char * icc_name = v3_cfg_val(cfg,"irq_bus"); + int i; + struct vm_device * icc = v3_find_dev(vm, icc_name); + + if (!icc) { + PrintError("Cannot find device %s\n", icc_name); + return -1; + } struct apic_state * apic = (struct apic_state *)V3_Malloc(sizeof(struct apic_state)); @@ -1137,11 +1118,16 @@ static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) { return -1; } - v3_register_intr_controller(vm, &intr_ops, dev); - v3_add_timer(vm, &timer_ops, dev); + for (i = 0; i < vm->num_cores; i++) + { + v3_register_intr_controller(&vm->cores[i], &intr_ops, dev); + v3_add_timer(&vm->cores[i], &timer_ops, dev); + } init_apic_state(apic); + v3_icc_register_apic(vm, icc, dev,apic->my_apic_id); + v3_hook_msr(vm, BASE_ADDR_MSR, read_apic_msr, write_apic_msr, dev); v3_hook_full_mem(vm, apic->base_addr, apic->base_addr + PAGE_SIZE_4KB, apic_read, apic_write, dev); diff --git a/palacios/src/devices/icc_bus.c b/palacios/src/devices/icc_bus.c new file mode 100644 index 0000000..fe76cff --- /dev/null +++ b/palacios/src/devices/icc_bus.c @@ -0,0 +1,146 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2008, Jack Lange + * Copyright (c) 2008, The V3VEE Project + * All rights reserved. + * + * Author: Jack Lange + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#include +#include +#include +#include +#include + +#define MAX_APIC 256 + +struct icc_bus_internal { + struct vm_device * apic[MAX_APIC]; +}; + +static struct v3_device_ops dev_ops = { + .free = NULL, + .reset = NULL, + .start = NULL, + .stop = NULL, +}; + +int v3_icc_register_apic(struct v3_vm_info *info, struct vm_device *icc_bus, struct vm_device *apic, uint32_t apic_num) +{ + struct icc_bus_internal * icc = (struct icc_bus_internal *)icc_bus->private_data; + + if (apic_num < MAX_APIC) { + if (icc->apic[apic_num]) { + PrintError("Attempt to re-register apic %u\n", apic_num); + return -1; + } else { + icc->apic[apic_num] = apic; + PrintDebug("Registered apic or ioapic %u\n", apic_num); + return 0; + } + } else { + PrintError("Too many apics for icc bus!"); + return -1; + } +} + +struct ipi_thunk_data { + struct vm_device *target; + uint64_t val; +} ; + +static void icc_force_exit(void *val) +{ + return; +} + +int v3_icc_send_ipi(struct vm_device * icc_bus, uint32_t apic_num, uint32_t val) { + struct icc_bus_internal * internal = (struct icc_bus_internal *)icc_bus->private_data; + + struct int_cmd_reg icr; + icr.lo = val; + + char *type = NULL, *dest = NULL; + char foo[8]; + + switch (icr.dst_shorthand) + { + case 0x0: + sprintf(foo, "%d", icr.dst); + dest = foo; + break; + case 0x1: + dest = "(self)"; + break; + case 0x2: + dest = "(broadcast inclusive)"; + break; + case 0x3: + dest = "(broadcast)"; + break; + } + switch (icr.msg_type) + { + case 0x0: + type = ""; + break; + case 0x4: + type = "(NMI)"; + break; + case 0x5: + type = "(INIT)"; + break; + case 0x6: + type = "(Startup)"; + break; + } + + + PrintDebug("Sending IPI of type %s and destination type %s from LAPIC %u to LAPIC %u.\n", type, dest, V3_Get_CPU(), apic_num); + + v3_apic_raise_intr(internal->apic[apic_num], val & 0xff); + + V3_Call_On_CPU(apic_num, icc_force_exit, (void *)(uint64_t)(val & 0xff)); + + return 0; +} + + +static int init_icc_bus_internal_state(struct icc_bus_internal* icc) { + int i; + for (i=0;iapic[i]=0; } + return 0; +} + +static int icc_bus_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) { + PrintDebug("Creating ICC_BUS\n"); + char * name = v3_cfg_val(cfg, "name"); + + struct icc_bus_internal * icc_bus = (struct icc_bus_internal *)V3_Malloc(sizeof(struct icc_bus_internal)); + + struct vm_device * dev = v3_allocate_device(name, &dev_ops, icc_bus); + + if (v3_attach_device(vm, dev) == -1) { + PrintError("Could not attach device %s\n", name); + return -1; + } + + init_icc_bus_internal_state(icc_bus); + + return 0; +} + + + +device_register("ICC_BUS", icc_bus_init) diff --git a/palacios/src/devices/io_apic.c b/palacios/src/devices/io_apic.c index 95392d0..dce46dd 100644 --- a/palacios/src/devices/io_apic.c +++ b/palacios/src/devices/io_apic.c @@ -281,7 +281,8 @@ static int ioapic_raise_irq(struct guest_info * info, void * private_data, int i if (irq_entry->mask == 0) { PrintDebug("IOAPIC Signalling APIC to raise INTR %d\n", irq_entry->vec); - v3_apic_raise_intr(info, ioapic->apic, irq_entry->vec); + //v3_apic_raise_intr(info, ioapic->apic, irq_entry->vec); + v3_apic_raise_intr(ioapic->apic, irq_entry->vec); } return 0; diff --git a/utils/guest_creator/default.xml b/utils/guest_creator/default.xml index 5635dbd..eb25cde 100644 --- a/utils/guest_creator/default.xml +++ b/utils/guest_creator/default.xml @@ -10,6 +10,8 @@ nested 100 + + @@ -42,10 +44,12 @@ +