From: Jack Lange Date: Fri, 29 Jul 2011 22:57:39 +0000 (-0400) Subject: added barrier synchronization and integrated it with pause/continue functionality X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=9a9c14099c09e9ed86d8ac2f73f7406d0ac8a90b added barrier synchronization and integrated it with pause/continue functionality --- diff --git a/linux_module/palacios.h b/linux_module/palacios.h index 8730346..633ede1 100644 --- a/linux_module/palacios.h +++ b/linux_module/palacios.h @@ -15,6 +15,9 @@ #define V3_VM_CONSOLE_CONNECT 20 #define V3_VM_STOP 22 +#define V3_VM_PAUSE 23 +#define V3_VM_CONTINUE 24 + #define V3_VM_INSPECT 30 diff --git a/linux_module/vm.c b/linux_module/vm.c index 852733c..967ecf6 100644 --- a/linux_module/vm.c +++ b/linux_module/vm.c @@ -129,10 +129,20 @@ static long v3_vm_ioctl(struct file * filp, switch (ioctl) { case V3_VM_STOP: { - printk("Stopping VM\n"); + printk("Stopping VM (%s)\n", guest->name); stop_palacios_vm(guest); break; } + case V3_VM_PAUSE: { + printk("Pausing VM (%s)\n", guest->name); + v3_pause_vm(guest->v3_ctx); + break; + } + case V3_VM_CONTINUE: { + printk("Continuing VM (%s)\n", guest->name); + v3_continue_vm(guest->v3_ctx); + break; + } default: { struct vm_ctrl * ctrl = get_ctrl(guest, ioctl); diff --git a/linux_usr/Makefile b/linux_usr/Makefile index f7ca219..1af1d8d 100644 --- a/linux_usr/Makefile +++ b/linux_usr/Makefile @@ -21,9 +21,6 @@ v3_serial : v3_serial.c v3_ctrl.h v3_monitor : v3_cons.c v3_ctrl.h gcc -static v3_monitor.c -o v3_monitor -v3_net : v3_net.c v3_ctrl.h - gcc -static v3_net.c -o v3_net - v3_user_host_dev_example: v3_user_host_dev_example.c v3_user_host_dev.h v3_user_host_dev.c gcc -static -I../linux_module v3_user_host_dev_example.c v3_user_host_dev.c -o v3_user_host_dev_example @@ -41,4 +38,4 @@ v3_user_keyed_stream_file: v3_user_keyed_stream_file.c v3_user_keyed_stream.h v3 clean: - rm -f v3_ctrl v3_cons v3_mem v3_monitor v3_serial v3_net v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file + rm -f v3_ctrl v3_cons v3_mem v3_monitor v3_serial v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file diff --git a/linux_usr/v3_net.c b/linux_usr/v3_net.c deleted file mode 100644 index 5449cd4..0000000 --- a/linux_usr/v3_net.c +++ /dev/null @@ -1,64 +0,0 @@ -/* - * V3 Control utility for Palacios network services - * (c) Lei Xia, 2010 - */ - - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "v3_ctrl.h" - -struct v3_network { - unsigned char socket; - unsigned char packet; - unsigned char vnet; -}; - -int main(int argc, char* argv[]) { - int v3_fd = 0; - struct v3_network net; - int i; - - if (argc <= 1) { - printf("Usage: ./v3_mem [socket] [packet] [vnet]\n"); - return -1; - } - - for (i = 1; i < argc; i++){ - if(!strcasecmp (argv[i], "packet")){ - net.packet = 1; - }else if(!strcasecmp (argv[i], "socket")){ - net.socket = 1; - }else if(!strcasecmp (argv[i], "vnet")){ - net.vnet = 1; - }else { - printf("unknown v3 network service: %s, ignored\n", argv[i]); - } - } - - printf("Network service: socket: %d, packet: %d, vnet: %d\n", net.socket, net.packet, net.vnet); - - v3_fd = open(v3_dev, O_RDONLY); - - if (v3_fd == -1) { - printf("Error opening V3Vee control device\n"); - return -1; - } - - ioctl(v3_fd, V3_START_NETWORK, &net); - - - /* Close the file descriptor. */ - close(v3_fd); - - - return 0; -} - diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index 4360a64..51ca2de 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -37,7 +37,7 @@ #include #include #include - +#include #ifdef V3_CONFIG_TELEMETRY @@ -170,7 +170,7 @@ struct v3_vm_info { v3_vm_operating_mode_t run_state; - + struct v3_barrier barrier; struct v3_extensions extensions; diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 337226f..708c331 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -331,6 +331,10 @@ void Shutdown_V3( void ); struct v3_vm_info * v3_create_vm(void * cfg, void * priv_data, char * name); int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask); int v3_stop_vm(struct v3_vm_info * vm); +int v3_pause_vm(struct v3_vm_info * vm); +int v3_continue_vm(struct v3_vm_info * vm); + + int v3_free_vm(struct v3_vm_info * vm); int v3_deliver_irq(struct v3_vm_info * vm, struct v3_interrupt * intr); diff --git a/palacios/include/palacios/vmm_barrier.h b/palacios/include/palacios/vmm_barrier.h index d4b7eda..19f54fc 100644 --- a/palacios/include/palacios/vmm_barrier.h +++ b/palacios/include/palacios/vmm_barrier.h @@ -22,22 +22,30 @@ #ifdef __V3VEE__ +#include +#include + -#include struct v3_barrier { - uint64_t cpus; int active; // If 1, barrier is active, everyone must wait // If 0, barrier is clear, can proceed + struct v3_bitmap cpu_map; + v3_lock_t lock; }; +struct v3_vm_info; +struct guest_info; + +int v3_init_barrier(struct v3_vm_info * vm_info); +int v3_deinit_barrier(struct v3_vm_info * vm_info); -int v3_init_barrier(struct v3_barrier * barrier); +int v3_raise_barrier(struct v3_vm_info * vm_info, struct guest_info * local_core); +int v3_lower_barrier(struct v3_vm_info * vm_info); -int v3_activate_barrier(struct guest_info * core); -int v3_check_barrier(struct guest_info * core); +int v3_wait_at_barrier(struct guest_info * core); #endif diff --git a/palacios/include/palacios/vmm_bitmap.h b/palacios/include/palacios/vmm_bitmap.h new file mode 100644 index 0000000..4842585 --- /dev/null +++ b/palacios/include/palacios/vmm_bitmap.h @@ -0,0 +1,48 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2011, Jack Lange + * Copyright (c) 2011, The V3VEE Project + * All rights reserved. + * + * Author: Jack Lange + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + + + +#ifndef __VMM_BITMAP_H__ +#define __VMM_BITMAP_H__ + +#ifdef __V3VEE__ +#include + + + +struct v3_bitmap { + int num_bits; // number of valid bit positions in the bitmap + uint8_t * bits; // actual bitmap. Dynamically allocated... ugly +}; + + +int v3_bitmap_init(struct v3_bitmap * bitmap, int num_bits); +void v3_bitmap_deinit(struct v3_bitmap * bitmap); +int v3_bitmap_reset(struct v3_bitmap * bitmap); + +int v3_bitmap_set(struct v3_bitmap * bitmap, int index); +int v3_bitmap_clear(struct v3_bitmap * bitmap, int index); +int v3_bitmap_check(struct v3_bitmap * bitmap, int index); + + + +#endif + +#endif diff --git a/palacios/src/palacios/Makefile b/palacios/src/palacios/Makefile index 4c55738..d067aae 100644 --- a/palacios/src/palacios/Makefile +++ b/palacios/src/palacios/Makefile @@ -35,7 +35,8 @@ obj-y := \ vmm_extensions.o \ vmm_mtrr.o \ vmm_multitree.o \ - + vmm_bitmap.o \ + vmm_barrier.o \ diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index f366a76..a781fe3 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -36,6 +36,8 @@ #include #include +#include + #include @@ -691,6 +693,8 @@ int v3_start_svm_guest(struct guest_info * info) { break; } + v3_wait_at_barrier(info); + if (info->vm_info->run_state == VM_STOPPED) { info->core_run_state = CORE_STOPPED; diff --git a/palacios/src/palacios/vm_guest.c b/palacios/src/palacios/vm_guest.c index 62d2a94..7bd275f 100644 --- a/palacios/src/palacios/vm_guest.c +++ b/palacios/src/palacios/vm_guest.c @@ -30,7 +30,7 @@ #include #include #include - +#include v3_cpu_mode_t v3_get_vm_cpu_mode(struct guest_info * info) { @@ -539,6 +539,8 @@ int v3_init_vm(struct v3_vm_info * vm) { v3_init_intr_routers(vm); v3_init_ext_manager(vm); + v3_init_barrier(vm); + // Initialize the memory map if (v3_init_mem_map(vm) == -1) { PrintError("Could not initialize shadow map\n"); @@ -637,6 +639,8 @@ int v3_free_vm_internal(struct v3_vm_info * vm) { v3_deinit_intr_routers(vm); v3_deinit_host_events(vm); + v3_deinit_barrier(vm); + v3_deinit_cpuid_map(vm); v3_deinit_msr_map(vm); v3_deinit_io_map(vm); diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index d60e55a..0597be1 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -404,6 +404,32 @@ int v3_stop_vm(struct v3_vm_info * vm) { } +int v3_pause_vm(struct v3_vm_info * vm) { + + if (vm->run_state != VM_RUNNING) { + PrintError("Tried to pause a VM that was not running\n"); + return -1; + } + + while (v3_raise_barrier(vm, NULL) == -1); + + return 0; +} + + +int v3_continue_vm(struct v3_vm_info * vm) { + + if (vm->run_state != VM_RUNNING) { + PrintError("Tried to continue a VM that was not already running\n"); + return -1; + } + + v3_lower_barrier(vm); + + return 0; +} + + int v3_free_vm(struct v3_vm_info * vm) { int i = 0; // deinitialize guest (free memory, etc...) diff --git a/palacios/src/palacios/vmm_barrier.c b/palacios/src/palacios/vmm_barrier.c index 614ceca..ab0aae4 100644 --- a/palacios/src/palacios/vmm_barrier.c +++ b/palacios/src/palacios/vmm_barrier.c @@ -18,22 +18,51 @@ */ -#include +#include +#include +#include +int v3_init_barrier(struct v3_vm_info * vm_info) { + struct v3_barrier * barrier = &(vm_info->barrier); - -int v3_init_barrier(struct v3_barrier * barrier) { memset(barrier, 0, sizeof(struct v3_barrier)); + v3_bitmap_init(&(barrier->cpu_map), vm_info->num_cores); v3_lock_init(&(barrier->lock)); return 0; } +int v3_deinit_barrier(struct v3_vm_info * vm_info) { + struct v3_barrier * barrier = &(vm_info->barrier); + + v3_bitmap_deinit(&(barrier->cpu_map)); + v3_lock_deinit(&(barrier->lock)); + + return 0; +} + + +/* Barrier synchronization primitive + * -- This call will block until all the guest cores are waiting at a common synchronization point + * in a yield loop. The core will block at the sync point until the barrier is lowered. + * + * ARGUMENTS: + * vm_info -- The VM for which the barrier is being activated + * local_core -- The core whose thread this function is being called from, or NULL + * if the calling thread is not associated with a VM's core context + */ -int v3_activate_barrier(struct guest_info * core, struct v3_barrier * barrier) { +int v3_raise_barrier(struct v3_vm_info * vm_info, struct guest_info * local_core) { + struct v3_barrier * barrier = &(vm_info->barrier); addr_t flag; int acquired = 0; - + int all_blocked = 0; + + int local_vcpu = -1; + int i = 0; + + + flag = v3_lock_irqsave(barrier->lock); if (barrier->active == 0) { @@ -44,11 +73,49 @@ int v3_activate_barrier(struct guest_info * core, struct v3_barrier * barrier) { v3_unlock_irqrestore(barrier->lock, flag); if (acquired == 0) { + /* If we are in a core context and the barrier has already been acquired + we'll be safe and let the other barrier proceed. We will still report an error + though to allow possible cleanups to occur at the call site. + */ + if (local_core != NULL) { + v3_wait_at_barrier(local_core); + } + return -1; } + // If we are raising the barrier from a core context + // we have to mark ourselves blocked first to avoid deadlock + if (local_core != NULL) { + local_vcpu = local_core->vcpu_id; + v3_bitmap_set(&(barrier->cpu_map), local_vcpu); + } + - // wait for barrier catch + // send out interrupts to force exits on all cores + for (i = 0; i < vm_info->num_cores; i++) { + if (vm_info->cores[i].vcpu_id != local_vcpu) { + v3_interrupt_cpu(vm_info, vm_info->cores[i].pcpu_id, 0); + } + } + + // wait for barrier catch on all cores + while (all_blocked == 0) { + all_blocked = 1; + + for (i = 0; i < vm_info->num_cores; i++) { + if (v3_bitmap_check(&(barrier->cpu_map), i) == 0) { + // There is still a core that is not waiting at the barrier + all_blocked = 0; + } + } + + if (all_blocked == 1) { + break; + } + + v3_yield(local_core); + } return 0; @@ -56,20 +123,52 @@ int v3_activate_barrier(struct guest_info * core, struct v3_barrier * barrier) { +/* Lowers a barrier that has already been raised + * guest cores will automatically resume execution + * once this has been called + * + * TODO: Need someway to check that the barrier is active + */ + +int v3_lower_barrier(struct v3_vm_info * vm_info) { + struct v3_barrier * barrier = &(vm_info->barrier); + + // Clear the active flag, so cores won't wait + barrier->active = 0; -int v3_deactivate_barrier(struct v3_barrier * barrier) { + // Clear all the cpu flags, so cores will proceed + v3_bitmap_reset(&(barrier->cpu_map)); + return 0; } -int v3_check_barrier(struct guest_info * core, struct v3_barrier * barrier) { +/* + * Syncronization point for guest cores + * -- called as part of the main VMM event loop for each core + * -- if a barrier has been activated then the core will signal + * it has reached the barrier and sit in a yield loop until the + * barrier has been lowered + */ +int v3_wait_at_barrier(struct guest_info * core) { + struct v3_barrier * barrier = &(core->vm_info->barrier); - if (barrier->activated == 0) { + if (barrier->active == 0) { return 0; } + + /* Barrier has been activated. + * Wait here until it's lowered + */ + - // set cpu bit + // set cpu bit in barrier bitmap + v3_bitmap_set(&(barrier->cpu_map), core->vcpu_id); // wait for cpu bit to clear + while (v3_bitmap_check(&(barrier->cpu_map), core->vcpu_id) == 1) { + v3_yield(core); + } + return 0; } diff --git a/palacios/src/palacios/vmm_bitmap.c b/palacios/src/palacios/vmm_bitmap.c new file mode 100644 index 0000000..c056a5b --- /dev/null +++ b/palacios/src/palacios/vmm_bitmap.c @@ -0,0 +1,101 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2011, Jack Lange + * Copyright (c) 2011, The V3VEE Project + * All rights reserved. + * + * Author: Jack Lange + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#include +#include + + +int v3_bitmap_init(struct v3_bitmap * bitmap, int num_bits) { + int num_bytes = (num_bits / 8) + ((num_bits % 8) > 0); + + bitmap->num_bits = num_bits; + bitmap->bits = V3_Malloc(num_bytes); + + if (bitmap->bits == NULL) { + PrintError("Could not allocate bitmap of %d bits\n", num_bits); + return -1; + } + + memset(bitmap->bits, 0, num_bytes); + + return 0; +} + + +void v3_bitmap_deinit(struct v3_bitmap * bitmap) { + V3_Free(bitmap->bits); +} + + +int v3_bitmap_reset(struct v3_bitmap * bitmap) { + int num_bytes = (bitmap->num_bits / 8) + ((bitmap->num_bits % 8) > 0); + + memset(bitmap->bits, 0, num_bytes); + + return 0; +} + +int v3_bitmap_set(struct v3_bitmap * bitmap, int index) { + int major = index / 8; + int minor = index % 8; + int old_val = 0; + + if (index > (bitmap->num_bits - 1)) { + PrintError("Index out of bitmap range: (pos = %d) (num_bits = %d)\n", + index, bitmap->num_bits); + return -1; + } + + old_val = (bitmap->bits[major] & (0x1 << minor)); + bitmap->bits[major] |= (0x1 << minor); + + return old_val; +} + + +int v3_bitmap_clear(struct v3_bitmap * bitmap, int index) { + int major = index / 8; + int minor = index % 8; + int old_val = 0; + + if (index > (bitmap->num_bits - 1)) { + PrintError("Index out of bitmap range: (pos = %d) (num_bits = %d)\n", + index, bitmap->num_bits); + return -1; + } + + old_val = (bitmap->bits[major] & (0x1 << minor)); + bitmap->bits[major] &= ~(0x1 << minor); + + return old_val; +} + +int v3_bitmap_check(struct v3_bitmap * bitmap, int index) { + int major = index / 8; + int minor = index % 8; + + if (index > (bitmap->num_bits - 1)) { + PrintError("Index out of bitmap range: (pos = %d) (num_bits = %d)\n", + index, bitmap->num_bits); + return -1; + } + + return (bitmap->bits[major] & (0x1 << minor)); +} +