From: Peter Dinda Date: Fri, 23 Sep 2011 20:12:58 +0000 (-0500) Subject: Merge branch 'devel' of newskysaw.cs.northwestern.edu:/home/palacios/palacios into... X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=ed0e255bee17901009000903d6059cf1e8335cfb;hp=ef9b4151c09c67502de3720b8fb8f42c10edd073 Merge branch 'devel' of newskysaw.cs.northwestern.edu:/home/palacios/palacios into devel --- diff --git a/Kconfig b/Kconfig index 5397fec..b7fc9a0 100644 --- a/Kconfig +++ b/Kconfig @@ -13,7 +13,6 @@ config KITTEN select BUILT_IN_STRCASECMP select BUILT_IN_ATOI select ALIGNED_PG_ALLOC - select MULTITHREAD_OS help This enables the necesary options to compile Palacios with Kitten @@ -108,13 +107,6 @@ endchoice menu "Supported host OS features" -config MULTITHREAD_OS - bool "Host support for multiple threads" - default y - help - Select this if your OS supports multiple threads of execution. This will enable features in Palacios - to require the creation of additional execution threads. - config ALIGNED_PG_ALLOC bool "Host support for aligned page allocations" @@ -169,6 +161,14 @@ config EXPERIMENTAL The purpose of this option is to allow features under development to be committed to the mainline to more easily track changes and provide access to multiple developers +config CHECKPOINT + bool "Enable Checkpointing" + default n + help + Temporary option to enable experimental checkpoint code + + + endmenu diff --git a/linux_module/Makefile b/linux_module/Makefile index d028d2e..0c20052 100644 --- a/linux_module/Makefile +++ b/linux_module/Makefile @@ -6,6 +6,10 @@ ifdef V3_CONFIG_SYMMOD LDFLAGS += --script=$(PWD)/ld.symmod.cmd endif +ifdef V3_CONFIG_CHECKPOINT +LDFLAGS += --script=$(PWD)/ld.chkpt.cmd +endif + EXTRA_CFLAGS += -I$(PWD)/../palacios/include/ -include autoconf.h -DMODULE=1 -D__KERNEL__=1 diff --git a/linux_module/iface-console.c b/linux_module/iface-console.c index 9eedbe5..ab66d5a 100644 --- a/linux_module/iface-console.c +++ b/linux_module/iface-console.c @@ -140,7 +140,7 @@ console_write(struct file * filp, const char __user * buf, size_t size, loff_t * for (i = 0; i < size; i++) { - if (copy_from_user(&(event.scan_code), buf++, 1)) { + if (copy_from_user(&(event.scan_code), buf + i, 1)) { printk("Console Write fault\n"); return -EFAULT; } @@ -228,7 +228,7 @@ static int console_connect(struct v3_guest * guest, unsigned int cmd, return -1; } - cons_fd = anon_inode_getfd("v3-cons", &cons_fops, cons, 0); + cons_fd = anon_inode_getfd("v3-cons", &cons_fops, cons, O_RDWR); if (cons_fd < 0) { printk("Error creating console inode\n"); diff --git a/linux_module/iface-file.c b/linux_module/iface-file.c index 396183b..965023a 100644 --- a/linux_module/iface-file.c +++ b/linux_module/iface-file.c @@ -63,6 +63,11 @@ static void * palacios_file_open(const char * path, int mode, void * private_dat } else if (mode & FILE_OPEN_MODE_WRITE) { pfile->mode = O_WRONLY; } + + if (mode & FILE_OPEN_MODE_CREATE) { + pfile->mode |= O_CREAT; + } + pfile->filp = filp_open(path, pfile->mode, 0); diff --git a/linux_module/ld.chkpt.cmd b/linux_module/ld.chkpt.cmd new file mode 100644 index 0000000..7a46435 --- /dev/null +++ b/linux_module/ld.chkpt.cmd @@ -0,0 +1,10 @@ +SECTIONS +{ + _v3_chkpt_stores : + { + __start__v3_chkpt_stores = .; + *(_v3_chkpt_stores); + __stop__v3_chkpt_stores = .; + } +} + diff --git a/linux_module/main.c b/linux_module/main.c index a6787ce..9317635 100644 --- a/linux_module/main.c +++ b/linux_module/main.c @@ -67,74 +67,54 @@ static long v3_dev_ioctl(struct file * filp, struct v3_guest * guest = kmalloc(sizeof(struct v3_guest), GFP_KERNEL); if (IS_ERR(guest)) { - printk("Error allocating Kernel guest_image\n"); + printk("Palacios: Error allocating Kernel guest_image\n"); return -EFAULT; } memset(guest, 0, sizeof(struct v3_guest)); - printk("Starting V3 Guest... (%p)\n", guest); + printk("Palacios: Starting V3 Guest...\n"); vm_minor = register_vm(guest); if (vm_minor == -1) { - printk("Too many VMs are currently running\n"); + printk("Palacios Error: Too many VMs are currently running\n"); return -EFAULT; } guest->vm_dev = MKDEV(v3_major_num, vm_minor); if (copy_from_user(&user_image, argp, sizeof(struct v3_guest_img))) { - printk("copy from user error getting guest image...\n"); + printk("Palacios Error: copy from user error getting guest image...\n"); return -EFAULT; } guest->img_size = user_image.size; - printk("Allocating kernel memory for guest image (%llu bytes)\n", user_image.size); + printk("Palacios: Allocating kernel memory for guest image (%llu bytes)\n", user_image.size); guest->img = vmalloc(guest->img_size); if (IS_ERR(guest->img)) { - printk("Error: Could not allocate space for guest image\n"); + printk("Palacios Error: Could not allocate space for guest image\n"); return -EFAULT; } if (copy_from_user(guest->img, user_image.guest_data, guest->img_size)) { - printk("Error loading guest data\n"); + printk("Palacios: Error loading guest data\n"); return -EFAULT; } strncpy(guest->name, user_image.name, 127); - printk("Launching VM\n"); + printk("Palacios: Launching VM\n"); INIT_LIST_HEAD(&(guest->exts)); - init_completion(&(guest->start_done)); - init_completion(&(guest->thread_done)); - - { - struct task_struct * launch_thread = NULL; - // At some point we're going to want to allow the user to specify a CPU mask - // But for now, well just launch from the local core, and rely on the global cpu mask - - preempt_disable(); - launch_thread = kthread_create(start_palacios_vm, guest, guest->name); - - if (IS_ERR(launch_thread)) { - preempt_enable(); - printk("Palacios error creating launch thread for vm (%s)\n", guest->name); - return -EFAULT; - } - - kthread_bind(launch_thread, smp_processor_id()); - preempt_enable(); - - wake_up_process(launch_thread); + if (start_palacios_vm(guest) == -1) { + printk("Palacios: Error starting guest\n"); + return -EFAULT; } - wait_for_completion(&(guest->start_done)); - return guest->vm_dev; break; } diff --git a/linux_module/palacios-stubs.c b/linux_module/palacios-stubs.c index 26ada0b..8eeeb7d 100644 --- a/linux_module/palacios-stubs.c +++ b/linux_module/palacios-stubs.c @@ -180,7 +180,7 @@ static int lnx_thread_target(void * arg) { /** * Creates a kernel thread. */ -static void +static void * palacios_start_kernel_thread( int (*fn) (void * arg), void * arg, @@ -192,8 +192,7 @@ palacios_start_kernel_thread( thread_info->arg = arg; thread_info->name = thread_name; - kthread_run( lnx_thread_target, thread_info, thread_name ); - return; + return kthread_run( lnx_thread_target, thread_info, thread_name ); } @@ -220,7 +219,7 @@ palacios_start_thread_on_cpu(int cpu_id, return NULL; } - kthread_bind(thread, cpu_id); + set_cpus_allowed_ptr(thread, cpumask_of(cpu_id)); wake_up_process(thread); return thread; diff --git a/linux_module/palacios.h b/linux_module/palacios.h index cece080..1ac8c03 100644 --- a/linux_module/palacios.h +++ b/linux_module/palacios.h @@ -10,6 +10,9 @@ /* Global Control IOCTLs */ #define V3_START_GUEST 10 #define V3_STOP_GUEST 11 +#define V3_CREATE_GUEST 12 +#define V3_FREE_GUEST 13 + #define V3_ADD_MEMORY 50 /* VM Specific IOCTLs */ @@ -19,6 +22,11 @@ #define V3_VM_PAUSE 23 #define V3_VM_CONTINUE 24 +#define V3_VM_LAUNCH 25 +#define V3_VM_STOP 26 +#define V3_VM_LOAD 27 +#define V3_VM_SAVE 28 + #define V3_VM_INSPECT 30 #define V3_VM_MOVE_CORE 33 @@ -35,17 +43,23 @@ struct v3_guest_img { unsigned long long size; void * guest_data; char name[128]; -}; +} __attribute__((packed)); struct v3_mem_region { unsigned long long base_addr; unsigned long long num_pages; -}; +} __attribute__((packed)); struct v3_core_move_cmd{ - uint16_t vcore_id; - uint16_t pcore_id; -}; + unsigned short vcore_id; + unsigned short pcore_id; +} __attribute__((packed)); + +struct v3_chkpt_info { + char store[128]; + char url[256]; /* This might need to be bigger... */ +} __attribute__((packed)); + void * trace_malloc(size_t size, gfp_t flags); @@ -64,9 +78,6 @@ struct v3_guest { struct rb_root vm_ctrls; struct list_head exts; - struct completion start_done; - struct completion thread_done; - dev_t vm_dev; struct cdev cdev; }; diff --git a/linux_module/vm.c b/linux_module/vm.c index 1e37587..abb798f 100644 --- a/linux_module/vm.c +++ b/linux_module/vm.c @@ -137,6 +137,48 @@ static long v3_vm_ioctl(struct file * filp, v3_continue_vm(guest->v3_ctx); break; } +#ifdef V3_CONFIG_CHECKPOINT + case V3_VM_SAVE: { + struct v3_chkpt_info chkpt; + void __user * argp = (void __user *)arg; + + memset(&chkpt, 0, sizeof(struct v3_chkpt_info)); + + if (copy_from_user(&chkpt, argp, sizeof(struct v3_chkpt_info))) { + printk("Copy from user error getting checkpoint info\n"); + return -EFAULT; + } + + printk("Saving Guest to %s:%s\n", chkpt.store, chkpt.url); + + if (v3_save_vm(guest->v3_ctx, chkpt.store, chkpt.url) == -1) { + printk("Error checkpointing VM state\n"); + return -EFAULT; + } + + break; + } + case V3_VM_LOAD: { + struct v3_chkpt_info chkpt; + void __user * argp = (void __user *)arg; + + memset(&chkpt, 0, sizeof(struct v3_chkpt_info)); + + if (copy_from_user(&chkpt, argp, sizeof(struct v3_chkpt_info))) { + printk("Copy from user error getting checkpoint info\n"); + return -EFAULT; + } + + printk("Loading Guest to %s:%s\n", chkpt.store, chkpt.url); + + if (v3_load_vm(guest->v3_ctx, chkpt.store, chkpt.url) == -1) { + printk("Error Loading VM state\n"); + return -EFAULT; + } + + break; + } +#endif case V3_VM_MOVE_CORE: { struct v3_core_move_cmd cmd; void __user * argp = (void __user *)arg; @@ -151,9 +193,9 @@ static long v3_vm_ioctl(struct file * filp, printk("moving guest %s vcore %d to CPU %d\n", guest->name, cmd.vcore_id, cmd.pcore_id); v3_move_vm_core(guest->v3_ctx, cmd.vcore_id, cmd.pcore_id); - } - break; + break; + } default: { struct vm_ctrl * ctrl = get_ctrl(guest, ioctl); @@ -204,22 +246,15 @@ extern u32 pg_frees; extern u32 mallocs; extern u32 frees; -int start_palacios_vm(void * arg) { - struct v3_guest * guest = (struct v3_guest *)arg; +int start_palacios_vm(struct v3_guest * guest) { int err; - - daemonize(guest->name); - // allow_signal(SIGKILL); - - init_vm_extensions(guest); guest->v3_ctx = v3_create_vm(guest->img, (void *)guest, guest->name); if (guest->v3_ctx == NULL) { printk("palacios: failed to create vm\n"); - complete(&(guest->start_done)); return -1; } @@ -238,7 +273,6 @@ int start_palacios_vm(void * arg) { if (err) { printk("Fails to add cdev\n"); v3_free_vm(guest->v3_ctx); - complete(&(guest->start_done)); return -1; } @@ -246,12 +280,9 @@ int start_palacios_vm(void * arg) { printk("Fails to create device\n"); cdev_del(&(guest->cdev)); v3_free_vm(guest->v3_ctx); - complete(&(guest->start_done)); return -1; } - complete(&(guest->start_done)); - printk("palacios: launching vm\n"); if (v3_start_vm(guest->v3_ctx, 0xffffffff) < 0) { @@ -262,8 +293,6 @@ int start_palacios_vm(void * arg) { return -1; } - complete(&(guest->thread_done)); - printk("palacios: vm completed. returning.\n"); return 0; @@ -277,7 +306,6 @@ int stop_palacios_vm(struct v3_guest * guest) { v3_stop_vm(guest->v3_ctx); - wait_for_completion(&(guest->thread_done)); v3_free_vm(guest->v3_ctx); diff --git a/linux_module/vm.h b/linux_module/vm.h index 3423d34..f8df6c7 100644 --- a/linux_module/vm.h +++ b/linux_module/vm.h @@ -8,7 +8,7 @@ #include "palacios.h" -int start_palacios_vm(void * arg); +int start_palacios_vm(struct v3_guest * guest); int stop_palacios_vm(struct v3_guest * guest); diff --git a/linux_usr/Makefile b/linux_usr/Makefile index 3aa822c..c36a212 100644 --- a/linux_usr/Makefile +++ b/linux_usr/Makefile @@ -1,17 +1,23 @@ -all: v3_ctrl v3_stop v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file v3_core_move +all: v3_ctrl v3_stop v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file v3_core_move v3_save v3_load -v3_ctrl : v3_ctrl.c v3_ctrl.h +v3_ctrl : v3_ctrl.c v3_pause.c v3_continue.c v3_ctrl.h gcc -static v3_ctrl.c -o v3_ctrl + gcc -static v3_pause.c -o v3_pause + gcc -static v3_continue.c -o v3_continue v3_stop : v3_stop.c v3_ctrl.h gcc -static v3_stop.c -o v3_stop +v3_save : v3_save.c v3_ctrl.h + gcc -static v3_save.c -o v3_save +v3_load : v3_load.c v3_ctrl.h + gcc -static v3_load.c -o v3_load + v3_mem : v3_mem.c v3_ctrl.h gcc -static v3_mem.c -o v3_mem - v3_cons : v3_cons.c v3_cons_sc.c v3_ctrl.h gcc v3_cons.c -o v3_cons -lcurses gcc v3_cons_sc.c -o v3_cons_sc -lcurses diff --git a/linux_usr/v3_continue.c b/linux_usr/v3_continue.c new file mode 100644 index 0000000..5165a6e --- /dev/null +++ b/linux_usr/v3_continue.c @@ -0,0 +1,51 @@ +/* + * V3 Control utility + * (c) Jack lange, 2010 + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v3_ctrl.h" + +int read_file(int fd, int size, unsigned char * buf); + +int main(int argc, char* argv[]) { + char * filename = argv[1]; + int vm_fd = 0; + + + if (argc <= 1) { + printf("Usage: ./v3_stop \n"); + return -1; + } + + printf("Stopping VM\n"); + + vm_fd = open(filename, O_RDONLY); + + if (vm_fd == -1) { + printf("Error opening V3Vee VM device\n"); + return -1; + } + + ioctl(vm_fd, 24, NULL); + + + + /* Close the file descriptor. */ + close(vm_fd); + + + + return 0; +} + + diff --git a/linux_usr/v3_core_move.c b/linux_usr/v3_core_move.c index d7439d8..29bea35 100644 --- a/linux_usr/v3_core_move.c +++ b/linux_usr/v3_core_move.c @@ -15,11 +15,6 @@ #include "v3_ctrl.h" -struct v3_core_move_cmd { - unsigned short vcore_id; - unsigned short pcore_id; -}; - int main(int argc, char* argv[]) { int vm_fd; diff --git a/linux_usr/v3_ctrl.h b/linux_usr/v3_ctrl.h index 550ab9a..0826825 100644 --- a/linux_usr/v3_ctrl.h +++ b/linux_usr/v3_ctrl.h @@ -8,8 +8,19 @@ #define V3_START_GUEST 10 #define V3_STOP_GUEST 11 +#define V3_CREATE_GUEST 12 +#define V3_FREE_GUEST 13 + + +#define V3_VM_PAUSE 23 +#define V3_VM_CONTINUE 24 + +#define V3_VM_LAUNCH 25 +#define V3_VM_STOP 26 +#define V3_VM_LOAD 27 +#define V3_VM_SAVE 28 + #define V3_ADD_MEMORY 50 -#define V3_START_NETWORK 60 #define V3_VM_CONSOLE_CONNECT 20 #define V3_VM_SERIAL_CONNECT 21 @@ -22,12 +33,24 @@ struct v3_guest_img { unsigned long long size; void * guest_data; char name[128]; -}; +} __attribute__((packed)); struct v3_mem_region { unsigned long long base_addr; unsigned long long num_pages; -}; +} __attribute__((packed)); + + +struct v3_core_move_cmd{ + unsigned short vcore_id; + unsigned short pcore_id; +} __attribute__((packed)); + + +struct v3_chkpt_info { + char store[128]; + char url[256]; /* This might need to be bigger... */ +} __attribute__((packed)); #endif diff --git a/linux_usr/v3_load.c b/linux_usr/v3_load.c new file mode 100644 index 0000000..019c8fa --- /dev/null +++ b/linux_usr/v3_load.c @@ -0,0 +1,70 @@ +/* + * V3 checkpoint save utility + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "v3_ctrl.h" + + +#define MAX_STORE_LEN 128 +#define MAX_URL_LEN 256 + + +struct v3_chkpt_info chkpt; + +int main(int argc, char* argv[]) { + int vm_fd; + char * vm_dev = NULL; + + if (argc < 4) { + printf("Usage: ./v3_load \n"); + return -1; + } + + vm_dev = argv[1]; + + if (strlen(argv[2]) >= MAX_STORE_LEN) { + printf("ERROR: Checkpoint store name longer than maximum size (%d)\n", MAX_STORE_LEN); + return -1; + } + + strncpy(chkpt.store, argv[2], MAX_STORE_LEN); + + + if (strlen(argv[3]) >= MAX_URL_LEN) { + printf("ERROR: Checkpoint URL longer than maximum size (%d)\n", MAX_URL_LEN); + return -1; + } + + strncpy(chkpt.url, argv[3], MAX_URL_LEN); + + vm_fd = open(vm_dev, O_RDONLY); + if (vm_fd == -1) { + printf("Error opening VM device: %s\n", vm_dev); + return -1; + } + + ioctl(vm_fd, V3_VM_LOAD, &chkpt); + + /* Close the file descriptor. */ + close(vm_fd); + + return 0; +} + + diff --git a/linux_usr/v3_pause.c b/linux_usr/v3_pause.c new file mode 100644 index 0000000..ddf7c47 --- /dev/null +++ b/linux_usr/v3_pause.c @@ -0,0 +1,51 @@ +/* + * V3 Control utility + * (c) Jack lange, 2010 + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v3_ctrl.h" + +int read_file(int fd, int size, unsigned char * buf); + +int main(int argc, char* argv[]) { + char * filename = argv[1]; + int vm_fd = 0; + + + if (argc <= 1) { + printf("Usage: ./v3_stop \n"); + return -1; + } + + printf("Stopping VM\n"); + + vm_fd = open(filename, O_RDONLY); + + if (vm_fd == -1) { + printf("Error opening V3Vee VM device\n"); + return -1; + } + + ioctl(vm_fd, 23, NULL); + + + + /* Close the file descriptor. */ + close(vm_fd); + + + + return 0; +} + + diff --git a/linux_usr/v3_save.c b/linux_usr/v3_save.c new file mode 100644 index 0000000..009e0f2 --- /dev/null +++ b/linux_usr/v3_save.c @@ -0,0 +1,70 @@ +/* + * V3 checkpoint save utility + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "v3_ctrl.h" + + +#define MAX_STORE_LEN 128 +#define MAX_URL_LEN 256 + + +struct v3_chkpt_info chkpt; + +int main(int argc, char* argv[]) { + int vm_fd; + char * vm_dev = NULL; + + if (argc < 4) { + printf("Usage: ./v3_save \n"); + return -1; + } + + vm_dev = argv[1]; + + if (strlen(argv[2]) >= MAX_STORE_LEN) { + printf("ERROR: Checkpoint store name longer than maximum size (%d)\n", MAX_STORE_LEN); + return -1; + } + + strncpy(chkpt.store, argv[2], MAX_STORE_LEN); + + + if (strlen(argv[3]) >= MAX_URL_LEN) { + printf("ERROR: Checkpoint URL longer than maximum size (%d)\n", MAX_URL_LEN); + return -1; + } + + strncpy(chkpt.url, argv[3], MAX_URL_LEN); + + vm_fd = open(vm_dev, O_RDONLY); + if (vm_fd == -1) { + printf("Error opening VM device: %s\n", vm_dev); + return -1; + } + + ioctl(vm_fd, V3_VM_SAVE, &chkpt); + + /* Close the file descriptor. */ + close(vm_fd); + + return 0; +} + + diff --git a/palacios/include/interfaces/vmm_file.h b/palacios/include/interfaces/vmm_file.h index 9776913..46b89b1 100644 --- a/palacios/include/interfaces/vmm_file.h +++ b/palacios/include/interfaces/vmm_file.h @@ -39,6 +39,7 @@ uint64_t v3_file_write(v3_file_t file, uint8_t * buf, uint64_t len, uint64_t off #define FILE_OPEN_MODE_READ (1 << 0) #define FILE_OPEN_MODE_WRITE (1 << 1) +#define FILE_OPEN_MODE_CREATE (1 << 2) struct v3_file_hooks { diff --git a/palacios/include/palacios/svm.h b/palacios/include/palacios/svm.h index 665b1c2..ed227fa 100644 --- a/palacios/include/palacios/svm.h +++ b/palacios/include/palacios/svm.h @@ -90,7 +90,8 @@ int v3_svm_enter(struct guest_info * info); int v3_start_svm_guest(struct guest_info *info); int v3_reset_svm_vm_core(struct guest_info * core, addr_t rip); - +int v3_svm_load_core(struct guest_info * core, void * ctx); +int v3_svm_save_core(struct guest_info * core, void * ctx); #endif diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 8a75498..ad3f952 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -182,15 +182,16 @@ struct guest_info; }) -#ifdef V3_CONFIG_MULTITHREAD_OS -#define V3_CREATE_THREAD(fn, arg, name) \ - do { \ - extern struct v3_os_hooks * os_hooks; \ - if ((os_hooks) && (os_hooks)->start_kernel_thread) { \ - (os_hooks)->start_kernel_thread(fn, arg, name); \ - } \ - } while (0) + +#define V3_CREATE_THREAD(fn, arg, name) ({ \ + void * thread = NULL; \ + extern struct v3_os_hooks * os_hooks; \ + if ((os_hooks) && (os_hooks)->start_kernel_thread) { \ + thread = (os_hooks)->start_kernel_thread(fn, arg, name); \ + } \ + thread; \ + }) @@ -223,7 +224,6 @@ struct guest_info; ret; \ }) -#endif /* ** */ @@ -260,10 +260,8 @@ void v3_yield(struct guest_info * info); void v3_yield_cond(struct guest_info * info); void v3_print_cond(const char * fmt, ...); - -#ifdef V3_CONFIG_MULTITHREAD_OS void v3_interrupt_cpu(struct v3_vm_info * vm, int logical_cpu, int vector); -#endif + v3_cpu_arch_t v3_get_cpu_type(int cpu_id); @@ -310,7 +308,7 @@ struct v3_os_hooks { - void (*start_kernel_thread)(int (*fn)(void * arg), void * arg, char * thread_name); + void * (*start_kernel_thread)(int (*fn)(void * arg), void * arg, char * thread_name); void (*interrupt_cpu)(struct v3_vm_info * vm, int logical_cpu, int vector); void (*call_on_cpu)(int logical_cpu, void (*fn)(void * arg), void * arg); void * (*start_thread_on_cpu)(int cpu_id, int (*fn)(void * arg), void * arg, char * thread_name); @@ -343,6 +341,9 @@ int v3_stop_vm(struct v3_vm_info * vm); int v3_pause_vm(struct v3_vm_info * vm); int v3_continue_vm(struct v3_vm_info * vm); +int v3_save_vm(struct v3_vm_info * vm, char * store, char * url); +int v3_load_vm(struct v3_vm_info * vm, char * store, char * url); + int v3_move_vm_core(struct v3_vm_info * vm, int vcore_id, int target_cpu); int v3_free_vm(struct v3_vm_info * vm); diff --git a/palacios/include/palacios/vmm_checkpoint.h b/palacios/include/palacios/vmm_checkpoint.h new file mode 100644 index 0000000..9bb5e81 --- /dev/null +++ b/palacios/include/palacios/vmm_checkpoint.h @@ -0,0 +1,56 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2011, Madhav Suresh + * Copyright (c) 2011, The V3VEE Project + * All rights reserved. + * + * Authors: Madhav Suresh + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#ifndef __VMM_CHECKPOINT_H__ +#define __VMM_CHECKPOINT_H__ + +#ifdef __V3VEE__ + +#include + + +struct v3_chkpt; + + +struct v3_chkpt_ctx { + struct v3_chkpt * chkpt; + struct v3_chkpt_ctx * parent; + void * store_ctx; +}; + +/* Temporary */ +#define V3_CHKPT_STD_SAVE(ctx,x) v3_chkpt_save(ctx,#x,sizeof(x),&(x)) +#define V3_CHKPT_STD_LOAD(ctx,x) v3_chkpt_load(ctx,#x,sizeof(x),&(x)) + + +int v3_chkpt_save(struct v3_chkpt_ctx * ctx, char * tag, uint64_t len, void * buf); +int v3_chkpt_load(struct v3_chkpt_ctx * ctx, char * tag, uint64_t len, void * buf); + +int v3_chkpt_close_ctx(struct v3_chkpt_ctx * ctx); +struct v3_chkpt_ctx * v3_chkpt_open_ctx(struct v3_chkpt * chkpt, struct v3_chkpt_ctx * parent, char * name); + +int v3_chkpt_save_vm(struct v3_vm_info * vm, char * store, char * url); +int v3_chkpt_load_vm(struct v3_vm_info * vm, char * store, char * url); + +int V3_init_checkpoint(); +int V3_deinit_checkpoint(); + +#endif + +#endif diff --git a/palacios/include/palacios/vmm_dev_mgr.h b/palacios/include/palacios/vmm_dev_mgr.h index 1f10874..156b560 100644 --- a/palacios/include/palacios/vmm_dev_mgr.h +++ b/palacios/include/palacios/vmm_dev_mgr.h @@ -30,6 +30,10 @@ #include #include +#ifdef V3_CONFIG_CHECKPOINT +#include +#endif + struct v3_vm_info; struct v3_device_ops; @@ -92,9 +96,10 @@ int v3_deinit_dev_mgr(struct v3_vm_info * vm); int v3_free_vm_devices(struct v3_vm_info * vm); - - - +#ifdef V3_CONFIG_CHECKPOINT +int v3_save_vm_devices(struct v3_vm_info * vm, struct v3_chkpt * chkpt); +int v3_load_vm_devices(struct v3_vm_info * vm, struct v3_chkpt * chkpt); +#endif @@ -102,16 +107,12 @@ int V3_init_devices(); int V3_deinit_devices(); -#ifdef V3_CONFIG_KEYED_STREAMS -#include -#endif - struct v3_device_ops { int (*free)(void * private_data); -#ifdef V3_CONFIG_KEYED_STREAMS - int (*checkpoint)(struct vm_device *dev, v3_keyed_stream_t stream); - int (*restore)(struct vm_device *dev, v3_keyed_stream_t stream); +#ifdef V3_CONFIG_CHECKPOINT + int (*save)(struct v3_chkpt_ctx * ctx, void * private_data); + int (*load)(struct v3_chkpt_ctx * ctx, void * privata_data); #endif }; diff --git a/palacios/include/palacios/vmx.h b/palacios/include/palacios/vmx.h index acadf22..06e4df2 100644 --- a/palacios/include/palacios/vmx.h +++ b/palacios/include/palacios/vmx.h @@ -218,12 +218,21 @@ int v3_is_vmx_capable(); void v3_init_vmx_cpu(int cpu_id); void v3_deinit_vmx_cpu(int cpu_id); +int v3_init_vmx_vmcs(struct guest_info * info, v3_vm_class_t vm_class); +int v3_deinit_vmx_vmcs(struct guest_info * core); + int v3_start_vmx_guest(struct guest_info* info); int v3_reset_vmx_vm_core(struct guest_info * core, addr_t rip); +void v3_flush_vmx_vm_core(struct guest_info * core); + int v3_vmx_enter(struct guest_info * info); -int v3_init_vmx_vmcs(struct guest_info * info, v3_vm_class_t vm_class); -int v3_deinit_vmx_vmcs(struct guest_info * core); +int v3_vmx_load_core(struct guest_info * core, void * ctx); +int v3_vmx_save_core(struct guest_info * core, void * ctx); + + + + #endif // ! __V3VEE__ diff --git a/palacios/src/devices/8254.c b/palacios/src/devices/8254.c index 9a9ee99..1a44399 100644 --- a/palacios/src/devices/8254.c +++ b/palacios/src/devices/8254.c @@ -682,65 +682,39 @@ static int pit_free(void * private_data) { return 0; } -#ifdef V3_CONFIG_KEYED_STREAMS -static int pit_checkpoint(struct vm_device *dev, v3_keyed_stream_t stream) -{ - struct pit *p = (struct pit *) (dev->private_data); - - v3_keyed_stream_key_t ks; - - ks = v3_keyed_stream_open_key(stream,dev->name); - - if (!ks) { - return -1; - } - - STD_SAVE(stream,ks,p->pit_counter); - STD_SAVE(stream,ks,p->pit_reload); - STD_SAVE(stream,ks,p->ch_0); - STD_SAVE(stream,ks,p->ch_1); - STD_SAVE(stream,ks,p->ch_2); - STD_SAVE(stream,ks,p->speaker); - - v3_keyed_stream_close_key(stream,ks); +#ifdef V3_CONFIG_CHECKPOINT +static int pit_save(struct v3_chkpt_ctx * ctx, void * private_data) { + struct pit * pit_state = (struct pit *)private_data; + + V3_CHKPT_STD_SAVE(ctx, pit_state->pit_counter); + V3_CHKPT_STD_SAVE(ctx, pit_state->pit_reload); + V3_CHKPT_STD_SAVE(ctx, pit_state->ch_0); + V3_CHKPT_STD_SAVE(ctx, pit_state->ch_1); + V3_CHKPT_STD_SAVE(ctx, pit_state->ch_2); + V3_CHKPT_STD_SAVE(ctx, pit_state->speaker); return 0; - - } -static int pit_restore(struct vm_device *dev, v3_keyed_stream_t stream) -{ - struct pit *p = (struct pit *) (dev->private_data); +static int pit_load(struct v3_chkpt_ctx * ctx, void * private_data) { + struct pit * pit_state = (struct pit *)private_data; - v3_keyed_stream_key_t ks; + V3_CHKPT_STD_LOAD(ctx, pit_state->pit_counter); + V3_CHKPT_STD_LOAD(ctx, pit_state->pit_reload); + V3_CHKPT_STD_LOAD(ctx, pit_state->ch_0); + V3_CHKPT_STD_LOAD(ctx, pit_state->ch_1); + V3_CHKPT_STD_LOAD(ctx, pit_state->ch_2); + V3_CHKPT_STD_LOAD(ctx, pit_state->speaker); - ks = v3_keyed_stream_open_key(stream,dev->name); - - if (!ks) { - return -1; - } - - STD_LOAD(stream,ks,p->pit_counter); - STD_LOAD(stream,ks,p->pit_reload); - STD_LOAD(stream,ks,p->ch_0); - STD_LOAD(stream,ks,p->ch_1); - STD_LOAD(stream,ks,p->ch_2); - STD_LOAD(stream,ks,p->speaker); - - v3_keyed_stream_close_key(stream,ks); - return 0; - - } #endif static struct v3_device_ops dev_ops = { .free = (int (*)(void *))pit_free, -#ifdef V3_CONFIG_KEYED_STREAMS - .checkpoint = pit_checkpoint, - .restore = pit_restore, +#ifdef V3_CONFIG_CHECKPOINT + .save = pit_save, + .load = pit_load, #endif }; diff --git a/palacios/src/devices/8259a.c b/palacios/src/devices/8259a.c index c24402d..5103791 100644 --- a/palacios/src/devices/8259a.c +++ b/palacios/src/devices/8259a.c @@ -737,10 +737,96 @@ static int pic_free(struct pic_internal * state) { return 0; } +#ifdef V3_CONFIG_CHECKPOINT +static int pic_save(struct v3_chkpt_ctx * ctx, void * private_data) { + struct pic_internal * pic = (struct pic_internal *)private_data; + + V3_CHKPT_STD_SAVE(ctx, pic->master_irr); + V3_CHKPT_STD_SAVE(ctx, pic->slave_irr); + + V3_CHKPT_STD_SAVE(ctx, pic->master_isr); + V3_CHKPT_STD_SAVE(ctx, pic->slave_isr); + + V3_CHKPT_STD_SAVE(ctx, pic->master_elcr); + V3_CHKPT_STD_SAVE(ctx, pic->slave_elcr); + V3_CHKPT_STD_SAVE(ctx, pic->master_elcr_mask); + V3_CHKPT_STD_SAVE(ctx, pic->slave_elcr_mask); + + V3_CHKPT_STD_SAVE(ctx, pic->master_icw1); + V3_CHKPT_STD_SAVE(ctx, pic->master_icw2); + V3_CHKPT_STD_SAVE(ctx, pic->master_icw3); + V3_CHKPT_STD_SAVE(ctx, pic->master_icw4); + + + V3_CHKPT_STD_SAVE(ctx, pic->slave_icw1); + V3_CHKPT_STD_SAVE(ctx, pic->slave_icw2); + V3_CHKPT_STD_SAVE(ctx, pic->slave_icw3); + V3_CHKPT_STD_SAVE(ctx, pic->slave_icw4); + + + V3_CHKPT_STD_SAVE(ctx, pic->master_imr); + V3_CHKPT_STD_SAVE(ctx, pic->slave_imr); + V3_CHKPT_STD_SAVE(ctx, pic->master_ocw2); + V3_CHKPT_STD_SAVE(ctx, pic->master_ocw3); + V3_CHKPT_STD_SAVE(ctx, pic->slave_ocw2); + V3_CHKPT_STD_SAVE(ctx, pic->slave_ocw3); + + V3_CHKPT_STD_SAVE(ctx, pic->master_state); + V3_CHKPT_STD_SAVE(ctx, pic->slave_state); + + + return 0; + +} + +static int pic_load(struct v3_chkpt_ctx * ctx, void * private_data) { + struct pic_internal * pic = (struct pic_internal *)private_data; + + V3_CHKPT_STD_LOAD(ctx, pic->master_irr); + V3_CHKPT_STD_LOAD(ctx, pic->slave_irr); + + V3_CHKPT_STD_LOAD(ctx, pic->master_isr); + V3_CHKPT_STD_LOAD(ctx, pic->slave_isr); + + V3_CHKPT_STD_LOAD(ctx, pic->master_elcr); + V3_CHKPT_STD_LOAD(ctx, pic->slave_elcr); + V3_CHKPT_STD_LOAD(ctx, pic->master_elcr_mask); + V3_CHKPT_STD_LOAD(ctx, pic->slave_elcr_mask); + + V3_CHKPT_STD_LOAD(ctx, pic->master_icw1); + V3_CHKPT_STD_LOAD(ctx, pic->master_icw2); + V3_CHKPT_STD_LOAD(ctx, pic->master_icw3); + V3_CHKPT_STD_LOAD(ctx, pic->master_icw4); + + + V3_CHKPT_STD_LOAD(ctx, pic->slave_icw1); + V3_CHKPT_STD_LOAD(ctx, pic->slave_icw2); + V3_CHKPT_STD_LOAD(ctx, pic->slave_icw3); + V3_CHKPT_STD_LOAD(ctx, pic->slave_icw4); + + + V3_CHKPT_STD_LOAD(ctx, pic->master_imr); + V3_CHKPT_STD_LOAD(ctx, pic->slave_imr); + V3_CHKPT_STD_LOAD(ctx, pic->master_ocw2); + V3_CHKPT_STD_LOAD(ctx, pic->master_ocw3); + V3_CHKPT_STD_LOAD(ctx, pic->slave_ocw2); + V3_CHKPT_STD_LOAD(ctx, pic->slave_ocw3); + + V3_CHKPT_STD_LOAD(ctx, pic->master_state); + V3_CHKPT_STD_LOAD(ctx, pic->slave_state); + + return 0; +} + +#endif + static struct v3_device_ops dev_ops = { .free = (int (*)(void *))pic_free, - +#ifdef V3_CONFIG_CHECKPOINT + .save = pic_save, + .load = pic_load +#endif }; diff --git a/palacios/src/devices/apic.c b/palacios/src/devices/apic.c index 508f221..d83f0bd 100644 --- a/palacios/src/devices/apic.c +++ b/palacios/src/devices/apic.c @@ -220,7 +220,7 @@ struct apic_state { uint32_t tmr_cur_cnt; uint32_t tmr_init_cnt; - + uint32_t missed_ints; struct local_vec_tbl_reg ext_intr_vec_tbl[4]; @@ -295,6 +295,7 @@ static void init_apic_state(struct apic_state * apic, uint32_t id) { apic->rem_rd_data = 0x00000000; apic->tmr_init_cnt = 0x00000000; apic->tmr_cur_cnt = 0x00000000; + apic->missed_ints = 0; apic->lapic_id.val = id; @@ -1562,10 +1563,33 @@ static int apic_begin_irq(struct guest_info * core, void * private_data, int irq } +/* Timer Functions */ +static void apic_inject_timer_intr(struct guest_info *core, + void * priv_data) { + struct apic_dev_state * apic_dev = (struct apic_dev_state *)(priv_data); + struct apic_state * apic = &(apic_dev->apics[core->vcpu_id]); + // raise irq + PrintDebug("apic %u: core %u: Raising APIC Timer interrupt (periodic=%d) (icnt=%d) (div=%d)\n", + apic->lapic_id.val, core->vcpu_id, + apic->tmr_vec_tbl.tmr_mode, apic->tmr_init_cnt, shift_num); + + if (apic_intr_pending(core, priv_data)) { + PrintDebug("apic %u: core %u: Overriding pending IRQ %d\n", + apic->lapic_id.val, core->vcpu_id, + apic_get_intr_number(core, priv_data)); + } + + if (activate_internal_irq(apic, APIC_TMR_INT) == -1) { + PrintError("apic %u: core %u: Could not raise Timer interrupt\n", + apic->lapic_id.val, core->vcpu_id); + } + + return; +} + -/* Timer Functions */ static void apic_update_time(struct guest_info * core, uint64_t cpu_cycles, uint64_t cpu_freq, @@ -1631,40 +1655,18 @@ static void apic_update_time(struct guest_info * core, if (tmr_ticks < apic->tmr_cur_cnt) { apic->tmr_cur_cnt -= tmr_ticks; + if (apic->missed_ints) { + apic_inject_timer_intr(core, priv_data); + apic->missed_ints--; + } } else { tmr_ticks -= apic->tmr_cur_cnt; apic->tmr_cur_cnt = 0; - // raise irq - PrintDebug("apic %u: core %u: Raising APIC Timer interrupt (periodic=%d) (icnt=%d) (div=%d)\n", - apic->lapic_id.val, core->vcpu_id, - apic->tmr_vec_tbl.tmr_mode, apic->tmr_init_cnt, shift_num); - - if (apic_intr_pending(core, priv_data)) { - PrintDebug("apic %u: core %u: Overriding pending IRQ %d\n", - apic->lapic_id.val, core->vcpu_id, - apic_get_intr_number(core, priv_data)); - } + apic_inject_timer_intr(core, priv_data); - if (activate_internal_irq(apic, APIC_TMR_INT) == -1) { - PrintError("apic %u: core %u: Could not raise Timer interrupt\n", - apic->lapic_id.val, core->vcpu_id); - } - if (apic->tmr_vec_tbl.tmr_mode == APIC_TMR_PERIODIC) { - static unsigned int nexits = 0; - static unsigned int missed_ints = 0; - - nexits++; - missed_ints += tmr_ticks / apic->tmr_init_cnt; - - if ((missed_ints > 0) && (nexits >= 5000)) { - V3_Print("apic %u: core %u: missed %u timer interrupts total in last %u exits.\n", - apic->lapic_id.val, core->vcpu_id, missed_ints, nexits); - missed_ints = 0; - nexits = 0; - } - + apic->missed_ints += tmr_ticks / apic->tmr_init_cnt; tmr_ticks = tmr_ticks % apic->tmr_init_cnt; apic->tmr_cur_cnt = apic->tmr_init_cnt - tmr_ticks; } @@ -1673,7 +1675,6 @@ static void apic_update_time(struct guest_info * core, return; } - static struct intr_ctrl_ops intr_ops = { .intr_pending = apic_intr_pending, .get_intr_number = apic_get_intr_number, @@ -1714,9 +1715,109 @@ static int apic_free(struct apic_dev_state * apic_dev) { return 0; } +#ifdef V3_CONFIG_CHECKPOINT +static int apic_save(struct v3_chkpt_ctx * ctx, void * private_data) { + struct apic_dev_state * apic_state = (struct apic_dev_state *)private_data; + int i = 0; + + V3_CHKPT_STD_SAVE(ctx, apic_state->num_apics); + + //V3_CHKPT_STD_SAVE(ctx,apic_state->state_lock); + for (i = 0; i < apic_state->num_apics; i++) { + + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].base_addr); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].base_addr_msr); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].lapic_id); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].apic_ver); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ext_apic_ctrl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].local_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_div_cfg); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].lint0_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].lint1_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].perf_ctr_loc_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].therm_loc_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].err_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].err_status); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].spurious_int); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_cmd); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].log_dst); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].dst_fmt); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].arb_prio); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].task_prio); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].proc_prio); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ext_apic_feature); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].spec_eoi); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_cur_cnt); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_init_cnt); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ext_intr_vec_tbl); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].rem_rd_data); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ipi_state); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_req_reg); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_svc_reg); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_en_reg); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].trig_mode_reg); + V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].eoi); + + } + + return 0; +} + +static int apic_load(struct v3_chkpt_ctx * ctx, void * private_data) { + struct apic_dev_state *apic_state = (struct apic_dev_state *)private_data; + int i = 0; + + V3_CHKPT_STD_LOAD(ctx,apic_state->num_apics); + + for (i = 0; i < apic_state->num_apics; i++) { + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].base_addr); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].base_addr_msr); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].lapic_id); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].apic_ver); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ext_apic_ctrl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].local_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_div_cfg); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].lint0_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].lint1_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].perf_ctr_loc_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].therm_loc_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].err_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].err_status); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].spurious_int); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_cmd); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].log_dst); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].dst_fmt); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].arb_prio); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].task_prio); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].proc_prio); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ext_apic_feature); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].spec_eoi); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_cur_cnt); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_init_cnt); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ext_intr_vec_tbl); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].rem_rd_data); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ipi_state); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_req_reg); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_svc_reg); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_en_reg); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].trig_mode_reg); + V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].eoi); + } + + + return 0; +} + +#endif static struct v3_device_ops dev_ops = { .free = (int (*)(void *))apic_free, +#ifdef V3_CONFIG_CHECKPOINT + .save = apic_save, + .load = apic_load +#endif }; diff --git a/palacios/src/devices/cga.c b/palacios/src/devices/cga.c index 7997818..0081609 100644 --- a/palacios/src/devices/cga.c +++ b/palacios/src/devices/cga.c @@ -1097,7 +1097,7 @@ int v3_cons_get_fb(struct vm_device * frontend_dev, uint8_t * dst, uint_t offset } } -static int free_cga(struct video_internal * video_state) { +static int cga_free(struct video_internal * video_state) { if (video_state->framebuf_pa) { PrintError("Freeing framebuffer PA %p\n", (void *)(video_state->framebuf_pa)); @@ -1112,8 +1112,89 @@ static int free_cga(struct video_internal * video_state) { } +#ifdef V3_CONFIG_CHECKPOINT +static int cga_save(struct v3_chkpt_ctx * ctx, void * private_data) { + struct video_internal * cga = (struct video_internal *)private_data; + + V3_CHKPT_STD_SAVE(ctx, cga->misc_outp_reg); + V3_CHKPT_STD_SAVE(ctx, cga->seq_index_reg); + V3_CHKPT_STD_SAVE(ctx, cga->seq_data_regs[SEQ_REG_COUNT]); + V3_CHKPT_STD_SAVE(ctx, cga->crtc_index_reg); + V3_CHKPT_STD_SAVE(ctx, cga->crtc_data_regs[CRTC_REG_COUNT]); + V3_CHKPT_STD_SAVE(ctx, cga->graphc_index_reg); + V3_CHKPT_STD_SAVE(ctx, cga->graphc_data_regs[GRAPHC_REG_COUNT]); + V3_CHKPT_STD_SAVE(ctx, cga->attrc_index_flipflop); + V3_CHKPT_STD_SAVE(ctx, cga->attrc_index_reg); + V3_CHKPT_STD_SAVE(ctx, cga->attrc_data_regs[ATTRC_REG_COUNT]); + V3_CHKPT_STD_SAVE(ctx, cga->dac_indexr_reg); + V3_CHKPT_STD_SAVE(ctx, cga->dac_indexr_color); + V3_CHKPT_STD_SAVE(ctx, cga->dac_indexw_reg); + V3_CHKPT_STD_SAVE(ctx, cga->dac_indexw_color); + V3_CHKPT_STD_SAVE(ctx, cga->dac_data_regs[DAC_REG_COUNT]); + + V3_CHKPT_STD_SAVE(ctx, cga->activefb_addr); + V3_CHKPT_STD_SAVE(ctx, cga->activefb_len); + V3_CHKPT_STD_SAVE(ctx, cga->iorange); + V3_CHKPT_STD_SAVE(ctx, cga->vres); + V3_CHKPT_STD_SAVE(ctx, cga->hres); + V3_CHKPT_STD_SAVE(ctx, cga->vchars); + V3_CHKPT_STD_SAVE(ctx, cga->hchars); + V3_CHKPT_STD_SAVE(ctx, cga->graphmode); + + V3_CHKPT_STD_SAVE(ctx, cga->dirty); + V3_CHKPT_STD_SAVE(ctx, cga->reschanged); + + V3_CHKPT_STD_SAVE(ctx, cga->passthrough); + + return 0; +} + +static int cga_load(struct v3_chkpt_ctx * ctx, void * private_data) { + struct video_internal * cga = (struct video_internal *)private_data; + + V3_CHKPT_STD_LOAD(ctx, cga->misc_outp_reg); + V3_CHKPT_STD_LOAD(ctx, cga->seq_index_reg); + V3_CHKPT_STD_LOAD(ctx, cga->seq_data_regs[SEQ_REG_COUNT]); + V3_CHKPT_STD_LOAD(ctx, cga->crtc_index_reg); + V3_CHKPT_STD_LOAD(ctx, cga->crtc_data_regs[CRTC_REG_COUNT]); + V3_CHKPT_STD_LOAD(ctx, cga->graphc_index_reg); + V3_CHKPT_STD_LOAD(ctx, cga->graphc_data_regs[GRAPHC_REG_COUNT]); + V3_CHKPT_STD_LOAD(ctx, cga->attrc_index_flipflop); + V3_CHKPT_STD_LOAD(ctx, cga->attrc_index_reg); + V3_CHKPT_STD_LOAD(ctx, cga->attrc_data_regs[ATTRC_REG_COUNT]); + V3_CHKPT_STD_LOAD(ctx, cga->dac_indexr_reg); + V3_CHKPT_STD_LOAD(ctx, cga->dac_indexr_color); + V3_CHKPT_STD_LOAD(ctx, cga->dac_indexw_reg); + V3_CHKPT_STD_LOAD(ctx, cga->dac_indexw_color); + V3_CHKPT_STD_LOAD(ctx, cga->dac_data_regs[DAC_REG_COUNT]); + + V3_CHKPT_STD_LOAD(ctx, cga->activefb_addr); + V3_CHKPT_STD_LOAD(ctx, cga->activefb_len); + V3_CHKPT_STD_LOAD(ctx, cga->iorange); + V3_CHKPT_STD_LOAD(ctx, cga->vres); + V3_CHKPT_STD_LOAD(ctx, cga->hres); + V3_CHKPT_STD_LOAD(ctx, cga->vchars); + V3_CHKPT_STD_LOAD(ctx, cga->hchars); + V3_CHKPT_STD_LOAD(ctx, cga->graphmode); + + V3_CHKPT_STD_LOAD(ctx, cga->dirty); + V3_CHKPT_STD_LOAD(ctx, cga->reschanged); + + V3_CHKPT_STD_LOAD(ctx, cga->passthrough); + + return 0; +} + +#endif + + static struct v3_device_ops dev_ops = { - .free = (int (*)(void *))free_cga, + .free = (int (*)(void *))cga_free, +#ifdef V3_CONFIG_CHECKPOINT + .save = cga_save, + .load = cga_load +#endif + }; static int cga_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) { diff --git a/palacios/src/devices/io_apic.c b/palacios/src/devices/io_apic.c index 3579fde..0817e9b 100644 --- a/palacios/src/devices/io_apic.c +++ b/palacios/src/devices/io_apic.c @@ -325,10 +325,42 @@ static int io_apic_free(struct io_apic_state * ioapic) { return 0; } +#ifdef V3_CONFIG_CHECKPOINT +static int io_apic_save(struct v3_chkpt_ctx * ctx, void * private_data) { + struct io_apic_state * io_apic = (struct io_apic_state *)private_data; + + V3_CHKPT_STD_SAVE(ctx, io_apic->base_addr); + V3_CHKPT_STD_SAVE(ctx, io_apic->index_reg); + V3_CHKPT_STD_SAVE(ctx, io_apic->ioapic_id); + V3_CHKPT_STD_SAVE(ctx, io_apic->ioapic_ver); + V3_CHKPT_STD_SAVE(ctx, io_apic->ioapic_arb_id); + V3_CHKPT_STD_SAVE(ctx, io_apic->redir_tbl); + + return 0; +} + +static int io_apic_load(struct v3_chkpt_ctx * ctx, void * private_data) { + struct io_apic_state * io_apic = (struct io_apic_state *)private_data; + + V3_CHKPT_STD_LOAD(ctx, io_apic->base_addr); + V3_CHKPT_STD_LOAD(ctx, io_apic->index_reg); + V3_CHKPT_STD_LOAD(ctx, io_apic->ioapic_id); + V3_CHKPT_STD_LOAD(ctx, io_apic->ioapic_ver); + V3_CHKPT_STD_LOAD(ctx, io_apic->ioapic_arb_id); + V3_CHKPT_STD_LOAD(ctx, io_apic->redir_tbl); + + return 0; +} +#endif + + static struct v3_device_ops dev_ops = { .free = (int (*)(void *))io_apic_free, - +#ifdef V3_CONFIG_CHECKPOINT + .save = io_apic_save, + .load = io_apic_load +#endif }; diff --git a/palacios/src/devices/keyboard.c b/palacios/src/devices/keyboard.c index f8786e2..5943fd3 100644 --- a/palacios/src/devices/keyboard.c +++ b/palacios/src/devices/keyboard.c @@ -1051,9 +1051,27 @@ static int keyboard_reset_device(struct keyboard_internal * kbd) { } +#ifdef V3_CONFIG_CHECKPOINT +static int keyboard_save(struct v3_chkpt_ctx * ctx, void * private_data) { + return 0; +} + + +static int keyboard_load(struct v3_chkpt_ctx * ctx, void * private_data) { + struct keyboard_internal * kbd = (struct keyboard_internal *)private_data; + keyboard_reset_device(kbd); + return 0; +} + +#endif + + static struct v3_device_ops dev_ops = { .free = (int (*)(void *))keyboard_free, - +#ifdef V3_CONFIG_CHECKPOINT + .save = keyboard_save, + .load = keyboard_load +#endif }; diff --git a/palacios/src/devices/os_debug.c b/palacios/src/devices/os_debug.c index 97cdabc..f821415 100644 --- a/palacios/src/devices/os_debug.c +++ b/palacios/src/devices/os_debug.c @@ -89,11 +89,36 @@ static int debug_free(struct debug_state * state) { return 0; }; +#ifdef V3_CONFIG_CHECKPOINT +static int debug_save(struct v3_chkpt_ctx * ctx, void * private_data) { + struct debug_state * dbg = (struct debug_state *)private_data; + + V3_CHKPT_STD_SAVE(ctx, dbg->debug_buf); + V3_CHKPT_STD_SAVE(ctx, dbg->debug_offset); + + return 0; +} + + +static int debug_load(struct v3_chkpt_ctx * ctx, void * private_data) { + struct debug_state * dbg = (struct debug_state *)private_data; + + V3_CHKPT_STD_LOAD(ctx, dbg->debug_buf); + V3_CHKPT_STD_LOAD(ctx, dbg->debug_offset); + + return 0; +} + +#endif static struct v3_device_ops dev_ops = { .free = (int (*)(void *))debug_free, +#ifdef V3_CONFIG_CHECKPOINT + .save = debug_save, + .load = debug_load +#endif }; diff --git a/palacios/src/palacios/Makefile b/palacios/src/palacios/Makefile index 63a822b..14a1c29 100644 --- a/palacios/src/palacios/Makefile +++ b/palacios/src/palacios/Makefile @@ -64,6 +64,7 @@ obj-$(V3_CONFIG_VMX) += vmx.o \ +obj-$(V3_CONFIG_CHECKPOINT) += vmm_checkpoint.o obj-$(V3_CONFIG_TELEMETRY) += vmm_telemetry.o diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index 1bf16cc..5319f90 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -37,6 +37,9 @@ #include #include +#ifdef V3_CONFIG_CHECKPOINT +#include +#endif #include @@ -339,6 +342,68 @@ int v3_deinit_svm_vmcb(struct guest_info * core) { } +#ifdef V3_CONFIG_CHECKPOINT +int v3_svm_save_core(struct guest_info * core, void * ctx){ + + v3_chkpt_save(ctx, "vmcb_data", PAGE_SIZE, core->vmm_data); + + return 0; +} + +int v3_svm_load_core(struct guest_info * core, void * chkpt_ctx){ + struct cr0_32 * shadow_cr0; + vmcb_saved_state_t * guest_state; + vmcb_ctrl_t * guest_ctrl; + + + + if (v3_chkpt_load(chkpt_ctx, "vmcb_data", PAGE_SIZE, core->vmm_data) == -1){ + return -1; + } + + guest_state = GET_VMCB_SAVE_STATE_AREA((vmcb_t *)(core->vmm_data)); + guest_ctrl = GET_VMCB_CTRL_AREA((vmcb_t *)(core->vmm_data)); + + + core->rip = guest_state->rip; + core->vm_regs.rsp = guest_state->rsp; + core->vm_regs.rax = guest_state->rax; + + core->cpl = guest_state->cpl; + + core->ctrl_regs.cr0 = guest_state->cr0; + core->ctrl_regs.cr2 = guest_state->cr2; + core->ctrl_regs.cr4 = guest_state->cr4; + core->dbg_regs.dr6 = guest_state->dr6; + core->dbg_regs.dr7 = guest_state->dr7; + core->ctrl_regs.cr8 = guest_ctrl->guest_ctrl.V_TPR; + core->ctrl_regs.rflags = guest_state->rflags; + core->ctrl_regs.efer = guest_state->efer; + + + shadow_cr0 = (struct cr0_32 *)&(core->ctrl_regs.cr0); + + + if (core->shdw_pg_mode == SHADOW_PAGING) { + if (v3_get_vm_mem_mode(core) == VIRTUAL_MEM) { + if (v3_activate_shadow_pt(core) == -1) { + PrintError("Failed to activate shadow page tables\n"); + return -1; + } + } else { + if (v3_activate_passthrough_pt(core) == -1) { + PrintError("Failed to activate passthrough page tables\n"); + return -1; + } + } + } + + + v3_get_vmcb_segments((vmcb_t *)(core->vmm_data), &(core->segments)); + return 0; +} +#endif + static int update_irq_exit_state(struct guest_info * info) { vmcb_ctrl_t * guest_ctrl = GET_VMCB_CTRL_AREA((vmcb_t*)(info->vmm_data)); @@ -545,7 +610,7 @@ int v3_svm_enter(struct guest_info * info) { #endif v3_time_enter_vm(info); - guest_ctrl->TSC_OFFSET = v3_tsc_host_offset(&info->time_state); + // guest_ctrl->TSC_OFFSET = v3_tsc_host_offset(&info->time_state); //V3_Print("Calling v3_svm_launch\n"); @@ -635,6 +700,9 @@ int v3_start_svm_guest(struct guest_info * info) { } PrintDebug("SVM core %u(on %u) initialized\n", info->vcpu_id, info->pcpu_id); + + // We'll be paranoid about race conditions here + v3_wait_at_barrier(info); } PrintDebug("SVM core %u(on %u): I am starting at CS=0x%x (base=0x%p, limit=0x%x), RIP=0x%p\n", diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index 961bbca..87746ef 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -33,6 +33,10 @@ #include #endif +#ifdef V3_CONFIG_CHECKPOINT +#include +#endif + v3_cpu_arch_t v3_cpu_types[V3_CONFIG_MAX_CPUS]; struct v3_os_hooks * os_hooks = NULL; @@ -119,9 +123,13 @@ void Init_V3(struct v3_os_hooks * hooks, int num_cpus) { V3_init_symmod(); #endif +#ifdef V3_CONFIG_CHECKPOINT + V3_init_checkpoint(); +#endif + + -#ifdef V3_CONFIG_MULTITHREAD_OS if ((hooks) && (hooks->call_on_cpu)) { for (i = 0; i < num_cpus; i++) { @@ -130,9 +138,7 @@ void Init_V3(struct v3_os_hooks * hooks, int num_cpus) { hooks->call_on_cpu(i, &init_cpu, (void *)(addr_t)i); } } -#else - init_cpu(0); -#endif + } @@ -149,8 +155,11 @@ void Shutdown_V3() { V3_deinit_symmod(); #endif +#ifdef V3_CONFIG_CHECKPOINT + V3_deinit_checkpoint(); +#endif + -#ifdef V3_CONFIG_MULTITHREAD_OS if ((os_hooks) && (os_hooks->call_on_cpu)) { for (i = 0; i < V3_CONFIG_MAX_CPUS; i++) { if (v3_cpu_types[i] != V3_INVALID_CPU) { @@ -159,9 +168,6 @@ void Shutdown_V3() { } } } -#else - deinit_cpu(0); -#endif } @@ -228,11 +234,7 @@ static int start_core(void * p) // For the moment very ugly. Eventually we will shift the cpu_mask to an arbitrary sized type... -#ifdef V3_CONFIG_MULTITHREAD_OS #define MAX_CORES 32 -#else -#define MAX_CORES 1 -#endif int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { @@ -264,9 +266,9 @@ int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { return -1; } -#ifdef V3_CONFIG_MULTITHREAD_OS - // spawn off new threads, for other cores - for (i = 0, vcore_id = 1; (i < MAX_CORES) && (vcore_id < vm->num_cores); i++) { + // Spawn off threads for each core. + // We work backwards, so that core 0 is always started last. + for (i = 0, vcore_id = vm->num_cores - 1; (i < MAX_CORES) && (vcore_id >= 0); i++) { int major = 0; int minor = 0; struct guest_info * core = &(vm->cores[vcore_id]); @@ -282,19 +284,12 @@ int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { i--; // We reset the logical core idx. Not strictly necessary I guess... } else { - - if (i == V3_Get_CPU()) { - // We skip the local CPU because it is reserved for vcore 0 - continue; - } - core_idx = i; } major = core_idx / 8; minor = core_idx % 8; - if ((core_mask[major] & (0x1 << minor)) == 0) { PrintError("Logical CPU %d not available for virtual core %d; not started\n", core_idx, vcore_id); @@ -316,7 +311,6 @@ int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { PrintDebug("run: core=%u, func=0x%p, arg=0x%p, name=%s\n", core_idx, start_core, core, core->exec_name); - // TODO: actually manage these threads instead of just launching them core->pcpu_id = core_idx; core->core_thread = V3_CREATE_THREAD_ON_CPU(core_idx, start_core, core, core->exec_name); @@ -326,18 +320,7 @@ int v3_start_vm(struct v3_vm_info * vm, unsigned int cpu_mask) { return -1; } - vcore_id++; - } -#endif - - sprintf(vm->cores[0].exec_name, "%s", vm->name); - - vm->cores[0].pcpu_id = V3_Get_CPU(); - - if (start_core(&(vm->cores[0])) != 0) { - PrintError("Error starting VM core 0\n"); - v3_stop_vm(vm); - return -1; + vcore_id--; } @@ -404,6 +387,20 @@ int v3_move_vm_core(struct v3_vm_info * vm, int vcore_id, int target_cpu) { V3_Print("Moving Core\n"); + +#ifdef V3_CONFIG_VMX + switch (v3_cpu_types[core->pcpu_id]) { + case V3_VMX_CPU: + case V3_VMX_EPT_CPU: + case V3_VMX_EPT_UG_CPU: + PrintDebug("Flushing VMX Guest CPU %d\n", core->vcpu_id); + V3_Call_On_CPU(core->pcpu_id, (void (*)(void *))v3_flush_vmx_vm_core, (void *)core); + break; + default: + break; + } +#endif + if (V3_MOVE_THREAD_TO_CPU(target_cpu, core->core_thread) != 0) { PrintError("Failed to move Vcore %d to CPU %d\n", core->vcpu_id, target_cpu); @@ -489,6 +486,19 @@ int v3_continue_vm(struct v3_vm_info * vm) { return 0; } +#ifdef V3_CONFIG_CHECKPOINT +#include + +int v3_save_vm(struct v3_vm_info * vm, char * store, char * url) { + return v3_chkpt_save_vm(vm, store, url); +} + + +int v3_load_vm(struct v3_vm_info * vm, char * store, char * url) { + return v3_chkpt_load_vm(vm, store, url); +} +#endif + int v3_free_vm(struct v3_vm_info * vm) { int i = 0; @@ -598,7 +608,6 @@ void v3_print_cond(const char * fmt, ...) { } -#ifdef V3_CONFIG_MULTITHREAD_OS void v3_interrupt_cpu(struct v3_vm_info * vm, int logical_cpu, int vector) { extern struct v3_os_hooks * os_hooks; @@ -607,7 +616,6 @@ void v3_interrupt_cpu(struct v3_vm_info * vm, int logical_cpu, int vector) { (os_hooks)->interrupt_cpu(vm, logical_cpu, vector); } } -#endif diff --git a/palacios/src/palacios/vmm_barrier.c b/palacios/src/palacios/vmm_barrier.c index ab0aae4..f7b863d 100644 --- a/palacios/src/palacios/vmm_barrier.c +++ b/palacios/src/palacios/vmm_barrier.c @@ -104,6 +104,14 @@ int v3_raise_barrier(struct v3_vm_info * vm_info, struct guest_info * local_core all_blocked = 1; for (i = 0; i < vm_info->num_cores; i++) { + + // Tricky: If a core is not running then it is safe to ignore it. + // Whenever we transition a core to the RUNNING state we MUST immediately wait on the barrier. + // TODO: Wrap the state transitions in functions that do this automatically + if (vm_info->cores[i].core_run_state != CORE_RUNNING) { + continue; + } + if (v3_bitmap_check(&(barrier->cpu_map), i) == 0) { // There is still a core that is not waiting at the barrier all_blocked = 0; @@ -157,6 +165,8 @@ int v3_wait_at_barrier(struct guest_info * core) { return 0; } + V3_Print("Core %d waiting at barrier\n", core->vcpu_id); + /* Barrier has been activated. * Wait here until it's lowered */ @@ -164,9 +174,10 @@ int v3_wait_at_barrier(struct guest_info * core) { // set cpu bit in barrier bitmap v3_bitmap_set(&(barrier->cpu_map), core->vcpu_id); + V3_Print("Core %d bit set as waiting\n", core->vcpu_id); // wait for cpu bit to clear - while (v3_bitmap_check(&(barrier->cpu_map), core->vcpu_id) == 1) { + while (v3_bitmap_check(&(barrier->cpu_map), core->vcpu_id)) { v3_yield(core); } diff --git a/palacios/src/palacios/vmm_bitmap.c b/palacios/src/palacios/vmm_bitmap.c index c056a5b..0f26473 100644 --- a/palacios/src/palacios/vmm_bitmap.c +++ b/palacios/src/palacios/vmm_bitmap.c @@ -96,6 +96,6 @@ int v3_bitmap_check(struct v3_bitmap * bitmap, int index) { return -1; } - return (bitmap->bits[major] & (0x1 << minor)); + return ((bitmap->bits[major] & (0x1 << minor)) != 0); } diff --git a/palacios/src/palacios/vmm_checkpoint.c b/palacios/src/palacios/vmm_checkpoint.c new file mode 100644 index 0000000..f33f64e --- /dev/null +++ b/palacios/src/palacios/vmm_checkpoint.c @@ -0,0 +1,573 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2011, Madhav Suresh + * Copyright (c) 2011, The V3VEE Project + * All rights reserved. + * + * Author: Madhav Suresh + * Arefin Huq + * + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + + +static struct hashtable * store_table = NULL; + +struct v3_chkpt; + +typedef enum {SAVE, LOAD} chkpt_mode_t; + +struct chkpt_interface { + char name[128]; + void * (*open_chkpt)(char * url, chkpt_mode_t mode); + int (*close_chkpt)(void * store_data); + + void * (*open_ctx)(void * store_data, void * parent_ctx, char * name); + int (*close_ctx)(void * store_data, void * ctx); + + int (*save)(void * store_data, void * ctx, char * tag, uint64_t len, void * buf); + int (*load)(void * store_data, void * ctx, char * tag, uint64_t len, void * buf); +}; + + +struct v3_chkpt { + struct v3_vm_info * vm; + + struct chkpt_interface * interface; + + void * store_data; +}; + + + + +static uint_t store_hash_fn(addr_t key) { + char * name = (char *)key; + return v3_hash_buffer((uint8_t *)name, strlen(name)); +} + +static int store_eq_fn(addr_t key1, addr_t key2) { + char * name1 = (char *)key1; + char * name2 = (char *)key2; + + return (strcmp(name1, name2) == 0); +} + + + +#include "vmm_chkpt_stores.h" + + +int V3_init_checkpoint() { + extern struct chkpt_interface * __start__v3_chkpt_stores[]; + extern struct chkpt_interface * __stop__v3_chkpt_stores[]; + struct chkpt_interface ** tmp_store = __start__v3_chkpt_stores; + int i = 0; + + store_table = v3_create_htable(0, store_hash_fn, store_eq_fn); + + while (tmp_store != __stop__v3_chkpt_stores) { + V3_Print("Registering Checkpoint Backing Store (%s)\n", (*tmp_store)->name); + + if (v3_htable_search(store_table, (addr_t)((*tmp_store)->name))) { + PrintError("Multiple instances of Checkpoint backing Store (%s)\n", (*tmp_store)->name); + return -1; + } + + if (v3_htable_insert(store_table, (addr_t)((*tmp_store)->name), (addr_t)(*tmp_store)) == 0) { + PrintError("Could not register Checkpoint backing store (%s)\n", (*tmp_store)->name); + return -1; + } + + tmp_store = &(__start__v3_chkpt_stores[++i]); + } + + return 0; +} + +int V3_deinit_checkpoint() { + v3_free_htable(store_table, 0, 0); + return 0; +} + + +static char svm_chkpt_header[] = "v3vee palacios checkpoint version: x.x, SVM x.x"; +static char vmx_chkpt_header[] = "v3vee palacios checkpoint version: x.x, VMX x.x"; + +static int chkpt_close(struct v3_chkpt * chkpt) { + chkpt->interface->close_chkpt(chkpt->store_data); + + V3_Free(chkpt); + + return 0; +} + + +static struct v3_chkpt * chkpt_open(struct v3_vm_info * vm, char * store, char * url, chkpt_mode_t mode) { + struct chkpt_interface * iface = NULL; + struct v3_chkpt * chkpt = NULL; + void * store_data = NULL; + + iface = (void *)v3_htable_search(store_table, (addr_t)store); + + if (iface == NULL) { + V3_Print("Error: Could not locate Checkpoint interface for store (%s)\n", store); + return NULL; + } + + store_data = iface->open_chkpt(url, mode); + + if (store_data == NULL) { + PrintError("Could not open url (%s) for backing store (%s)\n", url, store); + return NULL; + } + + + chkpt = V3_Malloc(sizeof(struct v3_chkpt)); + + if (!chkpt) { + PrintError("Could not allocate checkpoint state\n"); + return NULL; + } + + chkpt->interface = iface; + chkpt->vm = vm; + chkpt->store_data = store_data; + + return chkpt; +} + +struct v3_chkpt_ctx * v3_chkpt_open_ctx(struct v3_chkpt * chkpt, struct v3_chkpt_ctx * parent, char * name) { + struct v3_chkpt_ctx * ctx = V3_Malloc(sizeof(struct v3_chkpt_ctx)); + void * parent_store_ctx = NULL; + + memset(ctx, 0, sizeof(struct v3_chkpt_ctx)); + + ctx->chkpt = chkpt; + ctx->parent = parent; + + if (parent) { + parent_store_ctx = parent->store_ctx; + } + + ctx->store_ctx = chkpt->interface->open_ctx(chkpt->store_data, parent_store_ctx, name); + + return ctx; +} + +int v3_chkpt_close_ctx(struct v3_chkpt_ctx * ctx) { + struct v3_chkpt * chkpt = ctx->chkpt; + int ret = 0; + + ret = chkpt->interface->close_ctx(chkpt->store_data, ctx->store_ctx); + + V3_Free(ctx); + + return ret; +} + + + + + +int v3_chkpt_save(struct v3_chkpt_ctx * ctx, char * tag, uint64_t len, void * buf) { + struct v3_chkpt * chkpt = ctx->chkpt; + return chkpt->interface->save(chkpt->store_data, ctx->store_ctx, tag, len, buf); +} + + +int v3_chkpt_load(struct v3_chkpt_ctx * ctx, char * tag, uint64_t len, void * buf) { + struct v3_chkpt * chkpt = ctx->chkpt; + return chkpt->interface->load(chkpt->store_data, ctx->store_ctx, tag, len, buf); +} + + + +static int load_memory(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { + + void * guest_mem_base = NULL; + void * ctx = NULL; + uint64_t ret = 0; + + guest_mem_base = V3_VAddr((void *)vm->mem_map.base_region.host_addr); + + ctx = v3_chkpt_open_ctx(chkpt, NULL, "memory_img"); + + ret = v3_chkpt_load(ctx, "memory_img", vm->mem_size, guest_mem_base); + v3_chkpt_close_ctx(ctx); + + return ret; +} + + +static int save_memory(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { + void * guest_mem_base = NULL; + void * ctx = NULL; + uint64_t ret = 0; + + guest_mem_base = V3_VAddr((void *)vm->mem_map.base_region.host_addr); + + ctx = v3_chkpt_open_ctx(chkpt, NULL,"memory_img"); + + + ret = v3_chkpt_save(ctx, "memory_img", vm->mem_size, guest_mem_base); + v3_chkpt_close_ctx(ctx); + + return ret; +} + +int save_header(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { + v3_cpu_arch_t cpu_type = v3_get_cpu_type(V3_Get_CPU()); + void * ctx = NULL; + + ctx = v3_chkpt_open_ctx(chkpt, NULL, "header"); + + switch (cpu_type) { + case V3_SVM_CPU: + case V3_SVM_REV3_CPU: { + v3_chkpt_save(ctx, "header", strlen(svm_chkpt_header), svm_chkpt_header); + break; + } + case V3_VMX_CPU: + case V3_VMX_EPT_CPU: + case V3_VMX_EPT_UG_CPU: { + v3_chkpt_save(ctx, "header", strlen(vmx_chkpt_header), vmx_chkpt_header); + break; + } + default: + PrintError("checkpoint not supported on this architecture\n"); + v3_chkpt_close_ctx(ctx); + return -1; + } + + v3_chkpt_close_ctx(ctx); + + return 0; +} + +static int load_header(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { + v3_cpu_arch_t cpu_type = v3_get_cpu_type(V3_Get_CPU()); + void * ctx = NULL; + + ctx = v3_chkpt_open_ctx(chkpt, NULL, "header"); + + switch (cpu_type) { + case V3_SVM_CPU: + case V3_SVM_REV3_CPU: { + char header[strlen(svm_chkpt_header) + 1]; + + v3_chkpt_load(ctx, "header", strlen(svm_chkpt_header), header); + + break; + } + case V3_VMX_CPU: + case V3_VMX_EPT_CPU: + case V3_VMX_EPT_UG_CPU: { + char header[strlen(vmx_chkpt_header) + 1]; + + v3_chkpt_load(ctx, "header", strlen(vmx_chkpt_header), header); + + break; + } + default: + PrintError("checkpoint not supported on this architecture\n"); + v3_chkpt_close_ctx(ctx); + return -1; + } + + v3_chkpt_close_ctx(ctx); + + return 0; +} + + +static int load_core(struct guest_info * info, struct v3_chkpt * chkpt) { + v3_cpu_arch_t cpu_type = v3_get_cpu_type(V3_Get_CPU()); + void * ctx = NULL; + char key_name[16]; + memset(key_name, 0, 16); + + snprintf(key_name, 16, "guest_info%d", info->vcpu_id); + + ctx = v3_chkpt_open_ctx(chkpt, NULL, key_name); + + V3_CHKPT_STD_LOAD(ctx, info->vm_regs); + + V3_CHKPT_STD_LOAD(ctx, info->ctrl_regs.cr0); + V3_CHKPT_STD_LOAD(ctx, info->ctrl_regs.cr2); + V3_CHKPT_STD_LOAD(ctx, info->ctrl_regs.cr4); + V3_CHKPT_STD_LOAD(ctx, info->ctrl_regs.cr8); + V3_CHKPT_STD_LOAD(ctx, info->ctrl_regs.rflags); + V3_CHKPT_STD_LOAD(ctx, info->ctrl_regs.efer); + + V3_CHKPT_STD_LOAD(ctx, info->dbg_regs); + V3_CHKPT_STD_LOAD(ctx, info->segments); + V3_CHKPT_STD_LOAD(ctx, info->shdw_pg_state.guest_cr3); + V3_CHKPT_STD_LOAD(ctx, info->shdw_pg_state.guest_cr0); + V3_CHKPT_STD_LOAD(ctx, info->shdw_pg_state.guest_efer); + v3_chkpt_close_ctx(ctx); + + PrintDebug("Finished reading guest_info information\n"); + + info->cpu_mode = v3_get_vm_cpu_mode(info); + info->mem_mode = v3_get_vm_mem_mode(info); + + + switch (cpu_type) { + case V3_SVM_CPU: + case V3_SVM_REV3_CPU: { + char key_name[16]; + + snprintf(key_name, 16, "vmcb_data%d", info->vcpu_id); + ctx = v3_chkpt_open_ctx(chkpt, NULL, key_name); + + if (v3_svm_load_core(info, ctx) == -1) { + PrintError("Failed to patch core %d\n", info->vcpu_id); + return -1; + } + + v3_chkpt_close_ctx(ctx); + + break; + } + case V3_VMX_CPU: + case V3_VMX_EPT_CPU: + case V3_VMX_EPT_UG_CPU: { + char key_name[16]; + + snprintf(key_name, 16, "vmcs_data%d", info->vcpu_id); + ctx = v3_chkpt_open_ctx(chkpt, NULL, key_name); + + if (v3_vmx_load_core(info, ctx) < 0) { + PrintError("VMX checkpoint failed\n"); + return -1; + } + + v3_chkpt_close_ctx(ctx); + + break; + } + default: + PrintError("Invalid CPU Type (%d)\n", cpu_type); + return -1; + } + + return 0; +} + + +static int save_core(struct guest_info * info, struct v3_chkpt * chkpt) { + v3_cpu_arch_t cpu_type = v3_get_cpu_type(V3_Get_CPU()); + void * ctx = NULL; + char key_name[16]; + + memset(key_name, 0, 16); + + + snprintf(key_name, 16, "guest_info%d", info->vcpu_id); + + ctx = v3_chkpt_open_ctx(chkpt, NULL, key_name); + + V3_CHKPT_STD_SAVE(ctx, info->vm_regs); + + V3_CHKPT_STD_SAVE(ctx, info->ctrl_regs.cr0); + V3_CHKPT_STD_SAVE(ctx, info->ctrl_regs.cr2); + V3_CHKPT_STD_SAVE(ctx, info->ctrl_regs.cr4); + V3_CHKPT_STD_SAVE(ctx, info->ctrl_regs.cr8); + V3_CHKPT_STD_SAVE(ctx, info->ctrl_regs.rflags); + V3_CHKPT_STD_SAVE(ctx, info->ctrl_regs.efer); + + V3_CHKPT_STD_SAVE(ctx, info->dbg_regs); + V3_CHKPT_STD_SAVE(ctx, info->segments); + V3_CHKPT_STD_SAVE(ctx, info->shdw_pg_state.guest_cr3); + V3_CHKPT_STD_SAVE(ctx, info->shdw_pg_state.guest_cr0); + V3_CHKPT_STD_SAVE(ctx, info->shdw_pg_state.guest_efer); + + v3_chkpt_close_ctx(ctx); + + //Architechture specific code + switch (cpu_type) { + case V3_SVM_CPU: + case V3_SVM_REV3_CPU: { + char key_name[16]; + void * ctx = NULL; + + snprintf(key_name, 16, "vmcb_data%d", info->vcpu_id); + + ctx = v3_chkpt_open_ctx(chkpt, NULL, key_name); + + if (v3_svm_save_core(info, ctx) == -1) { + PrintError("VMCB Unable to be written\n"); + v3_chkpt_close_ctx(ctx); + return -1; + } + + v3_chkpt_close_ctx(ctx); + break; + } + case V3_VMX_CPU: + case V3_VMX_EPT_CPU: + case V3_VMX_EPT_UG_CPU: { + char key_name[16]; + void * ctx = NULL; + + snprintf(key_name, 16, "vmcs_data%d", info->vcpu_id); + + ctx = v3_chkpt_open_ctx(chkpt, NULL, key_name); + + if (v3_vmx_save_core(info, ctx) == -1) { + PrintError("VMX checkpoint failed\n"); + v3_chkpt_close_ctx(ctx); + return -1; + } + + v3_chkpt_close_ctx(ctx); + + break; + } + default: + PrintError("Invalid CPU Type (%d)\n", cpu_type); + return -1; + } + + return 0; +} + + +int v3_chkpt_save_vm(struct v3_vm_info * vm, char * store, char * url) { + struct v3_chkpt * chkpt = NULL; + int ret = 0;; + int i = 0; + + chkpt = chkpt_open(vm, store, url, SAVE); + + if (chkpt == NULL) { + PrintError("Error creating checkpoint store\n"); + return -1; + } + + /* If this guest is running we need to block it while the checkpoint occurs */ + if (vm->run_state == VM_RUNNING) { + while (v3_raise_barrier(vm, NULL) == -1); + } + + if ((ret = save_memory(vm, chkpt)) == -1) { + PrintError("Unable to save memory\n"); + goto out; + } + + + if ((ret = v3_save_vm_devices(vm, chkpt)) == -1) { + PrintError("Unable to save devices\n"); + goto out; + } + + + if ((ret = save_header(vm, chkpt)) == -1) { + PrintError("Unable to save header\n"); + goto out; + } + + for (i = 0; i < vm->num_cores; i++){ + if ((ret = save_core(&(vm->cores[i]), chkpt)) == -1) { + PrintError("chkpt of core %d failed\n", i); + goto out; + } + } + + out: + + /* Resume the guest if it was running */ + if (vm->run_state == VM_RUNNING) { + v3_lower_barrier(vm); + } + + chkpt_close(chkpt); + + return ret; + +} + +int v3_chkpt_load_vm(struct v3_vm_info * vm, char * store, char * url) { + struct v3_chkpt * chkpt = NULL; + int i = 0; + int ret = 0; + + chkpt = chkpt_open(vm, store, url, LOAD); + + if (chkpt == NULL) { + PrintError("Error creating checkpoint store\n"); + return -1; + } + + /* If this guest is running we need to block it while the checkpoint occurs */ + if (vm->run_state == VM_RUNNING) { + while (v3_raise_barrier(vm, NULL) == -1); + } + + if ((ret = load_memory(vm, chkpt)) == -1) { + PrintError("Unable to save memory\n"); + goto out; + } + + + if ((ret = v3_load_vm_devices(vm, chkpt)) == -1) { + PrintError("Unable to load devies\n"); + goto out; + } + + + if ((ret = load_header(vm, chkpt)) == -1) { + PrintError("Unable to load header\n"); + goto out; + } + + //per core cloning + for (i = 0; i < vm->num_cores; i++) { + if ((ret = load_core(&(vm->cores[i]), chkpt)) == -1) { + PrintError("Error loading core state (core=%d)\n", i); + goto out; + } + } + + out: + + /* Resume the guest if it was running and we didn't just trash the state*/ + if (vm->run_state == VM_RUNNING) { + + if (ret == -1) { + vm->run_state = VM_STOPPED; + } + + /* We check the run state of the VM after every barrier + So this will immediately halt the VM + */ + v3_lower_barrier(vm); + } + + chkpt_close(chkpt); + + return ret; +} + + + diff --git a/palacios/src/palacios/vmm_chkpt_stores.h b/palacios/src/palacios/vmm_chkpt_stores.h new file mode 100644 index 0000000..505ef8c --- /dev/null +++ b/palacios/src/palacios/vmm_chkpt_stores.h @@ -0,0 +1,186 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2011, Jack Lange + * Copyright (c) 2011, The V3VEE Project + * All rights reserved. + * + * Author: Jack Lange + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#ifndef __VMM_CHKPT_STORES_H__ +#define __VMM_CHKPT_STORES_H__ + +//#include + +/* + * This is a place holder to ensure that the _v3_extensions section gets created by gcc + */ +static struct {} null_store __attribute__((__used__)) \ + __attribute__((unused, __section__ ("_v3_chkpt_stores"), \ + aligned(sizeof(addr_t)))); + + +#define register_chkpt_store(store) \ + static struct chkpt_interface * _v3_store_##store \ + __attribute__((used)) \ + __attribute__((unused, __section__("_v3_chkpt_stores"), \ + aligned(sizeof(addr_t)))) \ + = &store; + + + + + +#include + + +static void * debug_open_chkpt(char * url, chkpt_mode_t mode) { + + if (mode == LOAD) { + V3_Print("Cannot load from debug store\n"); + return NULL; + } + + V3_Print("Opening Checkpoint: %s\n", url); + + return (void *)1; +} + + + +static int debug_close_chkpt(void * store_data) { + V3_Print("Closing Checkpoint\n"); + return 0; +} + +static void * debug_open_ctx(void * store_data, + void * parent_ctx, + char * name) { + V3_Print("[%s]\n", name); + return (void *)1; +} + +static int debug_close_ctx(void * store_data, void * ctx) { + V3_Print("[CLOSE]\n"); + return 0; +} + +static int debug_save(void * store_data, void * ctx, + char * tag, uint64_t len, void * buf) { + V3_Print("%s:\n", tag); + + if (len > 100) { + len = 100; + } + + v3_dump_mem(buf, len); + + return 0; +} + +static int debug_load(void * store_data, void * ctx, + char * tag, uint64_t len, void * buf) { + V3_Print("Loading not supported !!!\n"); + return 0; +} + + +static struct chkpt_interface debug_store = { + .name = "DEBUG", + .open_chkpt = debug_open_chkpt, + .close_chkpt = debug_close_chkpt, + .open_ctx = debug_open_ctx, + .close_ctx = debug_close_ctx, + .save = debug_save, + .load = debug_load +}; + +register_chkpt_store(debug_store); + + + + +#ifdef V3_CONFIG_KEYED_STREAMS +#include + +static void * keyed_stream_open_chkpt(char * url, chkpt_mode_t mode) { + if (mode == SAVE) { + return v3_keyed_stream_open(url, V3_KS_WR_ONLY_CREATE); + } else if (mode == LOAD) { + return v3_keyed_stream_open(url, V3_KS_RD_ONLY); + } + + // Shouldn't get here + return NULL; +} + + + +static int keyed_stream_close_chkpt(void * store_data) { + v3_keyed_stream_t stream = store_data; + + v3_keyed_stream_close(stream); + + return 0; +} + +static void * keyed_stream_open_ctx(void * store_data, + void * parent_ctx, + char * name) { + v3_keyed_stream_t stream = store_data; + + return v3_keyed_stream_open_key(stream, name); +} + +static int keyed_stream_close_ctx(void * store_data, void * ctx) { + v3_keyed_stream_t stream = store_data; + + v3_keyed_stream_close_key(stream, ctx); + + return 0; +} + +static int keyed_stream_save(void * store_data, void * ctx, + char * tag, uint64_t len, void * buf) { + return v3_keyed_stream_write_key(store_data, ctx, buf, len); +} + +static int keyed_stream_load(void * store_data, void * ctx, + char * tag, uint64_t len, void * buf) { + return v3_keyed_stream_read_key(store_data, ctx, buf, len); +} + + +static struct chkpt_interface keyed_stream_store = { + .name = "KEYED_STREAM", + .open_chkpt = keyed_stream_open_chkpt, + .close_chkpt = keyed_stream_close_chkpt, + .open_ctx = keyed_stream_open_ctx, + .close_ctx = keyed_stream_close_ctx, + .save = keyed_stream_save, + .load = keyed_stream_load +}; + +register_chkpt_store(keyed_stream_store); + + + +#endif + + + + + + + +#endif diff --git a/palacios/src/palacios/vmm_dev_mgr.c b/palacios/src/palacios/vmm_dev_mgr.c index a6f71e5..de6209b 100644 --- a/palacios/src/palacios/vmm_dev_mgr.c +++ b/palacios/src/palacios/vmm_dev_mgr.c @@ -22,6 +22,10 @@ #include #include +#ifdef V3_CONFIG_CHECKPOINT +#include +#endif + #ifndef V3_CONFIG_DEBUG_DEV_MGR #undef PrintDebug @@ -126,6 +130,108 @@ int v3_free_vm_devices(struct v3_vm_info * vm) { return 0; } +#ifdef V3_CONFIG_CHECKPOINT + +int v3_save_vm_devices(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { + struct vmm_dev_mgr * mgr = &(vm->dev_mgr); + struct vm_device * dev; + struct v3_chkpt_ctx * dev_mgr_ctx = NULL; + + uint32_t num_saved_devs = 0; + uint32_t table_len = mgr->num_devs * 32; + char * name_table = NULL; + uint32_t tbl_offset = 0; + + name_table = V3_Malloc(table_len); + + memset(name_table, 0, table_len); + + + dev_mgr_ctx = v3_chkpt_open_ctx(chkpt, NULL, "devices"); + + list_for_each_entry(dev, &(mgr->dev_list), dev_link) { + + if (dev->ops->save) { + struct v3_chkpt_ctx * dev_ctx = NULL; + + + dev_ctx = v3_chkpt_open_ctx(chkpt, dev_mgr_ctx, dev->name); + + dev->ops->save(dev_ctx, dev->private_data); + + v3_chkpt_close_ctx(dev_ctx); + + // Error checking?? + + strncpy(name_table + tbl_offset, dev->name, 32); + tbl_offset += 32; + num_saved_devs++; + } else { + PrintError("Error: %s save() not implemented\n", dev->name); + } + } + + + // Specify which devices were saved + v3_chkpt_save(dev_mgr_ctx, "num_devs", 4, &num_saved_devs); + v3_chkpt_save(dev_mgr_ctx, "names", table_len, name_table); + V3_Free(name_table); + + v3_chkpt_close_ctx(dev_mgr_ctx); + + return 0; +} + + +int v3_load_vm_devices(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { + struct vm_device * dev; + struct v3_chkpt_ctx * dev_mgr_ctx = NULL; + uint32_t num_devs = 0; + char * name_table = NULL; + int i = 0; + + dev_mgr_ctx = v3_chkpt_open_ctx(chkpt, NULL, "devices"); + + v3_chkpt_load(dev_mgr_ctx, "num_devs", 4, &num_devs); + + V3_Print("Loading State for %d devices\n", num_devs); + + name_table = V3_Malloc(32 * num_devs); + + v3_chkpt_load(dev_mgr_ctx, "names", 32 * num_devs, name_table); + + for (i = 0; i < num_devs; i++) { + char * name = &(name_table[i * 32]); + struct v3_chkpt_ctx * dev_ctx = NULL; + dev = v3_find_dev(vm, name); + + if (!dev) { + PrintError("Tried to load state into non existant device: %s\n", name); + continue; + } + + if (!dev->ops->load) { + PrintError("Error Device (%s) does not support load operation\n", name); + continue; + } + + dev_ctx = v3_chkpt_open_ctx(chkpt, dev_mgr_ctx, name); + + if (!dev_ctx) { + PrintError("Error missing device context (%s)\n", name); + continue; + } + + + dev->ops->load(dev_ctx, dev->private_data); + } + + return 0; +} + + +#endif + static int free_frontends(struct v3_vm_info * vm, struct vmm_dev_mgr * mgr); int v3_deinit_dev_mgr(struct v3_vm_info * vm) { diff --git a/palacios/src/palacios/vmm_time.c b/palacios/src/palacios/vmm_time.c index 781bae7..fa8f5cd 100644 --- a/palacios/src/palacios/vmm_time.c +++ b/palacios/src/palacios/vmm_time.c @@ -172,21 +172,21 @@ static int skew_guest_time(struct guest_info * info) { guest_time = v3_get_guest_time(time_state); if (guest_time < target_guest_time) { - uint64_t max_skew, desired_skew, skew; + sint64_t max_skew, desired_skew, skew; if (time_state->enter_time) { /* Limit forward skew to 10% of the amount the guest has * run since we last could skew time */ - max_skew = (guest_time - time_state->enter_time) / 10; + max_skew = ((sint64_t)guest_time - (sint64_t)time_state->enter_time) / 10; } else { max_skew = 0; } - desired_skew = target_guest_time - guest_time; + desired_skew = (sint64_t)target_guest_time - (sint64_t)guest_time; skew = desired_skew > max_skew ? max_skew : desired_skew; - PrintDebug("Guest %llu cycles behind where it should be.\n", + PrintDebug("Guest %lld cycles behind where it should be.\n", desired_skew); - PrintDebug("Limit on forward skew is %llu. Skewing forward %llu.\n", + PrintDebug("Limit on forward skew is %lld. Skewing forward %lld.\n", max_skew, skew); v3_offset_time(info, skew); @@ -228,10 +228,10 @@ v3_time_enter_vm( struct guest_info * info ) struct vm_time * time_state = &(info->time_state); uint64_t guest_time, host_time; - guest_time = v3_get_guest_time(time_state); host_time = v3_get_host_time(time_state); + guest_time = v3_get_guest_time(time_state); time_state->enter_time = host_time; - time_state->guest_host_offset = guest_time - host_time; + time_state->guest_host_offset = (sint64_t)guest_time - (sint64_t)host_time; return 0; } diff --git a/palacios/src/palacios/vmx.c b/palacios/src/palacios/vmx.c index 419b706..1b286ae 100644 --- a/palacios/src/palacios/vmx.c +++ b/palacios/src/palacios/vmx.c @@ -32,6 +32,11 @@ #include #include #include +#include + +#ifdef V3_CONFIG_CHECKPOINT +#include +#endif #include #include @@ -558,6 +563,63 @@ int v3_deinit_vmx_vmcs(struct guest_info * core) { } + +#ifdef V3_CONFIG_CHECKPOINT +/* + * JRL: This is broken + */ +int v3_vmx_save_core(struct guest_info * core, void * ctx){ + uint64_t vmcs_ptr = vmcs_store(); + + v3_chkpt_save(ctx, "vmcs_data", PAGE_SIZE, (void *)vmcs_ptr); + + return 0; +} + +int v3_vmx_load_core(struct guest_info * core, void * ctx){ + struct vmx_data * vmx_info = (struct vmx_data *)(core->vmm_data); + struct cr0_32 * shadow_cr0; + char vmcs[PAGE_SIZE_4KB]; + + v3_chkpt_load(ctx, "vmcs_data", PAGE_SIZE_4KB, vmcs); + + vmcs_clear(vmx_info->vmcs_ptr_phys); + vmcs_load((addr_t)vmcs); + + v3_vmx_save_vmcs(core); + + shadow_cr0 = (struct cr0_32 *)&(core->ctrl_regs.cr0); + + + /* Get the CPU mode to set the guest_ia32e entry ctrl */ + + if (core->shdw_pg_mode == SHADOW_PAGING) { + if (v3_get_vm_mem_mode(core) == VIRTUAL_MEM) { + if (v3_activate_shadow_pt(core) == -1) { + PrintError("Failed to activate shadow page tables\n"); + return -1; + } + } else { + if (v3_activate_passthrough_pt(core) == -1) { + PrintError("Failed to activate passthrough page tables\n"); + return -1; + } + } + } + + return 0; +} +#endif + + +void v3_flush_vmx_vm_core(struct guest_info * core) { + struct vmx_data * vmx_info = (struct vmx_data *)(core->vmm_data); + vmcs_clear(vmx_info->vmcs_ptr_phys); + vmx_info->state = VMX_UNLAUNCHED; +} + + + static int update_irq_exit_state(struct guest_info * info) { struct vmx_exit_idt_vec_info idt_vec_info; @@ -739,7 +801,7 @@ static void print_exit_log(struct guest_info * info) { */ int v3_vmx_enter(struct guest_info * info) { int ret = 0; - uint32_t tsc_offset_low, tsc_offset_high; + //uint32_t tsc_offset_low, tsc_offset_high; struct vmx_exit_info exit_info; struct vmx_data * vmx_info = (struct vmx_data *)(info->vmm_data); @@ -784,11 +846,10 @@ int v3_vmx_enter(struct guest_info * info) { // Perform last-minute time bookkeeping prior to entering the VM v3_time_enter_vm(info); - tsc_offset_high = (uint32_t)((v3_tsc_host_offset(&info->time_state) >> 32) & 0xffffffff); - tsc_offset_low = (uint32_t)(v3_tsc_host_offset(&info->time_state) & 0xffffffff); - check_vmcs_write(VMCS_TSC_OFFSET_HIGH, tsc_offset_high); - check_vmcs_write(VMCS_TSC_OFFSET, tsc_offset_low); - + // tsc_offset_high = (uint32_t)((v3_tsc_host_offset(&info->time_state) >> 32) & 0xffffffff); + // tsc_offset_low = (uint32_t)(v3_tsc_host_offset(&info->time_state) & 0xffffffff); + // check_vmcs_write(VMCS_TSC_OFFSET_HIGH, tsc_offset_high); + // check_vmcs_write(VMCS_TSC_OFFSET, tsc_offset_low); if (v3_update_vmcs_host_state(info)) { v3_enable_ints(); @@ -807,19 +868,22 @@ int v3_vmx_enter(struct guest_info * info) { ret = v3_vmx_resume(&(info->vm_regs), info, &(info->ctrl_regs)); } + + // PrintDebug("VMX Exit: ret=%d\n", ret); if (ret != VMX_SUCCESS) { uint32_t error = 0; - vmcs_read(VMCS_INSTR_ERR, &error); v3_enable_ints(); - PrintError("VMENTRY Error: %d\n", error); + PrintError("VMENTRY Error: %d (launch_ret = %d)\n", error, ret); return -1; } + + // Immediate exit from VM time bookkeeping v3_time_exit_vm(info); @@ -901,6 +965,9 @@ int v3_start_vmx_guest(struct guest_info * info) { } PrintDebug("VMX core %u initialized\n", info->vcpu_id); + + // We'll be paranoid about race conditions here + v3_wait_at_barrier(info); } @@ -954,6 +1021,7 @@ int v3_start_vmx_guest(struct guest_info * info) { return -1; } + v3_wait_at_barrier(info); if (info->vm_info->run_state == VM_STOPPED) { diff --git a/utils/output_analysis/bad_ports.pl b/utils/output_analysis/bad_ports.pl new file mode 100755 index 0000000..468e698 --- /dev/null +++ b/utils/output_analysis/bad_ports.pl @@ -0,0 +1,31 @@ +#!/usr/bin/perl -w + +$#ARGV==0 or die "Finds all unique unhandled I/O ports in a palacios output file\nusage: bad_ports.pl serial.out\n"; + +open(K,shift); + +while () { + if (/: (\S+) operation on unhooked IO port 0x(\S+)/) { + $dir=$1; + $port=$2; + + $p{$port} |= ($dir eq 'IN' ? 1 : 2); + $n{$port}++; + } +} + +close(K); + +@list = sort keys %p; + +foreach $port (@list) { + print $port,"\t",$n{$port}; + if ($p{$port} & 1) { + print "\tIN"; + } + if ($p{$port} & 2) { + print "\tOUT"; + } + print "\n"; +} + diff --git a/utils/output_analysis/guest_kernel_calls.pl b/utils/output_analysis/guest_kernel_calls.pl new file mode 100755 index 0000000..07a710b --- /dev/null +++ b/utils/output_analysis/guest_kernel_calls.pl @@ -0,0 +1,65 @@ +#!/usr/bin/perl -w + +$#ARGV==0 or die "Decodes guest RIP addresses against a guest kernel dissassembly file\nThis tells you which functions in the guest kernel are being used\nusage: guest_kernel_calls.pl disassmfile < RIPS\n"; + +open(K,shift); + +@k = ; + +close(K); + +while () { + if (/RIP Linear: (\S+)/) { + $addr=$1; + chomp($addr); + print join("\t",$addr,findit($addr)),"\n"; + } +} + + +sub findit { + my $addr=shift; + my $i; + my $line=-1; + my $funcline=-1; + my $funcaddr; + my $funcname; + + if (substr($addr,0,1) eq "0") { + return "USER"; + } else { + # search forward + for ($i=0;$i<=$#k;$i++) { + if ($k[$i] =~ /^(\S+):/) { + $x=$1; + if ($x eq $addr) { + $line=$i; + last; + } + } + } + if ($line<0) { + return "CANNOT FIND IN DISASSEMBLY"; + } else { + # search backward + for ($i=$line;$i>=0;$i--) { + if ($k[$i] =~ /^(\S+)\s\<(\S+)\>:/) { + $funcline=$i; + $funcname=$2; + $funcaddr=$1; + last; + } + } + if ($funcline<0) { + return "CANNOT FIND FUNCTION IN DISASSEMBLY"; + } else { + return $funcname." at ".$funcaddr; + } + } + } +} + + + + +