From: Madhav Suresh Date: Fri, 16 Sep 2011 20:36:31 +0000 (-0400) Subject: hooked checkpoint code up to the linux module and added user space tools X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=793f9cd53a472668dbeb16341085e5512484485c hooked checkpoint code up to the linux module and added user space tools --- diff --git a/linux_module/palacios.h b/linux_module/palacios.h index cece080..d96739c 100644 --- a/linux_module/palacios.h +++ b/linux_module/palacios.h @@ -10,6 +10,9 @@ /* Global Control IOCTLs */ #define V3_START_GUEST 10 #define V3_STOP_GUEST 11 +#define V3_CREATE_GUEST 12 +#define V3_FREE_GUEST 13 + #define V3_ADD_MEMORY 50 /* VM Specific IOCTLs */ @@ -19,6 +22,11 @@ #define V3_VM_PAUSE 23 #define V3_VM_CONTINUE 24 +#define V3_VM_LAUNCH 25 +#define V3_VM_STOP 26 +#define V3_VM_LOAD 27 +#define V3_VM_SAVE 28 + #define V3_VM_INSPECT 30 #define V3_VM_MOVE_CORE 33 @@ -35,17 +43,23 @@ struct v3_guest_img { unsigned long long size; void * guest_data; char name[128]; -}; +} __attribute__((packed)); struct v3_mem_region { unsigned long long base_addr; unsigned long long num_pages; -}; +} __attribute__((packed)); struct v3_core_move_cmd{ - uint16_t vcore_id; - uint16_t pcore_id; -}; + unsigned short vcore_id; + unsigned short pcore_id; +} __attribute__((packed)); + +struct v3_chkpt_info { + char store[128]; + char url[256]; /* This might need to be bigger... */ +} __attribute__((packed)); + void * trace_malloc(size_t size, gfp_t flags); diff --git a/linux_module/vm.c b/linux_module/vm.c index 1e37587..08ae7e0 100644 --- a/linux_module/vm.c +++ b/linux_module/vm.c @@ -137,6 +137,48 @@ static long v3_vm_ioctl(struct file * filp, v3_continue_vm(guest->v3_ctx); break; } +#ifdef V3_CONFIG_CHECKPOINT + case V3_VM_SAVE: { + struct v3_chkpt_info chkpt; + void __user * argp = (void __user *)arg; + + memset(&chkpt, 0, sizeof(struct v3_chkpt_info)); + + if (copy_from_user(&chkpt, argp, sizeof(struct v3_chkpt_info))) { + printk("Copy from user error getting checkpoint info\n"); + return -EFAULT; + } + + printk("Saving Guest to %s:%s\n", chkpt.store, chkpt.url); + + if (v3_save_vm(guest->v3_ctx, chkpt.store, chkpt.url) == -1) { + printk("Error checkpointing VM state\n"); + return -EFAULT; + } + + break; + } + case V3_VM_LOAD: { + struct v3_chkpt_info chkpt; + void __user * argp = (void __user *)arg; + + memset(&chkpt, 0, sizeof(struct v3_chkpt_info)); + + if (copy_from_user(&chkpt, argp, sizeof(struct v3_chkpt_info))) { + printk("Copy from user error getting checkpoint info\n"); + return -EFAULT; + } + + printk("Loading Guest to %s:%s\n", chkpt.store, chkpt.url); + + if (v3_load_vm(guest->v3_ctx, chkpt.store, chkpt.url) == -1) { + printk("Error Loading VM state\n"); + return -EFAULT; + } + + break; + } +#endif case V3_VM_MOVE_CORE: { struct v3_core_move_cmd cmd; void __user * argp = (void __user *)arg; @@ -151,9 +193,9 @@ static long v3_vm_ioctl(struct file * filp, printk("moving guest %s vcore %d to CPU %d\n", guest->name, cmd.vcore_id, cmd.pcore_id); v3_move_vm_core(guest->v3_ctx, cmd.vcore_id, cmd.pcore_id); - } - break; + break; + } default: { struct vm_ctrl * ctrl = get_ctrl(guest, ioctl); diff --git a/linux_usr/Makefile b/linux_usr/Makefile index 3aa822c..c74b4cc 100644 --- a/linux_usr/Makefile +++ b/linux_usr/Makefile @@ -1,4 +1,4 @@ -all: v3_ctrl v3_stop v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file v3_core_move +all: v3_ctrl v3_stop v3_cons v3_mem v3_monitor v3_stream v3_user_host_dev_example v3_os_debug v3_user_keyed_stream_example v3_user_keyed_stream_file v3_core_move v3_save v3_load @@ -8,6 +8,11 @@ v3_ctrl : v3_ctrl.c v3_ctrl.h v3_stop : v3_stop.c v3_ctrl.h gcc -static v3_stop.c -o v3_stop +v3_save : v3_save.c v3_ctrl.h + gcc -static v3_save.c -o v3_save +v3_load : v3_load.c v3_ctrl.h + gcc -static v3_load.c -o v3_load + v3_mem : v3_mem.c v3_ctrl.h gcc -static v3_mem.c -o v3_mem diff --git a/linux_usr/v3_core_move.c b/linux_usr/v3_core_move.c index d7439d8..29bea35 100644 --- a/linux_usr/v3_core_move.c +++ b/linux_usr/v3_core_move.c @@ -15,11 +15,6 @@ #include "v3_ctrl.h" -struct v3_core_move_cmd { - unsigned short vcore_id; - unsigned short pcore_id; -}; - int main(int argc, char* argv[]) { int vm_fd; diff --git a/linux_usr/v3_ctrl.h b/linux_usr/v3_ctrl.h index 550ab9a..0826825 100644 --- a/linux_usr/v3_ctrl.h +++ b/linux_usr/v3_ctrl.h @@ -8,8 +8,19 @@ #define V3_START_GUEST 10 #define V3_STOP_GUEST 11 +#define V3_CREATE_GUEST 12 +#define V3_FREE_GUEST 13 + + +#define V3_VM_PAUSE 23 +#define V3_VM_CONTINUE 24 + +#define V3_VM_LAUNCH 25 +#define V3_VM_STOP 26 +#define V3_VM_LOAD 27 +#define V3_VM_SAVE 28 + #define V3_ADD_MEMORY 50 -#define V3_START_NETWORK 60 #define V3_VM_CONSOLE_CONNECT 20 #define V3_VM_SERIAL_CONNECT 21 @@ -22,12 +33,24 @@ struct v3_guest_img { unsigned long long size; void * guest_data; char name[128]; -}; +} __attribute__((packed)); struct v3_mem_region { unsigned long long base_addr; unsigned long long num_pages; -}; +} __attribute__((packed)); + + +struct v3_core_move_cmd{ + unsigned short vcore_id; + unsigned short pcore_id; +} __attribute__((packed)); + + +struct v3_chkpt_info { + char store[128]; + char url[256]; /* This might need to be bigger... */ +} __attribute__((packed)); #endif diff --git a/linux_usr/v3_load.c b/linux_usr/v3_load.c new file mode 100644 index 0000000..019c8fa --- /dev/null +++ b/linux_usr/v3_load.c @@ -0,0 +1,70 @@ +/* + * V3 checkpoint save utility + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "v3_ctrl.h" + + +#define MAX_STORE_LEN 128 +#define MAX_URL_LEN 256 + + +struct v3_chkpt_info chkpt; + +int main(int argc, char* argv[]) { + int vm_fd; + char * vm_dev = NULL; + + if (argc < 4) { + printf("Usage: ./v3_load \n"); + return -1; + } + + vm_dev = argv[1]; + + if (strlen(argv[2]) >= MAX_STORE_LEN) { + printf("ERROR: Checkpoint store name longer than maximum size (%d)\n", MAX_STORE_LEN); + return -1; + } + + strncpy(chkpt.store, argv[2], MAX_STORE_LEN); + + + if (strlen(argv[3]) >= MAX_URL_LEN) { + printf("ERROR: Checkpoint URL longer than maximum size (%d)\n", MAX_URL_LEN); + return -1; + } + + strncpy(chkpt.url, argv[3], MAX_URL_LEN); + + vm_fd = open(vm_dev, O_RDONLY); + if (vm_fd == -1) { + printf("Error opening VM device: %s\n", vm_dev); + return -1; + } + + ioctl(vm_fd, V3_VM_LOAD, &chkpt); + + /* Close the file descriptor. */ + close(vm_fd); + + return 0; +} + + diff --git a/linux_usr/v3_save.c b/linux_usr/v3_save.c new file mode 100644 index 0000000..009e0f2 --- /dev/null +++ b/linux_usr/v3_save.c @@ -0,0 +1,70 @@ +/* + * V3 checkpoint save utility + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "v3_ctrl.h" + + +#define MAX_STORE_LEN 128 +#define MAX_URL_LEN 256 + + +struct v3_chkpt_info chkpt; + +int main(int argc, char* argv[]) { + int vm_fd; + char * vm_dev = NULL; + + if (argc < 4) { + printf("Usage: ./v3_save \n"); + return -1; + } + + vm_dev = argv[1]; + + if (strlen(argv[2]) >= MAX_STORE_LEN) { + printf("ERROR: Checkpoint store name longer than maximum size (%d)\n", MAX_STORE_LEN); + return -1; + } + + strncpy(chkpt.store, argv[2], MAX_STORE_LEN); + + + if (strlen(argv[3]) >= MAX_URL_LEN) { + printf("ERROR: Checkpoint URL longer than maximum size (%d)\n", MAX_URL_LEN); + return -1; + } + + strncpy(chkpt.url, argv[3], MAX_URL_LEN); + + vm_fd = open(vm_dev, O_RDONLY); + if (vm_fd == -1) { + printf("Error opening VM device: %s\n", vm_dev); + return -1; + } + + ioctl(vm_fd, V3_VM_SAVE, &chkpt); + + /* Close the file descriptor. */ + close(vm_fd); + + return 0; +} + + diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 8a75498..8364325 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -343,6 +343,9 @@ int v3_stop_vm(struct v3_vm_info * vm); int v3_pause_vm(struct v3_vm_info * vm); int v3_continue_vm(struct v3_vm_info * vm); +int v3_save_vm(struct v3_vm_info * vm, char * store, char * url); +int v3_load_vm(struct v3_vm_info * vm, char * store, char * url); + int v3_move_vm_core(struct v3_vm_info * vm, int vcore_id, int target_cpu); int v3_free_vm(struct v3_vm_info * vm); diff --git a/palacios/include/palacios/vmm_checkpoint.h b/palacios/include/palacios/vmm_checkpoint.h index cbb1b5d..4281d5e 100644 --- a/palacios/include/palacios/vmm_checkpoint.h +++ b/palacios/include/palacios/vmm_checkpoint.h @@ -41,8 +41,8 @@ int v3_chkpt_load(struct v3_chkpt_ctx * ctx, char * tag, uint64_t len, void * bu int v3_chkpt_close_ctx(struct v3_chkpt_ctx * ctx); struct v3_chkpt_ctx * v3_chkpt_open_ctx(struct v3_chkpt * chkpt, struct v3_chkpt_ctx * parent, char * name); -int v3_load_vm(struct v3_vm_info * vm, char * store, char * url); -int v3_save_vm(struct v3_vm_info * vm); +int v3_chkpt_save_vm(struct v3_vm_info * vm, char * store, char * url); +int v3_chkpt_load_vm(struct v3_vm_info * vm, char * store, char * url); diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index a6a6879..3e6a3f4 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -350,7 +350,7 @@ int v3_svm_save_core(struct guest_info * core, void * ctx){ return 0; } -int v3_svm_patch_core(struct guest_info * core, void * chkpt_ctx){ +int v3_svm_load_core(struct guest_info * core, void * chkpt_ctx){ struct cr0_32 * shadow_cr0; vmcb_saved_state_t * guest_state; vmcb_ctrl_t * guest_ctrl; diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index 961bbca..14a0d2f 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -489,6 +489,19 @@ int v3_continue_vm(struct v3_vm_info * vm) { return 0; } +#ifdef V3_CONFIG_CHECKPOINT +#include + +int v3_save_vm(struct v3_vm_info * vm, char * store, char * url) { + return v3_chkpt_save_vm(vm, store, url); +} + + +int v3_load_vm(struct v3_vm_info * vm, char * store, char * url) { + return v3_chkpt_load_vm(vm, store, url); +} +#endif + int v3_free_vm(struct v3_vm_info * vm) { int i = 0; diff --git a/palacios/src/palacios/vmm_checkpoint.c b/palacios/src/palacios/vmm_checkpoint.c index ded6171..ef4bec1 100644 --- a/palacios/src/palacios/vmm_checkpoint.c +++ b/palacios/src/palacios/vmm_checkpoint.c @@ -145,7 +145,7 @@ static int load_memory(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { return 0; } -/* + static int save_memory(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { void * guest_mem_base = NULL; void * ctx = NULL; @@ -195,7 +195,7 @@ int save_header(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { return 0; } -*/ + static int load_header(struct v3_vm_info * vm, struct v3_chkpt * chkpt) { v3_cpu_arch_t cpu_type = v3_get_cpu_type(V3_Get_CPU()); void * ctx = NULL; @@ -306,7 +306,7 @@ static int load_core(struct guest_info * info, struct v3_chkpt * chkpt) { return 0; } -/* + static int save_core(struct guest_info * info, struct v3_chkpt * chkpt) { v3_cpu_arch_t cpu_type = v3_get_cpu_type(V3_Get_CPU()); void * ctx = NULL; @@ -383,22 +383,85 @@ static int save_core(struct guest_info * info, struct v3_chkpt * chkpt) { return 0; } -*/ -int v3_load_vm(struct v3_vm_info * vm, char * store, char * url) { + +int v3_chkpt_save_vm(struct v3_vm_info * vm, char * store, char * url) { + struct v3_chkpt * chkpt = NULL; + int ret = 0;; int i = 0; + + chkpt = chkpt_open(store, url, SAVE); + + if (chkpt == NULL) { + PrintError("Error creating checkpoint store\n"); + return -1; + } + + /* If this guest is running we need to block it while the checkpoint occurs */ + if (vm->run_state == VM_RUNNING) { + while (v3_raise_barrier(vm, NULL) == -1); + } + + if ((ret = save_memory(vm, chkpt)) == -1) { + PrintError("Unable to save memory\n"); + goto out; + } + + /* + if ((ret = v3_chkpt_save_dev(vm)) == -1) { + PrintError("Unable to save devices\n"); + goto out; + } + */ + + if ((ret = save_header(vm, chkpt)) == -1) { + PrintError("Unable to save header\n"); + goto out; + } + + for (i = 0; i < vm->num_cores; i++){ + + if ((ret = save_core(&(vm->cores[i]), chkpt)) == -1) { + PrintError("chkpt of core %d failed\n", i); + goto out; + } + } + + out: + + /* Resume the guest if it was running */ + if (vm->run_state == VM_RUNNING) { + v3_lower_barrier(vm); + } + + chkpt_close(chkpt); + + return ret; + +} + +int v3_chkpt_load_vm(struct v3_vm_info * vm, char * store, char * url) { struct v3_chkpt * chkpt = NULL; + int i = 0; + int ret = 0; chkpt = chkpt_open(store, url, LOAD); + if (chkpt == NULL) { + PrintError("Error creating checkpoint store\n"); + return -1; + } - if (load_memory(vm, chkpt) == -1) { + /* If this guest is running we need to block it while the checkpoint occurs */ + if (vm->run_state == VM_RUNNING) { + while (v3_raise_barrier(vm, NULL) == -1); + } + + if ((ret = load_memory(vm, chkpt)) == -1) { PrintError("Unable to save memory\n"); - return -1; + goto out; } - //raising the barrier requires VM to be in VM_RUNNING state - vm->run_state = VM_RUNNING; /* Don't handle devices just yet if (v3_chkpt_load_dev(vm) == -1) { @@ -407,20 +470,37 @@ int v3_load_vm(struct v3_vm_info * vm, char * store, char * url) { */ - if (load_header(vm, chkpt) == -1) { + if ((ret = load_header(vm, chkpt)) == -1) { PrintError("Unable to load header\n"); + goto out; } //per core cloning for (i = 0; i < vm->num_cores; i++) { - if (load_core(&(vm->cores[i]), chkpt) == -1) { - return -1; + if ((ret = load_core(&(vm->cores[i]), chkpt)) == -1) { + PrintError("Error loading core state (core=%d)\n", i); + goto out; } } + out: + + /* Resume the guest if it was running and we didn't just trash the state*/ + if (vm->run_state == VM_RUNNING) { + + if (ret == -1) { + vm->run_state = VM_STOPPED; + } + + /* We check the run state of the VM after every barrier + So this will immediately halt the VM + */ + v3_lower_barrier(vm); + } + chkpt_close(chkpt); - return 0; + return ret; }