From: Peter Dinda Date: Thu, 14 Jul 2016 23:49:20 +0000 (-0500) Subject: Extensions to HVM ROS userspace library corresponding to HVM enhancements X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=15b65ca6d546fbd03cdd7ae72830555e0263e538 Extensions to HVM ROS userspace library corresponding to HVM enhancements - ability to mirror/unmirror GDT and other segment state from ROS process to HRT - ability to handle HRT events forwarded to ROS (syscalls, page faults, exceptions) - corrections to stack alignment issue in HRT->ROS signals - log file selection --- diff --git a/guest/linux/hvm-ros/Makefile b/guest/linux/hvm-ros/Makefile index f818a39..6f9aea7 100644 --- a/guest/linux/hvm-ros/Makefile +++ b/guest/linux/hvm-ros/Makefile @@ -1,17 +1,30 @@ +CC:=gcc +AR:=ar +ARFLAGS:=ruv + +CFLAGS:= -Wall -static + +ifeq ($(DEBUG),1) + CFLAGS += -DDEBUG_ENABLE=1 +endif + +ifeq ($(INFO),1) + CFLAGS += -DINFO_ENABLE=1 +endif + all: libv3_hvm_ros_user.a test libv3_hvm_ros_user.a: v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o - ar ruv libv3_hvm_ros_user.a v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o + $(AR) $(ARFLAGS) libv3_hvm_ros_user.a v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o v3_hvm_ros_user.o: v3_hvm_ros_user.c v3_hvm_ros_user.h - gcc -Wall -c v3_hvm_ros_user.c + $(CC) $(CFLAGS) -c v3_hvm_ros_user.c v3_hvm_ros_user_low_level.o: v3_hvm_ros_user_low_level.S - gcc -Wall -c v3_hvm_ros_user_low_level.S - + $(CC) $(CFLAGS) -c v3_hvm_ros_user_low_level.S test: test.c libv3_hvm_ros_user.a v3_hvm_ros_user.h - gcc -Wall -static test.c -L. -lv3_hvm_ros_user -o test + $(CC) $(CFLAGS) -static test.c -L. -lv3_hvm_ros_user -o test clean: rm -f libv3_hvm_ros_user.a v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o test diff --git a/guest/linux/hvm-ros/test.c b/guest/linux/hvm-ros/test.c index 2781610..461d8af 100644 --- a/guest/linux/hvm-ros/test.c +++ b/guest/linux/hvm-ros/test.c @@ -199,7 +199,7 @@ static void my_hvm_sig_handler(uint64_t code) int main(int argc, char *argv[]) { int rc; - + if (argc!=3 && argc!=5) { printf("usage: test simple|time sync|async num_merges num_calls\n"); return -1; diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user.c b/guest/linux/hvm-ros/v3_hvm_ros_user.c index c703ec1..9928ba9 100644 --- a/guest/linux/hvm-ros/v3_hvm_ros_user.c +++ b/guest/linux/hvm-ros/v3_hvm_ros_user.c @@ -2,11 +2,12 @@ Copyright (c) 2015 Peter Dinda */ -#include #include #include #include #include +#include +#include #include #include @@ -14,17 +15,21 @@ #include "v3_hvm_ros_user.h" -#define DEBUG_OUTPUT 0 -#define INFO_OUTPUT 0 +#define HRT_FAIL_VEC 0x1d + +static FILE *log = NULL; + -#if DEBUG_OUTPUT -#define DEBUG(...) fprintf(stderr,__VA_ARGS__) +#if DEBUG_ENABLE==1 +#define DEBUG(fmt, args...) if (log) { fprintf(log, "DEBUG: " fmt, ##args); } #else #define DEBUG(...) #endif -#if INFO_OUTPUT -#define INFO(...) fprintf(stdout,__VA_ARGS__) +#if INFO_ENABLE==1 +#define INFO(fmt, args...) if (log) { fprintf(log, fmt, ##args); } + fprintf(stdout, fmt, ##args); \ + } #else #define INFO(...) #endif @@ -119,10 +124,37 @@ typedef unsigned char uchar_t; #define HCALL(rc,id,a,b,c,d,e,f,g,h) HCALL32(rc,id,a,b,c,d,e,f,g,h) #endif +static inline uint64_t +Syscall (uint64_t num, + uint64_t a1, + uint64_t a2, + uint64_t a3, + uint64_t a4, + uint64_t a5, + uint64_t a6) +{ + uint64_t rc; + + __asm__ __volatile__ ("movq %1, %%rax; " + "movq %2, %%rdi; " + "movq %3, %%rsi; " + "movq %4, %%rdx; " + "movq %5, %%r10; " + "movq %6, %%r8; " + "movq %7, %%r9; " + "syscall; " + "movq %%rax, %0; " + : "=m"(rc) + : "m"(num), "m"(a1), "m"(a2), "m"(a3), "m"(a4), "m"(a5), "m"(a6) + : "%rax", "%rdi", "%rcx", "%rsi", "%rdx", "%r10", "%r8", "%r9", "%r11", "memory"); + + return rc; +} + /* This must match the definition in palacios/include/palacios/vmm_hvm.h" */ struct v3_ros_event { - enum { ROS_NONE=0, ROS_PAGE_FAULT=1, ROS_SYSCALL=2 } event_type; + enum { ROS_NONE=0, ROS_PAGE_FAULT=1, ROS_SYSCALL=2, HRT_EXCEPTION=3, HRT_THREAD_EXIT=4, ROS_DONE=5} event_type; uint64_t last_ros_event_result; // valid when ROS_NONE union { struct { // valid when ROS_PAGE_FAULT @@ -133,23 +165,130 @@ struct v3_ros_event { struct { // valid when ROS_SYSCALL uint64_t args[8]; } syscall; + struct { // valid when HRT_EXCEPTION + uint64_t rip; + uint64_t vector; + } excp; + struct { // valid when HRT_THREAD_EXIT + uint64_t nktid; + } thread_exit; }; }; int v3_hvm_ros_user_init() { - // currently nothing to do + log = stderr; + return 0; +} + + +int v3_hvm_ros_user_set_log(FILE *l) +{ + log = l; return 0; } int v3_hvm_ros_user_deinit() { - // currently nothing to do + log = NULL; return 0; } -static void handle_ros_event(struct v3_ros_event *event) + +int v3_hvm_handle_ros_event_nohcall(struct v3_ros_event *event) +{ + uint64_t rc; + char t; + + switch (event->event_type) { + case ROS_PAGE_FAULT: + // force the ros kernel to the PTE + if (event->page_fault.action==ROS_READ) { + DEBUG("EVENT [PF READ] : %p\n", (volatile char*)(event->page_fault.cr2)); + t=*(volatile char*)(event->page_fault.cr2); + t=t; // avoid wanting for this throwaway + } else if (event->page_fault.action==ROS_WRITE) { + DEBUG("EVENT [PF WRITE] : %p\n", (volatile char*)(event->page_fault.cr2)); + + // stash the old contents + char tmp = *((volatile char *)event->page_fault.cr2); + + // force a write + *((volatile char*)event->page_fault.cr2) = 0xa; + + // put the old value back + *((volatile char*)event->page_fault.cr2) = tmp; + + } else { + INFO("Huh?\n"); + } + + DEBUG("Done."); + event->last_ros_event_result = 0; + //event->event_type = ROS_DONE; + event->event_type = ROS_NONE; + // completed + DEBUG("[OK]\n"); + + break; + + case ROS_SYSCALL: + DEBUG("EVENT [SYSCALL] : syscall(0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx)\n", + event->syscall.args[0], + event->syscall.args[1], + event->syscall.args[2], + event->syscall.args[3], + event->syscall.args[4], + event->syscall.args[5], + event->syscall.args[6], + event->syscall.args[7]); + + rc = Syscall(event->syscall.args[0], + event->syscall.args[1], + event->syscall.args[2], + event->syscall.args[3], + event->syscall.args[4], + event->syscall.args[5], + event->syscall.args[6]); + + if ((long int)rc<0) { + DEBUG("Syscall (num=0x%llx) failed (rc=0x%llx)\n", (unsigned long long)event->syscall.args[0], (unsigned long long)rc); + } + + DEBUG("Return = 0x%lx...", rc); + event->last_ros_event_result = rc; + //event->event_type = ROS_DONE; + event->event_type = ROS_NONE; + DEBUG("[OK]\n"); + + break; + + case HRT_EXCEPTION: + DEBUG("EVENT [EXCEPTION] : (tid=%lx) at %p vector=0x%lx...", syscall(SYS_gettid), (void*)event->excp.rip, event->excp.vector); + if (event->excp.vector == HRT_FAIL_VEC) { + INFO("HRT execution failed due to exception in HRT\n"); + } else { + INFO("HRT execution failed due to exception in ROS (vec=0x%lx)\n", event->excp.vector); + } + + event->last_ros_event_result = 0; + //event->event_type = ROS_DONE; + event->event_type = ROS_NONE; + DEBUG("[OK]\n"); + exit(EXIT_FAILURE); + break; + + default: + DEBUG( "Unknown ROS event 0x%x\n", event->event_type); + break; + } + + return 0; +} + + +int v3_hvm_handle_ros_event(struct v3_ros_event *event) { unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0; char t; @@ -162,60 +301,105 @@ static void handle_ros_event(struct v3_ros_event *event) t=*(volatile char*)(event->page_fault.cr2); t=t; // avoid wanting for this throwaway } else if (event->page_fault.action==ROS_WRITE) { - DEBUG("Handling page fault writefor %p\n", (volatile char*)(event->page_fault.cr2)); - *(volatile char*)(event->page_fault.cr2) = *(volatile char *)(event->page_fault.cr2); + DEBUG("Handling page fault write for %p\n", (volatile char*)(event->page_fault.cr2)); + + // stash the old contents + char tmp = *((volatile char *)event->page_fault.cr2); + + // force a write + *((volatile char*)event->page_fault.cr2) = 0xa; + + // put the old value back + *((volatile char*)event->page_fault.cr2) = tmp; + } else { INFO("Huh?\n"); } - DEBUG("Done - doing hypercall\n"); + + DEBUG("Done - doing hypercall..."); num = 0xf00d; a1 = 0x1f; a2 = 0; // success HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); // completed - DEBUG("Completed.\n"); - + DEBUG("[OK]\n"); + break; case ROS_SYSCALL: DEBUG("Doing system call: syscall(0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx)\n", - event->syscall.args[0], - event->syscall.args[1], - event->syscall.args[2], - event->syscall.args[3], - event->syscall.args[4], - event->syscall.args[5], - event->syscall.args[6], - event->syscall.args[7]); - - rc = syscall(event->syscall.args[0], + event->syscall.args[0], + event->syscall.args[1], + event->syscall.args[2], + event->syscall.args[3], + event->syscall.args[4], + event->syscall.args[5], + event->syscall.args[6], + event->syscall.args[7]); + + rc = Syscall(event->syscall.args[0], event->syscall.args[1], event->syscall.args[2], event->syscall.args[3], event->syscall.args[4], event->syscall.args[5], - event->syscall.args[6], - event->syscall.args[7]); - - if ((int)rc<0) { - DEBUG("syscall failed"); + event->syscall.args[6]); + + if ((long int)rc<0) { + DEBUG("Syscall (num=0x%llx) failed (rc=0x%llx)\n", (unsigned long long)event->syscall.args[0], (unsigned long long)rc); } - - DEBUG("Return = 0x%llx, doing hypercall\n", rc); + + DEBUG("Return = 0x%llx, doing hypercall...", rc); num = 0xf00d; a1 = 0x1f; a2 = rc; HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); - DEBUG("Completed\n"); + DEBUG("[OK]\n"); + + break; + case HRT_EXCEPTION: + DEBUG("Handling exception (tid=%lx) at %p vector=0x%lx...", syscall(SYS_gettid), (void*)event->excp.rip, event->excp.vector); + if (event->excp.vector == HRT_FAIL_VEC) { + INFO("HRT execution failed due to exception in HRT\n"); + } else { + INFO("HRT execution failed due to exception in ROS (vec=0x%lx)\n", event->excp.vector); + } + + num = 0xf00d; + a1 = 0x1f; + a2 = rc; + HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); + DEBUG("[OK]\n"); + exit(EXIT_FAILURE); break; + default: DEBUG( "Unknown ROS event 0x%x\n", event->event_type); break; } + + return 0; } + +static void wait_for_merge_completion() +{ + unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0; + struct v3_ros_event event; + + memset(&event, 1, sizeof(event)); + rc = 1; + + while (rc) { + num = 0xf00d; + a1 = 0xf; + a2 = (unsigned long long) &event; + HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); + } +} + static void wait_for_completion() { @@ -234,7 +418,7 @@ static void wait_for_completion() if (rc) { // usleep(100); if (event.event_type != ROS_NONE) { - handle_ros_event(&event); + v3_hvm_handle_ros_event(&event); } } } @@ -308,7 +492,7 @@ int v3_hvm_ros_merge_address_spaces() INFO("Error in request to merge address spaces rc=0x%llx\n",rc); return -1; } else { - wait_for_completion(); + wait_for_merge_completion(); return 0; } } @@ -331,6 +515,44 @@ int v3_hvm_ros_unmerge_address_spaces() } +int v3_hvm_ros_mirror_gdt(uint64_t fsbase, uint64_t cpu) +{ + unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0; + unsigned long long rc; + + num=0xf00d; + a1 = 0x51; // mirror the ROS FS and GS on the HRT side + a2 = fsbase; + a3 = cpu; + HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); + if (rc) { + INFO("Error in request to mirror GDT (rc=0x%llx)\n", rc); + return -1; + } else { + wait_for_completion(); + return 0; + } + +} + +int v3_hvm_ros_unmirror_gdt() +{ + unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0; + unsigned long long rc; + + num=0xf00d; + a1 = 0x53; // unmirror GDT and the rest + HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); + if (rc) { + INFO("Error in request to unmirror GDT (rc=0x%llx)\n", rc); + return -1; + } else { + wait_for_completion(); + return 0; + } +} + + int v3_hvm_ros_invoke_hrt_async(void *buf, int par) { unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0; @@ -353,6 +575,22 @@ int v3_hvm_ros_invoke_hrt_async(void *buf, int par) } } +int v3_hvm_ros_invoke_hrt_async_nowait(void *buf, int par) +{ + unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0; + unsigned long long rc; + + num=0xf00d; + if (par) { + a1 = 0x21; // issue "function" in parallel + } else { + a1 = 0x20; // issue "function" sequentially + } + a2 = (unsigned long long) buf; + HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); + return 0; +} + @@ -402,7 +640,7 @@ static void wait_for_sync() if (rc!=4) { // usleep(100); if (event.event_type != ROS_NONE) { - handle_ros_event(&event); + v3_hvm_handle_ros_event(&event); } } } @@ -474,7 +712,7 @@ int v3_hvm_ros_invoke_hrt_sync(void *buf, int ros) a2 = (unsigned long long) &event; HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); if (event.event_type != ROS_NONE) { - handle_ros_event(&event); + v3_hvm_handle_ros_event(&event); } } } @@ -487,7 +725,7 @@ void (*__v3_hvm_ros_signal_handler)(uint64_t) = 0; static void *__v3_hvm_ros_signal_handler_stack = 0; static uint64_t __v3_hvm_ros_signal_handler_stack_size=0; -int v3_hvm_ros_register_signal(void (*h)(uint64_t), void *stack, uint64_t stack_size ) +int v3_hvm_ros_register_signal(void (*h)(uint64_t), void *stack, uint64_t stack_size) { unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0; @@ -507,7 +745,7 @@ int v3_hvm_ros_register_signal(void (*h)(uint64_t), void *stack, uint64_t stack_ num = 0xf00d; a1 = 0x40; a2 = (unsigned long long) &__v3_hvm_ros_signal_handler_stub; - a3 = (unsigned long long) stack + stack_size - 8; // put us at the top of the stack + a3 = (unsigned long long) stack + stack_size - 16; // put us at the top of the stack, and 16 byte aligned HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8); if (rc) { @@ -565,3 +803,5 @@ int v3_hvm_hrt_signal_ros(uint64_t code) return rc ? -1 : rc; } + + diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user.h b/guest/linux/hvm-ros/v3_hvm_ros_user.h index de3b2e1..6ac99a7 100644 --- a/guest/linux/hvm-ros/v3_hvm_ros_user.h +++ b/guest/linux/hvm-ros/v3_hvm_ros_user.h @@ -12,6 +12,10 @@ int v3_hvm_ros_user_init(); int v3_hvm_ros_user_deinit(); +// by default, debugging and other log output goes to STDERR +// you can change this here +int v3_hvm_ros_user_set_log(FILE *log); + // Establish function to be invoked by the VMM // to signal activity (basically an interrupt handler) // The handler can use the GPRs, but must save/restore @@ -25,13 +29,21 @@ int v3_hvm_ros_unregister_signal(); // - the intial image is the one given in the .pal config int v3_hvm_ros_install_hrt_image(void *image, uint64_t size); +// Reset diffent components of the HVM +// or the entire HVM. The component immediatly reboots typedef enum {RESET_HRT, RESET_ROS, RESET_BOTH} reset_type; - int v3_hvm_ros_reset(reset_type what); +// Merge address spaces between +// this ROS process and a compatible HRT int v3_hvm_ros_merge_address_spaces(); int v3_hvm_ros_unmerge_address_spaces(); +// Reflect this ROS process's GDT into a compatible +// HRT. This is typically necessary to run +// code out of the ROS address space in the HRT +int v3_hvm_ros_mirror_gdt(uint64_t fsbase, uint64_t cpu); +int v3_hvm_ros_unmiror_gdt(); // Asynchronosus invocation of the HRT using an // opaque pointer (typically this is a pointer @@ -39,7 +51,8 @@ int v3_hvm_ros_unmerge_address_spaces(); // arguments. The parallel flag indicates that // that it will be invoked simulatneously on all // cores. -int v3_hvm_ros_invoke_hrt_async(void *p, int parallel); +int v3_hvm_ros_invoke_hrt_async(void *p, int parallel); +int v3_hvm_ros_invoke_hrt_async_nowait(void *buf, int par); // synchronize with HRT via shared location @@ -50,8 +63,15 @@ int v3_hvm_ros_synchronize(); int v3_hvm_ros_invoke_hrt_sync(void *p, int handle_ros_events); int v3_hvm_ros_desynchronize(); -// Signal the HRT from the ROS -// The ROS can call this too, but it shouldn't be necesary +// Handle events occuring within the HRT +// that need to be be handled within the ROS +struct v3_ros_event; +int v3_hvm_handle_ros_event(struct v3_ros_event * event); +int v3_hvm_handle_ros_event_nohcall(struct v3_ros_event * event); + +// Signal the ROS process from the ROS process +// These signals usually are generated by the HRT, but +// The ROS can also signal itself using the same mechanism int v3_hvm_hrt_signal_ros(uint64_t code); #endif diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S b/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S index 3cbcc07..076b505 100644 --- a/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S +++ b/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S @@ -57,6 +57,9 @@ we can be interrupted, and deliverable interrupts automatically prioritize over us. + The start of the stack (before the VMM pushes the fake + interrupt frame is 16 byte aligned + 48bitsblank | return SS (16 bits) Return RSP Return RFLAGS @@ -67,22 +70,43 @@ We then need simply to do this: save regs + arrange 16 byte alignment at entry of call call the handler (if installed) restore regs iret */ +#define DEBUG_ENTRY 0 + .global __v3_hvm_ros_signal_handler_stub -__v3_hvm_ros_signal_handler_stub: - GPR_SAVE() - movabs __v3_hvm_ros_signal_handler, %rax - testq %rax, %rax - jz skip_handler - movq 120(%rsp), %rdi /* error code becomes argument */ - callq *%rax -skip_handler: - GPR_LOAD() - addq $8, %rsp /* nuke the error code */ - iretq +__v3_hvm_ros_signal_handler_stub: /* we are 16 byte aligned on entry 16 + 6*8 for interrupt frame */ + GPR_SAVE() /* push 15 words, all but rsp, now not aligned - need 1 more word */ + subq $8, %rsp /* make us 16 byte aligned */ + +#if DEBUG_ENTRY /* print out something if we are debugging - Note this is danerous code */ + pushq %rdi + pushq %rax + movabsq $printf, %rax + movabsq $string, %rdi + callq *%rax + popq %rax + popq %rdi +#endif + movq %rsp, %rbp /* give us a stack frame for any callee that needs it */ + movabs __v3_hvm_ros_signal_handler, %rax /* find the user-level handler */ + testq %rax, %rax /* return immediately if it doesn't exist */ + jz skip_handler /* " */ + movq 128(%rsp), %rdi /* error code becomes argument for user-level handler */ + callq *%rax /* call handelr - 16 byte aligned at call */ + jmp done +skip_handler: +done: + addq $8, %rsp /* get rid of stack alignment pad */ + GPR_LOAD() /* rbp is restored here */ + addq $8, %rsp /* nuke the error code */ + iretq /* restore rip, rsp, and rflags */ + +string: + .asciz "Got to signal handler stub\12"