Extensions to HVM ROS userspace library corresponding to HVM enhancements

- ability to mirror/unmirror GDT and other segment state from ROS process to HRT
- ability to handle HRT events forwarded to ROS (syscalls, page faults, exceptions)
- corrections to stack alignment issue in HRT->ROS signals
- log file selection

diff --git a/guest/linux/hvm-ros/Makefile b/guest/linux/hvm-ros/Makefile
index f818a39..6f9aea7 100644 (file)
--- a/guest/linux/hvm-ros/Makefile
+++ b/guest/linux/hvm-ros/Makefile
@@ -1,17 +1,30 @@
+CC:=gcc
+AR:=ar
+ARFLAGS:=ruv
+
+CFLAGS:= -Wall -static
+
+ifeq ($(DEBUG),1)
+       CFLAGS += -DDEBUG_ENABLE=1
+endif
+
+ifeq ($(INFO),1)
+       CFLAGS += -DINFO_ENABLE=1
+endif
+
 all: libv3_hvm_ros_user.a test
 
 libv3_hvm_ros_user.a: v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o
-       ar ruv libv3_hvm_ros_user.a v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o
+       $(AR) $(ARFLAGS) libv3_hvm_ros_user.a v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o
 
 v3_hvm_ros_user.o: v3_hvm_ros_user.c v3_hvm_ros_user.h 
-       gcc -Wall -c v3_hvm_ros_user.c
+       $(CC) $(CFLAGS) -c v3_hvm_ros_user.c
 
 v3_hvm_ros_user_low_level.o: v3_hvm_ros_user_low_level.S
-       gcc -Wall -c v3_hvm_ros_user_low_level.S
-
+       $(CC) $(CFLAGS) -c v3_hvm_ros_user_low_level.S
 
 test: test.c libv3_hvm_ros_user.a v3_hvm_ros_user.h
-       gcc -Wall -static test.c -L. -lv3_hvm_ros_user -o test
+       $(CC) $(CFLAGS) -static test.c -L. -lv3_hvm_ros_user -o test
 
 clean:
        rm -f libv3_hvm_ros_user.a v3_hvm_ros_user.o v3_hvm_ros_user_low_level.o test

diff --git a/guest/linux/hvm-ros/test.c b/guest/linux/hvm-ros/test.c
index 2781610..461d8af 100644 (file)
--- a/guest/linux/hvm-ros/test.c
+++ b/guest/linux/hvm-ros/test.c
@@ -199,7 +199,7 @@ static void my_hvm_sig_handler(uint64_t code)
 int main(int argc, char *argv[]) 
 {
     int rc;
-    
+
     if (argc!=3 && argc!=5) { 
        printf("usage: test simple|time sync|async num_merges num_calls\n");
        return -1;

diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user.c b/guest/linux/hvm-ros/v3_hvm_ros_user.c
index c703ec1..9928ba9 100644 (file)
--- a/guest/linux/hvm-ros/v3_hvm_ros_user.c
+++ b/guest/linux/hvm-ros/v3_hvm_ros_user.c
@@ -2,11 +2,12 @@
   Copyright (c) 2015 Peter Dinda
 */
 
-#include <sys/syscall.h>
 #include <unistd.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <syscall.h>
+#include <stdio.h>
 #include <string.h>
 
 #include <sys/mman.h>
@@ -14,17 +15,21 @@
 #include "v3_hvm_ros_user.h"
 
 
-#define DEBUG_OUTPUT 0
-#define INFO_OUTPUT 0
+#define HRT_FAIL_VEC 0x1d
+
+static FILE *log = NULL;
+
 
-#if DEBUG_OUTPUT
-#define DEBUG(...) fprintf(stderr,__VA_ARGS__)
+#if DEBUG_ENABLE==1
+#define DEBUG(fmt, args...) if (log) { fprintf(log, "DEBUG: " fmt, ##args); }
 #else
 #define DEBUG(...)
 #endif
 
-#if INFO_OUTPUT
-#define INFO(...) fprintf(stdout,__VA_ARGS__)
+#if INFO_ENABLE==1
+#define INFO(fmt, args...) if (log) {  fprintf(log, fmt, ##args); }
+        fprintf(stdout, fmt, ##args); \
+    }
 #else
 #define INFO(...)
 #endif
@@ -119,10 +124,37 @@ typedef unsigned char uchar_t;
 #define HCALL(rc,id,a,b,c,d,e,f,g,h)  HCALL32(rc,id,a,b,c,d,e,f,g,h)   
 #endif
 
+static inline uint64_t 
+Syscall (uint64_t num, 
+                uint64_t a1,
+                uint64_t a2,
+                uint64_t a3,
+                uint64_t a4,
+                uint64_t a5,
+                uint64_t a6)
+{
+    uint64_t rc;
+
+    __asm__ __volatile__ ("movq %1, %%rax; "
+                         "movq %2, %%rdi; "
+                         "movq %3, %%rsi; "
+                         "movq %4, %%rdx; "
+                         "movq %5, %%r10; "
+                         "movq %6, %%r8; "
+                         "movq %7, %%r9; "
+                         "syscall; "
+                         "movq %%rax, %0; "
+                         : "=m"(rc)
+                         : "m"(num), "m"(a1), "m"(a2), "m"(a3), "m"(a4), "m"(a5), "m"(a6)
+                         : "%rax", "%rdi", "%rcx", "%rsi", "%rdx", "%r10", "%r8", "%r9", "%r11", "memory");
+
+    return rc;
+}
+
 
 /* This must match the definition in palacios/include/palacios/vmm_hvm.h" */
 struct v3_ros_event {
-    enum { ROS_NONE=0, ROS_PAGE_FAULT=1, ROS_SYSCALL=2 } event_type;
+    enum { ROS_NONE=0, ROS_PAGE_FAULT=1, ROS_SYSCALL=2, HRT_EXCEPTION=3, HRT_THREAD_EXIT=4, ROS_DONE=5} event_type;
     uint64_t       last_ros_event_result; // valid when ROS_NONE
     union {
        struct {   // valid when ROS_PAGE_FAULT
@@ -133,23 +165,130 @@ struct v3_ros_event {
        struct { // valid when ROS_SYSCALL
            uint64_t args[8];
        } syscall;
+    struct { // valid when HRT_EXCEPTION
+        uint64_t rip;
+        uint64_t vector;
+    } excp;
+    struct { // valid when HRT_THREAD_EXIT
+        uint64_t nktid;
+    } thread_exit;
     };
 };
 
 
 int v3_hvm_ros_user_init()
 {
-    // currently nothing to do
+    log = stderr;
+    return 0;
+}
+
+
+int v3_hvm_ros_user_set_log(FILE *l)
+{
+    log = l;
     return 0;
 }
 
 int v3_hvm_ros_user_deinit()
 {
-    // currently nothing to do
+    log = NULL;
     return 0;
 }
 
-static void handle_ros_event(struct v3_ros_event *event)
+
+int v3_hvm_handle_ros_event_nohcall(struct v3_ros_event *event)
+{
+    uint64_t rc;
+    char t;
+    
+    switch (event->event_type) { 
+       case ROS_PAGE_FAULT: 
+           // force the ros kernel to the PTE
+           if (event->page_fault.action==ROS_READ) { 
+               DEBUG("EVENT [PF READ] : %p\n", (volatile char*)(event->page_fault.cr2));
+               t=*(volatile char*)(event->page_fault.cr2);
+               t=t; // avoid wanting for this throwaway
+           } else if (event->page_fault.action==ROS_WRITE) { 
+               DEBUG("EVENT [PF WRITE] : %p\n", (volatile char*)(event->page_fault.cr2));
+               
+               // stash the old contents
+               char tmp = *((volatile char *)event->page_fault.cr2);
+               
+               // force a write
+               *((volatile char*)event->page_fault.cr2) = 0xa;
+               
+               // put the old value back
+               *((volatile char*)event->page_fault.cr2) = tmp;
+               
+           } else {
+               INFO("Huh?\n");
+           }
+           
+           DEBUG("Done.");
+           event->last_ros_event_result = 0;
+           //event->event_type            = ROS_DONE;
+           event->event_type            = ROS_NONE;
+           // completed
+           DEBUG("[OK]\n");
+           
+           break;
+           
+       case ROS_SYSCALL:
+           DEBUG("EVENT [SYSCALL] : syscall(0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx)\n",
+                 event->syscall.args[0],
+                 event->syscall.args[1],
+                 event->syscall.args[2],
+                 event->syscall.args[3],
+                 event->syscall.args[4],
+                 event->syscall.args[5],
+                 event->syscall.args[6],
+                 event->syscall.args[7]);
+           
+           rc = Syscall(event->syscall.args[0],
+                        event->syscall.args[1],
+                        event->syscall.args[2],
+                        event->syscall.args[3],
+                        event->syscall.args[4],
+                        event->syscall.args[5],
+                        event->syscall.args[6]);
+           
+           if ((long int)rc<0) {
+               DEBUG("Syscall (num=0x%llx) failed (rc=0x%llx)\n", (unsigned long long)event->syscall.args[0], (unsigned long long)rc);
+           }
+           
+           DEBUG("Return = 0x%lx...", rc);
+           event->last_ros_event_result = rc;
+           //event->event_type            = ROS_DONE;
+           event->event_type            = ROS_NONE;
+           DEBUG("[OK]\n");
+           
+           break;
+           
+       case HRT_EXCEPTION:
+           DEBUG("EVENT [EXCEPTION] : (tid=%lx) at %p vector=0x%lx...", syscall(SYS_gettid), (void*)event->excp.rip, event->excp.vector);
+           if (event->excp.vector == HRT_FAIL_VEC) {
+               INFO("HRT execution failed due to exception in HRT\n");
+           } else {
+               INFO("HRT execution failed due to exception in ROS (vec=0x%lx)\n", event->excp.vector);
+           }
+           
+           event->last_ros_event_result = 0;
+           //event->event_type            = ROS_DONE;
+           event->event_type            = ROS_NONE;
+           DEBUG("[OK]\n");
+           exit(EXIT_FAILURE);
+           break;
+
+       default:
+           DEBUG( "Unknown ROS event 0x%x\n", event->event_type);
+           break;
+    }
+    
+    return 0;
+}
+
+
+int v3_hvm_handle_ros_event(struct v3_ros_event *event)
 {
     unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
     char t;
@@ -162,60 +301,105 @@ static void handle_ros_event(struct v3_ros_event *event)
                t=*(volatile char*)(event->page_fault.cr2);
                t=t; // avoid wanting for this throwaway
            } else if (event->page_fault.action==ROS_WRITE) { 
-               DEBUG("Handling page fault writefor %p\n", (volatile char*)(event->page_fault.cr2));
-               *(volatile char*)(event->page_fault.cr2) = *(volatile char *)(event->page_fault.cr2);
+               DEBUG("Handling page fault write for %p\n", (volatile char*)(event->page_fault.cr2));
+               
+               // stash the old contents
+               char tmp = *((volatile char *)event->page_fault.cr2);
+               
+               // force a write
+               *((volatile char*)event->page_fault.cr2) = 0xa;
+               
+               // put the old value back
+               *((volatile char*)event->page_fault.cr2) = tmp;
+               
            } else {
                INFO("Huh?\n");
            }
-           DEBUG("Done - doing hypercall\n");
+           
+           DEBUG("Done - doing hypercall...");
            num = 0xf00d;
            a1 = 0x1f;
            a2 = 0; // success
            HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
            // completed
-           DEBUG("Completed.\n");
-
+           DEBUG("[OK]\n");
+           
            break;
            
        case ROS_SYSCALL:
            DEBUG("Doing system call: syscall(0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx)\n",
-                   event->syscall.args[0],
-                   event->syscall.args[1],
-                   event->syscall.args[2],
-                   event->syscall.args[3],
-                   event->syscall.args[4],
-                   event->syscall.args[5],
-                   event->syscall.args[6],
-                   event->syscall.args[7]);
-
-           rc = syscall(event->syscall.args[0],
+                 event->syscall.args[0],
+                 event->syscall.args[1],
+                 event->syscall.args[2],
+                 event->syscall.args[3],
+                 event->syscall.args[4],
+                 event->syscall.args[5],
+                 event->syscall.args[6],
+                 event->syscall.args[7]);
+           
+           rc = Syscall(event->syscall.args[0],
                         event->syscall.args[1],
                         event->syscall.args[2],
                         event->syscall.args[3],
                         event->syscall.args[4],
                         event->syscall.args[5],
-                        event->syscall.args[6],
-                        event->syscall.args[7]);
-
-           if ((int)rc<0) {
-               DEBUG("syscall failed");
+                        event->syscall.args[6]);
+           
+           if ((long int)rc<0) {
+               DEBUG("Syscall (num=0x%llx) failed (rc=0x%llx)\n", (unsigned long long)event->syscall.args[0], (unsigned long long)rc);
            }
-
-           DEBUG("Return = 0x%llx, doing hypercall\n", rc);
+           
+           DEBUG("Return = 0x%llx, doing hypercall...", rc);
            num = 0xf00d;
            a1 = 0x1f;
            a2 = rc;
            HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
-           DEBUG("Completed\n");
+           DEBUG("[OK]\n");
+           
+           break;
 
+       case HRT_EXCEPTION:
+           DEBUG("Handling exception (tid=%lx) at %p vector=0x%lx...", syscall(SYS_gettid), (void*)event->excp.rip, event->excp.vector);
+           if (event->excp.vector == HRT_FAIL_VEC) {
+               INFO("HRT execution failed due to exception in HRT\n");
+           } else {
+               INFO("HRT execution failed due to exception in ROS (vec=0x%lx)\n", event->excp.vector);
+           }
+           
+           num = 0xf00d;
+           a1 = 0x1f;
+           a2 = rc;
+           HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+           DEBUG("[OK]\n");
+           exit(EXIT_FAILURE);
            break;
+           
        default:
            DEBUG( "Unknown ROS event 0x%x\n", event->event_type);
            break;
     }
+    
+    return 0;
 }
+
+static void wait_for_merge_completion()
+{
+    unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    struct v3_ros_event event;
+    
+    memset(&event, 1, sizeof(event));
     
+    rc = 1;
+    
+    while (rc) { 
+       num = 0xf00d;
+       a1 = 0xf;
+       a2 = (unsigned long long) &event;
+       HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    }
     
+}
+
 
 static void wait_for_completion()
 {
@@ -234,7 +418,7 @@ static void wait_for_completion()
     if (rc) { 
        //      usleep(100);
        if (event.event_type != ROS_NONE) { 
-           handle_ros_event(&event);
+           v3_hvm_handle_ros_event(&event);
        }
     }
   }
@@ -308,7 +492,7 @@ int v3_hvm_ros_merge_address_spaces()
       INFO("Error in request to merge address spaces rc=0x%llx\n",rc);
       return -1;
     } else {
-      wait_for_completion();
+      wait_for_merge_completion();
       return 0;
     }
 }
@@ -331,6 +515,44 @@ int v3_hvm_ros_unmerge_address_spaces()
 }
 
 
+int v3_hvm_ros_mirror_gdt(uint64_t fsbase, uint64_t cpu)
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    num=0xf00d;
+    a1 = 0x51; // mirror the ROS FS and GS on the HRT side
+    a2 = fsbase;
+    a3 = cpu;
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    if (rc) {
+      INFO("Error in request to mirror GDT (rc=0x%llx)\n", rc);
+      return -1;
+    } else {
+      wait_for_completion();
+      return 0;
+    }
+
+}
+
+int v3_hvm_ros_unmirror_gdt()
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    num=0xf00d;
+    a1 = 0x53; // unmirror GDT and the rest
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    if (rc) {
+      INFO("Error in request to unmirror GDT (rc=0x%llx)\n", rc);
+      return -1;
+    } else {
+      wait_for_completion();
+      return 0;
+    }
+}
+
+
 int v3_hvm_ros_invoke_hrt_async(void *buf, int par)
 {
     unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
@@ -353,6 +575,22 @@ int v3_hvm_ros_invoke_hrt_async(void *buf, int par)
     }
 }
 
+int v3_hvm_ros_invoke_hrt_async_nowait(void *buf, int par)
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    num=0xf00d;
+    if (par) { 
+       a1 = 0x21; // issue "function" in parallel
+    } else {
+       a1 = 0x20; // issue "function" sequentially
+    }
+    a2 = (unsigned long long) buf;
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    return 0;
+}
+
 
 
 
@@ -402,7 +640,7 @@ static void wait_for_sync()
     if (rc!=4) { 
        //      usleep(100);
        if (event.event_type != ROS_NONE) { 
-           handle_ros_event(&event);
+           v3_hvm_handle_ros_event(&event);
        }
     }
   }
@@ -474,7 +712,7 @@ int v3_hvm_ros_invoke_hrt_sync(void *buf, int ros)
            a2 = (unsigned long long) &event;
            HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
            if (event.event_type != ROS_NONE) { 
-               handle_ros_event(&event);
+               v3_hvm_handle_ros_event(&event);
            }
        }
     }   
@@ -487,7 +725,7 @@ void (*__v3_hvm_ros_signal_handler)(uint64_t) = 0;
 static void *__v3_hvm_ros_signal_handler_stack = 0;
 static uint64_t __v3_hvm_ros_signal_handler_stack_size=0;
 
-int v3_hvm_ros_register_signal(void (*h)(uint64_t), void *stack, uint64_t stack_size )
+int v3_hvm_ros_register_signal(void (*h)(uint64_t), void *stack, uint64_t stack_size)
 {
     unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
 
@@ -507,7 +745,7 @@ int v3_hvm_ros_register_signal(void (*h)(uint64_t), void *stack, uint64_t stack_
     num = 0xf00d;
     a1 = 0x40;
     a2 = (unsigned long long) &__v3_hvm_ros_signal_handler_stub;
-    a3 = (unsigned long long) stack + stack_size - 8; // put us at the top of the stack
+    a3 = (unsigned long long) stack + stack_size - 16; // put us at the top of the stack, and 16 byte aligned
 
     HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
     if (rc) {
@@ -565,3 +803,5 @@ int  v3_hvm_hrt_signal_ros(uint64_t code)
     return rc ? -1 : rc;
 
 }
+
+    

diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user.h b/guest/linux/hvm-ros/v3_hvm_ros_user.h
index de3b2e1..6ac99a7 100644 (file)
--- a/guest/linux/hvm-ros/v3_hvm_ros_user.h
+++ b/guest/linux/hvm-ros/v3_hvm_ros_user.h
@@ -12,6 +12,10 @@
 int v3_hvm_ros_user_init();
 int v3_hvm_ros_user_deinit();
 
+// by default, debugging and other log output goes to STDERR
+// you can change this here
+int v3_hvm_ros_user_set_log(FILE *log);
+
 // Establish function to be invoked by the VMM
 // to signal activity (basically an interrupt handler)
 // The handler can use the GPRs, but must save/restore
@@ -25,13 +29,21 @@ int v3_hvm_ros_unregister_signal();
 //  - the intial image is the one given in the .pal config
 int v3_hvm_ros_install_hrt_image(void *image, uint64_t size);
 
+// Reset diffent components of the HVM
+// or the entire HVM.  The component immediatly reboots
 typedef enum {RESET_HRT, RESET_ROS, RESET_BOTH} reset_type;
-
 int v3_hvm_ros_reset(reset_type what);
 
+// Merge address spaces between
+// this ROS process and a compatible HRT
 int v3_hvm_ros_merge_address_spaces();
 int v3_hvm_ros_unmerge_address_spaces();
 
+// Reflect this ROS process's GDT into a compatible
+// HRT.  This is typically necessary to run
+// code out of the ROS address space in the HRT
+int v3_hvm_ros_mirror_gdt(uint64_t fsbase, uint64_t cpu);
+int v3_hvm_ros_unmiror_gdt();
 
 // Asynchronosus invocation of the HRT using an
 // opaque pointer (typically this is a pointer
@@ -39,7 +51,8 @@ int v3_hvm_ros_unmerge_address_spaces();
 // arguments.  The parallel flag indicates that
 // that it will be invoked simulatneously on all
 // cores.  
-int  v3_hvm_ros_invoke_hrt_async(void *p, int parallel);
+int v3_hvm_ros_invoke_hrt_async(void *p, int parallel);
+int v3_hvm_ros_invoke_hrt_async_nowait(void *buf, int par);
 
 
 // synchronize with HRT via shared location
@@ -50,8 +63,15 @@ int  v3_hvm_ros_synchronize();
 int  v3_hvm_ros_invoke_hrt_sync(void *p, int handle_ros_events);
 int  v3_hvm_ros_desynchronize();
 
-// Signal the HRT from the ROS
-// The ROS can call this too, but it shouldn't be necesary
+// Handle events occuring within the HRT
+// that need to be be handled within the ROS
+struct v3_ros_event;
+int v3_hvm_handle_ros_event(struct v3_ros_event * event);
+int v3_hvm_handle_ros_event_nohcall(struct v3_ros_event * event);
+
+// Signal the ROS process from the ROS process
+// These signals usually are generated by the HRT, but
+// The ROS can also signal itself using the same mechanism
 int  v3_hvm_hrt_signal_ros(uint64_t code);
 
 #endif

diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S b/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S
index 3cbcc07..076b505 100644 (file)
--- a/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S
+++ b/guest/linux/hvm-ros/v3_hvm_ros_user_low_level.S
@@ -57,6 +57,9 @@
    we can be interrupted, and deliverable interrupts
    automatically prioritize over us.
 
+   The start of the stack (before the VMM pushes the fake
+   interrupt frame is  16 byte aligned
+
         48bitsblank | return SS (16 bits)
        Return RSP
        Return RFLAGS
@@ -67,22 +70,43 @@
        We then need simply to do this:
 
        save regs
+       arrange 16 byte alignment at entry of call
        call the handler (if installed)
        restore regs
        iret
        
 */
 
+#define DEBUG_ENTRY 0
+       
 .global __v3_hvm_ros_signal_handler_stub
-__v3_hvm_ros_signal_handler_stub:
-       GPR_SAVE()
-       movabs __v3_hvm_ros_signal_handler, %rax
-       testq %rax, %rax
-       jz skip_handler
-       movq 120(%rsp), %rdi /* error code becomes argument */
-       callq *%rax     
-skip_handler:  
-       GPR_LOAD()
-       addq $8, %rsp /* nuke the error code */
-       iretq
+__v3_hvm_ros_signal_handler_stub:   /* we are 16 byte aligned on entry 16 + 6*8 for interrupt frame */
+       GPR_SAVE()                  /* push 15 words, all but rsp, now not aligned - need 1 more word  */
+        subq $8, %rsp               /* make us 16 byte aligned */
+
+#if DEBUG_ENTRY                     /* print out something if we are debugging - Note this is danerous code */
+       pushq %rdi
+       pushq %rax
+       movabsq $printf, %rax
+       movabsq $string, %rdi
+       callq *%rax
+       popq %rax
+       popq %rdi
+#endif
+       movq %rsp, %rbp             /* give us a stack frame for any callee that needs it */
+       movabs __v3_hvm_ros_signal_handler, %rax  /* find the user-level handler */
+       testq %rax, %rax                          /* return immediately if it doesn't exist */
+       jz skip_handler                           /* " */
+       movq 128(%rsp), %rdi        /* error code becomes argument for user-level handler */
+       callq *%rax                 /* call handelr - 16 byte aligned at call */
+       jmp done
+skip_handler:
+done:
+       addq $8, %rsp               /* get rid of stack alignment pad */
+       GPR_LOAD()                  /* rbp is restored here */
+       addq $8, %rsp               /* nuke the error code */
+       iretq                       /* restore rip, rsp, and rflags */
        
+
+string:
+    .asciz "Got to signal handler stub\12"