From: Peter Dinda <pdinda@northwestern.edu>
Date: Wed, 9 Sep 2015 19:52:04 +0000 (-0500)
Subject: Linux guest user library and test application for ROS<->HRT interaction in an HVM
X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=495d40c7d02054399e118be6a10b0096c938d232

Linux guest user library and test application for ROS<->HRT interaction in an HVM
---

diff --git a/guest/linux/hvm-ros/Makefile b/guest/linux/hvm-ros/Makefile
new file mode 100644
index 0000000..96ce1f3
--- /dev/null
+++ b/guest/linux/hvm-ros/Makefile
@@ -0,0 +1,14 @@
+all: libv3_hvm_ros_user.a test
+
+libv3_hvm_ros_user.a: v3_hvm_ros_user.o
+	ar ruv libv3_hvm_ros_user.a v3_hvm_ros_user.o
+
+v3_hvm_ros_user.o: v3_hvm_ros_user.c v3_hvm_ros_user.h 
+	gcc -Wall -c v3_hvm_ros_user.c
+
+
+test: test.c libv3_hvm_ros_user.a v3_hvm_ros_user.h
+	gcc -Wall -static test.c -L. -lv3_hvm_ros_user -o test
+
+clean:
+	rm -f libv3_hvm_ros_user.a v3_hvm_ros_user.o test
diff --git a/guest/linux/hvm-ros/README b/guest/linux/hvm-ros/README
new file mode 100644
index 0000000..b16ce3a
--- /dev/null
+++ b/guest/linux/hvm-ros/README
@@ -0,0 +1,44 @@
+This is a library that encapsulates interaction between
+a ROS Linux application and an HRT that are both running
+in the same HVM.   A test application is included.
+
+ROS/HRT interaction is done via Palacios's HVM component
+and there is a detailed technical description in 
+palacios/include/palacios/vmm_hvm.h.  
+
+When the HVM is started, both the ROS (Linux) and the HRT
+(for example, Nautilus) are booted simultaneously.   
+
+A Linux application running in the ROS has control over the
+HRT and the ROS..  Specifically, it can:
+
+- reboot the HRT
+- reboot the ROS
+- reboot the whole HVM
+
+- merge/unmerge its address space with the HRT's address
+  space (provided the HRT can do so (e.g. Nautilus))
+
+- asynchronously invoke functions with the HRT
+  functions can be invoked on the HRT's first core 
+  or simultaneously on all cores of the HRT
+
+- synchronize with the HRT to decide on a common memory
+  location to use for future, synchronous communication
+  via that memory.   And teardown when this is done.
+
+The HRT can in turn call back to the ROS application
+for the following reasons, which are really only sensical
+for a merged address space.  These callbacks currently include:
+
+- system call forwarding
+
+- page fault forwarding (for the ROS user portion of 
+  the address space)
+
+Because on the ROS side, we do everything within user
+space, the ROS currently needs to poll.  It polls both 
+to determine completion of a request, and get event
+callbacks from the HRT.
+
+
diff --git a/guest/linux/hvm-ros/test.c b/guest/linux/hvm-ros/test.c
new file mode 100644
index 0000000..e6e14bf
--- /dev/null
+++ b/guest/linux/hvm-ros/test.c
@@ -0,0 +1,228 @@
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+
+#include "v3_hvm_ros_user.h"
+
+typedef unsigned char uchar_t;
+
+#define rdtscll(val)					\
+    do {						\
+	uint64_t tsc;					\
+	uint32_t a, d;					\
+	asm volatile("rdtsc" : "=a" (a), "=d" (d));	\
+	*(uint32_t *)&(tsc) = a;			\
+	*(uint32_t *)(((uchar_t *)&tsc) + 4) = d;	\
+	val = tsc;					\
+    } while (0)					
+
+
+
+int simple_test_sync()
+{
+    char buf[4096];  
+    
+    memset(buf,1,4096);
+    memset(buf,0,4096);
+    strcpy(buf,"hello world\n");
+
+    
+    printf("Merge\n");
+    if (v3_hvm_ros_merge_address_spaces()) { 
+	printf("failed to merge address spaces\n");
+	return -1;
+    }
+
+    printf("Synchronize\n");
+    if (v3_hvm_ros_synchronize()) { 
+	printf("failed to synchronize\n");
+	return -1;
+    }
+
+    printf("Invoke\n");
+    if (v3_hvm_ros_invoke_hrt_sync(buf,0)) { 
+	printf("failed to invoke HRT\n");
+	return -1;
+    }
+
+    printf("Desynchonize\n");
+    if (v3_hvm_ros_desynchronize()) { 
+	printf("failed to desynchronize\n");
+	return -1;
+    }
+
+    printf("Unmerge\n");
+    if (v3_hvm_ros_unmerge_address_spaces()) { 
+	printf("failed to merge address spaces\n");
+	return -1;
+    }
+
+    printf("Done.\n");
+
+    return 0;
+}
+
+int timing_test_sync(uint64_t num_merge, uint64_t num_call)
+{
+    char buf[4096];  
+    unsigned long long start,end,i;
+    
+    memset(buf,1,4096);
+    memset(buf,0,4096);
+    strcpy(buf,"hello world\n");
+
+    printf("Executing %lu address space merges\n",num_merge);
+    rdtscll(start);
+    for (i=0;i<num_merge;i++) {
+	if (v3_hvm_ros_merge_address_spaces()) { 
+	    return -1;
+	} 
+	//fprintf(stderr,"%llu\n",i+1);
+    }
+    rdtscll(end);
+    printf("Took %llu cycles, %llu cycles/iter, or %lf seconds/iter\n",end-start,(end-start)/num_merge,(((double)end-start)/num_merge)/2.1e9);
+    
+    printf("Setting up synchronous invocation\n");
+    
+    if (v3_hvm_ros_synchronize()) {
+	return -1;
+    }
+
+    printf("Executing %lu HRT calls synchronously\n",num_call);
+    rdtscll(start);
+    for (i=0;i<num_call;i++) {
+	if (v3_hvm_ros_invoke_hrt_sync(buf,0)) {
+	    return -1;
+	} 
+	//fprintf(stderr,"%llu\n",i+1);
+    }
+    rdtscll(end);
+    printf("Took %llu cycles, %llu cycles/iter, or %lf seconds/iter\n",end-start,(end-start)/num_call,(((double)end-start)/num_call)/2.1e9);
+
+    if (v3_hvm_ros_desynchronize()) { 
+	return -1;
+    }
+
+    if (v3_hvm_ros_unmerge_address_spaces()) { 
+	return -1;
+    } 
+
+    return 0;
+}
+
+int simple_test_async()
+{
+    char buf[4096];  
+    
+    memset(buf,1,4096);
+    memset(buf,0,4096);
+    strcpy(buf,"hello world\n");
+
+   printf("Merge\n");
+
+    if (v3_hvm_ros_merge_address_spaces()) { 
+	printf("failed to merge address spaces\n");
+	return -1;
+    }
+
+    printf("Invoke\n");
+
+    if (v3_hvm_ros_invoke_hrt_async(buf,0)) { 
+	printf("failed to invoke HRT\n");
+	return -1;
+    }
+
+    printf("Unmerge\n");
+
+    if (v3_hvm_ros_unmerge_address_spaces()) { 
+	printf("failed to unmerge address spaces\n");
+	return -1;
+    }
+    
+    
+    printf("Done.\n");
+    
+
+    return 0;
+}
+
+int timing_test_async(uint64_t num_merge, uint64_t num_call)
+{
+    char buf[4096];  
+    unsigned long long start,end,i;
+    
+    memset(buf,1,4096);
+    memset(buf,0,4096);
+    strcpy(buf,"hello world\n");
+
+    printf("Executing %lu address space merges\n",num_merge);
+    rdtscll(start);
+    for (i=0;i<num_merge;i++) {
+	if (v3_hvm_ros_merge_address_spaces()) { 
+	    return -1;
+	} 
+	//fprintf(stderr,"%llu\n",i+1);
+    }
+    rdtscll(end);
+    printf("Took %llu cycles, %llu cycles/iter, or %lf seconds/iter\n",end-start,(end-start)/num_merge,(((double)end-start)/num_merge)/2.1e9);
+    
+    printf("Executing %lu HRT calls\n",num_call);
+    rdtscll(start);
+    for (i=0;i<num_call;i++) {
+	if (v3_hvm_ros_invoke_hrt_async(buf,0)) {
+	    return -1;
+	} 
+	//fprintf(stderr,"%llu\n",i+1);
+    }
+    rdtscll(end);
+    printf("Took %llu cycles, %llu cycles/iter, or %lf seconds/iter\n",end-start,(end-start)/num_call,(((double)end-start)/num_call)/2.1e9);
+
+    if (v3_hvm_ros_unmerge_address_spaces()) { 
+	return -1;
+    } 
+
+    return 0;
+}
+
+int main(int argc, char *argv[]) 
+{
+    int rc;
+    
+    if (argc!=3 && argc!=5) { 
+	printf("usage: test simple|time sync|async num_merges num_calls\n");
+	return -1;
+    }
+    
+    v3_hvm_ros_user_init();
+    
+    if (argv[1][0]=='s') {
+	if (argv[2][0]=='s') { 
+	    rc=simple_test_sync();
+	} else if (argv[2][0]=='a') {
+	    rc=simple_test_async();
+	} else {
+	    printf("Unknown type %s\n",argv[2]);
+	    rc=-1;
+	}
+    } else if (argv[1][0]=='t') {
+	if (argv[2][0]=='s') { 
+	    rc=timing_test_sync(atoi(argv[3]),atoi(argv[4]));
+	} else if (argv[2][0]=='a') {
+	    rc=timing_test_async(atoi(argv[3]),atoi(argv[4]));
+	} else {
+	    printf("Unknown type %s\n",argv[2]);
+	    rc=-1;
+	}
+    } else {
+	printf("Unknown mode %s\n",argv[1]);
+	rc=-1;
+    }
+    
+    v3_hvm_ros_user_deinit();
+
+    return rc;
+}
diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user.c b/guest/linux/hvm-ros/v3_hvm_ros_user.c
new file mode 100644
index 0000000..f292626
--- /dev/null
+++ b/guest/linux/hvm-ros/v3_hvm_ros_user.c
@@ -0,0 +1,427 @@
+/*
+  Copyright (c) 2015 Peter Dinda
+*/
+
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+
+#include "v3_hvm_ros_user.h"
+
+
+#define DEBUG_OUTPUT 0
+#define INFO_OUTPUT 0
+
+#if DEBUG_OUTPUT
+#define DEBUG(...) fprintf(stderr,__VA_ARGS__)
+#else
+#define DEBUG(...)
+#endif
+
+#if INFO_OUTPUT
+#define INFO(...) fprintf(stdout,__VA_ARGS__)
+#else
+#define INFO(...)
+#endif
+
+
+
+typedef unsigned char uchar_t;
+
+#define rdtscll(val)					\
+    do {						\
+	uint64_t tsc;					\
+	uint32_t a, d;					\
+	asm volatile("rdtsc" : "=a" (a), "=d" (d));	\
+	*(uint32_t *)&(tsc) = a;			\
+	*(uint32_t *)(((uchar_t *)&tsc) + 4) = d;	\
+	val = tsc;					\
+    } while (0)					
+
+
+/*
+  This convention match the definition in palacios/include/palacios/vmm_hvm.h
+
+  Calling convention:
+
+64 bit:
+  rax = hcall number
+  rbx = 0x6464646464646464...
+  rcx = 1st arg
+  rdx = 2nd arg
+  rsi = 3rd arg
+  rdi = 4th arg
+  r8  = 5th arg
+  r9  = 6th arg
+  r10 = 7th arg
+  r11 = 8th arg
+
+32 bit:
+  eax = hcall number
+  ebx = 0x32323232
+  arguments on stack in C order (first argument is TOS)
+     arguments are also 32 bit
+*/
+#define HCALL64(rc,id,a,b,c,d,e,f,g,h)		      \
+  asm volatile ("movq %1, %%rax; "		      \
+		"pushq %%rbx; "			      \
+		"movq $0x6464646464646464, %%rbx; "   \
+		"movq %2, %%rcx; "		      \
+		"movq %3, %%rdx; "		      \
+		"movq %4, %%rsi; "		      \
+		"movq %5, %%rdi; "		      \
+		"movq %6, %%r8 ; "		      \
+		"movq %7, %%r9 ; "		      \
+		"movq %8, %%r10; "		      \
+		"movq %9, %%r11; "		      \
+		"vmmcall ;       "		      \
+		"movq %%rax, %0; "		      \
+		"popq %%rbx; "			      \
+		: "=m"(rc)			      \
+		: "m"(id),			      \
+                  "m"(a), "m"(b), "m"(c), "m"(d),     \
+		  "m"(e), "m"(f), "m"(g), "m"(h)      \
+		: "%rax","%rcx","%rdx","%rsi","%rdi", \
+		  "%r8","%r9","%r10","%r11"	      \
+		)
+
+#define HCALL32(rc,id,a,b,c,d,e,f,g,h)		      \
+  asm volatile ("movl %1, %%eax; "		      \
+		"pushl %%ebx; "			      \
+		"movl $0x32323232, %%ebx; "	      \
+		"pushl %9;"			      \
+		"pushl %8;"			      \
+		"pushl %7;"			      \
+		"pushl %6;"			      \
+		"pushl %5;"			      \
+		"pushl %4;"			      \
+		"pushl %3;"			      \
+		"pushl %2;"			      \
+		"vmmcall ;       "		      \
+		"movl %%eax, %0; "		      \
+		"addl $32, %%esp; "		      \
+		"popl %%ebx; "			      \
+		: "=r"(rc)			      \
+		: "m"(id),			      \
+		  "m"(a), "m"(b), "m"(c), "m"(d),     \
+		"m"(e), "m"(f), "m"(g), "m"(h)	      \
+		: "%eax"			      \
+		)
+
+#ifdef __x86_64__
+#define HCALL(rc,id,a,b,c,d,e,f,g,h)  HCALL64(rc,id,a,b,c,d,e,f,g,h)
+#else
+#define HCALL(rc,id,a,b,c,d,e,f,g,h)  HCALL32(rc,id,a,b,c,d,e,f,g,h)   
+#endif
+
+
+/* This must match the definition in palacios/include/palacios/vmm_hvm.h" */
+struct v3_ros_event {
+    enum { ROS_NONE=0, ROS_PAGE_FAULT=1, ROS_SYSCALL=2 } event_type;
+    uint64_t       last_ros_event_result; // valid when ROS_NONE
+    union {
+	struct {   // valid when ROS_PAGE_FAULT
+	    uint64_t rip;
+	    uint64_t cr2;
+	    enum {ROS_READ, ROS_WRITE} action;
+	} page_fault;
+	struct { // valid when ROS_SYSCALL
+	    uint64_t args[8];
+	} syscall;
+    };
+};
+
+
+int v3_hvm_ros_user_init()
+{
+    // currently nothing to do
+    return 0;
+}
+
+int v3_hvm_ros_user_deinit()
+{
+    // currently nothing to do
+    return 0;
+}
+
+static void handle_ros_event(struct v3_ros_event *event)
+{
+    unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    char t;
+
+    switch (event->event_type) { 
+	case ROS_PAGE_FAULT: 
+	    // force the ros kernel to the PTE
+	    if (event->page_fault.action==ROS_READ) { 
+		DEBUG("Handling page fault read for %p\n", (volatile char*)(event->page_fault.cr2));
+		t=*(volatile char*)(event->page_fault.cr2);
+		t=t; // avoid wanting for this throwaway
+	    } else if (event->page_fault.action==ROS_WRITE) { 
+		DEBUG("Handling page fault writefor %p\n", (volatile char*)(event->page_fault.cr2));
+		*(volatile char*)(event->page_fault.cr2) = *(volatile char *)(event->page_fault.cr2);
+	    } else {
+		INFO("Huh?\n");
+	    }
+	    DEBUG("Done - doing hypercall\n");
+	    num = 0xf00d;
+	    a1 = 0x1f;
+	    a2 = 0; // success
+	    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+	    // completed
+	    DEBUG("Completed.\n");
+
+	    break;
+	    
+	case ROS_SYSCALL:
+	    DEBUG("Doing system call: syscall(0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%lx)\n",
+		    event->syscall.args[0],
+		    event->syscall.args[1],
+		    event->syscall.args[2],
+		    event->syscall.args[3],
+		    event->syscall.args[4],
+		    event->syscall.args[5],
+		    event->syscall.args[6],
+		    event->syscall.args[7]);
+
+	    rc = syscall(event->syscall.args[0],
+			 event->syscall.args[1],
+			 event->syscall.args[2],
+			 event->syscall.args[3],
+			 event->syscall.args[4],
+			 event->syscall.args[5],
+			 event->syscall.args[6],
+			 event->syscall.args[7]);
+
+	    if ((int)rc<0) {
+		DEBUG("syscall failed");
+	    }
+
+	    DEBUG("Return = 0x%llx, doing hypercall\n", rc);
+	    num = 0xf00d;
+	    a1 = 0x1f;
+	    a2 = rc;
+	    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+	    DEBUG("Completed\n");
+
+	    break;
+	default:
+	    DEBUG( "Unknown ROS event 0x%x\n", event->event_type);
+	    break;
+    }
+}
+    
+    
+
+static void wait_for_completion()
+{
+  unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+  struct v3_ros_event event;
+
+  memset(&event, 1, sizeof(event));
+
+  rc = 1;
+
+  while (rc) { 
+    num = 0xf00d;
+    a1 = 0xf;
+    a2 = (unsigned long long) &event;
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    if (rc) { 
+	//      usleep(100);
+	if (event.event_type != ROS_NONE) { 
+	    handle_ros_event(&event);
+	}
+    }
+  }
+}
+
+
+int v3_hvm_ros_merge_address_spaces()
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    num=0xf00d;
+    a1 = 0x30; // merge address spaces
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    if (rc) {
+      INFO("Error in request to merge address spaces rc=0x%llx\n",rc);
+      return -1;
+    } else {
+      wait_for_completion();
+      return 0;
+    }
+}
+
+int v3_hvm_ros_unmerge_address_spaces()
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    num=0xf00d;
+    a1 = 0x31; // merge address spaces
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    if (rc) {
+      INFO("Error in request to unmerge address spaces rc=0x%llx\n",rc);
+      return -1;
+    } else {
+      wait_for_completion();
+      return 0;
+    }
+}
+
+
+int v3_hvm_ros_invoke_hrt_async(void *buf, int par)
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    num=0xf00d;
+    if (par) { 
+	a1 = 0x21; // issue "function" in parallel
+    } else {
+	a1 = 0x20; // issue "function" sequentially
+    }
+    a2 = (unsigned long long) buf;
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    if (rc) { 
+	INFO("Error in request to launch %s function rc=0x%llx\n", par ? "parallel" : "", rc);
+	return -1;
+    } else {
+	wait_for_completion();
+	return 0;
+    }
+}
+
+
+
+
+
+/*
+  Synchronous operation model 
+
+  On ROS:
+
+  [0] => issue count
+  [1] => completion count
+  [2] => function call ptr
+
+  1. merge
+  2. indicate this is the address for sync
+  3. ++[0]
+  4. wait for [1] to match
+  5. goto 3
+
+  On HRT:
+
+  1. merge
+  2. cnt=1;
+  3. wait for [0] to get to cnt
+  4. exec
+  5. ++[1]   ++cnt
+  6. goto 3
+*/
+
+static volatile unsigned long long sync_proto[3]={0,0,0};
+
+
+static void wait_for_sync()
+{
+  unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+  struct v3_ros_event event;
+
+  memset(&event, 1, sizeof(event));
+  
+  rc = 1;
+
+  while (rc!=4) { 
+    num = 0xf00d;
+    a1 = 0xf;
+    a2 = (unsigned long long) &event;
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    if (rc!=4) { 
+	//      usleep(100);
+	if (event.event_type != ROS_NONE) { 
+	    handle_ros_event(&event);
+	}
+    }
+  }
+}
+
+
+int v3_hvm_ros_synchronize()
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    // make sure this address is touched, then zero
+    sync_proto[0]=sync_proto[1]=sync_proto[2]=1;
+    sync_proto[0]=sync_proto[1]=sync_proto[2]=0;
+
+    num=0xf00d;
+    a1 = 0x28; // issue sync request setup
+    a2 = (unsigned long long) sync_proto;
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    
+    if (rc) { 
+	INFO("Synchronize call failed with rc=0x%llx\n",rc);
+	return -1;
+    } else {
+	wait_for_sync();
+	return 0;
+    }
+}
+
+
+int v3_hvm_ros_desynchronize()
+{
+    unsigned long long num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+    unsigned long long rc;
+
+    num=0xf00d;
+    a1 = 0x29; // issue sync request teardown
+    a2 = (unsigned long long) sync_proto;
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+
+    if (rc) { 
+	INFO("Desynchronize call failed with rc=0x%llx\n",rc);
+	return -1;
+    } else {
+	wait_for_completion();
+	return 0;
+    }
+}
+
+#define HOW_OFTEN 1000000
+
+int v3_hvm_ros_invoke_hrt_sync(void *buf, int ros)
+{
+    int i;
+    sync_proto[2]=(unsigned long long)buf;
+    sync_proto[0]++;
+
+    i=0;
+    while (sync_proto[1] != sync_proto[0]) {
+	i++;
+	if (ros && (!i%HOW_OFTEN)) { 
+	    unsigned long long rc, num, a1=0, a2=0, a3=0, a4=0, a5=0, a6=0, a7=0, a8=0;
+	    struct v3_ros_event event;
+
+	    memset(&event, 1, sizeof(event));
+
+	    num = 0xf00d;
+	    a1 = 0xf;
+	    a2 = (unsigned long long) &event;
+	    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+	    if (event.event_type != ROS_NONE) { 
+		handle_ros_event(&event);
+	    }
+	}
+    }   
+    return 0;
+}
+
diff --git a/guest/linux/hvm-ros/v3_hvm_ros_user.h b/guest/linux/hvm-ros/v3_hvm_ros_user.h
new file mode 100644
index 0000000..94858f4
--- /dev/null
+++ b/guest/linux/hvm-ros/v3_hvm_ros_user.h
@@ -0,0 +1,38 @@
+#ifndef __v3_hvm_ros_user
+#define __v3_hvm_ros_user
+
+/*
+  Copyright (c) 2015 Peter Dinda
+*/
+
+
+
+// setup and teardown
+// note that there is ONE HRT hence  no naming
+int v3_hvm_ros_user_init();
+int v3_hvm_ros_user_deinit();
+
+
+int v3_hvm_ros_merge_address_spaces();
+int v3_hvm_ros_unmerge_address_spaces();
+
+
+// Asynchronosus invocation of the HRT using an
+// opaque pointer (typically this is a pointer
+// to a structure containing a function pointer and
+// arguments.  The parallel flag indicates that
+// that it will be invoked simulatneously on all
+// cores.  
+int  v3_hvm_ros_invoke_hrt_async(void *p, int parallel);
+
+
+// synchronize with HRT via shared location
+// allow synchronous invokcations.  Note that
+// any parallelism is done internal to the HRT. 
+// Also the synchronous invocation always waits
+int  v3_hvm_ros_synchronize();   
+int  v3_hvm_ros_invoke_hrt_sync(void *p, int handle_ros_events);
+int  v3_hvm_ros_desynchronize();
+
+
+#endif
diff --git a/guest/linux/hypercall/Makefile b/guest/linux/hypercall/Makefile
new file mode 100644
index 0000000..daa560c
--- /dev/null
+++ b/guest/linux/hypercall/Makefile
@@ -0,0 +1,7 @@
+all: hypercall
+
+hypercall: hypercall.c hcall.h
+	gcc -Wall -static hypercall.c -o hypercall
+
+clean:
+	rm -f hypercall
diff --git a/guest/linux/hypercall/README b/guest/linux/hypercall/README
new file mode 100644
index 0000000..a035305
--- /dev/null
+++ b/guest/linux/hypercall/README
@@ -0,0 +1,3 @@
+Simple hypercall utility using a common calling convention.
+
+For use by Linux user space.
diff --git a/guest/linux/hypercall/hcall.h b/guest/linux/hypercall/hcall.h
new file mode 100644
index 0000000..483fa99
--- /dev/null
+++ b/guest/linux/hypercall/hcall.h
@@ -0,0 +1,77 @@
+#ifndef __HCALL__
+#define __HCALL__
+
+/*
+  Calling convention:
+
+64 bit:
+  rax = hcall number
+  rbx = 0x6464646464646464...
+  rcx = 1st arg
+  rdx = 2nd arg
+  rsi = 3rd arg
+  rdi = 4th arg
+  r8  = 5th arg
+  r9  = 6th arg
+  r10 = 7th arg
+  r11 = 8th arg
+
+32 bit:
+  eax = hcall number
+  ebx = 0x32323232
+  arguments on stack in C order (first argument is TOS)
+     arguments are also 32 bit
+*/
+#define HCALL64(rc,id,a,b,c,d,e,f,g,h)		      \
+  asm volatile ("movq %1, %%rax; "		      \
+		"pushq %%rbx; "			      \
+		"movq $0x6464646464646464, %%rbx; "   \
+		"movq %2, %%rcx; "		      \
+		"movq %3, %%rdx; "		      \
+		"movq %4, %%rsi; "		      \
+		"movq %5, %%rdi; "		      \
+		"movq %6, %%r8 ; "		      \
+		"movq %7, %%r9 ; "		      \
+		"movq %8, %%r10; "		      \
+		"movq %9, %%r11; "		      \
+		"vmmcall ;       "		      \
+		"movq %%rax, %0; "		      \
+		"popq %%rbx; "			      \
+		: "=m"(rc)			      \
+		: "m"(id),			      \
+                  "m"(a), "m"(b), "m"(c), "m"(d),     \
+		  "m"(e), "m"(f), "m"(g), "m"(h)      \
+		: "%rax","%rcx","%rdx","%rsi","%rdi", \
+		  "%r8","%r9","%r10","%r11"	      \
+		)
+
+#define HCALL32(rc,id,a,b,c,d,e,f,g,h)		      \
+  asm volatile ("movl %1, %%eax; "		      \
+		"pushl %%ebx; "			      \
+		"movl $0x32323232, %%ebx; "	      \
+		"pushl %9;"			      \
+		"pushl %8;"			      \
+		"pushl %7;"			      \
+		"pushl %6;"			      \
+		"pushl %5;"			      \
+		"pushl %4;"			      \
+		"pushl %3;"			      \
+		"pushl %2;"			      \
+		"vmmcall ;       "		      \
+		"movl %%eax, %0; "		      \
+		"addl $32, %%esp; "		      \
+		"popl %%ebx; "			      \
+		: "=r"(rc)			      \
+		: "m"(id),			      \
+		  "m"(a), "m"(b), "m"(c), "m"(d),     \
+		"m"(e), "m"(f), "m"(g), "m"(h)	      \
+		: "%eax"			      \
+		)
+
+#ifdef __x86_64__
+#define HCALL(rc,id,a,b,c,d,e,f,g,h)  HCALL64(rc,id,a,b,c,d,e,f,g,h)
+#else
+#define HCALL(rc,id,a,b,c,d,e,f,g,h)  HCALL32(rc,id,a,b,c,d,e,f,g,h)   
+#endif
+
+#endif
diff --git a/guest/linux/hypercall/hypercall.c b/guest/linux/hypercall/hypercall.c
new file mode 100644
index 0000000..c16d7d9
--- /dev/null
+++ b/guest/linux/hypercall/hypercall.c
@@ -0,0 +1,69 @@
+/* Simple hypercall utility for common convention */
+/* Copyright (c) 2015 Peter Dinda                 */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "hcall.h"
+
+int main(int argc, char *argv[])
+{
+    unsigned long long num, a1, a2, a3, a4, a5, a6, a7, a8;
+    unsigned long long rc;
+
+    if (argc<2 || argc>10) { 
+	printf("usage: hypercall <number> [args (up to 8)]\n");
+	printf("The hypercall number and arguments are in hex\n");
+	return -1;
+    }
+
+    num = strtoull(argv[1],0,16);
+
+    if (argc>2) { 
+	a1=strtoull(argv[2],0,16);
+    } else {
+	a1=0;
+    }
+    if (argc>3) { 
+	a2=strtoull(argv[3],0,16);
+    } else {
+	a2=0;
+    }
+    if (argc>5) { 
+	a3=strtoull(argv[4],0,16);
+    } else {
+	a3=0;
+    }
+    if (argc>6) { 
+	a4=strtoull(argv[5],0,16);
+    } else {
+	a4=0;
+    }
+    if (argc>7) { 
+	a5=strtoull(argv[6],0,16);
+    } else {
+	a5=0;
+    }
+    if (argc>8) { 
+	a6=strtoull(argv[7],0,16);
+    } else {
+	a6=0;
+    }
+    if (argc>9) { 
+	a7=strtoull(argv[8],0,16);
+    } else {
+	a7=0;
+    }
+    if (argc>10) { 
+	a8=strtoull(argv[9],0,16);
+    } else {
+	a8=0;
+    }
+    
+    printf("Executing hcall 0x%llx with arguments a1=0x%llx, a2=0x%llx, a3=0x%llx, a4=0x%llx, a5=0x%llx, a6=0x%llx, a7=0x%llx, a8=0x%llx\n", num, a1, a2, a3, a4, a5, a6, a7, a8);
+
+    HCALL(rc,num,a1,a2,a3,a4,a5,a6,a7,a8);
+    
+    printf("Return from hypercall was 0x%llx\n",rc);
+
+    return 0;
+}