#include <linux/uaccess.h>
#include <asm/irq_vectors.h>
#include <asm/io.h>
+#include <asm/thread_info.h>
+#include <asm/i387.h>
+#include <linux/version.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,7,0)
+#include <asm/fpu-internal.h>
+#endif
#include <linux/init.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/vmalloc.h>
+#include <asm/i387.h>
+
#include <palacios/vmm.h>
#include <palacios/vmm_host_events.h>
+
+#ifdef V3_CONFIG_HOST_LAZY_FPU_SWITCH
+#include <interfaces/vmm_lazy_fpu.h>
+#endif
+
#include "palacios.h"
+#include "util-hashtable.h"
+
#include "mm.h"
#include "memcheck.h"
#include "lockcheck.h"
-// The following can be used to track heap bugs
-// zero memory after allocation
-#define ALLOC_ZERO_MEM 0
-// pad allocations by this many bytes on both ends of block
+
+
+// The following can be used to track memory bugs
+// zero memory after allocation (now applies to valloc and page alloc as well)
+#define ALLOC_ZERO_MEM 1
+// pad allocations by this many bytes on both ends of block (heap only)
#define ALLOC_PAD 0
extern int cpu_list_len;
+extern struct hashtable *v3_thread_resource_map;
+
+
static char *print_buffer[NR_CPUS];
static void deinit_print_buffers(void)
if (!print_buffer[i]) {
ERROR("Cannot allocate print buffer for cpu %d\n",i);
deinit_print_buffers();
- return -1;
+ return -1;
}
memset(print_buffer[i],0,V3_PRINTK_BUF_SIZE);
}
* Allocates a contiguous region of pages of the requested size.
* Returns the physical address of the first page in the region.
*/
-void *palacios_allocate_pages(int num_pages, unsigned int alignment) {
+void *palacios_allocate_pages(int num_pages, unsigned int alignment, int node_id, int (*filter_func)(void *paddr, void *filter_state), void *filter_state) {
void * pg_addr = NULL;
+ v3_resource_control_t *r;
- pg_addr = (void *)alloc_palacios_pgs(num_pages, alignment);
+ if (num_pages<=0) {
+ ERROR("ALERT ALERT Attempt to allocate zero or fewer pages (%d pages, alignment %d, node %d, filter_func %p, filter_state %p)\n",num_pages, alignment, node_id, filter_func, filter_state);
+ return NULL;
+ }
+
+ if ((r=(v3_resource_control_t *)palacios_htable_search(v3_thread_resource_map,(addr_t)current))) {
+ // thread has a registered resource control structure
+ // these override any default values
+ // INFO("Overridden page search: (pre) alignment=%x, node_id=%x, filter_func=%p, filter_state=%p\n",alignment,node_id,filter_func,filter_state);
+ if (alignment==4096) {
+ alignment = r->pg_alignment;
+ }
+ if (node_id==-1) {
+ node_id = r->pg_node_id;
+ }
+ if (!filter_func) {
+ filter_func = r->pg_filter_func;
+ filter_state = r->pg_filter_state;
+ }
+ //INFO("Overridden page search: (post) alignment=%x, node_id=%x, filter_func=%p, filter_state=%p\n",alignment,node_id,filter_func,filter_state);
+ }
+
+ pg_addr = (void *)alloc_palacios_pgs(num_pages, alignment, node_id, filter_func, filter_state);
if (!pg_addr) {
- ERROR("ALERT ALERT Page allocation has FAILED Warning\n");
+ ERROR("ALERT ALERT Page allocation has FAILED Warning (%d pages, alignment %d, node %d, filter_func %p, filter_state %p)\n",num_pages, alignment, node_id, filter_func, filter_state);
return NULL;
}
pg_allocs += num_pages;
+#if ALLOC_ZERO_MEM
+ memset(__va(pg_addr),0,num_pages*4096);
+#endif
+
MEMCHECK_ALLOC_PAGES(pg_addr,num_pages*4096);
return pg_addr;
*/
void palacios_free_pages(void * page_paddr, int num_pages) {
+ if (!page_paddr) {
+ ERROR("Ignoring free pages: 0x%p (0x%lx)for %d pages\n", page_paddr, (uintptr_t)page_paddr, num_pages);
+ dump_stack();
+ return;
+ }
pg_frees += num_pages;
free_palacios_pgs((uintptr_t)page_paddr, num_pages);
MEMCHECK_FREE_PAGES(page_paddr,num_pages*4096);
void *
-palacios_alloc_extended(unsigned int size, unsigned int flags) {
+palacios_alloc_extended(unsigned int size, unsigned int flags, int node) {
void * addr = NULL;
- addr = kmalloc(size+2*ALLOC_PAD, flags);
+ if (size==0) {
+ // note that modern kernels will respond to a zero byte
+ // kmalloc and return the address 0x10... In Palacios,
+ // we will simply not allow 0 byte allocs at all, of any kind
+ ERROR("ALERT ALERT attempt to kmalloc zero bytes rejected\n");
+ return NULL;
+ }
+
+ if (node==-1) {
+ addr = kmalloc(size+2*ALLOC_PAD, flags);
+ } else {
+ addr = kmalloc_node(size+2*ALLOC_PAD, flags, node);
+ }
- if (!addr) {
+ if (!addr || IS_ERR(addr)) {
ERROR("ALERT ALERT kmalloc has FAILED FAILED FAILED\n");
return NULL;
}
memset(addr,0,size+2*ALLOC_PAD);
#endif
- MEMCHECK_KMALLOC(addr+ALLOC_PAD,size+2*ALLOC_PAD);
+ MEMCHECK_KMALLOC(addr,size+2*ALLOC_PAD);
return addr+ALLOC_PAD;
}
{
void * addr = NULL;
+ if (size==0) {
+ ERROR("ALERT ALERT attempt to vmalloc zero bytes rejected\n");
+ return NULL;
+ }
+
addr = vmalloc(size);
- if (!addr) {
+ if (!addr || IS_ERR(addr)) {
ERROR("ALERT ALERT vmalloc has FAILED FAILED FAILED\n");
return NULL;
}
vmallocs++;
+#if ALLOC_ZERO_MEM
+ memset(addr,0,size);
+#endif
+
MEMCHECK_VMALLOC(addr,size);
return addr;
void palacios_vfree(void *p)
{
+ if (!p) {
+ ERROR("Ignoring vfree: 0x%p\n",p);
+ dump_stack();
+ return;
+ }
vfree(p);
vfrees++;
MEMCHECK_VFREE(p);
// this function is used extensively throughout palacios and the linux
// module, both in places where interrupts are off and where they are on
// a GFP_KERNEL call, when done with interrupts off can lead to DEADLOCK
- if (irqs_disabled()) {
- return palacios_alloc_extended(size,GFP_ATOMIC);
+ if (irqs_disabled() || in_atomic()) {
+ return palacios_alloc_extended(size,GFP_ATOMIC,-1);
} else {
- return palacios_alloc_extended(size,GFP_KERNEL);
+ return palacios_alloc_extended(size,GFP_KERNEL,-1);
}
}
void * addr
)
{
+ if (!addr) {
+ ERROR("Ignoring free : 0x%p\n", addr);
+ dump_stack();
+ return;
+ }
frees++;
kfree(addr-ALLOC_PAD);
MEMCHECK_KFREE(addr-ALLOC_PAD);
/**
* Runs a function on the specified CPU.
*/
-static void
+void
palacios_xcall(
int cpu_id,
void (*fn)(void *arg),
struct lnx_thread_arg {
int (*fn)(void * arg);
void * arg;
+ v3_resource_control_t *resource_control;
char name[MAX_THREAD_NAME];
};
allow_signal(SIGKILL);
*/
+#ifdef V3_CONFIG_HOST_LAZY_FPU_SWITCH
+ // We are a kernel thread that needs FPU save/restore state
+ // vcores definitely need this, all the other threads get it too,
+ // but they just won't use it
- ret = thread_info->fn(thread_info->arg);
+ fpu_alloc(&(current->thread.fpu));
+#endif
+ palacios_htable_insert(v3_thread_resource_map,(addr_t)current,(addr_t)thread_info->resource_control);
+
+ ret = thread_info->fn(thread_info->arg);
INFO("Palacios Thread (%s) EXITING\n", thread_info->name);
+ palacios_htable_remove(v3_thread_resource_map,(addr_t)current,0);
+
palacios_free(thread_info);
// handle cleanup
+ // We rely on do_exit to free the fpu data
+ // since we could get switched at any point until the thread is done...
+
do_exit(ret);
-
+
return 0; // should not get here.
}
* Creates a kernel thread.
*/
void *
-palacios_start_kernel_thread(
+palacios_create_and_start_kernel_thread(
int (*fn) (void * arg),
void * arg,
- char * thread_name) {
+ char * thread_name,
+ v3_resource_control_t *resource_control) {
struct lnx_thread_arg * thread_info = palacios_alloc(sizeof(struct lnx_thread_arg));
thread_info->arg = arg;
strncpy(thread_info->name,thread_name,MAX_THREAD_NAME);
thread_info->name[MAX_THREAD_NAME-1] =0;
+ thread_info->resource_control = resource_control;
return kthread_run( lnx_thread_target, thread_info, thread_info->name );
}
* Starts a kernel thread on the specified CPU.
*/
void *
-palacios_start_thread_on_cpu(int cpu_id,
- int (*fn)(void * arg),
- void * arg,
- char * thread_name ) {
+palacios_create_thread_on_cpu(int cpu_id,
+ int (*fn)(void * arg),
+ void * arg,
+ char * thread_name,
+ v3_resource_control_t *resource_control) {
struct task_struct * thread = NULL;
struct lnx_thread_arg * thread_info = palacios_alloc(sizeof(struct lnx_thread_arg));
thread_info->arg = arg;
strncpy(thread_info->name,thread_name,MAX_THREAD_NAME);
thread_info->name[MAX_THREAD_NAME-1] =0;
+ thread_info->resource_control=resource_control;
thread = kthread_create( lnx_thread_target, thread_info, thread_info->name );
- if (IS_ERR(thread)) {
+ if (!thread || IS_ERR(thread)) {
WARNING("Palacios error creating thread: %s\n", thread_info->name);
palacios_free(thread_info);
return NULL;
return NULL;
}
- wake_up_process(thread);
-
return thread;
}
+void
+palacios_start_thread(void * th){
+
+ struct task_struct * thread = (struct task_struct *)th;
+ wake_up_process(thread);
+
+}
+
+/*
+ Convenience wrapper
+*/
+void *
+palacios_create_and_start_thread_on_cpu(int cpu_id,
+ int (*fn)(void * arg),
+ void * arg,
+ char * thread_name,
+ v3_resource_control_t *resource_control) {
+
+ void *t = palacios_create_thread_on_cpu(cpu_id, fn, arg, thread_name, resource_control);
+
+ if (t) {
+ palacios_start_thread(t);
+ }
+
+ return t;
+}
+
+
/**
* Rebind a kernel thread to the specified CPU
}
}
+void palacios_mutex_deinit(void *mutex)
+{
+ spinlock_t *lock = (spinlock_t*)mutex;
+
+ if (lock) {
+ // no actual spin_lock_deinit on linux
+ // our purpose here is to drive the lock checker
+ LOCKCHECK_FREE(lock);
+ }
+}
+
/**
* Frees a mutex.
*/
void
palacios_mutex_lock(void * mutex, int must_spin) {
+
+ LOCKCHECK_LOCK_PRE(mutex);
spin_lock((spinlock_t *)mutex);
- LOCKCHECK_LOCK(mutex);
+ LOCKCHECK_LOCK_POST(mutex);
}
unsigned long flags;
+ LOCKCHECK_LOCK_IRQSAVE_PRE(mutex,flags);
spin_lock_irqsave((spinlock_t *)mutex,flags);
- LOCKCHECK_LOCK_IRQSAVE(mutex,flags);
+ LOCKCHECK_LOCK_IRQSAVE_POST(mutex,flags);
return (void *)flags;
}
void * mutex
)
{
+ LOCKCHECK_UNLOCK_PRE(mutex);
spin_unlock((spinlock_t *)mutex);
- LOCKCHECK_UNLOCK(mutex);
+ LOCKCHECK_UNLOCK_POST(mutex);
}
void
palacios_mutex_unlock_irqrestore(void *mutex, void *flags)
{
+ LOCKCHECK_UNLOCK_IRQRESTORE_PRE(mutex,(unsigned long)flags);
// This is correct, flags is opaque
spin_unlock_irqrestore((spinlock_t *)mutex,(unsigned long)flags);
- LOCKCHECK_UNLOCK_IRQRESTORE(mutex,(unsigned long)flags);
+ LOCKCHECK_UNLOCK_IRQRESTORE_POST(mutex,(unsigned long)flags);
+}
+
+void palacios_used_fpu(void)
+{
+ // We assume we are not preemptible here...
+#ifndef TS_USEDFPU
+ struct task_struct *tsk = current;
+ tsk->thread.fpu.has_fpu = 1;
+#else
+ struct thread_info *cur = current_thread_info();
+ cur->status |= TS_USEDFPU;
+#endif
+ clts();
+ // After this, FP Save should be handled by Linux if it
+ // switches to a different task and that task uses FPU
+}
+
+inline int ists(void)
+{
+ return read_cr0() & X86_CR0_TS;
+
}
+void palacios_need_fpu(void)
+{
+ // We assume we are not preemptible here...
+ if (ists()) {
+ // we have been switched back to from somewhere else...
+ // Do a restore now - this will also do a clts()
+ math_state_restore();
+ }
+}
+
/**
* Structure used by the Palacios hypervisor to interface with the host kernel.
.print = palacios_print_scoped,
.allocate_pages = palacios_allocate_pages,
.free_pages = palacios_free_pages,
+ .vmalloc = palacios_valloc,
+ .vfree = palacios_vfree,
.malloc = palacios_alloc,
.free = palacios_free,
.vaddr_to_paddr = palacios_vaddr_to_paddr,
.hook_interrupt = palacios_hook_interrupt,
.ack_irq = palacios_ack_interrupt,
.get_cpu_khz = palacios_get_cpu_khz,
- .start_kernel_thread = palacios_start_kernel_thread,
+ .start_kernel_thread = palacios_create_and_start_kernel_thread,
.yield_cpu = palacios_yield_cpu,
.sleep_cpu = palacios_sleep_cpu,
.wakeup_cpu = palacios_wakeup_cpu,
.get_cpu = palacios_get_cpu,
.interrupt_cpu = palacios_interrupt_cpu,
.call_on_cpu = palacios_xcall,
- .start_thread_on_cpu = palacios_start_thread_on_cpu,
+ .create_thread_on_cpu = palacios_create_thread_on_cpu,
+ .start_thread = palacios_start_thread,
.move_thread_to_cpu = palacios_move_thread_to_cpu,
};
+#ifdef V3_CONFIG_HOST_LAZY_FPU_SWITCH
+// Note that this host interface is defined here since it's
+// intertwined with thread creation...
+static struct v3_lazy_fpu_iface palacios_fpu_hooks = {
+ .used_fpu = palacios_used_fpu,
+ .need_fpu = palacios_need_fpu
+};
+
+#endif
int palacios_vmm_init( char *options )
Init_V3(&palacios_os_hooks, cpu_mask, num_cpus, options);
+#ifdef V3_CONFIG_HOST_LAZY_FPU_SWITCH
+ V3_Init_Lazy_FPU(&palacios_fpu_hooks);
+#endif
+
return 0;
}
