Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


Linux kernel compatability enhancements (through 3.19)
[palacios.git] / linux_module / iface-pstate-ctrl.c
index 46ee834..2c49be9 100644 (file)
@@ -24,7 +24,9 @@
 #include <linux/cpufreq.h>
 #include <linux/kernel.h>
 #include <linux/kmod.h>
+#include <linux/module.h>
 #include <linux/string.h>
+#include <linux/interrupt.h>
 #include <asm/processor.h>
 #include <asm/msr.h>
 #include <asm/msr-index.h>
 #include "linux-exts.h"
 
 /*
-   This P-STATE control implementation includes:
+   This P-STATE control implementation includes the following modes.
+   You can switch between modes at any time.
 
-   - Direct control of Intel and AMD processor pstates
-   - External control of processor states via Linux (unimplemented)
    - Internal control of processor states in Palacios (handoff from Linux)
+     When Palacios acuires this control, this module disables Linux cpufreq control
+     and allows code within Palacios unfettered access to the DVFS hardware. 
+   - Direct control of Intel and AMD processor pstates using code in this module
+     When you acquire this control, this module disables Linux cpufreq control
+     and directly programs the processor itself in response to your requests
+   - External control of processor states via Linux 
+     When you acuire this control, this module uses the Linux cpufreq control
+     to program the processor on your behelf
+   - Host control of processor stastes
+     This is the normal mode of DVFS control (e.g., Linux cpufreq)
 
    Additionally, it provides a user-space interface for manipulating
    p-state regardless of the host's functionality.  This includes
    an ioctl for commanding the implementation and a /proc file for 
-   showing current status and capabilities.
+   showing current status and capabilities.  From user space, you can
+   use the Direct, External, and Host modes.  
 
-   What we mean by "pstate" here is the processor's internal
+   What we mean by "p-state" here is the processor's internal
    configuration.   For AMD, this is defined as being the same as
    the ACPI-defined p-state.  For Intel, it is not.  There, it is the 
-   contents of the perf ctl MSR, which, often, is the frequency id 
-   and voltage id (the multipliers).
+   contents of the perf ctl MSR, which is opaque.   We try hard to 
+   provide "p-states" that go from 0...max, by analogy or equivalence
+   to the ACPI p-states. 
 
 */
 
@@ -76,13 +89,14 @@ struct pstate_core_info {
     uint32_t mode;
 
     // Apply if we are under the DIRECT state
-    uint8_t cur_pstate;
-    uint8_t max_pstate;
-    uint8_t min_pstate;
+    uint64_t cur_pstate;
+    uint64_t max_pstate;
+    uint64_t min_pstate;
 
-    uint8_t cur_hw_pstate;
+    uint64_t cur_hw_pstate;
 
     // Apply if we are under the EXTERNAL state
+    uint64_t set_freq_khz; // this is the frequency we're hoping to get
     uint64_t cur_freq_khz;
     uint64_t max_freq_khz;
     uint64_t min_freq_khz;
@@ -192,6 +206,10 @@ struct p_state_ctl_reg_amd {
 /* CPUID Fn8000_0007_EDX[HwPstate(7)] = 1 */
 static uint8_t supports_pstates_amd (void)
 {
+    int i;
+    int mapwrong=0;
+    int amd_num_pstates;
+
     uint32_t eax, ebx, ecx, edx;
 
     cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
@@ -208,6 +226,20 @@ static uint8_t supports_pstates_amd (void)
             machine_state.have_feedback,
             machine_state.have_pstate_hw_coord);
 
+    amd_num_pstates = get_cpu_var(processors)->performance->state_count;
+    if (amd_num_pstates) { 
+       for (i=0;i<amd_num_pstates;i++) { 
+           INFO("P-State: %u: freq=%llu ctrl=%llx%s\n",
+                i, 
+                get_cpu_var(processors)->performance->states[i].core_frequency*1000,
+                get_cpu_var(processors)->performance->states[i].control,
+                get_cpu_var(processors)->performance->states[i].control != i ? (mapwrong=1, " ALERT - CTRL MAPPING NOT 1:1") : "");
+       }
+    }
+    if (mapwrong) { 
+       ERROR("P-State: AMD: mapping of pstate and control is not 1:1 on this processor - we will probably not work corrrectly\n");
+    }
+
     return machine_state.have_pstate;
 
 
@@ -242,6 +274,12 @@ static uint64_t get_pstate_amd(void)
 static void set_pstate_amd(uint64_t p)
 {
     struct p_state_ctl_reg_amd pctl;
+
+    if (p>get_cpu_var(core_state).max_pstate) { 
+       p=get_cpu_var(core_state).max_pstate;
+    }
+    put_cpu_var(core_state);
+
     pctl.val = 0;
     pctl.reg.cmd = p;
 
@@ -296,7 +334,7 @@ static struct pstate_core_funcs amd_funcs =
    This implementation uses SpeedStep, but does check
    to see if the other features (MPERF/APERF, Turbo/IDA, HWP)
    are available.
-   */
+*/
 
 /* Intel System Programmer's Manual Vol. 3B, 14-2 */
 #define MSR_MPERF_IA32         0x000000e7
@@ -422,6 +460,9 @@ static uint8_t supports_pstates_intel(void)
     machine_state.have_mwait_int =  !!(ecx & 1<<1);
 
 
+    // Note we test all the available hardware features documented as of August 2014
+    // We are only currently using speed_step, however.
+
     INFO("P-State: Intel: Speedstep=%d, PstateHWCoord=%d, Opportunistic=%d PolicyHint=%d HWP=%d HDC=%d, MwaitExt=%d MwaitInt=%d \n",
             machine_state.have_speedstep, 
             machine_state.have_pstate_hw_coord, 
@@ -550,12 +591,12 @@ static void set_pstate_intel(uint64_t p)
     // fid bits
 
     rdmsrl(MSR_PERF_CTL_IA32, val);
-    INFO("P-State: Pre-Set: 0x%llx\n", val);
+    //INFO("P-State: Pre-Set: 0x%llx\n", val);
 
     val &= ~0xffffULL;
     val |= ctrl & 0xffffULL;
 
-    INFO("P-State: Set: 0x%llx\n", val);
+    //INFO("P-State: Set: 0x%llx\n", val);
 
     wrmsrl(MSR_PERF_CTL_IA32, val);
 
@@ -667,6 +708,34 @@ static int pstate_arch_setup(void)
   Linux Interface
  *****************************************************************/
 
+static unsigned cpus_using_v3_governor;
+static DEFINE_MUTEX(v3_governor_mutex);
+
+/* KCH: this will tell us when there is an actual frequency transition */
+static int v3_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
+        void *data)
+{
+    struct cpufreq_freqs *freq = data;
+
+    if (per_cpu(core_state, freq->cpu).mode != V3_PSTATE_EXTERNAL_CONTROL) {
+        return 0;
+    }
+
+    if (val == CPUFREQ_POSTCHANGE) {
+        DEBUG("P-State: frequency change took effect on cpu %u (now %u kHz)\n",
+                freq->cpu, freq->new);
+        per_cpu(core_state, freq->cpu).cur_freq_khz = freq->new;
+    }
+
+    return 0;
+
+}
+
+
+static struct notifier_block v3_cpufreq_notifier_block = {
+    .notifier_call = v3_cpufreq_notifier
+};
+
 
 /* 
  * This stub governor is simply a placeholder for preventing 
@@ -675,18 +744,54 @@ static int pstate_arch_setup(void)
  */
 static int governor_run(struct cpufreq_policy *policy, unsigned int event)
 {
+    unsigned cpu = policy->cpu;
 
     switch (event) {
         /* we can't use cpufreq_driver_target here as it can result
-         * in a circular dependency, so we'll just do nothing.
+         * in a circular dependency, so we'll keep the current frequency as is
          */
         case CPUFREQ_GOV_START:
+            BUG_ON(!policy->cur);
+
+            mutex_lock(&v3_governor_mutex);
+
+            if (cpus_using_v3_governor == 0) {
+                cpufreq_register_notifier(&v3_cpufreq_notifier_block,
+                        CPUFREQ_TRANSITION_NOTIFIER);
+            }
+
+            cpus_using_v3_governor++;
+
+            per_cpu(core_state, cpu).set_freq_khz = policy->cur;
+            per_cpu(core_state, cpu).cur_freq_khz = policy->cur;
+            per_cpu(core_state, cpu).max_freq_khz = policy->max;
+            per_cpu(core_state, cpu).min_freq_khz = policy->min;
+
+            mutex_unlock(&v3_governor_mutex);
+            break;
         case CPUFREQ_GOV_STOP:
+            mutex_lock(&v3_governor_mutex);
+
+            cpus_using_v3_governor--;
+
+            if (cpus_using_v3_governor == 0) {
+                cpufreq_unregister_notifier(
+                        &v3_cpufreq_notifier_block,
+                        CPUFREQ_TRANSITION_NOTIFIER);
+            }
+
+            per_cpu(core_state, cpu).set_freq_khz = 0;
+            per_cpu(core_state, cpu).cur_freq_khz = 0;
+            per_cpu(core_state, cpu).max_freq_khz = 0;
+            per_cpu(core_state, cpu).min_freq_khz = 0;
+
+            mutex_unlock(&v3_governor_mutex);
+            break;
         case CPUFREQ_GOV_LIMITS:
             /* do nothing */
             break;
         default:
-            ERROR("Undefined governor command\n");
+            ERROR("Undefined governor command (%u)\n", event);
             return -1;
     }                          
 
@@ -702,6 +807,15 @@ static struct cpufreq_governor stub_governor =
 };
 
 
+static struct workqueue_struct *pstate_wq;
+
+typedef struct {
+    struct work_struct work;
+    uint64_t freq;
+} pstate_work_t;
+
+
+
 static inline void pstate_register_linux_governor(void)
 {
     cpufreq_register_governor(&stub_governor);
@@ -714,6 +828,31 @@ static inline void pstate_unregister_linux_governor(void)
 }
 
 
+static int pstate_linux_init(void)
+{
+    pstate_register_linux_governor();
+    pstate_wq = create_workqueue("v3vee_pstate_wq");
+    if (!pstate_wq) {
+        ERROR("Could not create work queue\n");
+        goto out_err;
+    }
+
+    return 0;
+
+out_err:
+    pstate_unregister_linux_governor();
+    return -1;
+}
+
+
+static void pstate_linux_deinit(void)
+{
+    pstate_unregister_linux_governor();
+    flush_workqueue(pstate_wq);
+    destroy_workqueue(pstate_wq);
+}
+
+
 static int get_current_governor(char **buf, unsigned int cpu)
 {
     struct cpufreq_policy * policy = palacios_alloc(sizeof(struct cpufreq_policy));
@@ -737,6 +876,7 @@ static int get_current_governor(char **buf, unsigned int cpu)
     }
 
     strncpy(govname, policy->governor->name, MAX_GOV_NAME_LEN);
+    govname[MAX_GOV_NAME_LEN-1] = 0;
 
     get_cpu_var(core_state).linux_governor = govname;
     put_cpu_var(core_state);
@@ -764,6 +904,8 @@ static void gov_switch_cleanup(struct subprocess_info * s)
 /* 
  * Switch governors
  * @s - the governor to switch to 
+ * TODO: this should probably be submitted to a work queue
+ * so we don't have to run it in interrupt context
  */
 static int governor_switch(char * s, unsigned int cpu)
 {
@@ -797,8 +939,22 @@ static int governor_switch(char * s, unsigned int cpu)
     argv[3] = NULL;
 
     /* KCH: we can't wait here to actually see if we succeeded, we're in interrupt context */
-    return call_usermodehelper_fns("/bin/sh", argv, envp, UMH_NO_WAIT, NULL, gov_switch_cleanup, NULL);
 
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(3,9,0)
+    return call_usermodehelper_fns("/bin/sh", argv, envp, UMH_NO_WAIT, NULL, gov_switch_cleanup, NULL);
+#else
+    {
+      struct subprocess_info *sp;
+      
+      sp = call_usermodehelper_setup("/bin/sh", argv, envp, GFP_ATOMIC, NULL, gov_switch_cleanup, NULL);
+      if (!sp) { 
+       goto out_freeargv;
+      }
+      
+      return call_usermodehelper_exec(sp,0);
+    }
+#endif
+      
 out_freeargv:
     palacios_free(argv);
     return -1;
@@ -816,6 +972,7 @@ static int linux_setup_palacios_governor(void)
 {
     char * gov;
     unsigned int cpu = get_cpu();
+    put_cpu();
 
     /* KCH:  we assume the v3vee governor is already 
      * registered with kernel by this point 
@@ -834,6 +991,7 @@ static int linux_setup_palacios_governor(void)
     DEBUG("setting the new governor (%s)\n", PALACIOS_GOVNAME);
 
     /* set the new one to ours */
+
     if (governor_switch(PALACIOS_GOVNAME, cpu) < 0) {
         ERROR("Could not set governor to (%s)\n", PALACIOS_GOVNAME);
         return -1;
@@ -844,13 +1002,15 @@ static int linux_setup_palacios_governor(void)
 
 
 
-static int linux_get_pstate(void)
+static uint64_t linux_get_pstate(void)
 {
     struct cpufreq_policy * policy = NULL;
     struct cpufreq_frequency_table *table;
-    int cpu = get_cpu(); 
     unsigned int i = 0;
     unsigned int count = 0;
+    unsigned int cpu = get_cpu(); 
+    put_cpu();
+
 
     policy = palacios_alloc(sizeof(struct cpufreq_policy));
     if (!policy) {
@@ -881,10 +1041,12 @@ static int linux_get_pstate(void)
 }
 
 
-static int linux_get_freq(void)
+static uint64_t linux_get_freq(void)
 {
+    uint64_t freq;
     struct cpufreq_policy * policy = NULL;
-    int cpu = get_cpu();
+    unsigned int cpu = get_cpu();
+    put_cpu();
 
     policy = palacios_alloc(sizeof(struct cpufreq_policy));
     if (!policy) {
@@ -897,19 +1059,68 @@ static int linux_get_freq(void)
         return -1;
     }
 
-    return policy->cur;
+    freq=policy->cur;
+
+    palacios_free(policy);
+
+    return freq;
 }
 
+static void  
+pstate_switch_workfn (struct work_struct *work)
+{
+    pstate_work_t * pwork = (pstate_work_t*)work;
+    struct cpufreq_policy * policy = NULL;
+    uint64_t freq; 
+    unsigned int cpu = get_cpu();
+    put_cpu();
+
+    mutex_lock(&v3_governor_mutex);
 
-static int linux_set_pstate(uint8_t p)
+    policy = palacios_alloc(sizeof(struct cpufreq_policy));
+    if (!policy) {
+        ERROR("Could not allocate space for cpufreq policy\n");
+        goto out;
+    }
+
+    if (cpufreq_get_policy(policy, cpu) != 0) {
+        ERROR("Could not get cpufreq policy\n");
+        goto out1;
+    }
+
+    freq = pwork->freq;
+    get_cpu_var(core_state).set_freq_khz = freq;
+
+    if (freq < get_cpu_var(core_state).min_freq_khz) {
+        freq = get_cpu_var(core_state).min_freq_khz;
+    }
+    if (freq > get_cpu_var(core_state).max_freq_khz) {
+        freq = get_cpu_var(core_state).max_freq_khz;
+    }
+    put_cpu_var(core_state);
+
+    INFO("P-state: requesting frequency change on core %u to %llu\n", cpu, freq);
+    __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L);
+
+out1:
+    palacios_free(policy);
+out:
+    palacios_free(work);
+    mutex_unlock(&v3_governor_mutex);
+} 
+
+
+static int linux_set_pstate(uint64_t p)
 {
     struct cpufreq_policy * policy = NULL;
     struct cpufreq_frequency_table *table;
-    int cpu = get_cpu();
+    pstate_work_t * work = NULL;
     unsigned int i = 0;
     unsigned int count = 0;
     int state_set = 0;
     int last_valid = 0;
+    unsigned int cpu = get_cpu();
+    put_cpu();
 
     policy = palacios_alloc(sizeof(struct cpufreq_policy));
     if (!policy) {
@@ -917,9 +1128,15 @@ static int linux_set_pstate(uint8_t p)
         return -1;
     }
 
+    work = (pstate_work_t*)palacios_alloc(sizeof(pstate_work_t));
+    if (!work) {
+        ERROR("Could not allocate work struct\n");
+        goto out_err;
+    }
+
     if (cpufreq_get_policy(policy, cpu)) {
         ERROR("Could not get current policy\n");
-        goto out_err;
+        goto out_err1;
     }
     table = cpufreq_frequency_get_table(cpu);
 
@@ -930,8 +1147,13 @@ static int linux_set_pstate(uint8_t p)
         }
 
         if (count == p) {
-            cpufreq_driver_target(policy, table[i].frequency, CPUFREQ_RELATION_H);
+
+            INIT_WORK((struct work_struct*)work, pstate_switch_workfn);
+            work->freq = table[i].frequency;
+            queue_work(pstate_wq, (struct work_struct*)work);
+
             state_set = 1;
+            break;
         }
 
         count++;
@@ -940,12 +1162,16 @@ static int linux_set_pstate(uint8_t p)
 
     /* we need to deal with the case in which we get a number > max pstate */
     if (!state_set) {
-        cpufreq_driver_target(policy, table[last_valid].frequency, CPUFREQ_RELATION_H);
+        INIT_WORK((struct work_struct*)work, pstate_switch_workfn);
+        work->freq = table[last_valid].frequency;
+        queue_work(pstate_wq, (struct work_struct*)work);
     }
 
     palacios_free(policy);
     return 0;
 
+out_err1: 
+    palacios_free(work);
 out_err:
     palacios_free(policy);
     return -1;
@@ -955,8 +1181,10 @@ out_err:
 static int linux_set_freq(uint64_t f)
 {
     struct cpufreq_policy * policy = NULL;
-    int cpu = get_cpu();
+    pstate_work_t * work = NULL;
     uint64_t freq;
+    unsigned int cpu = get_cpu();
+    put_cpu();
 
     policy = palacios_alloc(sizeof(struct cpufreq_policy));
     if (!policy) {
@@ -964,7 +1192,16 @@ static int linux_set_freq(uint64_t f)
         return -1;
     }
 
-    cpufreq_get_policy(policy, cpu);
+    work = (pstate_work_t*)palacios_alloc(sizeof(pstate_work_t));
+    if (!work) {
+        ERROR("Could not allocate work struct\n");
+        goto out_err;
+    }
+
+    if (cpufreq_get_policy(policy, cpu) != 0) {
+        ERROR("Could not get cpufreq policy\n");
+        goto out_err1;
+    }
 
     if (f < policy->min) {
         freq = policy->min;
@@ -974,17 +1211,26 @@ static int linux_set_freq(uint64_t f)
         freq = f;
     }
 
-    cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_H);
+    INIT_WORK((struct work_struct*)work, pstate_switch_workfn);
+    work->freq = freq;
+    queue_work(pstate_wq, (struct work_struct*)work);
 
     palacios_free(policy);
     return 0;
+
+out_err1:
+    palacios_free(work);
+out_err:
+    palacios_free(policy);
+    return -1;
 }
 
 
 static int linux_restore_defaults(void)
 {
-    unsigned int cpu = get_cpu();
     char * gov = NULL;
+    unsigned int cpu = get_cpu();
+    put_cpu();
 
     gov = get_cpu_var(core_state).linux_governor;
     put_cpu_var(core_state);
@@ -1015,10 +1261,9 @@ static void init_core(void)
 {
     unsigned cpu;
     struct cpufreq_policy *p;
-    unsigned int i;
 
 
-    DEBUG("P-State Core Init\n");
+    //DEBUG("P-State Core Init\n");
 
     get_cpu_var(core_state).mode = V3_PSTATE_HOST_CONTROL;
     get_cpu_var(core_state).cur_pstate = 0;
@@ -1045,13 +1290,10 @@ static void init_core(void)
         get_cpu_var(core_state).have_cpufreq = 1;
         get_cpu_var(core_state).min_freq_khz=p->min;
         get_cpu_var(core_state).max_freq_khz=p->max;
-        get_cpu_var(core_state).cur_freq_khz=p->cur;
-    }
-    
-    cpufreq_cpu_put(p);
-
+        get_cpu_var(core_state).cur_freq_khz=p->cur; } cpufreq_cpu_put(p); 
     put_cpu_var(core_state);
 
+    /*
     for (i=0;i<get_cpu_var(processors)->performance->state_count; i++) { 
         INFO("P-State: %u: freq=%llu ctrl=%llx",
                i, 
@@ -1059,6 +1301,7 @@ static void init_core(void)
                get_cpu_var(processors)->performance->states[i].control);
    }
    put_cpu_var(processors);
+    */
 }
 
 
@@ -1069,6 +1312,7 @@ static void deinit_core(void)
 {
     DEBUG("P-State Core Deinit\n");
     palacios_pstate_ctrl_release();
+
 }
 
 
@@ -1125,7 +1369,9 @@ void palacios_pstate_ctrl_set_pstate(uint64_t p)
     } else if (get_cpu_var(core_state).mode==V3_PSTATE_EXTERNAL_CONTROL) {
         put_cpu_var(core_state);
         linux_set_pstate(p);
-    } 
+    } else {
+        put_cpu_var(core_state);
+    }
 }
 
 
@@ -1152,8 +1398,9 @@ void palacios_pstate_ctrl_set_freq(uint64_t p)
     if (get_cpu_var(core_state).mode==V3_PSTATE_EXTERNAL_CONTROL) { 
         put_cpu_var(core_state);
         linux_set_freq(p);
-    } 
-    put_cpu_var(core_state);
+    } else {
+        put_cpu_var(core_state);
+    }
 }
 
 
@@ -1165,7 +1412,7 @@ static int switch_to_external(void)
         put_cpu_var(core_state);
         ERROR("No cpufreq  - cannot switch to external...\n");
         return -1;
-    }
+    } 
     put_cpu_var(core_state);
 
     linux_setup_palacios_governor();
@@ -1188,6 +1435,8 @@ static int switch_to_direct(void)
         // The implementation would set the policy and governor to peg cpu
         // regardless of load
         linux_setup_palacios_governor();
+    } else {
+        put_cpu_var(core_state);
     }
 
     if (machine_state.funcs && machine_state.funcs->arch_init) {
@@ -1210,6 +1459,8 @@ static int switch_to_internal(void)
         put_cpu_var(core_state);
         DEBUG("switch to internal on machine with cpu freq\n");
         linux_setup_palacios_governor();
+    } else {
+        put_cpu_var(core_state);
     }
 
     get_cpu_var(core_state).mode=V3_PSTATE_INTERNAL_CONTROL;
@@ -1227,15 +1478,18 @@ static int switch_from_external(void)
         ERROR("No cpufreq  - how did we get here... external...\n");
         return -1;
     }
+    put_cpu_var(core_state);
 
     DEBUG("Switching back to host control from external\n");
 
     if (get_cpu_var(core_state).have_cpufreq) { 
-       linux_restore_defaults();
+        put_cpu_var(core_state);
+        linux_restore_defaults();
+    } else {
+        put_cpu_var(core_state);
     }
 
     get_cpu_var(core_state).mode = V3_PSTATE_HOST_CONTROL;
-
     put_cpu_var(core_state);
 
     return 0;
@@ -1252,7 +1506,10 @@ static int switch_from_direct(void)
     machine_state.funcs->arch_deinit();
 
     if (get_cpu_var(core_state).have_cpufreq) { 
+        put_cpu_var(core_state);
         linux_restore_defaults();
+    } else {
+        put_cpu_var(core_state);
     }
 
     get_cpu_var(core_state).mode=V3_PSTATE_HOST_CONTROL;
@@ -1268,9 +1525,10 @@ static int switch_from_internal(void)
     DEBUG("Switching back to host control from internal\n");
 
     if (get_cpu_var(core_state).have_cpufreq) { 
-        // ERROR("Unimplemented: switch from internal on machine with cpu freq - will just pretend to do so\n");
-        // The implementation would switch back to default policy and governor
+        put_cpu_var(core_state);
         linux_restore_defaults();
+    } else {
+        put_cpu_var(core_state);
     }
 
     get_cpu_var(core_state).mode=V3_PSTATE_HOST_CONTROL;
@@ -1285,11 +1543,12 @@ static int switch_from_internal(void)
 void palacios_pstate_ctrl_acquire(uint32_t type)
 {
     if (get_cpu_var(core_state).mode != V3_PSTATE_HOST_CONTROL) { 
+        put_cpu_var(core_state);
         palacios_pstate_ctrl_release();
+    } else {
+        put_cpu_var(core_state);
     }
 
-    put_cpu_var(core_state);
-
     switch (type) { 
         case V3_PSTATE_EXTERNAL_CONTROL:
             switch_to_external();
@@ -1324,25 +1583,27 @@ void palacios_pstate_ctrl_release(void)
     if (get_cpu_var(core_state).mode == V3_PSTATE_HOST_CONTROL) { 
         put_cpu_var(core_state);
         return;
-    }
+    } 
+    put_cpu_var(core_state);
 
     switch (get_cpu_var(core_state).mode) { 
         case V3_PSTATE_EXTERNAL_CONTROL:
+            put_cpu_var(core_state);
             switch_from_external();
             break;
         case V3_PSTATE_DIRECT_CONTROL:
+            put_cpu_var(core_state);
             switch_from_direct();
             break;
         case V3_PSTATE_INTERNAL_CONTROL:
+            put_cpu_var(core_state);
             switch_from_internal();
             break;
         default:
+            put_cpu_var(core_state);
             ERROR("Unknown pstate control type %u\n",core_state.mode);
             break;
     }
-
-    put_cpu_var(core_state);
-
 }
 
 
@@ -1373,31 +1634,19 @@ static int pstate_show(struct seq_file * file, void * v)
         palacios_xcall(cpu,update_hw_pstate,0);
     }
 
-    seq_printf(file, "Arch:\t%s\nPStates:\t%s\n\n",
-            machine_state.arch==INTEL ? "Intel" : 
-            machine_state.arch==AMD ? "AMD" : "Other",
-            machine_state.supports_pstates ? "Yes" : "No");
-
     for (cpu=0;cpu<numcpus;cpu++) { 
         struct pstate_core_info *s = &per_cpu(core_state,cpu);
-        seq_printf(file,"pcore %u: hw pstate 0x%x mode %s of [ host ",cpu,
+        seq_printf(file,"pcore %u: hw pstate 0x%llx mode %s ",cpu,
                 s->cur_hw_pstate,
                 s->mode==V3_PSTATE_HOST_CONTROL ? "host" :
                 s->mode==V3_PSTATE_EXTERNAL_CONTROL ? "external" :
                 s->mode==V3_PSTATE_DIRECT_CONTROL ? "direct" : 
                 s->mode==V3_PSTATE_INTERNAL_CONTROL ? "internal" : "UNKNOWN");
-        if (s->have_cpufreq) { 
-            seq_printf(file,"external ");
-        }
-        if (machine_state.supports_pstates) {
-            seq_printf(file,"direct ");
-        }
-        seq_printf(file,"internal ] ");
         if (s->mode==V3_PSTATE_EXTERNAL_CONTROL) { 
             seq_printf(file,"(min=%llu max=%llu cur=%llu) ", s->min_freq_khz, s->max_freq_khz, s->cur_freq_khz);
         } 
         if (s->mode==V3_PSTATE_DIRECT_CONTROL) { 
-            seq_printf(file,"(min=%u max=%u cur=%u) ", (uint32_t)s->min_pstate, (uint32_t)s->max_pstate, (uint32_t)s->cur_pstate);
+            seq_printf(file,"(min=%llu max=%llu cur=%llu) ",s->min_pstate, s->max_pstate, s->cur_pstate);
         }
         seq_printf(file,"\n");
     }
@@ -1418,24 +1667,117 @@ static struct file_operations pstate_fops = {
     .release = seq_release
 };
 
+static int pstate_hw_show(struct seq_file * file, void * v)
+{
+    int numstates;
+
+    seq_printf(file, "V3VEE DVFS Hardware Info\n(all logical cores assumed identical)\n\n");
+
+    seq_printf(file, "Arch:   \t%s\n"
+                    "PStates:\t%s\n\n",
+            machine_state.arch==INTEL ? "Intel" : 
+            machine_state.arch==AMD ? "AMD" : "Other",
+            machine_state.supports_pstates ? "Yes" : "No");
+
+
+#define YN(x) ((x) ? "Y" : "N")
+
+    if (machine_state.arch==INTEL) {
+       seq_printf(file,"SpeedStep:           \t%s\n",YN(machine_state.have_speedstep));
+       seq_printf(file,"APERF/MPERF:         \t%s\n",YN(machine_state.have_pstate_hw_coord));
+       seq_printf(file,"IDA or TurboCore:    \t%s\n",YN(machine_state.have_opportunistic));
+       seq_printf(file,"Policy Hint:         \t%s\n",YN(machine_state.have_policy_hint));
+       seq_printf(file,"Hardware Policy:     \t%s\n",YN(machine_state.have_hwp));
+       seq_printf(file,"Hardware Duty Cycle: \t%s\n",YN(machine_state.have_hdc));
+       seq_printf(file,"MWAIT extensions:    \t%s\n",YN(machine_state.have_mwait_ext));
+       seq_printf(file,"MWAIT wake on intr:  \t%s\n",YN(machine_state.have_mwait_int));
+    } 
+
+    if (machine_state.arch==AMD) { 
+       seq_printf(file,"PState:              \t%s\n",YN(machine_state.have_pstate));
+       seq_printf(file,"APERF/MPERF:         \t%s\n",YN(machine_state.have_pstate_hw_coord));
+       seq_printf(file,"CoreBoost:           \t%s\n",YN(machine_state.have_coreboost));
+       seq_printf(file,"Feedback:            \t%s\n",YN(machine_state.have_feedback));
+    }
+
+
+    seq_printf(file,"\nPstate\tCtrl\tKHz\tmW\tuS(X)\tuS(B)\n");
+    numstates = get_cpu_var(processors)->performance->state_count;
+    if (!numstates) { 
+       seq_printf(file,"UNKNOWN\n");
+    } else {
+       int i;
+       for (i=0;i<numstates;i++) { 
+           seq_printf(file,
+                      "%u\t%llx\t%llu\t%llu\t%llu\t%llu\n",
+                      i, 
+                      get_cpu_var(processors)->performance->states[i].control,
+                      get_cpu_var(processors)->performance->states[i].core_frequency*1000,
+                      get_cpu_var(processors)->performance->states[i].power,
+                      get_cpu_var(processors)->performance->states[i].transition_latency,
+                      get_cpu_var(processors)->performance->states[i].bus_master_latency);
+       }
+    }
+    put_cpu_var(processors);
+
+    seq_printf(file,"\nAvailable Modes:");
+    seq_printf(file," host");
+    if (get_cpu_var(core_state).have_cpufreq) { 
+       seq_printf(file," external");
+    }
+    put_cpu_var(core_state);
+    if (machine_state.supports_pstates) {
+       seq_printf(file," direct");
+    }
+    seq_printf(file," internal\n");
+
+    return 0;
+}
+
+static int pstate_hw_open(struct inode * inode, struct file * file) 
+{
+    return single_open(file, pstate_hw_show, NULL);
+}
+
+
+static struct file_operations pstate_hw_fops = {
+    .owner = THIS_MODULE,
+    .open = pstate_hw_open, 
+    .read = seq_read,
+    .llseek = seq_lseek,
+    .release = seq_release
+};
+
+
 int pstate_proc_setup(void)
 {
     struct proc_dir_entry *proc;
+    struct proc_dir_entry *prochw;
 
-    proc = create_proc_entry("v3-dvfs",0444, palacios_get_procdir());
+    PAL_PROC_CREATE(proc,"v3-dvfs",0444,palacios_get_procdir(),&pstate_fops);
 
     if (!proc) { 
         ERROR("Failed to create proc entry for p-state control\n");
         return -1;
     }
 
-    proc->proc_fops = &pstate_fops;
+    INFO("/proc/v3vee/v3-dvfs successfully created\n");
+
+    PAL_PROC_CREATE(prochw,"v3-dvfs-hw",0444,palacios_get_procdir(),&pstate_hw_fops);
+
+    if (!prochw) { 
+        ERROR("Failed to create proc entry for p-state hw info\n");
+        return -1;
+    }
+
+    INFO("/proc/v3vee/v3-dvfs-hw successfully created\n");
 
     return 0;
 }
 
 void pstate_proc_teardown(void)
 {
+    remove_proc_entry("v3-dvfs-hw",palacios_get_procdir());
     remove_proc_entry("v3-dvfs",palacios_get_procdir());
 }
 
@@ -1540,7 +1882,7 @@ static int pstate_ctrl_init(void)
 
     pstate_user_setup();
 
-    pstate_register_linux_governor();
+    pstate_linux_init();
 
     INFO("P-State Control Initialized\n");
 
@@ -1552,7 +1894,7 @@ static int pstate_ctrl_deinit(void)
     unsigned int cpu;
     unsigned int numcpus=num_online_cpus();
 
-    pstate_unregister_linux_governor();
+    pstate_linux_deinit();
 
     pstate_user_teardown();