From: Kyle Hale <kh@u.northwestern.edu>
Date: Wed, 10 Apr 2013 21:07:37 +0000 (-0500)
Subject: Add tools for transforming a kernel module into a guarded module
X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=9fc08977f4954b9343fc2e38e57ffb11225ab24d

Add tools for transforming a kernel module into a guarded module
---

diff --git a/gears/guarded_modules/christo-s.pl b/gears/guarded_modules/christo-s.pl
new file mode 100755
index 0000000..e7f5348
--- /dev/null
+++ b/gears/guarded_modules/christo-s.pl
@@ -0,0 +1,313 @@
+#!/usr/bin/perl -w
+
+#
+#
+# Wrap a kernel module (.ko) so that all undefined functions
+# are resolved at compile time to wrapper that have the template
+# as given here
+#
+# TODO: parameterize hypercall numbers
+# TODO: remove printk special case
+#
+
+use Getopt::Long;
+use File::Basename;
+
+$user=0;
+$kern=0;
+
+$V3_BORDER_OUT_CALL_NR = 0x6003;
+$V3_BORDER_IN_RET_NR   = 0x6004;
+
+&GetOptions("user"=>\$user, "kern"=>\$kern);
+
+$#ARGV==0 or die "christo.pl [--user|--kern] <object-file>\n";
+
+$of=shift; chomp($of);
+($ofstem) = split(/\./,fileparse($of)); 
+
+if ($user) { 
+  
+  print "Wrapping $of to produce $ofstem-wrapped\n";
+  
+  # First, find all undefined symbols that are text
+  @funcs = ExtractUndefinedFunctions($of);
+  
+  print "Wrapping the following functions\n";
+  print join("\n", @funcs),"\n";
+  
+  print "Generating wrapper $ofstem\_wrapper.c\n";
+  
+  open(WRAPPER, ">$ofstem\_wrapper.c");
+  GenerateUserWrapper(WRAPPER, @funcs);
+  close(WRAPPER);
+
+  print "Compiling wrapper\n";
+  
+  CompileUserWrapper($ofstem);
+
+
+  print "Linking with your supplied file $of to form $ofstem\_wrapped\n";
+
+  LinkUserWrapped($of,"$ofstem\_wrapper.o","$ofstem\_wrapped",@funcs);
+
+  print "Done.\n"
+
+} else {
+
+  print "Wrapping $of to produce $ofstem-wrapped\n";
+
+  @funcs = KernUndefFuncs($of);
+
+  
+  print "Wrapping the following functions\n";
+  print join("\n", @funcs), "\n";
+
+  print "Generating wrapper $ofstem\_wrapper.S\n";
+
+  open(WRAPPER,">$ofstem\_wrapper.S");
+  GenerateKernWrapper($ofstem, WRAPPER, @funcs);
+  close(WRAPPER);
+
+}
+
+sub CompileUserWrapper {
+  my $stem=shift;
+  
+  return system("gcc -c -fPIC $stem.c -o $stem.o");
+}
+
+
+sub LinkUserWrapped {
+  my ($of,$wrapper,$wrapped,@funcs) = @_;
+  
+  $cmd = "gcc ".join(" ", 
+		     join(" ",
+			  map {" -Xlinker --wrap=$_ "} @funcs
+			 ), 
+		     $of, $wrapper)." -o $wrapped";
+  
+  print $cmd;
+  return system $cmd;
+
+}
+
+sub KernUndefFuncs {
+    my $of = shift;
+    my $line;
+    my @funcs;
+    my @lines;
+    my @final_funcs;
+    my %sysmap;
+
+    print STDERR "Acquiring all undefined symbols from file\n";
+
+    open(OBJDUMP, "objdump -t $of |") or die "Cannot open file $of\n";
+
+    while ($line=<OBJDUMP>) {
+        if ($line =~ /.*\*UND\*\s+\S+\s+(\S+)$/) {
+            push @funcs, $1;
+        }
+    }
+
+    close(OBJDUMP);
+
+    print STDERR "Finding all functions listed in System.map\n";
+
+    open(MAP,"System.map") or die "Cannot open System.map\n";
+
+    while ($line=<MAP>) { 
+	chomp($line);
+	my ($addr,$type,$name) = split(/\s+/,$line);
+	if (($type eq "t") or ($type eq "T")) { 
+	    $sysmap{$name}=1;
+	}
+    }
+
+    close(MAP);
+
+    foreach $func (@funcs) { 
+	if (defined($sysmap{$func}) and ($func ne "printk") and ($func ne "mcount")
+        and ($func ne "__cyg_profile_func_enter") and ($func ne "__cyg_profile_func_exit")) {
+	    print STDERR "ACCEPT  $func\n";
+	    push @final_funcs, $func;
+	} else {
+	    print STDERR "DISCARD $func\n";
+	}
+    }
+
+    return @final_funcs;
+}
+
+sub ExtractUndefinedFunctions {
+  my $of=shift;
+  my $line;
+  my @funcs;
+
+  open(OBJDUMP,"objdump -t $of |") or die "Cannot open file $of\n";
+
+  while ($line=<OBJDUMP>) { 
+    if ($line =~ /.*\*UND\*\s+\S+\s+(\S+)$/) { 
+      push @funcs, $1;
+    }
+  }
+
+  close(OBJDUMP);
+
+  return @funcs;
+
+}
+
+sub GenerateUserWrapper  {
+  my $file=shift;
+  
+
+  while ($func=shift) {
+    print $file <<ENDF;
+
+int __wrap_$func(void *a1, 
+		 void *a2,
+		 void *a3,
+		 void *a4,
+		 void *a5,
+		 void *a6,
+		 void *a7,
+		 void *a8) 
+{
+ asm volatile("vmmcall"
+        :"=a" (ret)
+        :"0" ($V3_GUARD_EXIT_HCALL_NR));
+  return __real_$func(a1,a2,a3,a4,a5,a6,a7,a8);
+ asm volatile("vmmcall"
+        :"=a" (ret)
+        :"0" ($V3_GUARD_ENTER_HCALL_NR));
+}
+
+ENDF
+
+  }
+}
+
+sub GenerateKernWrapper {
+    my $fname=shift;
+    my $file=shift;
+    my @funcs=@_;
+
+  open(MKFILE,">Makefile");
+  open(CFILE, ">wrap_impl.c");
+
+    print MKFILE "### THIS FILE IS AUTOMATICALLY GENERATED ###\n";
+    print MKFILE "EXTRA_LDFLAGS = ";
+
+    print $file "/* THIS FILE IS AUTOMATICALLY GENERATED */\n";
+    print $file ".text\n.align 4\n";
+    print $file map { ".globl __wrap_$_\n" } @funcs;
+    print $file map { ".globl __wrap_impl_$_\n" } @funcs;
+ 
+    print CFILE "/* THIS FILE IS AUTOMATICALLY GENERATED */\n";
+    print CFILE "#include <linux/kernel.h>\n\n";
+    #print CFILE "extern int wrapper_count;\n\n";
+    #print CFILE "static int border_count = 0;\n\n";
+    print CFILE map { "void __wrap_impl_$_(void);\n\n" } @funcs;
+
+    foreach $func (@funcs) {
+#__wrap_$func:
+        #nop
+        #nop
+        #nop
+        #nop
+        #nop
+        #nop
+        #nop
+        #pushq %rbp
+        #movq  %rsp, %rbp
+        #pushq %rax
+        #pushq %rbx
+        #pushq %rcx
+        #pushq %rdx
+        #pushq %rdi
+        #pushq %rsi
+        #pushq %r8
+        #pushq %r9
+        #pushq %r10
+        #pushq %r11
+        #pushq %r12
+        #pushq %r13
+        #pushq %r14
+        #pushq %r15
+        #callq __wrap_impl_$func
+        #popq %r15
+        #popq %r14
+        #popq %r13
+        #popq %r12
+        #popq %r11 
+        #popq %r10
+        #popq %r9
+        #popq %r8 
+        #popq %rsi
+        #popq %rdi
+        #popq %rdx
+        #popq %rcx
+        #popq %rbx
+        #popq %rax
+        #popq %rbp
+        #jmp __real_$func
+        #nop
+        #nop
+        #nop
+        #nop
+        #nop
+
+    print $file <<ENDF;
+
+__wrap_$func:
+        popq  %r11
+        pushq %rax
+        movq  \$$V3_BORDER_OUT_CALL_NR, %rax
+        vmmcall
+        popq  %rax
+        callq __real_$func
+        pushq %rax
+        movq \$$V3_BORDER_IN_RET_NR, %rax
+        vmmcall
+        popq  %rax
+        pushq %r11
+        ret
+ENDF
+
+    print MKFILE "--wrap $func ";
+
+    print CFILE <<ENDF;
+void __wrap_impl_$func(void) {
+
+        //wrapper_count++;
+        //printk("exit: $func\\n");
+        //printk("wc: %d (%s)\\n", wrapper_count,__func__);
+}
+
+ENDF
+
+  }
+  print MKFILE "\n\nldflags-y := \$(EXTRA_LDFLAGS)\n";
+
+# TODO: change this to not be hardcoded!
+  my $kerndir = "/v-test/guard_modules/gm_guest/kyle_gl/";
+  print MKFILE <<ENDF;
+KERNDIR=$kerndir
+  
+obj-m := $fname\_wrapped.o
+    
+$fname\_wrapped-objs := $fname.ko $fname\_wrapper.o wrap_impl.o
+
+$fname\_wrapped.ko: $fname.ko $fname\_wrapper.o wrap_impl.o
+\tmake -C \$(KERNDIR) M=\$(PWD) modules
+
+clean:
+\tmake -C \$(KERNDIR) M=\$(PWD) clean
+
+ENDF
+
+  close(MKFILE);
+  close(CFILE);
+
+}
diff --git a/gears/guarded_modules/guard_callbacks_new.rb b/gears/guarded_modules/guard_callbacks_new.rb
new file mode 100755
index 0000000..ed60c4f
--- /dev/null
+++ b/gears/guarded_modules/guard_callbacks_new.rb
@@ -0,0 +1,402 @@
+#!/usr/bin/ruby
+
+# Kyle C. Hale (c) 2012/2013
+#
+# Ruby implementation of a tool to identify the callbacks
+# in a driver given the source for the driver and it's compiled kernel
+# object. It does this by finding all static functions in the module and scanning
+# the source to identify sites where a static function is being assigned to a variable.
+#
+# Dependencies: 
+#               json ruby gem
+#
+# By default, all callbacks will be added as valid entry points into the module.
+# For now, entries must manually be added and deleted by module author
+#
+# TODO:     
+#           - remove hardcoded directories from this and chirsto script
+#
+#
+# NOTES:
+#
+# - .i files are generated to confirm callback functions
+#
+# - source files and makefiles must be in same dir. Wont work for big
+#   multi-level hierarchical source trees
+#
+# - this produces JSON output, including valid entry points, which 
+#   (both entering into callback functions and returning from calls
+#   to the kernel) can be modified by driver developer
+#
+# - this modifies the original driver source, as well as the original
+#   driver Makefile
+#
+# - You must provide your **GUEST'S** System.map file for this script to reference
+
+require 'optparse'
+require 'rubygems'
+require 'json'
+
+CHRISTO = "/root/kyle/kyle/bnx/christo-s.pl"
+GM_INIT = "v3_guard_mod_init"
+GM_INIT_INTERNAL = "__guard_mod_init_internal"
+GM_MACRO = "GM_ENTRY_REQUEST"
+WRAP_FILE = "entry_wrapper.S"
+
+GUARD_INIT_NR  = 0x6000
+GUARD_ENTER_NR = 0x6001
+GUARD_EXIT_NR  = 0x6002
+
+
+def replace_callback_assigns(callbacks, file)
+
+end
+
+# ================== OPTIONS ========================= 
+
+options = {}
+
+optparse = OptionParser.new do|opts|
+
+  opts.banner = "Usage: guard_modules.rb [options] ..."
+
+  options[:verbose] = false
+  opts.on( '-v', '--verbose', 'Allow verbose output') do
+      options[:verbose] = true
+  end
+  
+  opts.on( '-h', '--help', 'Show this information') do
+      puts opts
+      exit
+  end
+
+  opts.on('-s', '--source <driver,src,files,...,>', '(C) source code for driver module') do |file|
+      options[:srcfile] = file.split(',')
+  end
+  
+  opts.on('-p', '--privs <requested,privileges>', 'Privileges requested. Currently numerical') do |privs|
+      options[:privs] = privs.split(',')
+  end
+
+  opts.on('-k', '--ko DRIVER KO', 'The compiled and linked kernel object for the driver') do |file|
+      options[:ko] = file
+  end
+  
+  opts.on('-m', '--makefile MAKEFILE', 'The driver\'s makefile') do |file|
+        options[:mf] = file
+  end
+
+  opts.on('-o', '--output OUTFILE', 'The file to which static module info will be written') do |file|
+    options[:out] = file
+  end
+
+  opts.on('-n', '--name NAME', 'The name of the guarded module') do |name|
+    options[:name] = name
+  end
+
+  options[:instr] = false
+  opts.on('i', '--instrument', 'Dump out the functions which are *NOT* to be instrumented, i.e. those that aren\'t callbacks')  do
+    options[:instr] = true
+  end
+end
+
+
+begin
+  optparse.parse!
+  mand = [:ko, :srcfile, :mf, :name]
+  missing = mand.select{ |parm| options[parm].nil? }
+  if not missing.empty?
+    puts "Missing options: #{missing.join(', ')}"
+    puts optparse
+    exit
+  end
+rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+  puts $!.to_s
+  puts optparse
+  exit
+end
+
+# ===================== END OPTIONS ==============================
+
+$mkdir   = File.expand_path(File.dirname(options[:mf]))
+$mkbase  = File.basename(options[:mf])
+#TODO: this needs to be specified explicitly
+$srcpath = options[:srcfile][0]
+$srcdir  = File.expand_path(File.dirname(options[:srcfile][0]))
+$srcbase = File.basename(options[:srcfile][0])
+
+# global list of functions
+# TODO: THIS IS NOT WORKING!!!! e.g., on bnx2_find_max_ring, objdump is not finding static funs
+funcs = `objdump -t #{options[:ko]} | awk '\$3==\"F\" {print \$6}'`
+
+# only static functions
+stats = `objdump -t #{options[:ko]} | awk '\$2==\"l\"&&\$3==\"F\" {print \$6}'`
+
+callbacks = []
+
+puts "Running preprocessor..."
+prep_files = {}
+
+Dir.chdir($mkdir) do
+
+    # generate .i files (preprocessing)
+    ENV['PWD'] = Dir.pwd
+
+    abort("Problem cleaning") if not system "make clean > /dev/null"
+
+    #TODO parameterize this!
+    kerndir = "/v-test/guard_modules/gm_guest/kyle_gl/"
+
+    # run the kernel make process, and repeat the build of each .o with preprocessing directives
+    cmds = `make V=1 2>&1 | grep -P "gcc.*?\\s+\\-o\\s+.*?[^mod]\\.c" | sed 's/\\s\\+-c/ -E/g'| sed 's/\\(-o\\s\\+.*\\?\\)\\.o/\\1.i/g'`
+
+    Dir.chdir(kerndir) do
+        ENV['PWD'] = Dir.pwd
+
+        cmds.each do |i| 
+            abort ("Problem compiling.") if not system i
+        end
+    end
+
+    ENV['PWD'] = Dir.pwd
+
+    # read in preprocessed files
+    Dir.glob("#{$mkdir}/*.i", File::FNM_DOTMATCH).each do |fname|
+            prep_files[fname] = IO.readlines(fname)
+            puts "Reading in preprocessed file: #{fname}"
+            if not prep_files[fname]
+                puts "could not open #{fname}"
+                exit
+            end
+    end
+end
+
+ENV['PWD'] = Dir.pwd
+
+stats.each do |stat|
+  s = stat.chomp
+  defs = []
+  
+  # look for callback assignments, this tells us that this
+  # static function is indeed being used as a callback
+  prep_files.each_value do |src|
+    tmp_defs = src.grep(/.*=\s*#{s}\s*(;|,)/)
+    defs.concat(tmp_defs) if not tmp_defs.empty?
+  end
+
+  callbacks.push(s) if not defs.empty?
+
+  if (options[:verbose]) 
+    puts "Possible callback assignment for #{s} : " if not defs.empty?
+    defs.each {|i| print "\t#{i}\n"}
+  end
+  
+end
+
+if (options[:instr])
+    f = funcs.split("\n")
+    
+    puts "Searching for driver's module_init routine..."
+    mod_init = nil
+
+    # TODO: make this work for multiple files
+    File.open($srcpath, 'r') do |file|
+        new_str = file.read
+        if new_str =~ /module_init\((.*?)\)/ 
+            puts "Found it, it's #{$1}" 
+            mod_init = $1
+        end
+    end
+    
+    # back up (TODO: ALL) source/makefiles
+    `cp #{$mkdir}/#{$mkbase} #{$mkdir}/#{$mkbase}.bak`
+    `cp #{options[:srcfile][0]} #{$srcdir}/#{$srcbase}.bak`
+
+    # generate a restore script
+    shell = `which sh`
+    restore = <<-eos
+#!#{shell}
+for i in `ls *.bak | sed 's/\.bak//g'`; do mv $i.bak $i; done
+make clean && make
+rm -f *.i
+eos
+
+    # TODO: WARNING WARNING: assumes source files and makefiles are in same place
+    File.open("#{$srcdir}/gm_restore.sh", "w", 0744) do |file|
+        file.puts restore 
+    end
+
+    str = <<-eos
+/* BEGIN GENERATED */
+#define V3_GUARD_ENTER_HCALL_NR #{GUARD_ENTER_NR}
+#define V3_GUARD_EXIT_HCALL_NR  #{GUARD_EXIT_NR}
+#define V3_GUARD_INIT_HCALL_NR  #{GUARD_INIT_NR}
+#define V3_GUARDED_MODULE_ID   0xA3AEEA3AEEBADBADULL
+#define #{GM_MACRO}(fun) _gm_entry_req_##fun
+int __init #{GM_INIT} (void);
+eos
+
+# for assembly linkage
+callbacks.each do |c| 
+    str +=<<-eos "int _gm_entry_req_#{c}(void);\n"
+__asm__(".globl _gm_entry_req_#{c};\\
+   _gm_entry_req_#{c}:\\
+   popq  %r11;          \\
+   pushq %rax;\\
+   movq  $#{GUARD_ENTER_NR}, %rax;\\
+   vmmcall;\\
+   popq  %rax;\\
+   callq #{c};\\
+   pushq %rax;\\
+   movq  $#{GUARD_EXIT_NR}, %rax;\\
+   vmmcall;\\
+   popq  %rax;\\
+   pushq %r11;\\
+   ret;");
+eos
+end
+str += "/* END GENREATED */\n"
+
+end_str = <<-eos
+/* BEGIN GENERATED */
+int #{GM_INIT_INTERNAL} (void) __attribute__((section (".text"),no_instrument_function));
+
+int #{GM_INIT_INTERNAL} (void) {
+        int ret, ret_orig;
+
+        /* GUARD INIT */
+        __asm__ __volatile__ ("vmmcall"
+                      :"=a" (ret)
+                      :"0"  (V3_GUARD_INIT_HCALL_NR), "b" (V3_GUARDED_MODULE_ID));
+
+        if (ret < 0) {
+                printk("Guest GM: error initializing guarded module\\n");
+        } else {
+                printk("Guest GM: successfully initialized guarded module\\n");
+        }
+
+        ret_orig = #{mod_init}();
+
+        if (ret_orig < 0) { 
+                printk("Guest GM: error calling original init\\n");
+        } else {
+                printk("Guest GM: successfully called original driver init\\n");
+        }
+
+        /* GUARD EXIT */
+        __asm__ __volatile__("vmmcall" : "=a" (ret) : "0" (V3_GUARD_EXIT_HCALL_NR));
+
+        if (ret < 0) {
+                printk("Guest GM: error doing initial exit\\n");
+                return ret;
+        }
+
+        return ret_orig;
+}
+
+int __init #{GM_INIT} (void) {
+        return #{GM_INIT_INTERNAL}();
+}
+
+/* END GENERATED */
+eos
+
+# GENERATE THE CALLBACK WRAPPERS
+#
+# statements like 
+# .callback = myfunc;
+#
+# will be replaced with 
+# .callback = GM_ENTRY_REQUEST(myfunc);
+#
+# which expands to 
+# .callback = __gm_entry_req_myfunc;  (this is an assembly stub)
+#
+
+    callbacks.each do |c|
+      end_str += <<-eos
+      void  (*#{c}_ptr)() = #{c};
+eos
+    end
+
+    end_str += "/* END GENERATED */"
+
+    # append instrumentation functions to end of source file
+    new_str = ""
+    File.open($srcpath, "r") do |file|
+        new_str = file.read
+    end
+
+    # fixup module init calls with our own
+    new_str = new_str.gsub(/__init #{mod_init}/, "#{mod_init}")
+    new_str = new_str.gsub(/module_init\(.*?\);/, "module_init(#{GM_INIT});")
+
+    # fixup callback assignments with our macro
+    # TODO: make this work for multiple source files
+    callbacks.each do |c|
+        new_str = new_str.gsub(/(.*=\s*)#{c}(\s*(;|,))/) {
+            "#{$1}#{GM_MACRO}(#{c})#{$2}"
+        }
+    end
+
+    # put these at the top of the file, after the includes
+    new_str = new_str.insert(new_str.rindex(/#include.*?[>"]/), str)
+    new_str += end_str
+    
+    File.open($srcpath, "w+") do |file|
+        file.puts new_str
+    end
+
+    Dir.chdir($mkdir) do
+        abort ("Error cleaning") if not system "make clean > /dev/null 2>&1"
+        abort("Error rebuilding module") if not system "make > /dev/null 2>&1"
+    end
+
+end
+
+# if an outfile is specified, output static module info
+# in JSON format
+if (options[:out])
+  # First generate wrapped module
+
+  ko = File.expand_path(options[:ko])
+  puts "Running christo with Kernel object: " + ko
+  christo_output = `#{CHRISTO} --kern #{ko} 2>&1 | grep ACCEPT`
+  out_arr = []
+  christo_output.each_line { |x| out_arr.push(x.chomp.gsub(/\s*ACCEPT\s*/, '')) }
+
+  wrap_name = File.basename(ko).sub(/\.ko$/,'_wrapped.ko')
+
+  ret_points = []
+  out_arr.each do |x|
+      ret_points.push Array[x, `objdump -d #{wrap_name} | sed -n '/__wrap_#{x}>/,/ret/p' | grep vmmcall | tail -1 | cut -f1 -d':' | sed 's/\s//g'`.to_i(16)]
+  end
+
+  abort("Problem building wrapped module") if not system "make > /dev/null 2>&1"
+  static_info = {}
+  static_info['module_name'] = options[:name]
+
+  # get the content hash of .text
+  static_info['content_hash'] = `objdump -h #{wrap_name} | grep -P "\s+\.text\s+" | awk '{print "dd if=#{wrap_name} bs=1 count=$[0x" $3 "] skip=$[0x" $6 "] 2>/dev/null"}' | bash | md5sum - | awk '{print $1}'`.chomp
+  static_info['size'] = `objdump -h #{wrap_name} | grep -P "\s+\.text\s+" | awk '{print $3}'`.to_i(16)
+
+  # find offset to hcall (we've ensured it's in .text)
+  static_info['hcall_offset'] = `objdump -d #{wrap_name} | sed -n '/#{GM_INIT_INTERNAL}/,/ret/p'| grep vmmcall | cut -f1 -d':'|sed 's/\s//g'`.to_i(16)
+
+  # each callback function will have ONE valid entry point, the address of the VMMCALL
+  # in its function body
+  static_info['entry_points'] = callbacks.collect do |c|
+      c_off = `objdump -d #{wrap_name} | sed -n '/_gm_entry_req_#{c}>/,/vmmcall/p' | tail -1 | cut -f1 -d':' | sed 's/\s//g'`.to_i(16)
+      Array[c, c_off]
+  end
+
+  static_info['ret_points'] = ret_points
+  static_info['privileges'] = options[:privs]
+
+  File.open(options[:out], "w") do |file|
+      file.write(JSON.pretty_generate(static_info))
+  end
+
+else
+  callbacks.each {|c| puts c }
+end