--- /dev/null
+#include <lwk/linkage.h>
+#include <lwk/errno.h>
+#include <arch/ptrace.h>
+#include <arch/asm-offsets.h>
+#include <arch/idt_vectors.h>
+#include <arch/dwarf2.h>
+#include <arch/task.h>
+
+
+/**
+ * This performs the architecture-specific portion of a context switch.
+ * Normally, this is called in the context of prev and returns in the
+ * context of next. However, new tasks are handled differently. Since
+ * new tasks do not yet have a kernel context (rather, their kernel
+ * stack just has the pt_regs to use for the new task), execution returns
+ * directly to the new task, rather than context_switch().
+ *
+ * Input Registers:
+ * RDI = prev
+ * RSI = next
+ *
+ * Output Registers::
+ * RAX = prev (same value as on entry)
+ *
+ * C Prototype:
+ * struct task_struct *arch_context_switch(struct task_struct *prev,
+ * struct task_struct *next);
+ * arch_context_switch() returns prev
+ *
+ * NOTE: External interrupts are disabled on entry.
+ */
+ENTRY(arch_context_switch)
+ /* Save prev's callee saved registers (others saved by caller) */
+ pushf
+ pushq %rbp
+ pushq %rbx
+ pushq %r12
+ pushq %r13
+ pushq %r14
+ pushq %r15
+
+ /* Switch to next's stack */
+ movq %rsp, tsk_arch_rsp(%rdi)
+ movq tsk_arch_rsp(%rsi), %rsp
+
+ /* Call C code to do more stuff (save/restore FPU, update PDA, ...) */
+ call __arch_context_switch
+ /* returns with %rax set to prev */
+ movq %rax, %rdi
+ movq %gs:pda_pcurrent, %rsi
+
+ /* New tasks need to be kick-started */
+ lock btr $_TF_NEW_TASK_BIT, tsk_arch_flags(%rsi)
+ jc kickstart_new_task
+
+ /* Restore next's callee saved registers */
+ popq %r15
+ popq %r14
+ popq %r13
+ popq %r12
+ popq %rbx
+ popq %rbp
+ popf;
+
+ /* Return to context_switch(), with new task active */
+ retq
+
+kickstart_new_task:
+ call schedule_new_task_tail /* Finish up schedule(), drop locks, etc. */
+ testl $3, CS(%rsp) /* Sets ZF=1 if returning to kernel-space */
+ je 1f /* If ZF=1, leave kernel PDA in place */
+ swapgs /* Install the user PDA */
+ movq $0, %rax /* Zero all of the segment registers */
+ movl %eax, %ds
+ movl %eax, %es
+ movl %eax, %fs
+ movl %eax, %gs
+1:
+ movq (%rsp), %r15 /* Unpack the pt_regs struct that */
+ movq 1*8(%rsp), %r14 /* __arch_context_switch() put at the top */
+ movq 2*8(%rsp), %r13 /* of the new task's kernel stack. */
+ movq 3*8(%rsp), %r12
+ movq 4*8(%rsp), %rbp
+ movq 5*8(%rsp), %rbx
+ movq 6*8(%rsp), %r11
+ movq 7*8(%rsp), %r10
+ movq 8*8(%rsp), %r9
+ movq 9*8(%rsp), %r8
+ movq 10*8(%rsp), %rax
+ movq 11*8(%rsp), %rcx
+ movq 12*8(%rsp), %rdx
+ movq 13*8(%rsp), %rsi
+ movq 14*8(%rsp), %rdi
+ add $128, %rsp /* Bump to point to RIP slot in pt_regs */
+ iretq /* Start the new task running */
+
+END(arch_context_switch)
+
+
+/**
+ * This is the entry point for system calls. Upon entry we are still running
+ * with the user-level stack and the x86_64 CPU control unit has stashed the
+ * user-level RIP in RCX and RFLAGS in R11. External interrupts are diabled.
+ *
+ * The first thing this function does is generate a partial stack frame
+ * containing all caller-saved registers. After this is done, the system call
+ * number (stored in RAX by user-level) is used to index into the system call
+ * table (sys_call_table) and call the handler function. The handler function
+ * is responsible for saving all callee-saved registers... if it is a C
+ * function, callee-saved registers are saved automatically by the compiler.
+ *
+ * Immediately before calling the handler function, the kernel stack looks
+ * like:
+ *
+ * RIP = user-space RIP
+ * ORIG_RAX = system call number, passed from user-space
+ * RDI = ARG0, passed from user-space
+ * RSI = ARG1, passed from user-space
+ * RDX = ARG2, passed from user-space
+ * (junk) = normally RCX, but RCX is clobbered by SYSCALL
+ * RAX = system call number, passed from user-space
+ * R8 = ARG4, passed from user-space
+ * R9 = ARG5, passed from user-space
+ * R10 = ARG3, passed from user-space
+ * RSP -> R11 = user-space RFLAGS
+ *
+ * And the registers are setup as follows:
+ *
+ * RDI = ARG0
+ * RSI = ARG1
+ * RDX = ARG2
+ * RCX = ARG3 (was stored on R10 on entry)
+ * R8 = ARG4
+ * R9 = ARG5
+ * RAX = System call number
+ *
+ * NOTE: RCX and R11 are clobbered by system calls. This is due to the SYSCALL
+ * instruction using RCX and R11 to store RIP and RFLAGS before
+ * transfering control to the kernel. User-level will observe different
+ * values of RCX and R11 after SYSCALL than before.
+ *
+ * NOTE: External interrupts are disabled on entry.
+ */
+ENTRY(asm_syscall)
+ /*
+ * Enter from user-space
+ */
+ swapgs /* Load GS.base with kernel PDA addr */
+ movq %rsp, %gs:pda_oldrsp /* Backup user-space RSP */
+ movq %gs:pda_kernelstack, %rsp /* Load kernel stack */
+
+ /*
+ * Save registers to kernel-stack
+ */
+ subq $10*8, %rsp /* Make room on the stack */
+ movq %rcx, 10*8(%rsp) /* Save user-space RIP */
+ movq %rax, 9*8(%rsp) /* Save syscall # in ORIG_RAX slot */
+ movq %rdi, 8*8(%rsp) /* Save user-space RDI (ARG0) */
+ movq %rsi, 7*8(%rsp) /* Save user-space RSI (ARG1) */
+ movq %rdx, 6*8(%rsp) /* Save user-space RDX (ARG2) */
+ movq %rcx, 5*8(%rsp) /* RCX is clobbered, save anyways */
+ movq %rax, 4*8(%rsp) /* Save user-space RAX (syscall #) */
+ movq %r8, 3*8(%rsp) /* Save user-space R8 (ARG4) */
+ movq %r9, 2*8(%rsp) /* Save user-space R9 (ARG5) */
+ movq %r10, 1*8(%rsp) /* Save user-space R10 (ARG3) */
+ movq %r11, (%rsp) /* Save user-space RFLAGS */
+ sti /* Enable external interrupts */
+
+ /*
+ * Call the system call handler
+ */
+ movq %r10, %rcx /* Per x86_64 C ABI, RCX holds ARG3 */
+ cmp $__NR_syscall_max, %rax /* Make sure syscall # is in range */
+ jg 1f
+ call *sys_call_table(,%rax,8) /* Call the system call handler */
+ jmp 2f
+1:
+ call syscall_not_implemented /* Print error and return */
+2:
+ movq %rax, 4*8(%rsp) /* Save return code in stack frame */
+
+ /* Reschedule, since we're returning to user space */
+ call schedule
+
+ /*
+ * Return to user-space
+ */
+ cli /* Disable external interrupts */
+ movq (%rsp), %r11 /* Restore RFLAGS for SYSRET */
+ movq 1*8(%rsp), %r10 /* Restore user-space R10 (ARG3) */
+ movq 2*8(%rsp), %r9 /* Restore user-space R9 (ARG5) */
+ movq 3*8(%rsp), %r8 /* Restore user-space R8 (ARG4) */
+ movq 4*8(%rsp), %rax /* Return syscall return code */
+ movq 6*8(%rsp), %rdx /* Restore user-space RDX (ARG2) */
+ movq 7*8(%rsp), %rsi /* Restore user-space RSI (ARG1) */
+ movq 8*8(%rsp), %rdi /* Restore user-space RDI (ARG0) */
+ movq 10*8(%rsp), %rcx /* Restore RIP for SYSRET */
+ movq %gs:pda_oldrsp, %rsp /* Restore user-space RSP */
+ swapgs /* Restore user-space GS.base */
+ sysretq /* Return to user-space */
+END(asm_syscall)
+
+
+/**
+ * This is a handler for SYSCALL instructions issued from compatibility mode...
+ * we don't support them.
+ */
+ENTRY(asm_syscall_ignore)
+ mov $-ENOSYS,%eax
+ sysret
+END(asm_syscall_ignore)
+
+
+/**
+ * This is the common entry point for all interrupts.
+ *
+ * Before calling the C handler function, the kernel stack looks like:
+ *
+ * [...]
+ * SS (stack segment selector)
+ * RSP (stack pointer)
+ * RFLAGS (flags register)
+ * CS (code segment selector)
+ * RIP (instruction pointer)
+ * ERROR_CODE (0 for interrupts with no error code)
+ * RDI (this was the vector # on entry, we move to %rsi/ARG1)
+ * RSI
+ * RDX
+ * RCX
+ * RAX
+ * R8
+ * R9
+ * R10
+ * R11
+ * RBX
+ * RBP
+ * R12
+ * R13
+ * R14
+ * RSP -> R15
+ *
+ * And the registers are setup as follows:
+ *
+ * RDI = ARG0: A fully populated 'struct pt_regs *'
+ * RSI = ARG1: The interrupt vector number
+ *
+ * NOTE: External interrupts are disabled on entry.
+ */
+ENTRY(asm_interrupt)
+ cld /* Clear direction flag */
+
+ /*
+ * Save registers to kernel-stack
+ */
+ subq $14*8, %rsp /* Make room on the stack */
+ movq %rsi, 13*8(%rsp)
+ movq 14*8(%rsp), %rsi /* ARG1: the interrupt vector number */
+ movq %rdi, 14*8(%rsp)
+ movq %rdx, 12*8(%rsp)
+ movq %rcx, 11*8(%rsp)
+ movq %rax, 10*8(%rsp)
+ movq %r8, 9*8(%rsp)
+ movq %r9, 8*8(%rsp)
+ movq %r10, 7*8(%rsp)
+ movq %r11, 6*8(%rsp)
+ movq %rbx, 5*8(%rsp)
+ movq %rbp, 4*8(%rsp)
+ movq %r12, 3*8(%rsp)
+ movq %r13, 2*8(%rsp)
+ movq %r14, 1*8(%rsp)
+ movq %r15, (%rsp)
+
+ /*
+ * Load kernel GS if we're coming from user-space
+ */
+ testl $3, CS(%rsp) /* Sets ZF=1 if coming from kspace */
+ je 1f /* If ZF=1, skip installing the PDA */
+ swapgs /* Install the PDA */
+1:
+ /*
+ * Call C code interrupt handler entry point
+ */
+ movq %rsp, %rdi /* ARG0: pointer to 'struct pt_regs' */
+ sti /* Enable external interrupts */
+ call do_interrupt /* Call common C handler */
+ cli /* Disable external interrupts */
+
+ /*
+ * If returning to user-space, reschedule and restore user-space GS
+ */
+ testl $3, CS(%rsp) /* Sets ZF=1 if returning to kspace */
+ je 2f /* If ZF=1, jump forward to 2: below */
+ sti /* Enable external interrupts */
+ call schedule /* Reschedule */
+ cli /* Disable external interrupts */
+ swapgs /* Restore uspace GS register */
+2:
+ /*
+ * Restore registers and return to interrupted program
+ */
+ movq (%rsp), %r15
+ movq 1*8(%rsp), %r14
+ movq 2*8(%rsp), %r13
+ movq 3*8(%rsp), %r12
+ movq 4*8(%rsp), %rbp
+ movq 5*8(%rsp), %rbx
+ movq 6*8(%rsp), %r11
+ movq 7*8(%rsp), %r10
+ movq 8*8(%rsp), %r9
+ movq 9*8(%rsp), %r8
+ movq 10*8(%rsp), %rax
+ movq 11*8(%rsp), %rcx
+ movq 12*8(%rsp), %rdx
+ movq 13*8(%rsp), %rsi
+ movq 14*8(%rsp), %rdi
+ addq $16*8, %rsp
+ iretq
+END(asm_interrupt)
+
+
+/**
+ * This table contains an initial entry point function for each IDT vector.
+ * When an interrupt vector fires, the first instruction executed is at
+ * table[vector].
+ *
+ * This table scheme is necessary because some x86_64 interrupts push an
+ * error code onto the stack and others do not. Additionally, there is no way
+ * for an interrupt handler to determine the interrupt vector that triggered
+ * it. Therefore, the functions in this table push a dummy error code onto
+ * the stack when necessary, always push the vector number, and then call a
+ * common handler (asm_interrupt).
+ *
+ * WARNING: Each function/entry in this table must be <= 16 bytes.
+ * Be very careful when adding instructions.
+ */
+.align 16
+ENTRY(asm_idtvec_table)
+ vector=0
+ .rept NUM_IDT_ENTRIES
+ .if vector<=7||vector==9||vector==15||vector==16||vector>=18
+ pushq $0 /* push dummy error_code */
+ .endif
+ pushq $vector /* push vector # into RDI slot */
+ jmp asm_interrupt /* call common handler */
+
+ /* Move onto next entry in table*/
+ .align 16
+ vector=vector+1
+ .endr
+END(asm_idtvec_table)
+
+
+/**
+ * Reload gs selector with exception handling.
+ * edi: new selector
+ */
+ENTRY(load_gs_index)
+ CFI_STARTPROC
+ pushf
+ CFI_ADJUST_CFA_OFFSET 8
+ cli
+ swapgs
+gs_change:
+ movl %edi,%gs
+2: mfence /* workaround */
+ swapgs
+ popf
+ CFI_ADJUST_CFA_OFFSET -8
+ ret
+ CFI_ENDPROC
+ENDPROC(load_gs_index)
+
+ .section __ex_table,"a"
+ .align 8
+ .quad gs_change,bad_gs
+ .previous
+ .section .fixup,"ax"
+ /* running with kernelgs */
+bad_gs:
+ swapgs /* switch back to user gs */
+ xorl %eax,%eax
+ movl %eax,%gs
+ jmp 2b
+ .previous
+
+
