selftest: support x2apic and tsc_deadline MSRs

All selftests should now be compatible with both legacy xapic MMIO and x2apic
MSR addressing modes via libsgxstep support for privileged rdmsr/wrmsr call
gates.

cf issue #72

app/selftest/README.md

@@ -6,11 +6,11 @@ This directory contains bare-metal applications designed to test various functio
 
 | Directory  | Description                                                                                               |
 |------------|-----------------------------------------------------------------------------------------------------------|
-| `apic`     | Microbenchmarks for timer interrupts via various Advanced Programmable Interrupt Controller (APIC) modes. |
-| `apic-ipi` | Microbenchmarks for APIC inter-processor interrupts.                                                      |
-| `cpl`      | Changing the Current Privilege Level (CPL) through calland interrupt GDT/IDT gates.                       |
+| `cpl`      | Changing the Current Privilege Level (CPL) through call and interrupt GDT/IDT gates.                      |
 | `idt`      | Custom handlers for software and timer interrupts via the Interrupt Descriptor Table (IDT).               |
 | `idt_isr_map` | Stresstest to demonstrate custom IDT handlers can be called across processes.                          |
+| `apic`     | Microbenchmarks for timer interrupts via various Advanced Programmable Interrupt Controller (APIC) modes. |
+| `apic-ipi` | Microbenchmarks for APIC inter-processor interrupts.                                                      |
 
 ## Usage
 
@@ -28,4 +28,5 @@ To run the selftest applications:
 
 Reference output is available in the respective README files within each subdirectory.
 
-**Note:** Any crashes during execution may indicate that your machine is not set up correctly for using SGX-Step. 
+**Note:** For troubleshooting, recommended is to execute the selftests in the order specified in the table above. Any crashes or assertion errors during execution may indicate that your machine is not set up correctly for using SGX-Step.
+

app/selftest/apic-ipi/asm.S

@@ -1,11 +1,21 @@
 #define APIC_ICR_LOW                0x300
 #define APIC_ICR_HIGH               0x310
+#define APIC_ICR_MSR                0x830
+#include "libsgxstep/config.h"
 
 .macro apic_write_icr
+#if !X2APIC
     mov apic_base(%rip), %rax
     movl %esi, APIC_ICR_HIGH(%rax)
     /* SDM: The act of writing to the low doubleword of the ICR causes the IPI to be sent. */
     movl %edi, APIC_ICR_LOW(%rax)
+#else /* X2APIC */
+    /* SDM: A single MSR write to the Interrupt Command Register is required for dispatching an interrupt in x2APIC mode. */
+    shl $32, %rsi
+    or  %rdi, %rsi
+    mov $APIC_ICR_MSR, %rdi
+    int $WRMSR_GATE_VECTOR
+#endif /* X2APIC */
 .endm
 
     .text

app/selftest/apic-ipi/main.c

@@ -15,8 +15,8 @@
 #define SELF_IPI        0
 
 #define MY_VICTIM_CPU   1
-//#define MY_SPY_CPU      3       // no hyperthreading
-#define MY_SPY_CPU      37       // hyperthreading
+#define MY_SPY_CPU      3       // no hyperthreading
+//#define MY_SPY_CPU      37       // hyperthreading
 
 #if SELF_IPI
     #define DUMP_FILE       "ipi-self.csv"
@@ -63,7 +63,12 @@ void spy_func(int eid)
             apic_write_icr_ret(APIC_ICR_VECTOR(IRQ_VECTOR) |
                                APIC_ICR_DELIVERY_FIXED | APIC_ICR_LEVEL_ASSERT |
                                APIC_ICR_DEST_PHYSICAL,
-                               (apic_id_victim << APIC_ID_SHIFT) & APIC_ICR_DEST_MASK);
+                               #if !X2APIC
+                                 (apic_id_victim << APIC_ID_SHIFT) & APIC_ICR_DEST_MASK
+                               #else
+                                 apic_id_victim
+                               #endif
+                              );
 
             spy_ready = 0;
             while( !__ss_irq_fired );

app/selftest/apic/Makefile

@@ -71,7 +71,7 @@ run: clean all
 	cat out.txt
 
 parse: run
-	SGX_STEP_PLATFORM=$(SGX_STEP_PLATFORM) ./parse_$(PARSE).py $(NUM)
+	./parse.py
 
 $(OUTPUT): $(BUILDDIRS) $(OBJECTS)
 	echo "$(INDENT)[LD]" $(OBJECTS) $(LIBS) -o $(OUTPUT) 

app/selftest/apic/config.h

@@ -5,13 +5,10 @@
 
 #define TEST_ITERATIONS                 100001
 #define APIC_ONESHOT_TICKS              30*2	// NOTE: sgx-step uses APIC timer divide 2(!)
-#define APIC_TSC_INTERVAL               5750
+#define CPU_TSC_CLOCK_RATIO             256     // as reported by CPUID
+#define APIC_TSC_INTERVAL               (APIC_ONESHOT_TICKS * CPU_TSC_CLOCK_RATIO)
 
 #define USE_IRQ_GATE                    1
 #define WRMSR_ON_CPU_LATENCY            0 	// is about 8000
 
-#ifndef IA32_TSC_DEADLINE_MSR
-    #define IA32_TSC_DEADLINE_MSR       0x6e0
-#endif
-
 #endif

app/selftest/apic/main.c

@@ -36,12 +36,8 @@ unsigned long long test_tsc_results[TEST_ITERATIONS];
 unsigned long long test_inc_results[TEST_ITERATIONS];
 unsigned long long test_deadline_results[TEST_ITERATIONS];
 
+extern uint64_t __ss_irq_rax;
 void incs(void);
-void __apic_irq_handler_timer(void);
-void __apic_priv_irq_gate(void);
-extern unsigned long long __apic_irq_tsc;
-extern unsigned long long __apic_deadline_tsc;
-extern unsigned long long __apic_irq_rax;
 
 /* ================== ATTACKER INIT/SETUP ================= */
 
@@ -68,15 +64,11 @@ int main( int argc, char **argv )
     idt_t idt = {0};
     attacker_config_runtime();
     map_idt(&idt);
-    install_kernel_irq_handler(&idt, __apic_irq_handler_timer, IRQ_VECTOR);
+    install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR);
 
     #if TSC_DEADLINE
         const char *filename = "deadline_results.txt";
         apic_timer_deadline(IRQ_VECTOR);
-
-        #if USE_IRQ_GATE
-            install_kernel_irq_handler(&idt, __apic_priv_irq_gate, IRQ_PRIV_VECTOR);
-        #endif
     #else
         const char *filename = "oneshot_results.txt";
         apic_timer_oneshot(IRQ_VECTOR);
@@ -85,32 +77,31 @@ int main( int argc, char **argv )
     for (int i = 0; i < TEST_ITERATIONS; ++i)
     {
         __ss_irq_fired = 0;
-        unsigned long long begin_time = __rdtsc();
+        g_apic_deadline_tsc_begin = rdtsc_begin();
 
         #if TSC_DEADLINE && USE_IRQ_GATE
-            __apic_deadline_tsc = begin_time + APIC_TSC_INTERVAL;
-            asm("int %0\n\t" ::"i"(IRQ_PRIV_VECTOR):);
+            apic_timer_deadline_irq(APIC_TSC_INTERVAL);
         #elif TSC_DEADLINE
-            __apic_deadline_tsc = begin_time + APIC_TSC_INTERVAL + WRMSR_ON_CPU_LATENCY;
-            wrmsr_on_cpu(IA32_TSC_DEADLINE_MSR, get_cpu(), __apic_deadline_tsc);
+            wrmsr_on_cpu(IA32_TSC_DEADLINE_MSR, get_cpu(),
+                         g_apic_deadline_tsc_begin + APIC_TSC_INTERVAL + WRMSR_ON_CPU_LATENCY);
             //unsigned long long wrmsr_time = __rdtsc();
         #else
             apic_timer_irq(APIC_ONESHOT_TICKS);
         #endif
 
         incs();
-        ASSERT(__ss_irq_fired);
+        ASSERT(__ss_irq_fired == 1);
 
-	test_tsc_results[i] = __apic_irq_tsc - begin_time;
-	test_inc_results[i] = __apic_irq_rax;
-        test_deadline_results[i] = __apic_irq_tsc - __apic_deadline_tsc;
+	test_tsc_results[i] = nemesis_tsc_aex - g_apic_deadline_tsc_begin ;
+	test_inc_results[i] = __ss_irq_rax;
+        test_deadline_results[i] = nemesis_tsc_aex - (g_apic_deadline_tsc_begin + APIC_TSC_INTERVAL);
     }
 
     // record results
     FILE *fp = fopen(filename, "w");
     fprintf(fp, "#tsc_diff,inc_count,tsc_deadline_drift\n");
     for (int i = 1; i < TEST_ITERATIONS; ++i) {
-        fprintf(fp, "%llu,%llu,%llu\n", test_tsc_results[i], test_inc_results[i], test_deadline_results[i]);
+        fprintf(fp, "%llu,%llu,%lld\n", test_tsc_results[i], test_inc_results[i], test_deadline_results[i]);
     }
     fclose(fp);
 

app/selftest/cpl/README.md

@@ -1,67 +1,67 @@
 Sample output:
 
 ```
-[idt.c] locking IRQ handlers..
 [sched.c] continuing on CPU 1
 
 --------------------------------------------------------------------------------
 [main.c] Establishing user space GDT mapping
 --------------------------------------------------------------------------------
 
-[gdt.c] DTR.base=0xfffffe0000034000/size=127 (16 entries)
+[gdt.c] DTR.base=0xfffffe45202b3000/size=127 (16 entries)
 [pt.c] /dev/sgx-step opened!
 [pt.c] /dev/mem opened!
-[gdt.c] established user space GDT mapping at 0x7faee1a13000
+[gdt.c] established user space GDT mapping at 0x7f1924083000
 [gdt.c] --------------------------------------------------------------------------------
-[gdt.c] GDT[  0] @0x7faee1a13000=0x00000000 / 0x00000 (seg sel 0x00); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[  1] @0x7faee1a13008=0x00000000 / 0xfffff (seg sel 0x08); p=1; dpl=0; type=11; g=1
-[gdt.c] GDT[  2] @0x7faee1a13010=0x00000000 / 0xfffff (seg sel 0x10); p=1; dpl=0; type=11; g=1
-[gdt.c] GDT[  3] @0x7faee1a13018=0x00000000 / 0xfffff (seg sel 0x18); p=1; dpl=0; type= 3; g=1
-[gdt.c] GDT[  4] @0x7faee1a13020=0x00000000 / 0xfffff (seg sel 0x23); p=1; dpl=3; type=11; g=1
-[gdt.c] GDT[  5] @0x7faee1a13028=0x00000000 / 0xfffff (seg sel 0x2b); p=1; dpl=3; type= 3; g=1
-[gdt.c] GDT[  6] @0x7faee1a13030=0x00000000 / 0xfffff (seg sel 0x33); p=1; dpl=3; type=11; g=1
-[gdt.c] GDT[  7] @0x7faee1a13038=0x00000000 / 0x00000 (seg sel 0x38); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[  8] @0x7faee1a13040=0x00036000 / 0x0206f (seg sel 0x40); p=1; dpl=0; type=11; g=0
-[gdt.c] GDT[  9] @0x7faee1a13048=0x0000ffff / 0x0fe00 (seg sel 0x48); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[ 10] @0x7faee1a13050=0x00000000 / 0x00000 (seg sel 0x50); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[ 11] @0x7faee1a13058=0x00000000 / 0x00000 (seg sel 0x58); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[ 12] @0x7faee1a13060=0x00000000 / 0x00000 (seg sel 0x60); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[ 13] @0x7faee1a13068=0x00000000 / 0x00000 (seg sel 0x68); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[ 14] @0x7faee1a13070=0x00000000 / 0x00000 (seg sel 0x70); p=0; dpl=0; type= 0; g=0
-[gdt.c] GDT[ 15] @0x7faee1a13078=0x00000000 / 0x00001 (seg sel 0x7b); p=1; dpl=3; type= 5; g=0
+[gdt.c] GDT[  0] @0x7f1924083000=0x00000000 / 0x00000 (seg sel 0x00); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[  1] @0x7f1924083008=0x00000000 / 0xfffff (seg sel 0x08); p=1; dpl=0; type=11; g=1
+[gdt.c] GDT[  2] @0x7f1924083010=0x00000000 / 0xfffff (seg sel 0x10); p=1; dpl=0; type=11; g=1
+[gdt.c] GDT[  3] @0x7f1924083018=0x00000000 / 0xfffff (seg sel 0x18); p=1; dpl=0; type= 3; g=1
+[gdt.c] GDT[  4] @0x7f1924083020=0x00000000 / 0xfffff (seg sel 0x23); p=1; dpl=3; type=11; g=1
+[gdt.c] GDT[  5] @0x7f1924083028=0x00000000 / 0xfffff (seg sel 0x2b); p=1; dpl=3; type= 3; g=1
+[gdt.c] GDT[  6] @0x7f1924083030=0x00000000 / 0xfffff (seg sel 0x33); p=1; dpl=3; type=11; g=1
+[gdt.c] GDT[  7] @0x7f1924083038=0x00000000 / 0x00000 (seg sel 0x38); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[  8] @0x7f1924083040=0x202b5000 / 0x04087 (seg sel 0x40); p=1; dpl=0; type=11; g=0
+[gdt.c] GDT[  9] @0x7f1924083048=0x0000ffff / 0x0fe45 (seg sel 0x48); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[ 10] @0x7f1924083050=0x00000000 / 0x00000 (seg sel 0x50); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[ 11] @0x7f1924083058=0x00000000 / 0x00000 (seg sel 0x58); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[ 12] @0x7f1924083060=0x00000000 / 0x00000 (seg sel 0x60); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[ 13] @0x7f1924083068=0x00000000 / 0x00000 (seg sel 0x68); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[ 14] @0x7f1924083070=0x00000000 / 0x00000 (seg sel 0x70); p=0; dpl=0; type= 0; g=0
+[gdt.c] GDT[ 15] @0x7f1924083078=0x00000000 / 0x00001 (seg sel 0x7b); p=1; dpl=3; type= 5; g=0
 [gdt.c] --------------------------------------------------------------------------------
 
 --------------------------------------------------------------------------------
 [main.c] Installing and calling ring0 call gate
 --------------------------------------------------------------------------------
 
-[idt.c] IDT[ 13] @0x7faee1a13068 = 0x55f1625064d7 (seg sel 0x10); p=1; dpl=3; type=12; ist=0
-[main.c] back from call gate w CPL prev/cur=0/3; RDMSR=0xfee00800
+[idt.c] IDT[ 13] @0x7f1924083068 = 0x557a76cc84bc (seg sel 0x10); p=1; dpl=3; type=12; ist=0
+[main.c] back from call gate w CPL prev/cur=0/3; RDMSR=0x2
 
 --------------------------------------------------------------------------------
 [main.c] Establishing user space IDT mapping
 --------------------------------------------------------------------------------
 
 [idt.c] DTR.base=0xfffffe0000000000/size=4095 (256 entries)
-[idt.c] established user space IDT mapping at 0x7faee1a12000
+[idt.c] established user space IDT mapping at 0x7f1924082000
 
 --------------------------------------------------------------------------------
 [main.c] Installing and calling ring0 irq gate
 --------------------------------------------------------------------------------
 
-[idt.c] installed asm IRQ handler at 10:0x55f1625064de
-[idt.c] IDT[ 45] @0x7faee1a122d0 = 0x55f1625064de (seg sel 0x10); p=1; dpl=3; type=14; ist=0
-[main.c] back from irq gate w CPL prev/cur=0/3; RDMSR=0xfee00800
+[idt.c] installed asm IRQ handler at 10:0x557a76cc84c3
+[idt.c] IDT[ 45] @0x7f19240822d0 = 0x557a76cc84c3 (seg sel 0x10); p=1; dpl=3; type=14; ist=0
+[main.c] back from irq gate w CPL prev/cur=0/3; RDMSR=0x2
 
 --------------------------------------------------------------------------------
 [main.c] Calling ring0 function on user page with `exec_priv()`
 --------------------------------------------------------------------------------
 
-[idt.c] installing and calling ring0 irq gate
+[idt.c] locking user-space IRQ gate handler page at 0x557a76ccb000
+[idt.c] installing ring-0 IRQ gate
 [sched.c] continuing on CPU 1
 [idt.c] DTR.base=0xfffffe0000000000/size=4095 (256 entries)
-[idt.c] established user space IDT mapping at 0x7faee1a11000
-[idt.c] installed asm IRQ handler at 10:0x55f16250905a
-[idt.c] IDT[ 49] @0x7faee1a11310 = 0x55f16250905a (seg sel 0x10); p=1; dpl=3; type=15; ist=0
-[main.c] back from my_ring0_func w CPL=0 and IA32_TIME_STAMP_COUNTER=0x9e6bb6308a6/0x9e6bb630a8e/488
+[idt.c] established user space IDT mapping at 0x7f1924081000
+[idt.c] installed asm IRQ handler at 10:0x557a76ccb000
+[idt.c] IDT[ 49] @0x7f1924081310 = 0x557a76ccb000 (seg sel 0x10); p=1; dpl=3; type=14; ist=0
+[main.c] back from my_ring0_func w CPL prev/cur=0/3; RDMSR=0x2
 ```

app/selftest/cpl/asm.S

@@ -1,17 +1,19 @@
 #define IA32_TIME_STAMP_COUNTER 0x10
-#define IA32_APIC_BASE          0x1b
+#define IA32_APIC_ID            0x802
 
     .text
+    .global do_rdmsr
 do_rdmsr:
     # store CPL (i.e., ring 0)
     call get_cpl
     mov %rax, gate_cpl(%rip)
 
     # store privileged RDMSR result
-    mov $IA32_APIC_BASE, %rdi
-    call rdmsr
+    mov $IA32_APIC_ID, %ecx
+    rdmsr
+    shl $32, %rdx
+    or  %rdx, %rax
     mov %rax, gate_msr(%rip)
-
     ret
 
     .global call_gate_func

app/selftest/cpl/main.c

@@ -30,20 +30,19 @@
 
 int gate_cpl = -1;
 uint64_t gate_msr = -1;
+uint64_t my_msr = -1;
 void call_gate_func(void);
 void irq_gate_func(void);
+uint64_t do_rdmsr(void);
 
 int my_cpl = -1;
-uint64_t my_tsc1 = -1;
-uint64_t my_tsc2 = -1;
 
 /* ------------------------------------------------------------ */
 /* This code will execute with ring0 privileges :) */
 void my_ring0_func(void)
 {
-    my_tsc1 = rdmsr(IA32_TIME_STAMP_COUNTER);
     my_cpl = get_cpl();
-    my_tsc2 = rdmsr(IA32_TIME_STAMP_COUNTER);
+    my_msr = do_rdmsr();
 }
 /* ------------------------------------------------------------ */
 
@@ -85,8 +84,8 @@ int main( int argc, char **argv )
 
     info_event("Calling ring0 function on user page with `exec_priv()`");
     exec_priv(my_ring0_func);
-    info("back from my_ring0_func w CPL=%d and IA32_TIME_STAMP_COUNTER=%p/%p/%d",
-            my_cpl, my_tsc1, my_tsc2, (int) my_tsc2-my_tsc1);
+    info("back from my_ring0_func w CPL prev/cur=%d/%d; RDMSR=%p",
+        my_cpl, get_cpl(), my_msr);
 #endif
 
     return 0;