app/apic: Add APIC precision microbenchmark experiments

As described in Appendix A of the AEX-Notify paper.

app/apic/.gitignore

@@ -0,0 +1,4 @@
+*.txt
+*.pdf
+log
+copy_icx.sh

app/apic/Makefile

@@ -0,0 +1,98 @@
+LIBSGXSTEP_DIR       = ../..
+LIBSGXSTEP           = $(LIBSGXSTEP_DIR)/libsgxstep
+-include $(LIBSGXSTEP)/Makefile.config
+
+LIBSGXSTEP_SILENT    = 1
+
+ifeq ($(SGX_SDK),)
+    SGX_SDK          = /opt/intel/sgxsdk
+endif
+export SGX_SDK
+ifneq ($(SGX_SDK), /opt/intel/sgxsdk)
+    URTS_LD_LIBRARY_PATH = LD_LIBRARY_PATH=$(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux
+endif
+
+SUBDIRS              = $(LIBSGXSTEP)
+
+CC                   = gcc
+AS                   = gcc
+LD                   = gcc
+
+ifeq ($(M32), 1)
+	ASFLAGS  = -m32 -DM32=$(M32)
+	CFLAGS   = -m32 -DM32=$(M32)
+	LDFLAGS  = -m32
+else
+	LIB_SUFX = 64
+endif
+
+CFLAGS              += -fPIC -fno-stack-protector -fno-builtin -fno-jump-tables \
+                       -fno-common -Wno-attributes -g -D_GNU_SOURCE -O0
+INCLUDE              = -I$(SGX_SDK)/include/ -I$(LIBSGXSTEP_DIR)
+LDFLAGS             += -lsgx-step -lsgx_urts \
+                       -lsgx_uae_service -pthread $(SUBDIRS:%=-L %) -L$(SGX_SDK)/lib$(LIB_SUFX)/ \
+                       -L$(LIBSGXSTEP_DIR)/linux-sgx/psw/urts/linux
+
+C_SOURCES            = $(shell ls *.c)
+ASM_SOURCES          = $(shell ls *.S)
+C_OBJECTS            = $(C_SOURCES:.c=.o)
+ASM_OBJECTS          = $(ASM_SOURCES:.S=.o)
+OBJECTS              = $(C_OBJECTS) $(ASM_OBJECTS)
+OUTPUT               = app
+
+BUILDDIRS            = $(SUBDIRS:%=build-%)
+CLEANDIRS            = $(SUBDIRS:%=clean-%)
+
+ATTACK = 1
+PARSE  = nop
+ifeq ($(STRLEN), 1)
+    ATTACK = 2
+    PARSE  = strlen
+endif
+ifeq ($(ZIGZAG), 1)
+    ATTACK = 3
+    PARSE  = zz
+endif
+
+ifeq ($(NUM),)
+    NUM             = 100
+endif
+export NUM
+
+CFLAGS              += -DATTACK_SCENARIO=$(ATTACK) -DNUM_RUNS=$(NUM)
+
+MAKEFLAGS           += --silent
+
+#.SILENT:
+all: $(OUTPUT)
+
+run: clean all
+	sudo $(URTS_LD_LIBRARY_PATH) ./app > out.txt
+	cat out.txt
+
+parse: run
+	SGX_STEP_PLATFORM=$(SGX_STEP_PLATFORM) ./parse_$(PARSE).py $(NUM)
+
+$(OUTPUT): $(BUILDDIRS) $(OBJECTS)
+	echo "$(INDENT)[LD]" $(OBJECTS) $(LIBS) -o $(OUTPUT) 
+	$(LD) $(OBJECTS) $(LDFLAGS) -o $(OUTPUT) 
+
+%.o : %.c
+	echo "$(INDENT)[CC] " $<
+	$(CC) $(CFLAGS) $(INCLUDE) -c $<
+
+%.o : %.S
+	echo "$(INDENT)[AS] " $<
+	$(AS) $(ASFLAGS) $(INCLUDE) -c $< -o $@
+
+clean: $(CLEANDIRS)
+	echo "$(INDENT)[RM]" $(OBJECTS) $(OUTPUT)
+	rm -f $(OBJECTS) $(OUTPUT)
+
+$(BUILDDIRS):
+	echo "$(INDENT)[===] $(@:build-%=%) [===]"
+	$(MAKE) -C $(@:build-%=%) INDENT+="$(INDENT_STEP)" M32=$(M32) curr-dir=$(curr-dir)/$(@:build-%=%)
+
+$(CLEANDIRS):
+	echo "$(INDENT)[===] $(@:clean-%=%) [===]"
+	$(MAKE) clean -C $(@:clean-%=%) INDENT+="$(INDENT_STEP)" curr-dir=$(curr-dir)/$(@:build-%=%)

app/apic/README.md

@@ -0,0 +1,30 @@
+# APIC Precision Microbenchmarks
+
+This directory contains the experiments to determine the accuracy of the local
+APIC timer, as described in Appendix A of the [AEX-Notify
+paper](https://jovanbulck.github.io/files/usenix23-aexnotify.pdf).
+
+The local APIC timer can be configured in one-shot or periodic mode to send an
+interrupt when an MMIO counter register reaches zero, or in TSC-deadline mode
+when the processor's timestamp counter exceeds a value specified in the
+dedicated `IA32_TSC_DEADLINE_MSR` model-specific register.  Depending on the
+processor model, the APIC timer operates at the frequency of either the
+processor’s bus clock or its core crystal clock. Neither time source is
+synchronized with the core clock, which typically operates at a much higher
+frequency (e.g., more than 100x higher).
+
+**TSC-Deadline Mode.** As a contribution of independent interest, we devised an
+improved setup to considerably reduce the noise from additional kernel context
+switches. For this, we registered a custom ring-0 interrupt gate in the
+processor's interrupt-descriptor table. This allows to essentially bypass the
+OS kernel altogether by directly invoking a minimal assembly handler that
+programs a specified value in `IA32_TSC_DEADLINE_MSR` using the privileged
+`WRMSR` instruction via a software interrupt. 
+
+**Comparison Results.** We conclude that our novel interrupt-gate TSC-deadline
+timer configuration technique yields a considerable improvement in terms of
+standard deviation of elapsed cycles for both the existing one-shot technique
+that SGX-Step currently uses, as well as naive kernel-level configuration
+approaches (cf. as is also visually evident in the figure below)
+
+![APIC distribution histogram](apic-hist.png)

app/apic/asm.S

@@ -0,0 +1,15 @@
+.text
+.global incs
+incs:
+    xor %rax, %rax
+    .rept 1000000
+       inc %rax 
+    .endr
+    ret
+
+.data
+.align 0x1000
+.global my_page
+my_page:
+.word 0xdead
+.space 0x1000

app/apic/config.h

@@ -0,0 +1,17 @@
+#ifndef CONFIG_H_INC
+#define CONFIG_H_INC
+
+#define TSC_DEADLINE                    1
+
+#define TEST_ITERATIONS                 100001
+#define APIC_ONESHOT_TICKS              30*2	// NOTE: sgx-step uses APIC timer divide 2(!)
+#define APIC_TSC_INTERVAL               5750
+
+#define USE_IRQ_GATE                    1
+#define WRMSR_ON_CPU_LATENCY            0 	// is about 8000
+
+#ifndef IA32_TSC_DEADLINE_MSR
+    #define IA32_TSC_DEADLINE_MSR       0x6e0
+#endif
+
+#endif

app/apic/irq_entry.S

@@ -0,0 +1,58 @@
+#include "config.h"
+
+/* ********************************************************************** */
+    .section isr_section,"awx",@progbits
+    .global __apic_irq_tsc, __apic_irq_rax, __apic_deadline_tsc
+__apic_irq_tsc:
+    .quad 0x0
+__apic_deadline_tsc:
+    .quad 0x0
+__apic_irq_rax:
+    .quad 0x0
+__apic_irq_rcx:
+    .quad 0x0
+__apic_irq_rdx:
+    .quad 0x0
+
+/* ********************************************************************** */
+    .section isr_section,"awx",@progbits
+    .global __apic_irq_handler_timer
+__apic_irq_handler_timer:
+    mov %rax, __apic_irq_rax(%rip)
+    mov %rdx, __apic_irq_rdx(%rip)
+    rdtsc
+    mov %eax, __apic_irq_tsc(%rip)
+    mov %edx, __apic_irq_tsc+4(%rip)
+
+    /* IRQ bookkeeping */
+    incl __ss_irq_fired(%rip)
+
+    /* apic_write(APIC_EOI, 0x0); */
+    lea apic_base(%rip), %rax
+    mov (%rax),%rax
+    test %rax, %rax
+    jz 1f
+    add $0xb0, %rax
+    movl $0x0, (%rax)
+1:
+    mov __apic_irq_rax(%rip), %rax
+    mov __apic_irq_rdx(%rip), %rdx
+    iretq
+
+    .section isr_section,"awx",@progbits
+    .global __apic_priv_irq_gate
+__apic_priv_irq_gate:
+    push %rax
+    push %rcx
+    push %rdx
+
+    /* wrmsr(IA32_TSC_DEADLINE_MSR, __apic_deadline_tsc) */
+    mov __apic_deadline_tsc(%rip), %eax
+    mov __apic_deadline_tsc+4(%rip), %edx
+    mov $IA32_TSC_DEADLINE_MSR, %ecx
+    wrmsr
+
+    pop %rdx
+    pop %rcx
+    pop %rax
+    iretq

app/apic/main.c

@@ -0,0 +1,118 @@
+/*
+ *  This file is part of the SGX-Step enclave execution control framework.
+ *
+ *  Copyright (C) 2017 Jo Van Bulck <[email protected]>,
+ *                     Raoul Strackx <[email protected]>
+ *
+ *  SGX-Step is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  SGX-Step is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with SGX-Step. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <signal.h>
+#include <unistd.h>
+#include <x86intrin.h>
+#include "libsgxstep/apic.h"
+#include "libsgxstep/cpu.h"
+#include "libsgxstep/pt.h"
+#include "libsgxstep/sched.h"
+#include "libsgxstep/enclave.h"
+#include "libsgxstep/debug.h"
+#include "libsgxstep/config.h"
+#include "libsgxstep/idt.h"
+#include "libsgxstep/config.h"
+#include "config.h"
+
+unsigned long long test_tsc_results[TEST_ITERATIONS];
+unsigned long long test_inc_results[TEST_ITERATIONS];
+unsigned long long test_deadline_results[TEST_ITERATIONS];
+
+void incs(void);
+void __apic_irq_handler_timer(void);
+void __apic_priv_irq_gate(void);
+extern unsigned long long __apic_irq_tsc;
+extern unsigned long long __apic_deadline_tsc;
+extern unsigned long long __apic_irq_rax;
+
+/* ================== ATTACKER INIT/SETUP ================= */
+
+/* Configure and check attacker untrusted runtime environment. */
+void attacker_config_runtime(void)
+{
+    ASSERT( !claim_cpu(VICTIM_CPU) );
+    ASSERT( !prepare_system_for_benchmark(PSTATE_PCT) );
+    print_system_settings();
+
+    if (isatty(fileno(stdout)))
+    {
+        info("WARNING: interactive terminal detected; known to cause ");
+        info("unstable timer intervals! Use stdout file redirection for ");
+        info("precise single-stepping results...");
+    }
+}
+
+/* ================== ATTACKER MAIN ================= */
+
+/* Untrusted main function to create/enter the trusted enclave. */
+int main( int argc, char **argv )
+{
+    idt_t idt = {0};
+    attacker_config_runtime();
+    map_idt(&idt);
+    install_kernel_irq_handler(&idt, __apic_irq_handler_timer, IRQ_VECTOR);
+
+    #if TSC_DEADLINE
+        const char *filename = "deadline_results.txt";
+        apic_timer_deadline(IRQ_VECTOR);
+
+        #if USE_IRQ_GATE
+            install_kernel_irq_handler(&idt, __apic_priv_irq_gate, IRQ_PRIV_VECTOR);
+        #endif
+    #else
+        const char *filename = "oneshot_results.txt";
+        apic_timer_oneshot(IRQ_VECTOR);
+    #endif
+
+    for (int i = 0; i < TEST_ITERATIONS; ++i)
+    {
+        __ss_irq_fired = 0;
+        unsigned long long begin_time = __rdtsc();
+
+        #if TSC_DEADLINE && USE_IRQ_GATE
+            __apic_deadline_tsc = begin_time + APIC_TSC_INTERVAL;
+            asm("int %0\n\t" ::"i"(IRQ_PRIV_VECTOR):);
+        #elif TSC_DEADLINE
+            __apic_deadline_tsc = begin_time + APIC_TSC_INTERVAL + WRMSR_ON_CPU_LATENCY;
+            wrmsr_on_cpu(IA32_TSC_DEADLINE_MSR, get_cpu(), __apic_deadline_tsc);
+            //unsigned long long wrmsr_time = __rdtsc();
+        #else
+            apic_timer_irq(APIC_ONESHOT_TICKS);
+        #endif
+
+        incs();
+        ASSERT(__ss_irq_fired);
+
+	test_tsc_results[i] = __apic_irq_tsc - begin_time;
+	test_inc_results[i] = __apic_irq_rax;
+        test_deadline_results[i] = __apic_irq_tsc - __apic_deadline_tsc;
+    }
+
+    // record results
+    FILE *fp = fopen(filename, "w");
+    fprintf(fp, "#tsc_diff,inc_count,tsc_deadline_drift\n");
+    for (int i = 1; i < TEST_ITERATIONS; ++i) {
+        fprintf(fp, "%llu,%llu,%llu\n", test_tsc_results[i], test_inc_results[i], test_deadline_results[i]);
+    }
+    fclose(fp);
+
+    return 0;
+}