libsgxstep/aep: Trap ENCLU via LD_LIBRARY_PATH

Trap any ENCLU occurences in shared libraries before loading using
LD_LIBRARY_PATH and redirect them transparently to SGX-Step at runtime (cf.
issue #28).

This should allow to get rid of the custom patched SDK dependency and make
SGX-Step inherently SDK-agnostic. In principle, SGX-Step could in this way also
attached as a LD_PRELOAD shared library to a precompiled SGX application (eg in
case of a libOS like Gramine).

app/aep-redirect/Makefile

@@ -44,7 +44,8 @@ MAKEFLAGS           += --silent
 all: $(OUTPUT)
 
 run: clean all
-	sudo $(URTS_LD_LIBRARY_PATH) ./$(OUTPUT)
+	./patch.sh ./$(OUTPUT)
+	sudo LD_LIBRARY_PATH=./patched-libs ./$(OUTPUT)
 
 debug: clean all
 	sudo `which sgx-gdb` ./$(OUTPUT)

app/aep-redirect/patch.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# ENCLU        = 0F 01 D7
+# UD2; NOP     = 0F 0B 90
+# INT3;NOP;NOP = CC 90 90
+enclu_replacement='\x0f\x0b\x90'
+enclu_replacement='\xcc\x90\x90'
+patch_dir="./patched-libs"
+
+if [[ -z $1 ]];
+then 
+    echo "Usage:" $0 "./app"
+    exit 1
+fi
+
+printf ".. parsing shared library dependencies for '%s'\n" $1
+SHLIBS=$(ldd $1 | sed -En 's/(.*) => (.*) \(.*\)/\2/p')
+
+for lib in $SHLIBS
+do
+    printf ".. checking shared library dependency %-50s" $lib
+    ENCLU=$(objdump -d $lib | grep enclu | sed -En 's/\s*([[:xdigit:]]*):.*/\1/p')
+    if [[ -z $ENCLU ]]; then
+        echo -e " [OK]"
+    else
+        echo -e " [Found ENCLU]"
+
+        echo -e "\t.. creating local patched library copy"
+        mkdir -p $patch_dir
+        cp $lib $patch_dir
+        dest_lib=$patch_dir/$(basename $lib)
+
+        for hexAdrs in $ENCLU
+        do
+            decAdrs=$(( 16#$hexAdrs ))
+            printf "\t.. patching enclu at adrs 0x%s (decimal %d)\n" $hexAdrs $decAdrs
+            printf $enclu_replacement | dd of=$dest_lib bs=1 seek=$decAdrs count=3 conv=notrunc
+        done
+        printf "\t.. local patched library copy available at '%s'\n" $dest_lib
+    fi
+done
+printf ".. patching done; run with LD_LIBRARY_PATH=%s %s\n" $patch_dir $1

libsgxstep/aep.c

@@ -0,0 +1,88 @@
+#include "debug.h"
+#include "aep.h"
+#define __USE_GNU
+#include <signal.h>
+#include <string.h>
+
+/* See aep_trampoline.S to see how these are used. */
+extern void xs_enclu_stub(void);
+extern void sgx_step_aep_eresume(void);
+uint64_t *xs_enclu_cont_inst = NULL;
+
+void* xs_aep_pt = sgx_step_aep_eresume;
+void* xs_tcs_pt = NULL;
+
+void* xs_get_aep(void)
+{
+    return xs_aep_pt;
+}
+
+void xs_set_aep(void* aep)
+{
+    xs_aep_pt = aep;
+}
+
+void* xs_get_tcs(void)
+{
+    return xs_tcs_pt;
+}
+
+/*
+ * TODO 
+ *      1/ integrate this with unified signal handler <https://github.com/jovanbulck/sgx-step/pull/85>
+ *      2/ make sure this also works when the application defines its own (trap) signal handlers
+ */
+void xs_sigtrap_handler(int signo, siginfo_t * si, void  *ctx)
+{
+    /* /usr/include/x86_64-linux-gnu/sys/ucontext.h */
+    ucontext_t *uctx = (ucontext_t *) ctx;
+    void *rip = (void*) uctx->uc_mcontext.gregs[REG_RIP];
+    void *rax = (void*) uctx->uc_mcontext.gregs[REG_RAX];
+    void *rbx = (void*) uctx->uc_mcontext.gregs[REG_RBX];
+    void *rcx = (void*) uctx->uc_mcontext.gregs[REG_RCX];
+
+    info("signal handler rip %p; rax=%p; rbx=%p; rcx=%p", rip, rax, rbx, rcx);
+    switch ( signo )
+    {
+      case SIGTRAP:
+        uint8_t trap_inst = *(((uint8_t*) ((uint8_t*)rip)-1));
+        uint8_t next_inst = *((uint8_t*) rip);
+        xs_enclu_cont_inst = rip;
+        char *reason;
+        switch ((uint64_t) rax)
+        {
+            case 2:
+                reason = "EENTER";
+                break;
+            case 3:
+                reason = "ERESUME";
+                break;
+            default:
+                reason = "OTHER";
+        }
+        xs_tcs_pt = (uint64_t*) rbx;
+
+        info("Caught trap at %p (disasm: %#x %#x) tcs=%p; aep=%p; reason=%s",
+            rip, trap_inst, next_inst, rbx, rcx, reason);
+        info("\t.. redirecting instruction pointer to enclu_stub at %p", &xs_enclu_stub);
+        uctx->uc_mcontext.gregs[REG_RIP] = (uint64_t) &xs_enclu_stub;
+        return;
+
+      default:
+        info("Caught unexpected signal '%d'", signo);
+        abort();
+    }
+}
+
+void __attribute__((constructor)) xs_register_fault_handler(void)
+{
+    struct sigaction act, old_act;
+    memset(&act, 0, sizeof(sigaction));
+
+    act.sa_sigaction = xs_sigtrap_handler;
+    act.sa_flags = SA_RESTART | SA_SIGINFO;
+
+    /* Block all signals while the signal is being handled */
+    sigfillset(&act.sa_mask);
+    ASSERT(!sigaction(SIGTRAP, &act, &old_act));
+}

libsgxstep/aep.h

@@ -0,0 +1,8 @@
+#ifndef SGX_STEP_AEP_INC
+#define SGX_STEP_AEP_INC
+
+void* xs_get_aep(void);
+void xs_set_aep(void* aep);
+void* xs_get_tcs(void);
+
+#endif

libsgxstep/aep_trampoline.S

@@ -92,3 +92,10 @@ sgx_step_aep_trampoline:
 sgx_step_aep_eresume:
     .byte 0x0f, 0x01, 0xd7              /* ENCLU */
     ud2
+
+    .global xs_enclu_stub
+xs_enclu_stub:
+    mov xs_aep_pt(%rip), %rcx
+    enclu
+    jmp *xs_enclu_cont_inst(%rip)
+

libsgxstep/enclave.c

@@ -27,11 +27,7 @@
 #include "pt.h"
 #include <fcntl.h>
 #include <string.h>
-
-/* Custom AEP get/set functions from patched SGX SDK urts. */
-void* sgx_get_aep(void);
-void sgx_set_aep(void* aep);
-void* sgx_get_tcs(void);
+#include "aep.h"
 
 /* See aep_trampoline.S to see how these are used. */
 extern void sgx_step_aep_trampoline(void);
@@ -47,7 +43,7 @@ int fd_self_mem = -1;
 
 void register_aep_cb(aep_cb_t cb)
 {
-    sgx_set_aep(sgx_step_aep_trampoline);
+    xs_set_aep(sgx_step_aep_trampoline);
     sgx_step_aep_cb = cb;
 }
 
@@ -109,8 +105,8 @@ void register_enclave_info(void)
     }
     ASSERT( victim.drv && "no enclave found in /proc/self/maps");
 
-    victim.tcs = (uint64_t) sgx_get_tcs();
-    victim.aep = (uint64_t) sgx_get_aep();
+    victim.tcs = (uint64_t) xs_get_tcs();
+    victim.aep = (uint64_t) xs_get_aep();
     info("tcs at %lx; aep at %lx", victim.tcs, victim.aep);
     ASSERT( victim.tcs >= victim.base && victim.tcs < victim.limit);
     ioctl_init = 1;
@@ -190,7 +186,7 @@ void* get_enclave_ssa_gprsgx_adrs(void)
 {
     uint64_t ossa = 0x0;
     uint32_t cssa = 0x0;
-    void *tcs_addr = sgx_get_tcs();
+    void *tcs_addr = xs_get_tcs();
     edbgrd(tcs_addr + SGX_TCS_OSSA_OFFSET, &ossa, sizeof(ossa));
     edbgrd(tcs_addr + SGX_TCS_CSSA_OFFSET, &cssa, sizeof(cssa));
 
@@ -199,7 +195,7 @@ void* get_enclave_ssa_gprsgx_adrs(void)
 
 void set_debug_optin(void) 
 {
-    void *tcs_addr = sgx_get_tcs();
+    void *tcs_addr = xs_get_tcs();
     uint64_t flags;
     edbgrd(tcs_addr + SGX_TCS_FLAGS_OFFSET, &flags, sizeof(flags));
     flags |= SGX_FLAGS_DBGOPTIN;
@@ -215,12 +211,12 @@ void print_enclave_info(void)
     printf( "    Base:   %p\n", get_enclave_base() );
     printf( "    Limit:  %p\n", get_enclave_limit());
     printf( "    Size:   %d\n", get_enclave_size() );
-    printf( "    TCS:    %p\n", sgx_get_tcs() );
+    printf( "    TCS:    %p\n", xs_get_tcs() );
     printf( "    SSA:    %p\n", get_enclave_ssa_gprsgx_adrs() );
-    printf( "    AEP:    %p\n", sgx_get_aep() );
+    printf( "    AEP:    %p\n", xs_get_aep() );
 
     /* First 8 bytes of TCS must be zero */
-    int rv = edbgrd( sgx_get_tcs(), &read, 8);
+    int rv = edbgrd( xs_get_tcs(), &read, 8);
     printf( "    EDBGRD: %s\n", rv < 0 ? "production" : "debug");
 }