app/{bench,memcmp}: flush PTE to improve single-stepping

This further increases the latency of the ucode-assisted page-table walk for
the first instruction following ERESUME and thus the landing space for
SGX-Step's timer interrupt.

README.md

@@ -318,6 +318,7 @@ Some different microcode versions are provided for reference in the table below.
 | Coffee Lake HR | [i7-9750H](https://ark.intel.com/content/www/us/en/ark/products/191045/intel-core-i7-9750h-processor-12m-cache-up-to-4-50-ghz.html)  | 2.6 GHz        | 0xf4 (2023-02-23) | 37                  |
 | Ice Lake       | [i5-1035G1](https://ark.intel.com/content/www/us/en/ark/products/196603/intel-core-i5-1035g1-processor-6m-cache-up-to-3-60-ghz.html) | 1.00 GHz       | 0x32 (2019-07-05) | 135                 |
 | Ice Lake       | [i5-1035G1](https://ark.intel.com/content/www/us/en/ark/products/196603/intel-core-i5-1035g1-processor-6m-cache-up-to-3-60-ghz.html) | 1.00 GHz       | 0xb0 (2022-03-09) | 255                 |
+| Emerald Rapids | [Xeon Gold 5515+](https://ark.intel.com/content/www/us/en/ark/products/237562/intel-xeon-gold-5515-processor-22-5m-cache-3-20-ghz.html) | 3.2 GHz | 0x21000230 (2024-02-05) | 32 |
 
 **Note (calibration).**
 Currently, the easiest way to configure a reliable timer interval is to

app/bench/main.c

@@ -31,6 +31,7 @@
 #include "libsgxstep/idt.h"
 #include "libsgxstep/pt.h"
 #include "libsgxstep/sched.h"
+#include "libsgxstep/cache.h"
 
 #ifndef NUM_RUNS
 #define NUM_RUNS 100
@@ -85,14 +86,16 @@ void aep_cb_func(void) {
     /*
      * Configure APIC timer interval for next interrupt.
      *
-     * On our evaluation platforms, we explicitly clear the enclave's
-     * _unprotected_ PMD "accessed" bit below, so as to slightly slow down
-     * ERESUME such that the interrupt reliably arrives in the first subsequent
-     * enclave instruction.
-     *
+     * NOTE: Clearing the PMD "accessed" bit forces the CPU to take a
+     * ucode-assisted page-table walk for the first instruction following
+     * ERESUME, which causes that instruction to be much longer. We
+     * additionally flush this PMD from the cache to further delay the
+     * page-table walk and increase the landing space for the timer interrupt.
      */
     if (do_irq) {
         *pmd_encl = MARK_NOT_ACCESSED(*pmd_encl);
+        flush(pmd_encl);
+        flush(pte_encl);
         apic_timer_irq(SGX_STEP_TIMER_INTERVAL);
     }
 }
@@ -189,6 +192,11 @@ int main(int argc, char **argv) {
     install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR);
     apic_timer_oneshot(IRQ_VECTOR);
 
+    __ss_irq_fired = 0;
+    apic_timer_irq( SGX_STEP_TIMER_INTERVAL );
+    while (!__ss_irq_fired);
+    info("APIC timer IRQ handler seems to be working");
+
     /* 2. Single-step enclaved execution. */
     info_event("calling enclave: attack=%d; num_runs=%d; timer=%d",
                ATTACK_SCENARIO, NUM_RUNS, SGX_STEP_TIMER_INTERVAL);

app/memcmp/main.c

@@ -12,11 +12,12 @@
 #include "libsgxstep/config.h"
 #include "libsgxstep/idt.h"
 #include "libsgxstep/config.h"
+#include "libsgxstep/cache.h"
 #include "jsh-colors.h"
 #include <sys/mman.h>
 
 #define MAX_LEN            15
-#define DO_TIMER_STEP      0
+#define DO_TIMER_STEP      1
 #define DEBUG              0
 #define DBG_ENCL           1
 #if DO_TIMER_STEP
@@ -74,9 +75,16 @@ void aep_cb_func(void)
      * NOTE: We explicitly clear the "accessed" bit of the _unprotected_ PTE
      * referencing the enclave code page about to be executed, so as to be able
      * to filter out "zero-step" results that won't set the accessed bit.
+     *
+     * Clearing the PTE "accessed" bit forces the CPU to take a ucode-assisted
+     * page-table walk for the first instruction following ERESUME, which
+     * causes that instruction to be much longer. We additionally flush this
+     * PTE from the cache to further delay the page-table walk and increase the
+     * landing space for the timer interrupt.
      */
     if (do_irq && ACCESSED(*pte_encl)) step_cnt++;
     *pte_encl = MARK_NOT_ACCESSED( *pte_encl );
+    flush(pte_encl);
     *pte_trigger = MARK_NOT_ACCESSED(*pte_trigger);
 
     /*
@@ -118,7 +126,10 @@ void fault_handler(int signo, siginfo_t * si, void  *ctx)
             #endif
             ASSERT(!mprotect(trigger_adrs, 4096, PROT_READ | PROT_WRITE));
             do_irq = 1;
-            sgx_step_do_trap = 1;
+
+            #if !DO_TIMER_STEP
+                sgx_step_do_trap = 1;
+            #endif
         }
         else
         {
@@ -228,6 +239,11 @@ int main( int argc, char **argv )
     map_idt(&idt);
     install_kernel_irq_handler(&idt, __ss_irq_handler, IRQ_VECTOR);
     apic_timer_oneshot(IRQ_VECTOR);
+
+    __ss_irq_fired = 0;
+    apic_timer_irq( SGX_STEP_TIMER_INTERVAL );
+    while (!__ss_irq_fired);
+    info("APIC timer IRQ handler seems to be working");
 #else
     register_signal_handler( SIGTRAP );
     set_debug_optin();

libsgxstep/config.h

@@ -58,6 +58,6 @@
  *       suitable timer intervals on our evaluation platforms by
  *       tweaking and observing the NOP microbenchmark erip results.
  */
-#define SGX_STEP_TIMER_INTERVAL 53
+#define SGX_STEP_TIMER_INTERVAL 32
 
 #endif