SPRR/GXF emulation for the HV

Makefile

@@ -144,7 +144,7 @@ OBJECTS := \
 	firmware.o \
 	gxf.o gxf_asm.o \
 	heapblock.o \
-	hv.o hv_vm.o hv_exc.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_virtio.o \
+	hv.o hv_vm.o hv_exc.o hv_sprr.o hv_vuart.o hv_wdt.o hv_asm.o hv_aic.o hv_virtio.o \
 	i2c.o \
 	iodev.o \
 	iova.o \

proxyclient/m1n1/hv/__init__.py

@@ -15,6 +15,7 @@
 from .types import *
 from .virtutils import *
 from .virtio import *
+from .sprr import patch_text_sprr_emu
 
 __all__ = ["HV"]
 
@@ -107,6 +108,7 @@ def __init__(self, iface, proxy, utils):
         self.wdt_cpu = None
         self.smp = True
         self.hook_exceptions = False
+        self.emulate_sprr = None
         self.started_cpus = {}
         self.started = False
         self.ctx = None
@@ -1360,6 +1362,11 @@ def set_logfile(self, fd):
         self.iface.tty_log = fd
 
     def init(self):
+        if self.emulate_sprr is None:
+            self.emulate_sprr = not self.u.cpu_features.apple_sysregs_unlocked
+        if self.emulate_sprr:
+            print("SPRR/GXF: emulating in EL2 (guest image will be patched)")
+
         self.adt = load_adt(self.u.get_adt())
         self.iodev = self.p.iodev_whoami()
         self.tba = self.u.ba.copy()
@@ -1615,7 +1622,10 @@ def start_secondary(self, die, cluster, cpu):
     def setup_adt(self):
         self.adt["product"].product_name += " on m1n1 hypervisor"
         self.adt["product"].product_description += " on m1n1 hypervisor"
-        soc_name = "Virtual " + self.adt["product"].product_soc_name + " on m1n1 hypervisor"
+        if self.emulate_sprr:
+            soc_name = "Virtual " + self.adt["product"].product_soc_name + " on cursed m1n1 hypervisor"
+        else:
+            soc_name = "Virtual " + self.adt["product"].product_soc_name + " on m1n1 hypervisor"
         self.adt["product"].product_soc_name = soc_name
 
         # change default serial to uart0 instead of UART-via-dockchannel
@@ -1827,39 +1837,8 @@ def load_macho(self, data, symfile=None):
 
         self._load_macho_symbols()
 
-        def load_hook(data, segname, size, fileoff, dest):
-            if segname != "__TEXT_EXEC":
-                return data
-
-            print(f"Patching segment {segname}...")
-
-            a = array.array("I", data)
-
-            output = []
-
-            p = 0
-            while (p := data.find(b"\x20\x00", p)) != -1:
-                if (p & 3) != 2:
-                    p += 1
-                    continue
-
-                opcode = a[p // 4]
-                inst = self.hvc((opcode & 0xffff))
-                off = fileoff + (p & ~3)
-                if off >= 0xbfcfc0:
-                    print(f"  0x{off:x}: 0x{opcode:04x} -> hvc 0x{opcode:x} (0x{inst:x})")
-                    a[p // 4] = inst
-                p += 4
-
-            print("Done.")
-            return a.tobytes()
-
-        def load_hook_m3(data, segname, size, fileoff, dest):
-            if segname != "__TEXT_EXEC":
-                return data
-
+        def patch_m3_regs(data, fileoff):
             inst = 0xd503201f # noop
-            print(f"Patching segment {segname}...")
 
             a = array.array("I", data)
 
@@ -1898,13 +1877,25 @@ def load_hook_m3(data, segname, size, fileoff, dest):
                         a[p // 4] = inst
                 p += 4
 
-            print("Done.")
             return a.tobytes()
 
-        #image = macho.prepare_image(load_hook)
         chip_id = self.u.adt["/chosen"].chip_id
-        if chip_id in (0x8122, 0x6030, 0x6031, 0x6032, 0x6034):
-            image = macho.prepare_image(load_hook_m3)
+        do_patch_m3_regs = chip_id in (0x8122, 0x6030, 0x6031, 0x6032, 0x6034)
+
+        def load_hook(data, segname, size, fileoff, dest):
+            if segname != "__TEXT_EXEC":
+                return data
+
+            print(f"Patching segment {segname}...")
+            if self.emulate_sprr:
+                data = patch_text_sprr_emu(data, log=print)
+            if do_patch_m3_regs:
+                data = patch_m3_regs(data, fileoff)
+            print("Done.")
+            return data
+
+        if self.emulate_sprr or do_patch_m3_regs:
+            image = macho.prepare_image(load_hook)
         else:
             image = macho.prepare_image()
         self.load_raw(image, entryoffset=(macho.entry - macho.vmin), use_xnu_symbols=self.xnu_mode and symfile is not None, vmin=macho.vmin)
@@ -1987,7 +1978,10 @@ def start(self):
         print("Shutting down framebuffer...")
         self.p.fb_shutdown(True)
 
-        if self.u.cpu_features.apple_sysregs_unlocked:
+        if self.emulate_sprr:
+            print("Enabling emulated SPRR/GXF...")
+            self.p.hv_sprr_set_active(True)
+        elif self.u.cpu_features.apple_sysregs_unlocked:
             print("Enabling SPRR...")
             self.u.msr(SPRR_CONFIG_EL1, 1)
 

proxyclient/m1n1/hv/sprr.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: MIT
+import array
+
+from .. import sysreg
+
+__all__ = ["patch_text_sprr_emu", "HV_VREGS"]
+
+# Must be kept in sync with enum hv_vreg in src/hv_sprr.h (vreg_id = index).
+HV_VREGS = [
+    sysreg.SPRR_CONFIG_EL1,
+    sysreg.GXF_CONFIG_EL1,
+    sysreg.GXF_STATUS_EL1,
+    sysreg.GXF_ENTRY_EL1,
+    sysreg.GXF_PABENTRY_EL1,
+    sysreg.SPRR_UPERM_EL0,
+    sysreg.SPRR_PPERM_EL1,
+    sysreg.TPIDR_GL1,
+    sysreg.VBAR_GL1,
+    sysreg.SPSR_GL1,
+    sysreg.ASPSR_GL1,
+    sysreg.ESR_GL1,
+    sysreg.ELR_GL1,
+    sysreg.FAR_GL1,
+    sysreg.AFSR1_GL1,
+    sysreg.ASPSR_EL1,
+    sysreg.SPRR_UMPRR_EL1,
+    sysreg.TTBR0_EL1,
+    sysreg.TTBR1_EL1,
+    sysreg.TCR_EL1,
+    sysreg.SCTLR_EL1,
+]
+
+GENTER = 0x00201420
+GEXIT = 0x00201400
+
+HVC_SYSREG_FLAG = 0x8000
+
+
+def _msr(enc, read):
+    op0, op1, crn, crm, op2 = enc
+    return (0xd5000000 | (read << 21) | ((op0 & 3) << 19) | (op1 << 16) | (crn << 12) |
+            (crm << 8) | (op2 << 5))
+
+
+def _hvc(imm):
+    return 0xd4000002 | (imm << 5)
+
+
+# MSR/MRS opcode (Rt masked) -> (vreg_id, read)
+_SYSREG = {_msr(enc, rd): (i, rd) for i, enc in enumerate(HV_VREGS) for rd in (0, 1)}
+
+
+def patch_text_sprr_emu(data, log=None):
+    a = array.array("I", data)
+    genter = gexit = sysreg_ = 0
+
+    for i, w in enumerate(a):
+        if w >> 16 == GENTER >> 16:
+            if w & 0xfffffff0 == GENTER:  # genter #imm; bit5 set distinguishes gexit
+                a[i] = _hvc(w & 0xffff)
+                genter += 1
+            elif w == GEXIT:
+                a[i] = _hvc(GEXIT & 0xffff)
+                gexit += 1
+            continue
+        if w >> 24 != 0xd5:
+            continue
+        hit = _SYSREG.get(w & ~0x1f)
+        if hit is not None:
+            vreg, rd = hit
+            a[i] = _hvc(HVC_SYSREG_FLAG | (vreg << 6) | (rd << 5) | (w & 0x1f))
+            sysreg_ += 1
+
+    if log:
+        log(f"  {genter} genter, {gexit} gexit, {sysreg_} sysreg patched")
+
+    return a.tobytes()

proxyclient/m1n1/proxy.py

@@ -661,6 +661,7 @@ class M1N1Proxy(Reloadable):
     P_VIRTIO_PUT_BUFFER = 0xc0e
     P_HV_EXIT_CPU = 0xc0f
     P_HV_ADD_TIME = 0xc10
+    P_HV_SPRR_SET_ACTIVE = 0xc11
 
     P_FB_INIT = 0xd00
     P_FB_SHUTDOWN = 0xd01
@@ -1151,6 +1152,8 @@ def hv_exit_cpu(self, cpu=-1):
         return self.request(self.P_HV_EXIT_CPU, cpu)
     def hv_add_time(self, time):
         return self.request(self.P_HV_ADD_TIME, time)
+    def hv_sprr_set_active(self, active):
+        return self.request(self.P_HV_SPRR_SET_ACTIVE, int(bool(active)))
 
     def fb_init(self):
         return self.request(self.P_FB_INIT)

proxyclient/tools/run_guest.py

@@ -23,6 +23,8 @@ def volumespec(s):
 parser.add_argument('-r', '--raw', action="store_true")
 parser.add_argument('-E', '--entry-point', action="store", type=int, help="Entry point for the raw image", default=0x800)
 parser.add_argument('-a', '--append-payload', type=pathlib.Path, action="append", default=[])
+parser.add_argument('-P', '--emulate-sprr', action="store_true",
+                    help='Force the EL2 SPRR/GXF emulation on.')
 parser.add_argument('-v', '--volume', type=volumespec, action='append',
                     help='Attach a 9P virtio device for file export to the guest. The argument is a host path to the '
                          'exported tree, joined by colon (\':\') with a tag under which the tree will be advertised '
@@ -43,7 +45,13 @@ def volumespec(s):
 iface = UartInterface()
 p = M1N1Proxy(iface, debug=False)
 bootstrap_port(iface, p)
-u = ProxyUtils(p, heap_size = 128 * 1024 * 1024)
+
+if args.emulate_sprr or not p.get_cpu_features().apple_sysregs_unlocked:
+    m1n1_heap_mb = 1024
+else:
+    m1n1_heap_mb = 128
+u = ProxyUtils(p, heap_size = 128 * 1024 * 1024,
+               m1n1_heap = m1n1_heap_mb * 1024 * 1024)
 
 # Setup counter redirect / AHCR_EL2 as expected by macOS for macho payloads
 if not args.raw:
@@ -56,6 +64,9 @@ def volumespec(s):
 
 hv.hook_exceptions = args.hook_exceptions
 
+if args.emulate_sprr:
+    hv.emulate_sprr = True
+
 hv.init()
 
 if args.cpus:

src/cpu_regs.h

@@ -719,6 +719,7 @@
 #define SYS_IMP_APL_VBAR_GL1  sys_reg(3, 6, 15, 10, 2)
 #define SYS_IMP_APL_SPSR_GL1  sys_reg(3, 6, 15, 10, 3)
 #define SYS_IMP_APL_ASPSR_GL1 sys_reg(3, 6, 15, 10, 4)
+#define ASPSR_GUARDED         BIT(0)
 #define SYS_IMP_APL_ESR_GL1   sys_reg(3, 6, 15, 10, 5)
 #define SYS_IMP_APL_ELR_GL1   sys_reg(3, 6, 15, 10, 6)
 #define SYS_IMP_APL_FAR_GL1   sys_reg(3, 6, 15, 10, 7)

src/hv.h

@@ -63,6 +63,7 @@ bool hv_pt_is_ram(u64 ipa);
 int hv_pt_set_writable(u64 ipa, bool writable);
 u64 hv_translate(u64 addr, bool s1only, bool w, u64 *par_out);
 u64 hv_pt_walk(u64 addr);
+bool hv_emulate_pt_store(struct exc_info *ctx, u64 far, u64 ipa, u64 *bytes);
 bool hv_handle_dabort(struct exc_info *ctx);
 bool hv_pa_write(struct exc_info *ctx, u64 addr, u64 *val, int width);
 bool hv_pa_read(struct exc_info *ctx, u64 addr, u64 *val, int width);

src/hv_exc.c

@@ -4,6 +4,7 @@
 #include "assert.h"
 #include "cpu_regs.h"
 #include "exception.h"
+#include "hv_sprr.h"
 #include "smp.h"
 #include "string.h"
 #include "uart.h"
@@ -203,6 +204,11 @@ static bool hv_handle_msr_unlocked(struct exc_info *ctx, u64 iss)
 
     regs[31] = 0;
 
+    /* Don't handle any TLB maintenance here if we're running in emulated SPRR mode */
+    if (!is_read && FIELD_GET(ESR_ISS_MSR_OP0, reg) == 1 && FIELD_GET(ESR_ISS_MSR_CRn, reg) == 8 &&
+        hv_sprr_traps_tlbi())
+        return false;
+
     switch (reg) {
         SYSREG_PASS(SYS_IMP_APL_CORE_NRG_ACC_DAT);
         SYSREG_PASS(SYS_IMP_APL_CORE_SRM_NRG_ACC_DAT);
@@ -388,7 +394,7 @@ static bool hv_handle_msr(struct exc_info *ctx, u64 iss)
 #endif
     }
 
-    return false;
+    return hv_sprr_handle_msr(ctx, reg, rt, is_read);
 }
 
 static void hv_get_context(struct exc_info *ctx)
@@ -460,6 +466,13 @@ void hv_exc_sync(struct exc_info *ctx)
                     break;
             }
             break;
+        case ESR_EC_HVC:
+            hv_wdt_breadcrumb('h');
+            if (hv_hvc_dispatch_unlocked(ctx, FIELD_GET(ESR_ISS, ctx->esr))) {
+                hv_wdt_breadcrumb('s');
+                return;
+            }
+            break;
     }
 
     if (handled) {
@@ -490,11 +503,17 @@ void hv_exc_sync(struct exc_info *ctx)
                     break;
             }
             break;
+        case ESR_EC_HVC:
+            hv_wdt_breadcrumb('H');
+            handled = hv_hvc_dispatch(ctx, FIELD_GET(ESR_ISS, ctx->esr));
+            break;
     }
 
     if (handled) {
         hv_wdt_breadcrumb('+');
-        ctx->elr += 4;
+        // HVC alread leaves ELR past the instruction
+        if (ec != ESR_EC_HVC)
+            ctx->elr += 4;
     } else {
         hv_wdt_breadcrumb('-');
         // VM code can forward a nested SError exception here