8370947: Mitigate Neoverse-N1 erratum 1542419 negative impact on GenZGC performance

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp

@@ -862,7 +862,7 @@ static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint
   instr |= imm << start;
 }
 
-void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
+void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format, bool defer_icache_invalidation) {
   const uint16_t value = patch_barrier_relocation_value(format);
   uint32_t* const patch_addr = (uint32_t*)addr;
 
@@ -879,6 +879,12 @@ void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
     ShouldNotReachHere();
   }
 
+  if (defer_icache_invalidation) {
+    // Instruction cache invalidation per barrier can be expensive, e.g. on Neoverse N1 having erratum 1542419.
+    // Defer the ICache invalidation to a later point where multiple patches can be handled together.
+    return;
+  }
+
   OrderAccess::fence();
   ICache::invalidate_word((address)patch_addr);
 }

src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp

@@ -154,7 +154,7 @@ class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
 
   virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; }
 
-  void patch_barrier_relocation(address addr, int format);
+  void patch_barrier_relocation(address addr, int format, bool defer_icache_invalidation = false);
 
   void patch_barriers() {}
 

src/hotspot/cpu/aarch64/globals_aarch64.hpp

@@ -127,6 +127,8 @@ define_pd_global(intx, InlineSmallCode,          1000);
           "Branch Protection to use: none, standard, pac-ret")          \
   product(bool, AlwaysMergeDMB, true, DIAGNOSTIC,                       \
           "Always merge DMB instructions in code emission")             \
+  product(bool, NeoverseN1Errata1542419, false, DIAGNOSTIC,             \
+          "Enable workaround for Neoverse N1 erratum 1542419")          \
 
 // end of ARCH_FLAGS
 

src/hotspot/cpu/aarch64/icache_aarch64.hpp

@@ -28,4 +28,40 @@
 
 #include OS_CPU_HEADER(icache)
 
+#include "utilities/globalDefinitions.hpp"
+
+#define PD_ICACHE_INVALIDATION_CONTEXT
+
+inline void ICacheInvalidationContext::pd_init(nmethod* nm) {
+  if (NeoverseN1Errata1542419) {
+    _nm = nm;
+  }
+}
+
+inline void ICacheInvalidationContext::pd_invalidate_icache() {
+  if (_nm != nullptr) {
+    assert(NeoverseN1Errata1542419, "Should only be set for Neoverse N1 erratum");
+    // Neoverse-N1 implementation mitigates erratum 1542419 with a workaround:
+    // - Disable coherent icache.
+    // - Trap IC IVAU instructions.
+    // - Execute:
+    //   - tlbi vae3is, xzr
+    //   - dsb sy
+    //
+    // `tlbi vae3is, xzr` invalidates translations for all address spaces (global for address).
+    //  It waits for all memory accesses using in-scope old translation information to complete
+    //  before it is considered complete.
+    //
+    // As this workaround has significant overhead, Arm Neoverse N1 (MP050) Software Developer
+    // Errata Notice version 29.0 suggests:
+    //
+    // "Since one TLB inner-shareable invalidation is enough to avoid this erratum, the number
+    // of injected TLB invalidations should be minimized in the trap handler to mitigate
+    // the performance impact due to this workaround."
+    //
+    // As the addrress for icache invalidation is not relevant, we use the nmethod's code start address.
+    ICache::invalidate_word(_nm->code_begin());
+  }
+}
+
 #endif // CPU_AARCH64_ICACHE_AARCH64_HPP

src/hotspot/cpu/aarch64/relocInfo_aarch64.cpp

@@ -54,6 +54,13 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
     bytes = MacroAssembler::pd_patch_instruction_size(addr(), x);
     break;
   }
+
+  if (binding() != nullptr && binding()->deferred_icache_invalidation()) {
+    // Instruction cache invalidation per relocation can be expensive, e.g. on Neoverse N1 having erratum 1542419.
+    // Defer the ICache invalidation to a later point where multiple patches can be handled together.
+    return;
+  }
+
   ICache::invalidate_range(addr(), bytes);
 }
 

src/hotspot/cpu/aarch64/vm_version_aarch64.cpp

@@ -641,6 +641,17 @@ void VM_Version::initialize() {
     clear_feature(CPU_SVE);
   }
 
+  // Neoverse N1: 0xd0c
+  if (_cpu == CPU_ARM && model_is(0xd0c) && FLAG_IS_DEFAULT(NeoverseN1Errata1542419)) {
+    const int major_rev_num = cpu_variant();
+    const int minor_rev_num = cpu_revision();
+    if (!(major_rev_num >= 4 && minor_rev_num >= 1)) {
+      // As Neoverse N1 r4p1 and later are not affected by the erratum,
+      // enable the workaround by default for earlier revisions.
+      FLAG_SET_DEFAULT(NeoverseN1Errata1542419, true);
+    }
+  }
+
   // Construct the "features" string
   stringStream ss(512);
   ss.print("0x%02x:0x%x:0x%03x:%d", _cpu, _variant, _model, _revision);

src/hotspot/share/code/nmethod.cpp

@@ -2032,9 +2032,10 @@ void nmethod::copy_values(GrowableArray<Metadata*>* array) {
   }
 }
 
-void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates) {
+void nmethod::fix_oop_relocations(address begin, address end, bool initialize_immediates, bool defer_icache_invalidation) {
   // re-patch all oop-bearing instructions, just in case some oops moved
   RelocIterator iter(this, begin, end);
+  AARCH64_ONLY(iter.set_deferred_icache_invalidation(defer_icache_invalidation);)
   while (iter.next()) {
     if (iter.type() == relocInfo::oop_type) {
       oop_Relocation* reloc = iter.oop_reloc();

src/hotspot/share/code/nmethod.hpp

@@ -799,7 +799,7 @@ class nmethod : public CodeBlob {
 
   // Relocation support
 private:
-  void fix_oop_relocations(address begin, address end, bool initialize_immediates);
+  void fix_oop_relocations(address begin, address end, bool initialize_immediates, bool defer_icache_invalidation = false);
   inline void initialize_immediate_oop(oop* dest, jobject handle);
 
 protected:
@@ -808,6 +808,7 @@ class nmethod : public CodeBlob {
 public:
   void fix_oop_relocations(address begin, address end) { fix_oop_relocations(begin, end, false); }
   void fix_oop_relocations()                           { fix_oop_relocations(nullptr, nullptr, false); }
+  void fix_oop_relocations(bool defer_icache_invalidation) { fix_oop_relocations(nullptr, nullptr, false, defer_icache_invalidation); }
 
   bool is_at_poll_return(address pc);
   bool is_at_poll_or_poll_return(address pc);

src/hotspot/share/code/relocInfo.hpp

@@ -569,6 +569,8 @@ class RelocIterator : public StackObj {
   short*          _data;    // pointer to the relocation's data
   short           _datalen; // number of halfwords in _data
 
+  AARCH64_ONLY(bool _deferred_icache_invalidation;)
+
   // Base addresses needed to compute targets of section_word_type relocs.
   address _section_start[SECT_LIMIT];
   address _section_end  [SECT_LIMIT];
@@ -639,6 +641,11 @@ class RelocIterator : public StackObj {
   bool     has_current()      const { return _datalen >= 0; }
   bool   addr_in_const()      const;
 
+#ifdef AARCH64
+  bool deferred_icache_invalidation() const { return _deferred_icache_invalidation;  }
+  void set_deferred_icache_invalidation(bool b) { _deferred_icache_invalidation = b; }
+#endif
+
   address section_start(int n) const {
     assert(_section_start[n], "section %d must be initialized", n);
     return _section_start[n];

src/hotspot/share/gc/z/zMark.cpp

@@ -60,6 +60,7 @@
 #include "runtime/atomicAccess.hpp"
 #include "runtime/continuation.hpp"
 #include "runtime/handshake.hpp"
+#include "runtime/icache.hpp"
 #include "runtime/javaThread.hpp"
 #include "runtime/prefetch.inline.hpp"
 #include "runtime/safepointMechanism.hpp"
@@ -753,10 +754,6 @@ class ZMarkYoungNMethodClosure : public NMethodClosure {
     if (_bs_nm->is_armed(nm)) {
       const uintptr_t prev_color = ZNMethod::color(nm);
 
-      // Heal oops
-      ZUncoloredRootMarkYoungOopClosure cl(prev_color);
-      ZNMethod::nmethod_oops_do_inner(nm, &cl);
-
       // Disarm only the young marking, not any potential old marking cycle
 
       const uintptr_t old_marked_mask = ZPointerMarkedMask ^ (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
@@ -767,9 +764,17 @@ class ZMarkYoungNMethodClosure : public NMethodClosure {
       // Check if disarming for young mark, completely disarms the nmethod entry barrier
       const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr);
 
-      if (complete_disarm) {
-        // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming
-        ZNMethod::nmethod_patch_barriers(nm);
+      {
+        ICacheInvalidationContext icic(nm);
+
+        if (complete_disarm) {
+          // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming
+          ZNMethod::nmethod_patch_barriers(nm, icic.deferred_invalidation());
+        }
+
+        // Heal oops
+        ZUncoloredRootMarkYoungOopClosure cl(prev_color);
+        ZNMethod::nmethod_oops_do_inner(nm, &cl, icic.deferred_invalidation());
       }
 
       _bs_nm->guard_with(nm, (int)untype(new_disarm_value_ptr));

src/hotspot/share/gc/z/zNMethod.cpp

@@ -244,11 +244,11 @@ void ZNMethod::set_guard_value(nmethod* nm, int value) {
   bs->guard_with(nm, value);
 }
 
-void ZNMethod::nmethod_patch_barriers(nmethod* nm) {
+void ZNMethod::nmethod_patch_barriers(nmethod* nm, bool defer_icache_invalidation) {
   ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler();
   ZArrayIterator<ZNMethodDataBarrier> iter(gc_data(nm)->barriers());
   for (ZNMethodDataBarrier barrier; iter.next(&barrier);) {
-    bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format);
+    bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format AARCH64_ONLY(COMMA defer_icache_invalidation));
   }
 }
 
@@ -257,7 +257,7 @@ void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) {
   ZNMethod::nmethod_oops_do_inner(nm, cl);
 }
 
-void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
+void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, bool defer_icache_invalidation) {
   // Process oops table
   {
     oop* const begin = nm->oops_begin();
@@ -283,7 +283,7 @@ void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
 
   // Process non-immediate oops
   if (data->has_non_immediate_oops()) {
-    nm->fix_oop_relocations();
+    nm->fix_oop_relocations(defer_icache_invalidation);
   }
 }
 

src/hotspot/share/gc/z/zNMethod.hpp

@@ -55,10 +55,10 @@ class ZNMethod : public AllStatic {
   static void disarm(nmethod* nm);
   static void set_guard_value(nmethod* nm, int value);
 
-  static void nmethod_patch_barriers(nmethod* nm);
+  static void nmethod_patch_barriers(nmethod* nm, bool defer_icache_invalidation = false);
 
   static void nmethod_oops_do(nmethod* nm, OopClosure* cl);
-  static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl);
+  static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl, bool defer_icache_invalidation = false);
 
   static void nmethods_do_begin(bool secondary);
   static void nmethods_do_end(bool secondary);

src/hotspot/share/runtime/icache.hpp

@@ -71,6 +71,32 @@ class AbstractICache : AllStatic {
   static void invalidate_range(address start, int nbytes);
 };
 
+class nmethod;
+
+class ICacheInvalidationContext : StackObj {
+  NONCOPYABLE(ICacheInvalidationContext);
+
+private:
+  nmethod* _nm;
+
+  void pd_init(nmethod* nm);
+  void pd_invalidate_icache();
+
+public:
+  ICacheInvalidationContext(nmethod* nm) : _nm(nullptr) {
+    pd_init(nm);
+  }
+
+  ~ICacheInvalidationContext() {
+    if (_nm != nullptr) {
+      pd_invalidate_icache();
+    }
+  }
+
+  bool deferred_invalidation() const {
+    return _nm != nullptr;
+  }
+};
 
 // Must be included before the definition of ICacheStubGenerator
 // because ICacheStubGenerator uses ICache definitions.
@@ -129,4 +155,10 @@ class ICacheStubGenerator : public StubCodeGenerator {
   void generate_icache_flush(ICache::flush_icache_stub_t* flush_icache_stub);
 };
 
+#ifndef PD_ICACHE_INVALIDATION_CONTEXT
+  // Default implementation: do nothing
+  inline void ICacheInvalidationContext::pd_init(nmethod*) {}
+  inline void ICacheInvalidationContext::pd_invalidate_icache() {}
+#endif
+
 #endif // SHARE_RUNTIME_ICACHE_HPP