8361376: Regressions 1-6% in several Renaissance in 26-b4 only MacOSX aarch64

src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp

@@ -117,8 +117,22 @@ class NativeNMethodBarrier {
     return Atomic::load_acquire(guard_addr());
   }
 
-  void set_value(int value) {
-    Atomic::release_store(guard_addr(), value);
+  void set_value(int value, int bit_mask) {
+    if (bit_mask == ~0) {
+      Atomic::release_store(guard_addr(), value);
+      return;
+    }
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = Atomic::load(guard_addr());
+    while (true) {
+      // Only bits in the mask are changed
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
   }
 
   bool check_barrier(err_msg& msg) const;
@@ -181,7 +195,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
   new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
 }
 
-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
   if (!supports_entry_barrier(nm)) {
     return;
   }
@@ -198,7 +212,7 @@ void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
   }
 
   NativeNMethodBarrier barrier(nm);
-  barrier.set_value(value);
+  barrier.set_value(value, bit_mask);
 }
 
 int BarrierSetNMethod::guard_value(nmethod* nm) {

src/hotspot/cpu/arm/gc/shared/barrierSetNMethod_arm.cpp

@@ -51,8 +51,22 @@ class NativeNMethodBarrier: public NativeInstruction {
     return Atomic::load_acquire(guard_addr());
   }
 
-  void set_value(int value) {
-    Atomic::release_store(guard_addr(), value);
+  void set_value(int value, int bit_mask) {
+    if (bit_mask == ~0) {
+      Atomic::release_store(guard_addr(), value);
+      return;
+    }
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = Atomic::load(guard_addr());
+    while (true) {
+      // Only bits in the mask are changed
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
   }
 
   void verify() const;
@@ -115,15 +129,15 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
   new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
 }
 
-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
   if (!supports_entry_barrier(nm)) {
     return;
   }
 
   // Disarms the nmethod guard emitted by BarrierSetAssembler::nmethod_entry_barrier.
   // Symmetric "LDR; DMB ISHLD" is in the nmethod barrier.
   NativeNMethodBarrier* barrier = native_nmethod_barrier(nm);
-  barrier->set_value(value);
+  barrier->set_value(value, bit_mask);
 }
 
 int BarrierSetNMethod::guard_value(nmethod* nm) {

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp

@@ -183,12 +183,9 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register t
   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
   assert_different_registers(tmp, R0);
 
-  __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");
+  __ align(8); // must align the following block which requires atomic updates
 
-  // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
-  __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
-                                       true, true, false); // 2 instructions
-  __ mtctr(tmp);
+  __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");
 
   // This is a compound instruction. Patching support is provided by NativeMovRegMem.
   // Actual patching is done in (platform-specific part of) BarrierSetNMethod.
@@ -198,6 +195,11 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register t
   __ ld(R0, in_bytes(bs_nm->thread_disarmed_guard_value_offset()), R16_thread);
   __ cmpw(CR0, R0, tmp);
 
+  // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
+  __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
+                                       true, true, false); // 2 instructions
+  __ mtctr(tmp);
+
   __ bnectrl(CR0);
 
   // Oops may have been changed. Make those updates observable.

src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp

@@ -38,7 +38,7 @@ class NativeNMethodBarrier: public NativeInstruction {
 
   NativeMovRegMem* get_patchable_instruction_handle() const {
     // Endianness is handled by NativeMovRegMem
-    return reinterpret_cast<NativeMovRegMem*>(get_barrier_start_address() + 3 * 4);
+    return reinterpret_cast<NativeMovRegMem*>(get_barrier_start_address());
   }
 
 public:
@@ -47,16 +47,45 @@ class NativeNMethodBarrier: public NativeInstruction {
     return get_patchable_instruction_handle()->offset();
   }
 
-  void release_set_guard_value(int value) {
+  void release_set_guard_value(int value, int bit_mask) {
     // Patching is not atomic.
     // Stale observations of the "armed" state is okay as invoking the barrier stub in that case has no
     // unwanted side effects. Disarming is thus a non-critical operation.
     // The visibility of the "armed" state must be ensured by safepoint/handshake.
 
     OrderAccess::release(); // Release modified oops
 
-    // Set the guard value (naming of 'offset' function is misleading).
-    get_patchable_instruction_handle()->set_offset(value);
+    if (bit_mask == ~0) {
+      // Set the guard value (naming of 'offset' function is misleading).
+      get_patchable_instruction_handle()->set_offset(value);
+      return;
+    }
+
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+
+    NativeMovRegMem* mov = get_patchable_instruction_handle();
+    assert(align_up(mov->instruction_address(), sizeof(uint64_t)) ==
+           align_down(mov->instruction_address(), sizeof(uint64_t)), "instruction not aligned");
+    uint64_t *instr = (uint64_t*)mov->instruction_address();
+    assert(NativeMovRegMem::instruction_size == sizeof(*instr), "must be");
+    union {
+      u_char buf[NativeMovRegMem::instruction_size];
+      uint64_t u64;
+    } new_mov_instr, old_mov_instr;
+    new_mov_instr.u64 = old_mov_instr.u64 = Atomic::load(instr);
+    while (true) {
+      // Only bits in the mask are changed
+      int old_value = nativeMovRegMem_at(old_mov_instr.buf)->offset();
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) return; // skip icache flush if nothing changed
+      nativeMovRegMem_at(new_mov_instr.buf)->set_offset(new_value, false /* no icache flush */);
+      // Swap in the new value
+      uint64_t v = Atomic::cmpxchg(instr, old_mov_instr.u64, new_mov_instr.u64, memory_order_relaxed);
+      if (v == old_mov_instr.u64) break;
+      old_mov_instr.u64 = v;
+    }
+    ICache::ppc64_flush_icache_bytes(addr_at(0), NativeMovRegMem::instruction_size);
   }
 
   void verify() const {
@@ -66,12 +95,6 @@ class NativeNMethodBarrier: public NativeInstruction {
 
     uint* current_instruction = reinterpret_cast<uint*>(get_barrier_start_address());
 
-    // calculate_address_from_global_toc (compound instruction)
-    verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
-    verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
-
-    verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
-
     get_patchable_instruction_handle()->verify();
     current_instruction += 2;
 
@@ -80,6 +103,12 @@ class NativeNMethodBarrier: public NativeInstruction {
     // cmpw (mnemonic)
     verify_op_code(current_instruction, Assembler::CMP_OPCODE);
 
+    // calculate_address_from_global_toc (compound instruction)
+    verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
+    verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
+
+    verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
+
     // bnectrl (mnemonic) (weak check; not checking the exact type)
     verify_op_code(current_instruction, Assembler::BCCTR_OPCODE);
 
@@ -117,13 +146,13 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
   // Thus, there's nothing to do here.
 }
 
-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
   if (!supports_entry_barrier(nm)) {
     return;
   }
 
   NativeNMethodBarrier* barrier = get_nmethod_barrier(nm);
-  barrier->release_set_guard_value(value);
+  barrier->release_set_guard_value(value, bit_mask);
 }
 
 int BarrierSetNMethod::guard_value(nmethod* nm) {

src/hotspot/cpu/ppc/nativeInst_ppc.hpp

@@ -462,7 +462,7 @@ class NativeMovRegMem: public NativeInstruction {
     return ((*hi_ptr) << 16) | ((*lo_ptr) & 0xFFFF);
   }
 
-  void set_offset(intptr_t x) {
+  void set_offset(intptr_t x, bool flush_icache = true) {
 #ifdef VM_LITTLE_ENDIAN
     short *hi_ptr = (short*)(addr_at(0));
     short *lo_ptr = (short*)(addr_at(4));
@@ -472,7 +472,9 @@ class NativeMovRegMem: public NativeInstruction {
 #endif
     *hi_ptr = x >> 16;
     *lo_ptr = x & 0xFFFF;
-    ICache::ppc64_flush_icache_bytes(addr_at(0), NativeMovRegMem::instruction_size);
+    if (flush_icache) {
+      ICache::ppc64_flush_icache_bytes(addr_at(0), NativeMovRegMem::instruction_size);
+    }
   }
 
   void add_offset_in_bytes(intptr_t radd_offset) {

src/hotspot/cpu/riscv/gc/shared/barrierSetNMethod_riscv.cpp

@@ -111,8 +111,22 @@ class NativeNMethodBarrier {
     return Atomic::load_acquire(guard_addr());
   }
 
-  void set_value(int value) {
-    Atomic::release_store(guard_addr(), value);
+  void set_value(int value, int bit_mask) {
+    if (bit_mask == ~0) {
+      Atomic::release_store(guard_addr(), value);
+      return;
+    }
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = Atomic::load(guard_addr());
+    while (true) {
+      // Only bits in the mask are changed
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = Atomic::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
   }
 
   bool check_barrier(err_msg& msg) const;
@@ -194,7 +208,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
   new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
 }
 
-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
   if (!supports_entry_barrier(nm)) {
     return;
   }
@@ -211,7 +225,7 @@ void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
   }
 
   NativeNMethodBarrier barrier(nm);
-  barrier.set_value(value);
+  barrier.set_value(value, bit_mask);
 }
 
 int BarrierSetNMethod::guard_value(nmethod* nm) {

src/hotspot/cpu/s390/gc/shared/barrierSetNMethod_s390.cpp

@@ -53,11 +53,26 @@ class NativeMethodBarrier: public NativeInstruction {
       return *((int32_t*)data_addr);
     }
 
-    void set_guard_value(int value) {
-      int32_t* data_addr = (int32_t*)get_patchable_data_address();
+    void set_guard_value(int value, int bit_mask) {
+      if (bit_mask == ~0) {
+        int32_t* data_addr = (int32_t*)get_patchable_data_address();
 
-      // Set guard instruction value
-      *data_addr = value;
+        // Set guard instruction value
+        *data_addr = value;
+        return;
+      }
+      assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+      value &= bit_mask;
+      int32_t* data_addr = (int32_t*)get_patchable_data_address();
+      int old_value = Atomic::load(data_addr);
+      while (true) {
+        // Only bits in the mask are changed
+        int new_value = value | (old_value & ~bit_mask);
+        if (new_value == old_value) break;
+        int v = Atomic::cmpxchg(data_addr, old_value, new_value, memory_order_release);
+        if (v == old_value) break;
+        old_value = v;
+      }
     }
 
     #ifdef ASSERT
@@ -100,13 +115,13 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
   return;
 }
 
-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
   if (!supports_entry_barrier(nm)) {
     return;
   }
 
   NativeMethodBarrier* barrier = get_nmethod_barrier(nm);
-  barrier->set_guard_value(value);
+  barrier->set_guard_value(value, bit_mask);
 }
 
 int BarrierSetNMethod::guard_value(nmethod* nm) {

src/hotspot/cpu/x86/gc/shared/barrierSetNMethod_x86.cpp

@@ -50,8 +50,31 @@ class NativeNMethodCmpBarrier: public NativeInstruction {
   address instruction_address() const { return addr_at(0); }
   address immediate_address() const { return addr_at(imm_offset); }
 
+  NativeNMethodCmpBarrier* nativeNMethodCmpBarrier_at(address a) { return (NativeNMethodCmpBarrier*)a; }
+
   jint get_immediate() const { return int_at(imm_offset); }
-  void set_immediate(jint imm) { set_int_at(imm_offset, imm); }
+  void set_immediate(jint imm, int bit_mask) {
+    if (bit_mask == ~0) {
+      set_int_at(imm_offset, imm);
+      return;
+    }
+
+    assert((imm & ~bit_mask) == 0, "trying to set bits outside the mask");
+    imm &= bit_mask;
+
+    assert(align_up(immediate_address(), sizeof(jint)) ==
+           align_down(immediate_address(), sizeof(jint)), "immediate not aligned");
+    jint* data_addr = (jint*)immediate_address();
+    jint old_value = Atomic::load(data_addr);
+    while (true) {
+      // Only bits in the mask are changed
+      jint new_value = imm | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      jint v = Atomic::cmpxchg(data_addr, old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
+  }
   bool check_barrier(err_msg& msg) const;
   void verify() const {
 #ifdef ASSERT
@@ -159,13 +182,13 @@ static NativeNMethodCmpBarrier* native_nmethod_barrier(nmethod* nm) {
   return barrier;
 }
 
-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
   if (!supports_entry_barrier(nm)) {
     return;
   }
 
   NativeNMethodCmpBarrier* cmp = native_nmethod_barrier(nm);
-  cmp->set_immediate(value);
+  cmp->set_immediate(value, bit_mask);
 }
 
 int BarrierSetNMethod::guard_value(nmethod* nm) {

src/hotspot/cpu/zero/gc/shared/barrierSetNMethod_zero.cpp

@@ -29,7 +29,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
   ShouldNotReachHere();
 }
 
-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
   ShouldNotReachHere();
 }