8362193: Re-work MacOS/AArch64 SpinPause to handle SB

src/hotspot/cpu/aarch64/globals_aarch64.hpp

@@ -117,7 +117,8 @@ define_pd_global(intx, InlineSmallCode,          1000);
   product(ccstr, OnSpinWaitInst, "yield", DIAGNOSTIC,                   \
           "The instruction to use to implement "                        \
           "java.lang.Thread.onSpinWait()."                              \
-          "Options: none, nop, isb, yield, sb.")                        \
+          "Valid values are: none, nop, isb, yield, sb.")               \
+          constraint(OnSpinWaitInstNameConstraintFunc, AtParse)         \
   product(uint, OnSpinWaitInstCount, 1, DIAGNOSTIC,                     \
           "The number of OnSpinWaitInst instructions to generate."      \
           "It cannot be used with OnSpinWaitInst=none.")                \

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -6816,6 +6816,7 @@ void MacroAssembler::spin_wait() {
         yield();
         break;
       case SpinWait::SB:
+        assert(VM_Version::supports_sb(), "current CPU does not support SB instruction");
         sb();
         break;
       default:

src/hotspot/cpu/aarch64/spin_wait_aarch64.cpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "spin_wait_aarch64.hpp"
+#include "utilities/debug.hpp"
+
+#include <string.h>
+
+bool SpinWait::supports(const char *name) {
+  return name != nullptr &&
+         (strcmp(name, "nop")   == 0 ||
+          strcmp(name, "isb")   == 0 ||
+          strcmp(name, "yield") == 0 ||
+          strcmp(name, "sb")    == 0 ||
+          strcmp(name, "none")  == 0);
+}
+
+SpinWait::Inst SpinWait::from_name(const char* name) {
+  assert(supports(name), "checked by OnSpinWaitInstNameConstraintFunc");
+
+  if (strcmp(name, "nop") == 0) {
+    return SpinWait::NOP;
+  } else if (strcmp(name, "isb") == 0) {
+    return SpinWait::ISB;
+  } else if (strcmp(name, "yield") == 0) {
+    return SpinWait::YIELD;
+  } else if (strcmp(name, "sb") == 0) {
+    return SpinWait::SB;
+  }
+
+  return SpinWait::NONE;
+}

src/hotspot/cpu/aarch64/spin_wait_aarch64.hpp

@@ -19,31 +19,39 @@
  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  * or visit www.oracle.com if you need additional information or have any
  * questions.
- *
  */
 
 #ifndef CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
 #define CPU_AARCH64_SPIN_WAIT_AARCH64_HPP
 
 class SpinWait {
 public:
+  // Non-zero values are chosen to have only one bit set.
+  // This simplifies testing values in assembly code.
+  // This limits us to 64 possible implementation.
+  // Value 1 is used for the default implementation.
   enum Inst {
-    NONE = -1,
-    NOP,
-    ISB,
-    YIELD,
-    SB
+    NONE  = 0,
+    YIELD = (1 << 0),
+    ISB   = (1 << 1),
+    SB    = (1 << 2),
+    NOP   = (1 << 3)
   };
 
 private:
   Inst _inst;
   int _count;
 
+  Inst from_name(const char *name);
+
 public:
-  SpinWait(Inst inst = NONE, int count = 0) : _inst(inst), _count(count) {}
+  SpinWait(Inst inst = NONE, int count = 0) : _inst(inst), _count(inst == NONE ? 0 : count) {}
+  SpinWait(const char *name, int count) : SpinWait(from_name(name), count) {}
 
   Inst inst() const { return _inst; }
   int inst_count() const { return _count; }
+
+  static bool supports(const char *name);
 };
 
 #endif // CPU_AARCH64_SPIN_WAIT_AARCH64_HPP

src/hotspot/cpu/aarch64/vm_version_aarch64.cpp

@@ -51,26 +51,12 @@ uintptr_t VM_Version::_pac_mask;
 SpinWait VM_Version::_spin_wait;
 
 static SpinWait get_spin_wait_desc() {
-  if (strcmp(OnSpinWaitInst, "nop") == 0) {
-    return SpinWait(SpinWait::NOP, OnSpinWaitInstCount);
-  } else if (strcmp(OnSpinWaitInst, "isb") == 0) {
-    return SpinWait(SpinWait::ISB, OnSpinWaitInstCount);
-  } else if (strcmp(OnSpinWaitInst, "yield") == 0) {
-    return SpinWait(SpinWait::YIELD, OnSpinWaitInstCount);
-  } else if (strcmp(OnSpinWaitInst, "sb") == 0) {
-    if (!VM_Version::supports_sb()) {
-      vm_exit_during_initialization("OnSpinWaitInst is SB but current CPU does not support SB instruction");
-    }
-    return SpinWait(SpinWait::SB, OnSpinWaitInstCount);
-  } else if (strcmp(OnSpinWaitInst, "none") != 0) {
-    vm_exit_during_initialization("The options for OnSpinWaitInst are nop, isb, yield, sb, and none", OnSpinWaitInst);
-  }
-
-  if (!FLAG_IS_DEFAULT(OnSpinWaitInstCount) && OnSpinWaitInstCount > 0) {
-    vm_exit_during_initialization("OnSpinWaitInstCount cannot be used for OnSpinWaitInst 'none'");
+  SpinWait spin_wait(OnSpinWaitInst, OnSpinWaitInstCount);
+  if (spin_wait.inst() == SpinWait::SB && !VM_Version::supports_sb()) {
+    vm_exit_during_initialization("OnSpinWaitInst is SB but current CPU does not support SB instruction");
   }
 
-  return SpinWait{};
+  return spin_wait;
 }
 
 void VM_Version::initialize() {

src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp

@@ -49,6 +49,7 @@
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 #include "runtime/timer.hpp"
+#include "runtime/vm_version.hpp"
 #include "signals_posix.hpp"
 #include "utilities/align.hpp"
 #include "utilities/events.hpp"
@@ -524,39 +525,56 @@ static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
 }
 
 extern "C" {
-  // needs local assembler label '1:' to avoid trouble when using linktime optimization
+  // needs local assembler label '4:' to avoid trouble when using linktime optimization
   int SpinPause() {
     // We don't use StubRoutines::aarch64::spin_wait stub in order to
     // avoid a costly call to os::current_thread_enable_wx() on MacOS.
     // We should return 1 if SpinPause is implemented, and since there
-    // will be a sequence of 11 instructions for NONE and YIELD and 12
-    // instructions for NOP and ISB, SpinPause will always return 1.
-    uint64_t br_dst;
-    const int instructions_per_case = 2;
-    int64_t off = VM_Version::spin_wait_desc().inst() * instructions_per_case * Assembler::instruction_size;
-
-    assert(VM_Version::spin_wait_desc().inst() >= SpinWait::NONE &&
-           VM_Version::spin_wait_desc().inst() <= SpinWait::YIELD, "must be");
-    assert(-1 == SpinWait::NONE,  "must be");
-    assert( 0 == SpinWait::NOP,   "must be");
-    assert( 1 == SpinWait::ISB,   "must be");
-    assert( 2 == SpinWait::YIELD, "must be");
-
+    // will be always a sequence of instructions, SpinPause will always return 1.
+
+    assert(SpinWait::NONE  == 0, "SpinWait::Inst value 0 reserved to indicate no implementation");
+    assert(SpinWait::YIELD == 1, "SpinWait::Inst value 1 reserved for 'yield' instruction");
+    assert(SpinWait::ISB   == 2, "SpinWait::Inst value 2 reserved for 'isb' instruction");
+    assert(SpinWait::SB    == 4, "SpinWait::Inst value 4 reserved for 'sb' instruction");
+    assert(SpinWait::NOP   == 8, "SpinWait::Inst value 8 reserved for 'nop' instruction");
+
+    const uint64_t inst_id = VM_Version::spin_wait_desc().inst();
+    assert(inst_id == 0 || is_power_of_2(inst_id), "Values of SpinWait::Inst must be 0 or power of 2");
+    assert(inst_id != SpinWait::SB || VM_Version::supports_sb(), "current CPU does not support SB instruction");
+    assert(inst_id <= SpinWait::NOP, "Unsupported type of SpinWait::Inst: %llu", inst_id);
+
+    // The assembly code below is equivalent to the following:
+    //
+    // if (inst_id == 1) {
+    //   exec_yield_inst();
+    // } else if (inst_id == 2) {
+    //   exec_isb_inst();
+    // } else if (inst_id == 4) {
+    //   exec_sb_inst();
+    // } else if (inst_id == 8) {
+    //   exec_nop_inst();
+    // }
     asm volatile(
-        "  adr  %[d], 20          \n" // 20 == PC here + 5 instructions => address
-                                      // to entry for case SpinWait::NOP
-        "  add  %[d], %[d], %[o]  \n"
-        "  br   %[d]              \n"
-        "  b    1f                \n" // case SpinWait::NONE  (-1)
-        "  nop                    \n" // padding
-        "  nop                    \n" // case SpinWait::NOP   ( 0)
-        "  b    1f                \n"
-        "  isb                    \n" // case SpinWait::ISB   ( 1)
-        "  b    1f                \n"
-        "  yield                  \n" // case SpinWait::YIELD ( 2)
-        "1:        \n"
-        : [d]"=&r"(br_dst)
-        : [o]"r"(off)
+        "  tbz %[id], 0, 0f      \n" // The default instruction for SpinWait is YIELD.
+                                     // We check it first before going to switch.
+        "  yield                 \n"
+        "  b    4f               \n"
+        "0:                      \n"
+        "  tbnz %[id], 1, 1f     \n"
+        "  tbnz %[id], 2, 2f     \n"
+        "  tbnz %[id], 3, 3f     \n"
+        "  b    4f               \n"
+        "1:                      \n"
+        "  isb                   \n"
+        "  b    4f               \n"
+        "2:                      \n"
+        "  .inst 0xd50330ff      \n" // SB instruction, explicitly encoded not to rely on a compiler
+        "  b    4f               \n"
+        "3:                      \n"
+        "  nop                   \n"
+        "4:                      \n"
+        :
+        : [id]"r"(inst_id)
         : "memory");
     return 1;
   }

src/hotspot/share/runtime/flags/jvmFlagConstraintsRuntime.cpp

@@ -127,3 +127,25 @@ JVMFlag::Error NUMAInterleaveGranularityConstraintFunc(size_t value, bool verbos
 
   return JVMFlag::SUCCESS;
 }
+
+JVMFlag::Error OnSpinWaitInstNameConstraintFunc(ccstr value, bool verbose) {
+#ifdef AARCH64
+  if (value == nullptr) {
+    JVMFlag::printError(verbose, "OnSpinWaitInst cannot be empty\n");
+    return JVMFlag::VIOLATES_CONSTRAINT;
+  }
+
+  if (strcmp(value, "nop")   != 0 &&
+      strcmp(value, "isb")   != 0 &&
+      strcmp(value, "yield") != 0 &&
+      strcmp(value, "sb")    != 0 &&
+      strcmp(value, "none")  != 0) {
+    JVMFlag::printError(verbose,
+                        "Unrecognized value %s for OnSpinWaitInst. Must be one of the following: "
+                        "nop, isb, yield, sb, none\n",
+                        value);
+    return JVMFlag::VIOLATES_CONSTRAINT;
+  }
+#endif
+  return JVMFlag::SUCCESS;
+}

src/hotspot/share/runtime/flags/jvmFlagConstraintsRuntime.hpp

@@ -40,7 +40,8 @@
   f(int,    ObjectAlignmentInBytesConstraintFunc)     \
   f(int,    ContendedPaddingWidthConstraintFunc)      \
   f(size_t, VMPageSizeConstraintFunc)                 \
-  f(size_t, NUMAInterleaveGranularityConstraintFunc)
+  f(size_t, NUMAInterleaveGranularityConstraintFunc)  \
+  f(ccstr,  OnSpinWaitInstNameConstraintFunc)
 
 RUNTIME_CONSTRAINTS(DECLARE_CONSTRAINT)
 

test/hotspot/gtest/aarch64/test_spin_pause.cpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#if defined(AARCH64) && !defined(ZERO)
+
+#include "utilities/spinYield.hpp"
+#include "unittest.hpp"
+
+TEST_VM(SpinPause, sanity) {
+  ASSERT_EQ(SpinPause(), 1);
+}
+
+#endif  // AARCH64

test/hotspot/jtreg/gtest/TestSpinPauseAArch64.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test id=default_armv8_0
+ * @bug 8362193
+ * @summary Run SpinPause gtest using different instructions for SpinPause
+ * @library /test/lib
+ * @requires vm.flagless
+ * @requires os.arch=="aarch64"
+ * @run main/native GTestWrapper --gtest_filter=SpinPause*
+ * @run main/native GTestWrapper --gtest_filter=SpinPause* -XX:+UnlockDiagnosticVMOptions -XX:OnSpinWaitInst=none
+ * @run main/native GTestWrapper --gtest_filter=SpinPause* -XX:+UnlockDiagnosticVMOptions -XX:OnSpinWaitInst=nop
+ * @run main/native GTestWrapper --gtest_filter=SpinPause* -XX:+UnlockDiagnosticVMOptions -XX:OnSpinWaitInst=isb
+ * @run main/native GTestWrapper --gtest_filter=SpinPause* -XX:+UnlockDiagnosticVMOptions -XX:OnSpinWaitInst=yield
+ */
+
+/*
+ * @test id=sb_armv8_5
+ * @bug 8362193
+ * @summary Run SpinPause gtest using SB instruction for SpinPause
+ * @library /test/lib
+ * @requires vm.flagless
+ * @requires (os.arch=="aarch64" & vm.cpu.features ~= ".*sb.*")
+ * @run main/native GTestWrapper --gtest_filter=SpinPause* -XX:+UnlockDiagnosticVMOptions -XX:OnSpinWaitInst=sb
+ */