8336845: [lworld] Virtual threads don't support the value class calling convention

src/hotspot/cpu/aarch64/continuationFreezeThaw_aarch64.inline.hpp

@@ -67,6 +67,12 @@ inline frame FreezeBase::sender(const frame& f) {
 
   int slot = 0;
   CodeBlob* sender_cb = CodeCache::find_blob_and_oopmap(sender_pc, slot);
+
+  // Repair the sender sp if the frame has been extended
+  if (sender_cb->is_nmethod()) {
+    sender_sp = f.repair_sender_sp(sender_sp, link_addr);
+  }
+
   return sender_cb != nullptr
     ? frame(sender_sp, sender_sp, *link_addr, sender_pc, sender_cb,
             slot == -1 ? nullptr : sender_cb->oop_map_for_slot(slot, sender_pc),
@@ -75,7 +81,7 @@ inline frame FreezeBase::sender(const frame& f) {
 }
 
 template<typename FKind>
-frame FreezeBase::new_heap_frame(frame& f, frame& caller) {
+frame FreezeBase::new_heap_frame(frame& f, frame& caller, int size_adjust) {
   assert(FKind::is_instance(f), "");
   assert(!caller.is_interpreted_frame()
     || caller.unextended_sp() == (intptr_t*)caller.at(frame::interpreter_frame_last_sp_offset), "");
@@ -106,8 +112,8 @@ frame FreezeBase::new_heap_frame(frame& f, frame& caller) {
     fp = *(intptr_t**)(f.sp() - frame::sender_sp_offset);
 
     int fsize = FKind::size(f);
-    sp = caller.unextended_sp() - fsize;
-    if (caller.is_interpreted_frame()) {
+    sp = caller.unextended_sp() - fsize - size_adjust;
+    if (caller.is_interpreted_frame() && size_adjust == 0) {
       // If the caller is interpreted, our stackargs are not supposed to overlap with it
       // so we make more room by moving sp down by argsize
       int argsize = FKind::stack_argsize(f);
@@ -180,11 +186,12 @@ inline void FreezeBase::set_top_frame_metadata_pd(const frame& hf) {
                                        : (intptr_t)hf.fp();
 }
 
-inline void FreezeBase::patch_pd(frame& hf, const frame& caller) {
+inline void FreezeBase::patch_pd(frame& hf, const frame& caller, bool is_bottom_frame) {
   if (caller.is_interpreted_frame()) {
     assert(!caller.is_empty(), "");
     patch_callee_link_relative(caller, caller.fp());
-  } else {
+  } else if (is_bottom_frame && caller.pc() != nullptr) {
+    assert(caller.is_compiled_frame(), "");
     // If we're the bottom-most frame frozen in this freeze, the caller might have stayed frozen in the chunk,
     // and its oop-containing fp fixed. We've now just overwritten it, so we must patch it back to its value
     // as read from the chunk.
@@ -215,7 +222,7 @@ inline frame ThawBase::new_entry_frame() {
   return frame(sp, sp, _cont.entryFP(), _cont.entryPC()); // TODO PERF: This finds code blob and computes deopt state
 }
 
-template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom) {
+template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame& caller, bool bottom, int size_adjust) {
   assert(FKind::is_instance(hf), "");
   // The values in the returned frame object will be written into the callee's stack in patch.
 
@@ -243,24 +250,23 @@ template<typename FKind> frame ThawBase::new_stack_frame(const frame& hf, frame&
     return f;
   } else {
     int fsize = FKind::size(hf);
-    intptr_t* frame_sp = caller.unextended_sp() - fsize;
+    intptr_t* frame_sp = caller.unextended_sp() - fsize - size_adjust;
     if (bottom || caller.is_interpreted_frame()) {
-      int argsize = FKind::stack_argsize(hf);
-
-      fsize += argsize;
-      frame_sp   -= argsize;
-      caller.set_sp(caller.sp() - argsize);
-      assert(caller.sp() == frame_sp + (fsize-argsize), "");
-
+      if (size_adjust == 0) {
+        int argsize = FKind::stack_argsize(hf);
+        frame_sp -= argsize;
+      }
       frame_sp = align(hf, frame_sp, caller, bottom);
     }
+    caller.set_sp(frame_sp + fsize);
+    assert(is_aligned(frame_sp, frame::frame_alignment), "");
 
     assert(hf.cb() != nullptr, "");
     assert(hf.oop_map() != nullptr, "");
     intptr_t* fp;
     if (PreserveFramePointer) {
       // we need to recreate a "real" frame pointer, pointing into the stack
-      fp = frame_sp + FKind::size(hf) - frame::sender_sp_offset;
+      fp = frame_sp + fsize - frame::sender_sp_offset;
     } else {
       fp = FKind::stub || FKind::native
         ? frame_sp + fsize - frame::sender_sp_offset // fp always points to the address below the pushed return pc. We need correct address.
@@ -275,16 +281,16 @@ inline intptr_t* ThawBase::align(const frame& hf, intptr_t* frame_sp, frame& cal
   if (((intptr_t)frame_sp & 0xf) != 0) {
     assert(caller.is_interpreted_frame() || (bottom && hf.compiled_frame_stack_argsize() % 2 != 0), "");
     frame_sp--;
-    caller.set_sp(caller.sp() - 1);
   }
   assert(is_aligned(frame_sp, frame::frame_alignment), "");
 #endif
-
   return frame_sp;
 }
 
 inline void ThawBase::patch_pd(frame& f, const frame& caller) {
-  patch_callee_link(caller, caller.fp());
+  if (caller.is_interpreted_frame() || PreserveFramePointer) {
+    patch_callee_link(caller, caller.fp());
+  }
 }
 
 inline void ThawBase::patch_pd(frame& f, intptr_t* caller_sp) {

src/hotspot/cpu/aarch64/continuationHelper_aarch64.inline.hpp

@@ -141,8 +141,8 @@ inline address ContinuationHelper::Frame::real_pc(const frame& f) {
   return pauth_strip_pointer(*pc_addr);
 }
 
-inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) {
-  address* pc_addr = &(((address*) f.sp())[-1]);
+inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc, bool callee_augmented) {
+  address* pc_addr = &(((address*) (callee_augmented ? f.unextended_sp() : f.sp()))[-1]);
   *pc_addr = pauth_sign_return_address(pc);
 }
 

src/hotspot/cpu/aarch64/frame_aarch64.cpp

@@ -670,6 +670,9 @@ void frame::describe_pd(FrameValues& values, int frame_no) {
     } else {
       ret_pc_loc = real_fp() - return_addr_offset;
       fp_loc = real_fp() - sender_sp_offset;
+      if (cb()->is_nmethod() && cb()->as_nmethod_or_null()->needs_stack_repair()) {
+        values.describe(frame_no, fp_loc - 1, err_msg("fsize for #%d", frame_no), 1);
+      }
     }
     address ret_pc = *(address*)ret_pc_loc;
     values.describe(frame_no, ret_pc_loc,
@@ -841,6 +844,28 @@ intptr_t* frame::repair_sender_sp(intptr_t* sender_sp, intptr_t** saved_fp_addr)
   return sender_sp;
 }
 
+intptr_t* frame::repair_sender_sp(nmethod* nm, intptr_t* sp, intptr_t** saved_fp_addr) {
+  assert(nm != nullptr && nm->needs_stack_repair(), "");
+  // The stack increment resides just below the saved FP on the stack and
+  // records the total frame size excluding the two words for saving FP and LR.
+  intptr_t* real_frame_size_addr = (intptr_t*) (saved_fp_addr - 1);
+  int real_frame_size = (*real_frame_size_addr / wordSize) + 2;
+  assert(real_frame_size >= nm->frame_size() && real_frame_size <= 1000000, "invalid frame size");
+  return sp + real_frame_size;
+}
+
+bool frame::was_augmented_on_entry(int& real_size) const {
+  assert(is_compiled_frame(), "");
+  if (_cb->as_nmethod_or_null()->needs_stack_repair()) {
+    intptr_t* real_frame_size_addr = unextended_sp() + _cb->frame_size() - sender_sp_offset - 1;
+    log_trace(continuations)("real_frame_size is addr is " INTPTR_FORMAT, p2i(real_frame_size_addr));
+    real_size = (*real_frame_size_addr / wordSize) + 2;
+    return real_size != _cb->frame_size();
+  }
+  real_size = _cb->frame_size();
+  return false;
+}
+
 void JavaFrameAnchor::make_walkable() {
   // last frame set?
   if (last_Java_sp() == nullptr) return;

src/hotspot/cpu/aarch64/frame_aarch64.hpp

@@ -152,8 +152,13 @@
     return (intptr_t*) addr_at(offset);
   }
 
+ public:
   // Support for scalarized inline type calling convention
   intptr_t* repair_sender_sp(intptr_t* sender_sp, intptr_t** saved_fp_addr) const;
+  static intptr_t* repair_sender_sp(nmethod* nm, intptr_t* sp, intptr_t** saved_fp_addr);
+  bool was_augmented_on_entry(int& real_size) const;
+
+ private:
 
 #ifdef ASSERT
   // Used in frame::sender_for_{interpreter,compiled}_frame

src/hotspot/cpu/aarch64/stackChunkFrameStream_aarch64.inline.hpp

@@ -35,8 +35,30 @@ template <ChunkFrames frame_kind>
 inline bool StackChunkFrameStream<frame_kind>::is_in_frame(void* p0) const {
   assert(!is_done(), "");
   intptr_t* p = (intptr_t*)p0;
-  int argsize = is_compiled() ? (_cb->as_nmethod()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord : 0;
-  int frame_size = _cb->frame_size() + argsize;
+  int frame_size = _cb->frame_size();
+  if (is_compiled()) {
+    nmethod* nm = _cb->as_nmethod_or_null();
+    if (nm->needs_stack_repair() && nm->is_compiled_by_c2()) {
+      frame f = to_frame();
+      bool augmented = f.was_augmented_on_entry(frame_size);
+      if (!augmented) {
+        // Fix: C2 caller, so frame was not extended and thus the
+        // size read from the frame does not include the arguments.
+        // Ideally we have to count the arg size for the scalarized
+        // convention. For now we include the size of the caller frame
+        // which would at least be equal to that.
+        RegisterMap map(nullptr,
+                        RegisterMap::UpdateMap::skip,
+                        RegisterMap::ProcessFrames::skip,
+                        RegisterMap::WalkContinuation::skip);
+        frame caller = to_frame().sender(&map);
+        assert(caller.is_compiled_frame() && caller.cb()->as_nmethod()->is_compiled_by_c2(), "needs stack repair but was not extended with c1/interpreter caller");
+        frame_size += (caller.real_fp() - caller.sp());
+      }
+    } else {
+      frame_size += _cb->as_nmethod()->num_stack_arg_slots() * VMRegImpl::stack_slot_size >> LogBytesPerWord;
+    }
+  }
   return p == sp() - frame::sender_sp_offset || ((p - unextended_sp()) >= 0 && (p - unextended_sp()) < frame_size);
 }
 #endif
@@ -46,15 +68,21 @@ inline frame StackChunkFrameStream<frame_kind>::to_frame() const {
   if (is_done()) {
     return frame(_sp, _sp, nullptr, nullptr, nullptr, nullptr, true);
   } else {
-    return frame(sp(), unextended_sp(), fp(), pc(), cb(), _oopmap, true);
+    frame f = frame(sp(), unextended_sp(), fp(), pc(), cb(), _oopmap, true);
+    // If caller tries to get the sender of this frame and PreserveFramePointer
+    // is set, fp() will be used which contains the old value at the time of
+    // freeze (fp is reconstructed again during thaw). Setting sp as trusted
+    // causes the sender code to use _unextended_sp instead (see sender_for_compiled_frame()).
+    f.set_sp_is_trusted();
+    return f;
   }
 }
 
 template <ChunkFrames frame_kind>
 inline address StackChunkFrameStream<frame_kind>::get_pc() const {
   assert(!is_done(), "");
   // Just strip it for frames on the heap.
-  return pauth_strip_pointer(*(address*)(_sp - 1));
+  return pauth_strip_pointer(*(address*)((_callee_augmented ? _unextended_sp : _sp) - 1));
 }
 
 template <ChunkFrames frame_kind>

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -10480,6 +10480,30 @@ class StubGenerator: public StubCodeGenerator {
   }
 #endif // LINUX
 
+  static void save_return_registers(MacroAssembler* masm) {
+    if (InlineTypeReturnedAsFields) {
+      masm->push(RegSet::range(r0, r7), sp);
+      masm->sub(sp, sp, 4 * wordSize);
+      masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
+      masm->sub(sp, sp, 4 * wordSize);
+      masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
+    } else {
+      masm->fmovd(rscratch1, v0);
+      masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
+    }
+  }
+
+  static void restore_return_registers(MacroAssembler* masm) {
+    if (InlineTypeReturnedAsFields) {
+      masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
+      masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
+      masm->pop(RegSet::range(r0, r7), sp);
+    } else {
+      masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
+      masm->fmovd(v0, rscratch1);
+    }
+  }
+
   address generate_cont_thaw(Continuation::thaw_kind kind) {
     bool return_barrier = Continuation::is_thaw_return_barrier(kind);
     bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
@@ -10494,8 +10518,7 @@ class StubGenerator: public StubCodeGenerator {
 
     if (return_barrier) {
       // preserve possible return value from a method returning to the return barrier
-      __ fmovd(rscratch1, v0);
-      __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
+      save_return_registers(_masm);
     }
 
     __ movw(c_rarg1, (return_barrier ? 1 : 0));
@@ -10504,8 +10527,7 @@ class StubGenerator: public StubCodeGenerator {
 
     if (return_barrier) {
       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
-      __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
-      __ fmovd(v0, rscratch1);
+      restore_return_registers(_masm);
     }
     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
 
@@ -10524,8 +10546,7 @@ class StubGenerator: public StubCodeGenerator {
 
     if (return_barrier) {
       // save original return value -- again
-      __ fmovd(rscratch1, v0);
-      __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
+      save_return_registers(_masm);
     }
 
     // If we want, we can templatize thaw by kind, and have three different entries
@@ -10536,8 +10557,7 @@ class StubGenerator: public StubCodeGenerator {
 
     if (return_barrier) {
       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
-      __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
-      __ fmovd(v0, rscratch1);
+      restore_return_registers(_masm);
     } else {
       __ mov(r0, zr); // return 0 (success) from doYield
     }

src/hotspot/cpu/arm/continuationHelper_arm.inline.hpp

@@ -107,7 +107,7 @@ inline address ContinuationHelper::Frame::real_pc(const frame& f) {
   return nullptr;
 }
 
-inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) {
+inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc, bool callee_augmented) {
   Unimplemented();
 }
 

src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp

@@ -110,7 +110,7 @@ inline address ContinuationHelper::Frame::real_pc(const frame& f) {
   return (address)f.own_abi()->lr;
 }
 
-inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) {
+inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc, bool callee_augmented) {
   f.own_abi()->lr = (uint64_t)pc;
 }
 

src/hotspot/cpu/riscv/continuationHelper_riscv.inline.hpp

@@ -128,7 +128,7 @@ inline address ContinuationHelper::Frame::real_pc(const frame& f) {
   return *pc_addr;
 }
 
-inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) {
+inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc, bool callee_augmented) {
   address* pc_addr = &(((address*) f.sp())[-1]);
   *pc_addr = pc;
 }

src/hotspot/cpu/s390/continuationHelper_s390.inline.hpp

@@ -107,7 +107,7 @@ inline address ContinuationHelper::Frame::real_pc(const frame& f) {
   return nullptr;
 }
 
-inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) {
+inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc, bool callee_augmented) {
   Unimplemented();
 }