perf(runtime): #6804 — eager ShapeId stamping + canonical typed_feedback tokens

changelog.d/6807-eager-shape-stamping.md

@@ -0,0 +1 @@
+perf(runtime): #6804 first half — shape-cached literals birth-stamp their stable ShapeId (zero extra probes per allocation), fresh dynamic shapes stamp at birth, and typed_feedback observation tokens canonicalize on ids with self-healing at first observation — one logical shape yields one token per site, stable across grow-reallocs and GC moves. Also decouples token-as-address consumers, including the typed-feedback GC scanner, which no longer visits id tokens as heap pointers.

changelog.d/6808-pic-shape-id-compare.md

@@ -0,0 +1 @@
+perf(codegen): #6804 second half — the generic property-get PIC compares a discriminated shape token: header-stamped ShapeIds (lifted above the 48-bit pointer space) for plain objects — stable across grow-reallocs and GC moves, immune to address-recycling, and re-enabling PIC caching for owned keys arrays — with keys-pointer tokens retained for class instances. The hit path also bounds the cached slot by the receiver's inline capacity, closing a latent over-read for same-shape siblings with different physical allocations.

crates/perry-codegen/src/expr/property_get/generic_dispatch.rs

@@ -327,23 +327,47 @@ pub(crate) fn lower_generic_property_get(
     let keys_ptr_p = ctx.block().inttoptr(I64, &keys_addr);
     let keys_val = ctx.block().load(I64, &keys_ptr_p);
 
-    // Load cached keys_array from the per-site global.
+    // #6804: the receiver's shape TOKEN. A plain object stamped with a
+    // runtime ShapeId (`parent_class_id` ∈ [0x8000_0000, 0xC000_0000) —
+    // see `shapes::SHAPE_ID_BASE/END`; a real parent class id can never
+    // fall in that range) compares by id: stable across keys grow-reallocs
+    // and GC moves, and immune to address recycling (ids are never
+    // reused). Everything else (class instances, unstamped receivers)
+    // keeps the keys-pointer compare. Id tokens are lifted above the
+    // 48-bit pointer space (bit 62, `shapes::PIC_ID_TOKEN_BIT`) so the
+    // two token kinds can never collide numerically — one compare, no
+    // discriminant word. `parent_class_id` is a u32 at offset 8 on every
+    // target (the four leading u32s precede the pointer fields).
+    let pcid_addr = ctx.block().add(I64, &safe_obj_handle, "8");
+    let pcid_ptr = ctx.block().inttoptr(I64, &pcid_addr);
+    let pcid = ctx.block().load(I32, &pcid_ptr);
+    // In-range test via wrapping add + ult: (pcid - 0x8000_0000) < 0x4000_0000.
+    // (-2147483648 is the i32 spelling of the 0x8000_0000 subtrahend.)
+    let pcid_rel = ctx.block().add(I32, &pcid, "-2147483648");
+    let is_stamp = ctx.block().icmp_ult(I32, &pcid_rel, "1073741824");
+    let pcid64 = ctx.block().zext(I32, &pcid, I64);
+    // PIC_ID_TOKEN_BIT = 1 << 62.
+    let id_token = ctx.block().or(I64, &pcid64, "4611686018427387904");
+    let token = ctx.block().select(I1, &is_stamp, I64, &id_token, &keys_val);
+
+    // Load the cached token from the per-site global.
     let cache_keys_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]);
-    let cached_keys = ctx.block().load(I64, &cache_keys_ptr);
-    let keys_eq = ctx.block().icmp_eq(I64, &keys_val, &cached_keys);
+    let cached_token = ctx.block().load(I64, &cache_keys_ptr);
+    let token_eq = ctx.block().icmp_eq(I64, &token, &cached_token);
     // #809: an object with `keys_array == null` (e.g. an
     // `Object.create(proto)` result, or any object with no own
     // string props) has no cacheable own-slot. The per-site cache
-    // global is zero-initialized, so `keys_val (0) == cached_keys
-    // (0)` spuriously "hits" and the hit path returns the empty
-    // slot[0] — never invoking the miss handler, so the runtime's
-    // prototype-chain walk in `js_object_get_field_by_name` is
-    // skipped and `Object.create(P).m()` reads `undefined`. Require
-    // a non-null keys_array for a hit so keyless receivers fall to
-    // the slow path (which resolves inherited props correctly).
-    let keys_nonnull = ctx.block().icmp_ne(I64, &keys_val, "0");
-    let hit_keys = ctx.block().and(I1, &is_object, &keys_eq);
-    let hit = ctx.block().and(I1, &hit_keys, &keys_nonnull);
+    // global is zero-initialized, so a zero token would spuriously
+    // "hit" and the hit path would return the empty slot[0] — never
+    // invoking the miss handler, so the runtime's prototype-chain
+    // walk in `js_object_get_field_by_name` is skipped and
+    // `Object.create(P).m()` reads `undefined`. Require a non-zero
+    // token for a hit so keyless receivers fall to the slow path
+    // (which resolves inherited props correctly). Id tokens always
+    // carry bit 62, so they are never zero.
+    let token_nonnull = ctx.block().icmp_ne(I64, &token, "0");
+    let hit_token = ctx.block().and(I1, &is_object, &token_eq);
+    let hit = ctx.block().and(I1, &hit_token, &token_nonnull);
 
     let hit_idx = ctx.new_block("pic.hit");
     let miss_idx = ctx.new_block("pic.miss");
@@ -353,14 +377,36 @@ pub(crate) fn lower_generic_property_get(
     let merge_label = ctx.block_label(merge_idx);
     ctx.block().cond_br(&hit, &hit_label, &miss_label);
 
-    // PIC hit: direct field load.
+    // PIC hit: bounds-check the cached slot, then direct field load.
     ctx.current_block = hit_idx;
+    let cache_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]);
+    let slot = ctx.block().load(I64, &cache_slot_ptr);
+    // #6804: bound the cached slot by THIS receiver's inline capacity.
+    // Same-shape siblings can differ in physical allocation (an object
+    // built at a small alloc site adopts a shared keys array whose later
+    // slots live in its OVERFLOW map) — a slot primed from a
+    // larger-capacity sibling must not drive a raw load past this
+    // receiver's field region. `alloc_limit = max(field_count,
+    // INLINE_SLOT_FLOOR=4)` mirrors the miss handler's cacheability
+    // rule; an out-of-bounds slot falls to the miss path, which reads
+    // the overflow map correctly (and records the guard failure —
+    // `record_guard_pass` only fires after the bounds check passes).
+    let fc_addr = ctx.block().add(I64, &safe_obj_handle, "12");
+    let fc_ptr = ctx.block().inttoptr(I64, &fc_addr);
+    let fc = ctx.block().load(I32, &fc_ptr);
+    let fc64 = ctx.block().zext(I32, &fc, I64);
+    let fc_floor = ctx.block().icmp_ult(I64, &fc64, "4"); // INLINE_SLOT_FLOOR
+    let limit = ctx.block().select(I1, &fc_floor, I64, "4", &fc64);
+    let slot_in_bounds = ctx.block().icmp_ult(I64, &slot, &limit);
+    let bounds_hit = ctx.new_block("pic.hit.load");
+    let bounds_hit_label = ctx.block_label(bounds_hit);
+    ctx.block()
+        .cond_br(&slot_in_bounds, &bounds_hit_label, &miss_label);
+    ctx.current_block = bounds_hit;
     ctx.block().call_void(
         "js_typed_feedback_record_guard_pass",
         &[(I64, &feedback_site_id)],
     );
-    let cache_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]);
-    let slot = ctx.block().load(I64, &cache_slot_ptr);
     let offset = ctx.block().shl(I64, &slot, "3");
     // arm64_32 watchOS: the object fields region begins at
     // `size_of::<ObjectHeader>()` past the user pointer — 24 on 64-bit, 20 on

crates/perry-runtime/src/object/alloc.rs

@@ -604,9 +604,9 @@ pub extern "C" fn js_object_alloc_with_shape(
         crate::gc::layout_init_pointer_free(obj_ptr as *mut u8);
     }
 
-    let cached = shape_cache_get(shape_id);
-    let keys_arr = if !cached.is_null() {
-        cached
+    let (cached, cached_runtime_id) = shape_cache_get_with_id(shape_id);
+    let (keys_arr, runtime_shape_id) = if !cached.is_null() {
+        (cached, cached_runtime_id)
     } else {
         let keys_bytes =
             unsafe { std::slice::from_raw_parts(packed_keys, packed_keys_len as usize) };
@@ -633,11 +633,18 @@ pub extern "C" fn js_object_alloc_with_shape(
             }
         }
         shape_cache_insert(shape_id, arr);
-        arr
+        (arr, shape_cache_get_with_id(shape_id).1)
     };
 
     unsafe {
         set_object_keys_array(obj_ptr, keys_arr);
+        // #6804: birth-stamp the runtime ShapeId (see `ShapeCacheEntry`) —
+        // newborn literals carry their stable identity immediately, so
+        // typed_feedback tokens and the id-keyed FIELD_CACHE never see a
+        // pre-stamp window for shape-cached objects.
+        if runtime_shape_id != 0 {
+            (*obj_ptr).parent_class_id = runtime_shape_id;
+        }
     }
 
     obj_ptr

crates/perry-runtime/src/object/field_get_set/ic_miss.rs

@@ -424,6 +424,18 @@ pub extern "C" fn js_object_get_field_ic_miss(
             let keys_data = (keys as *const u8).add(8) as *const f64;
             let alloc_limit =
                 std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize;
+            // #6804: stamp the receiver's stable ShapeId at PIC-miss
+            // resolution, so the id-keyed FIELD_CACHE (and the future
+            // id-comparing PIC) see a stamped object from its first read.
+            if (*obj).class_id == 0
+                && !crate::object::shapes::is_shape_id((*obj).parent_class_id)
+                && !crate::regex::regex_header_has_magic(obj as *const crate::regex::RegExpHeader)
+            {
+                let id = crate::object::shapes::shape_id_for_keys_ensure(keys, key_count as u32);
+                if id != 0 {
+                    (*(obj as *mut ObjectHeader)).parent_class_id = id;
+                }
+            }
             for i in 0..key_count {
                 let k_bits = (*keys_data.add(i)).to_bits();
                 let k_ptr = (k_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader;
@@ -447,16 +459,24 @@ pub extern "C" fn js_object_get_field_ic_miss(
                     // ~900k times per run (40% inclusive samples per
                     // perfcomp.profile).
                     //
-                    // #6759 C3c: only prime the cache for SHAPE-SHARED keys
-                    // arrays (literal shapes, class-keys arrays — both
-                    // shape-cache-resident, process-rooted, address-stable).
-                    // An OWNED keys array can die and have its address
-                    // recycled under a DIFFERENT shape, and this compare is
-                    // the one unvalidated fast path in the system — a stale
-                    // hit reads the wrong slot. Owned/wide receivers are
-                    // served by the validated, shape-id-keyed FIELD_CACHE
-                    // instead.
-                    if keys_cacheable_for_pic(keys) {
+                    // #6804: a stamped plain receiver primes an ID token
+                    // (`stamp | PIC_ID_TOKEN_BIT`, matching the emitted
+                    // PIC's discriminated compare). Ids are never reused,
+                    // so id tokens are immune to the address-recycling ABA
+                    // that keys-pointer tokens have — which also makes
+                    // OWNED keys arrays safely cacheable again for plain
+                    // objects. #6759 C3c: keys-POINTER tokens stay
+                    // restricted to SHAPE-SHARED arrays (literal shapes,
+                    // class-keys arrays — shape-cache-resident,
+                    // process-rooted, address-stable), because that compare
+                    // is unvalidated and a recycled owned-array address
+                    // would read the wrong slot.
+                    let stamp = (*obj).parent_class_id;
+                    if (*obj).class_id == 0 && crate::object::shapes::is_shape_id(stamp) {
+                        (*cache)[0] =
+                            (stamp as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT) as i64;
+                        (*cache)[1] = i as i64;
+                    } else if keys_cacheable_for_pic(keys) {
                         (*cache)[0] = keys as i64;
                         (*cache)[1] = i as i64;
                     }

crates/perry-runtime/src/object/field_set_by_name.rs

@@ -1336,6 +1336,14 @@ pub extern "C" fn js_object_set_field_by_name(
             // fast path above instead of allocating a fresh 4-elem
             // keys_array here.
             transition_cache_insert(0, interned_key, new_keys as usize, 0);
+            // #6804: birth-stamp the new dynamic shape (once per shape
+            // birth — the transition edge above serves the siblings).
+            if (*obj).class_id == 0 {
+                let id = super::shapes::shape_id_for_keys_ensure(new_keys, 1);
+                if id != 0 {
+                    (*obj).parent_class_id = id;
+                }
+            }
             return;
         }
 

crates/perry-runtime/src/object/mod.rs

@@ -258,8 +258,9 @@ pub(crate) struct ObjectHotTables {
     /// and keys_array == null.
     pub(crate) shape_inline_cache:
         std::cell::UnsafeCell<[ShapeCacheEntry; SHAPE_INLINE_CACHE_SIZE]>,
-    /// Overflow map for shape_ids that collide in the inline cache.
-    pub(crate) shape_cache_overflow: RefCell<HashMap<u32, *mut ArrayHeader>>,
+    /// Overflow map for shape_ids that collide in the inline cache. Values
+    /// are `(keys_array, runtime_shape_id)` — see [`ShapeCacheEntry`].
+    pub(crate) shape_cache_overflow: RefCell<HashMap<u32, (*mut ArrayHeader, u32)>>,
     /// Per-thread shape-transition cache for the dynamic-key write path;
     /// see the doc block above `with_transition_cache`. HEAP-allocated
     /// (`Box`) — oversized inline storage overflowed the arm64_32 ILP32
@@ -276,6 +277,7 @@ impl ObjectHotTables {
             shape_inline_cache: std::cell::UnsafeCell::new(
                 [ShapeCacheEntry {
                     shape_id: 0,
+                    runtime_shape_id: 0,
                     keys_array: std::ptr::null_mut(),
                 }; SHAPE_INLINE_CACHE_SIZE],
             ),
@@ -650,6 +652,12 @@ const SHAPE_INLINE_CACHE_SIZE: usize = 256;
 #[derive(Clone, Copy)]
 pub(crate) struct ShapeCacheEntry {
     shape_id: u32,
+    /// #6804: the RUNTIME ShapeId (`shapes::shape_id_for_keys_ensure`) of
+    /// `keys_array`, computed once at insert so the literal-allocation path
+    /// can stamp newborn plain objects without a per-allocation table
+    /// probe. Distinct from `shape_id`, which is the CODEGEN packed-keys
+    /// hash used as this cache's lookup key.
+    runtime_shape_id: u32,
     keys_array: *mut ArrayHeader,
 }
 
@@ -675,21 +683,28 @@ thread_local! {
 /// Hot-path: ~3 ALU ops + 1 load + 1 cmp + 1 branch (no RefCell, no HashMap).
 #[inline(always)]
 fn shape_cache_get(shape_id: u32) -> *mut ArrayHeader {
+    shape_cache_get_with_id(shape_id).0
+}
+
+/// #6804: `shape_cache_get` plus the keys array's RUNTIME ShapeId (0 on
+/// miss), so the literal-allocation birth-stamp costs no extra probe.
+#[inline(always)]
+fn shape_cache_get_with_id(shape_id: u32) -> (*mut ArrayHeader, u32) {
     let st = crate::state::state();
     let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1);
     // Safety: the state is per-thread by construction; the UnsafeCell
     // allows zero-overhead reads on the hot path.
     let entry = unsafe { (*st.object_hot.shape_inline_cache.get())[slot] };
     if entry.shape_id == shape_id {
-        return entry.keys_array;
+        return (entry.keys_array, entry.runtime_shape_id);
     }
     // Miss — check the overflow map.
     st.object_hot
         .shape_cache_overflow
         .borrow()
         .get(&shape_id)
         .copied()
-        .unwrap_or(std::ptr::null_mut())
+        .unwrap_or((std::ptr::null_mut(), 0))
 }
 
 /// Insert a keys_array into the cache. Updates the inline slot
@@ -711,18 +726,27 @@ fn shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) {
             (*gc_header).gc_flags |= crate::gc::GC_FLAG_SHAPE_SHARED;
         }
     }
+    // #6804: bind the runtime ShapeId once at insert (one probe per shape
+    // BIRTH), so every later allocation of this shape reads it from the
+    // cache entry it already touches.
+    let runtime_shape_id = if keys_array.is_null() {
+        0
+    } else {
+        shapes::shape_id_for_keys_ensure(keys_array, unsafe { (*keys_array).length })
+    };
     let st = crate::state::state();
     let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1);
     unsafe {
         // GC_STORE_AUDIT(ROOT): shape_inline_cache entries are scanned by scan_shape_cache_roots_mut.
         let entry = &mut (*st.object_hot.shape_inline_cache.get())[slot];
         entry.shape_id = shape_id;
+        entry.runtime_shape_id = runtime_shape_id;
         crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.keys_array, keys_array);
     }
     st.object_hot
         .shape_cache_overflow
         .borrow_mut()
-        .insert(shape_id, keys_array);
+        .insert(shape_id, (keys_array, runtime_shape_id));
     crate::gc::runtime_write_barrier_root_raw_ptr(keys_array);
 }
 
@@ -1047,7 +1071,7 @@ pub fn scan_shape_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_
     }
     {
         let mut cache = st.object_hot.shape_cache_overflow.borrow_mut();
-        for arr_ptr in cache.values_mut() {
+        for (arr_ptr, _runtime_shape_id) in cache.values_mut() {
             visitor.visit_raw_mut_ptr_slot(arr_ptr);
         }
     }
@@ -1262,7 +1286,7 @@ pub(crate) fn test_seed_shape_cache_root(shape_id: u32, keys_array: *mut ArrayHe
     {
         let mut cache = st.object_hot.shape_cache_overflow.borrow_mut();
         cache.clear();
-        cache.insert(shape_id, keys_array);
+        cache.insert(shape_id, (keys_array, 0));
     }
     crate::gc::runtime_write_barrier_root_raw_ptr(keys_array);
 }
@@ -1277,7 +1301,7 @@ pub(crate) fn test_shape_cache_root(shape_id: u32) -> (usize, usize) {
         .shape_cache_overflow
         .borrow()
         .get(&shape_id)
-        .map(|ptr| *ptr as usize)
+        .map(|(ptr, _)| *ptr as usize)
         .unwrap_or(0);
     (inline, overflow)
 }