Ensure log bits are correctly maintained

src/plan/barriers.rs

@@ -171,13 +171,13 @@ impl<S: BarrierSemantics> ObjectBarrier<S> {
         Self { semantics }
     }
 
-    /// Attepmt to atomically log an object.
+    /// Attempt to atomically log an object.
     /// Returns true if the object is not logged previously.
     fn object_is_unlogged(&self, object: ObjectReference) -> bool {
         unsafe { S::UNLOG_BIT_SPEC.load::<S::VM, u8>(object, None) != 0 }
     }
 
-    /// Attepmt to atomically log an object.
+    /// Attempt to atomically log an object.
     /// Returns true if the object is not logged previously.
     fn log_object(&self, object: ObjectReference) -> bool {
         #[cfg(all(feature = "vo_bit", feature = "extreme_assertions"))]

src/plan/generational/gc_work.rs

@@ -108,22 +108,19 @@ impl<E: ProcessEdgesWork> ProcessModBuf<E> {
 
 impl<E: ProcessEdgesWork> GCWork<E::VM> for ProcessModBuf<E> {
     fn do_work(&mut self, worker: &mut GCWorker<E::VM>, mmtk: &'static MMTK<E::VM>) {
-        // Flip the per-object unlogged bits to "unlogged" state.
-        for obj in &self.modbuf {
-            <E::VM as VMBinding>::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<E::VM, u8>(
-                *obj,
-                1,
-                None,
-                Ordering::SeqCst,
-            );
-        }
-        // scan modbuf only if the current GC is a nursery GC
-        if mmtk
-            .get_plan()
-            .generational()
-            .unwrap()
-            .is_current_gc_nursery()
-        {
+        // Process and scan modbuf only if the current GC is a nursery GC
+        let gen = mmtk.get_plan().generational().unwrap();
+        if gen.is_current_gc_nursery() {
+            // Flip the per-object unlogged bits to "unlogged" state.
+            for obj in &self.modbuf {
+                debug_assert!(!gen.is_object_in_nursery(*obj), "{} was logged but is not mature", *obj);
+                <E::VM as VMBinding>::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<E::VM, u8>(
+                    *obj,
+                    1,
+                    None,
+                    Ordering::SeqCst,
+                );
+            }
             // Scan objects in the modbuf and forward pointers
             let modbuf = std::mem::take(&mut self.modbuf);
             GCWork::do_work(

src/plan/generational/global.rs

@@ -216,6 +216,10 @@ impl<VM: VMBinding> CommonGenPlan<VM> {
 
         // Evacuate nursery objects
         if self.nursery.in_space(object) {
+            debug_assert!(VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.is_unlogged::<VM>(
+                object,
+                Ordering::SeqCst
+            ));
             return self.nursery.trace_object::<Q>(
                 queue,
                 object,
@@ -227,6 +231,13 @@ impl<VM: VMBinding> CommonGenPlan<VM> {
         if self.common.get_los().in_space(object) {
             return self.common.get_los().trace_object::<Q>(queue, object);
         }
+
+        // Otherwise it is a mature object
+        debug_assert!(VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.is_unlogged::<VM>(
+            object,
+            Ordering::SeqCst
+        ));
+
         object
     }
 

src/plan/marksweep/global.rs

@@ -57,7 +57,7 @@ impl<VM: VMBinding> Plan for MarkSweep<VM> {
 
     fn prepare(&mut self, tls: VMWorkerThread) {
         self.common.prepare(tls, true);
-        self.ms.prepare();
+        self.ms.prepare(true);
     }
 
     fn release(&mut self, tls: VMWorkerThread) {

src/plan/sticky/immix/global.rs

@@ -252,8 +252,17 @@ impl<VM: VMBinding> crate::plan::generational::global::GenerationalPlanExt<VM> f
             if !self.is_object_in_nursery(object) {
                 // Mature object
                 trace!("Immix mature object {}, skip", object);
+                debug_assert!(VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.is_unlogged::<VM>(
+                    object,
+                    Ordering::SeqCst
+                ));
                 return object;
             } else {
+                // Nursery object
+                debug_assert!(!VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.is_unlogged::<VM>(
+                    object,
+                    Ordering::SeqCst
+                ));
                 let object = if KIND == TRACE_KIND_TRANSITIVE_PIN || KIND == TRACE_KIND_FAST {
                     trace!(
                         "Immix nursery object {} is being traced without moving",

src/policy/immix/immixspace.rs

@@ -389,6 +389,16 @@ impl<VM: VMBinding> ImmixSpace<VM> {
                 unimplemented!("cyclic mark bits is not supported at the moment");
             }
 
+            if self.common.needs_log_bit {
+                if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC {
+                    for chunk in self.chunk_map.all_chunks() {
+                        side.bzero_metadata(chunk.start(), Chunk::BYTES);
+                    }
+                } else {
+                    unimplemented!("in header log bit is not supported");
+                }
+            }
+
             // Prepare defrag info
             if super::DEFRAG {
                 self.defrag.prepare(self, plan_stats);

src/policy/immortalspace.rs

@@ -213,6 +213,15 @@ impl<VM: VMBinding> ImmortalSpace<VM> {
             object
         );
         if self.mark_state.test_and_mark::<VM>(object) {
+            // Set the unlog bit if required
+            if self.common.needs_log_bit {
+                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<VM, u8>(
+                    object,
+                    1,
+                    None,
+                    Ordering::SeqCst,
+                );
+            }
             queue.enqueue(object);
         }
         object

src/policy/largeobjectspace.rs

@@ -269,7 +269,9 @@ impl<VM: VMBinding> LargeObjectSpace<VM> {
                 trace!("LOS object {} is being marked now", object);
                 self.treadmill.copy(object, nursery_object);
                 // We just moved the object out of the logical nursery, mark it as unlogged.
-                if nursery_object && self.common.needs_log_bit {
+                // We also unlog mature objects as their unlog bit may have been unset before the
+                // full-heap GC
+                if self.common.needs_log_bit {
                     VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC
                         .mark_as_unlogged::<VM>(object, Ordering::SeqCst);
                 }
@@ -288,6 +290,10 @@ impl<VM: VMBinding> LargeObjectSpace<VM> {
         let sweep = |object: ObjectReference| {
             #[cfg(feature = "vo_bit")]
             crate::util::metadata::vo_bit::unset_vo_bit(object);
+            // Clear log bits for dead objects to prevent a new nursery object having the unlog bit set
+            if self.common.needs_log_bit {
+                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::<VM>(object, Ordering::SeqCst);
+            }
             self.pr
                 .release_pages(get_super_page(object.to_object_start::<VM>()));
         };

src/policy/marksweepspace/malloc_ms/global.rs

@@ -488,7 +488,7 @@ impl<VM: VMBinding> MallocSpace<VM> {
         }
     }
 
-    pub fn prepare(&mut self) {}
+    pub fn prepare(&mut self, _full_heap: bool) {}
 
     pub fn release(&mut self) {
         use crate::scheduler::WorkBucketStage;

src/policy/marksweepspace/native_ms/global.rs

@@ -405,7 +405,17 @@ impl<VM: VMBinding> MarkSweepSpace<VM> {
         self.chunk_map.set(block.chunk(), ChunkState::Allocated);
     }
 
-    pub fn prepare(&mut self) {
+    pub fn prepare(&mut self, full_heap: bool) {
+        if self.common.needs_log_bit && full_heap {
+            if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC {
+                for chunk in self.chunk_map.all_chunks() {
+                    side.bzero_metadata(chunk.start(), Chunk::BYTES);
+                }
+            } else {
+                unimplemented!("in header log bit is not supported");
+            }
+        }
+
         #[cfg(debug_assertions)]
         self.abandoned_in_gc.lock().unwrap().assert_empty();
 

src/policy/vmspace.rs

@@ -283,6 +283,17 @@ impl<VM: VMBinding> VMSpace<VM> {
         );
         debug_assert!(self.in_space(object));
         if self.mark_state.test_and_mark::<VM>(object) {
+            // Flip the per-object unlogged bits to "unlogged" state for objects inside the
+            // bootimage
+            #[cfg(feature = "set_unlog_bits_vm_space")]
+            if self.common.needs_log_bit {
+                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<VM, u8>(
+                    object,
+                    1,
+                    None,
+                    Ordering::SeqCst,
+                );
+            }
             queue.enqueue(object);
         }
         object

src/util/metadata/log_bit.rs

@@ -6,6 +6,11 @@ use std::sync::atomic::Ordering;
 use super::MetadataSpec;
 
 impl VMGlobalLogBitSpec {
+    /// Clear the unlog bit to log object (0 means logged)
+    pub fn clear<VM: VMBinding>(&self, object: ObjectReference, order: Ordering) {
+        self.store_atomic::<VM, u8>(object, 0, None, order)
+    }
+
     /// Mark the log bit as unlogged (1 means unlogged)
     pub fn mark_as_unlogged<VM: VMBinding>(&self, object: ObjectReference, order: Ordering) {
         self.store_atomic::<VM, u8>(object, 1, None, order)