Don't run retention for backfilling indexes (#24036)

It is not safe to run retention while index is backfilling for longer than the retention period. To fix the correctness issue, simply don't run retention. We will send a separate PR where index backfill is responsible for catching up retention before enabling the index for reads.

GitOrigin-RevId: 86b14762d5a9ba7323309f7617c42e7669dde08c

Author: Preslav Le

crates/database/src/retention.rs

@@ -24,7 +24,10 @@ use async_trait::async_trait;
 use common::{
     backoff::Backoff,
     bootstrap_model::index::{
-        database_index::IndexedFields,
+        database_index::{
+            DatabaseIndexState,
+            IndexedFields,
+        },
         IndexConfig,
         IndexMetadata,
     },
@@ -854,11 +857,22 @@ impl<RT: Runtime> LeaderRetentionManager<RT> {
         let index: ParsedDocument<IndexMetadata<TableId>> = doc.try_into()?;
         let index = index.into_value();
         let IndexConfig::Database {
-            developer_config, ..
+            developer_config,
+            on_disk_state,
         } = index.config
         else {
             return Ok(());
         };
+
+        // Don't run retention for indexes that are currently backfilling. This
+        // is important for correctness since IndexBackfilling and retention
+        // interact poorly. NOTE that accumulate only adds indexes. Thus we won't
+        // stop running retention if index is deleted or goes from Enabled to
+        // Backfilling.
+        if let DatabaseIndexState::Backfilling { .. } = on_disk_state {
+            return Ok(());
+        }
+
         all_indexes.insert(index_id, (index.name, developer_config.fields));
         Ok(())
     }