Keep and backfill Usage Event records of running apps, tasks, and service instances

app/jobs/runtime/events_cleanup.rb

@@ -9,7 +9,7 @@ def initialize(cutoff_age_in_days)
         end
 
         def perform
-          Database::OldRecordCleanup.new(Event, cutoff_age_in_days).delete
+          Database::OldRecordCleanup.new(Event, cutoff_age_in_days:).delete
         end
 
         def job_name_in_configuration

app/models/runtime/app_usage_event.rb

@@ -9,6 +9,23 @@ class AppUsageEvent < Sequel::Model
                       :buildpack_guid, :buildpack_name,
                       :package_state, :previous_package_state, :parent_app_guid,
                       :parent_app_name, :process_type, :task_name, :task_guid
+
+    def self.usage_lifecycles
+      [
+        {
+          beginning_states: [ProcessModel::STARTED, Repositories::AppUsageEventRepository::WAS_RUNNING_EVENT_STATE],
+          ending_state: ProcessModel::STOPPED,
+          guid_column: :app_guid
+        },
+        {
+          beginning_states: [Repositories::AppUsageEventRepository::TASK_STARTED_EVENT_STATE,
+                             Repositories::AppUsageEventRepository::TASK_WAS_RUNNING_EVENT_STATE],
+          ending_state: Repositories::AppUsageEventRepository::TASK_STOPPED_EVENT_STATE,
+          guid_column: :task_guid
+        }
+      ].freeze
+    end
+
     AppUsageEvent.dataset_module do
       def supports_window_functions?
         false

app/models/runtime/task_model.rb

@@ -137,9 +137,16 @@ def create_start_event
     def create_stop_event_if_needed
       app_usage_repo = Repositories::AppUsageEventRepository.new
 
-      start_event = app_usage_repo.find_by_task_and_state(task: self, state: 'TASK_STARTED')
-      existing_stop_event = app_usage_repo.find_by_task_and_state(task: self, state: 'TASK_STOPPED')
-      return if start_event.nil? || existing_stop_event.present?
+      return if app_usage_repo.find_by_task_and_state(task: self, state: 'TASK_STOPPED').present?
+
+      # Record the stop only when there is recorded evidence that the task
+      # started: the TASK_STARTED event, or the TASK_WAS_RUNNING baseline seeded
+      # for tasks that were already running when the keep-running cleanup was
+      # introduced. Without either, no consumer ever saw the task start, so a
+      # stop event would be unmatched noise.
+      started = app_usage_repo.find_by_task_and_state(task: self, state: 'TASK_STARTED') ||
+                app_usage_repo.find_by_task_and_state(task: self, state: 'TASK_WAS_RUNNING')
+      return if started.nil?
 
       create_stop_event
     end

app/models/services/service_usage_event.rb

@@ -7,5 +7,17 @@ class ServiceUsageEvent < Sequel::Model
                       :service_plan_guid, :service_plan_name,
                       :service_guid, :service_label,
                       :service_broker_name, :service_broker_guid
+
+    def self.usage_lifecycles
+      [
+        {
+          beginning_states: [Repositories::ServiceUsageEventRepository::CREATED_EVENT_STATE,
+                             Repositories::ServiceUsageEventRepository::UPDATED_EVENT_STATE,
+                             Repositories::ServiceUsageEventRepository::WAS_RUNNING_EVENT_STATE],
+          ending_state: Repositories::ServiceUsageEventRepository::DELETED_EVENT_STATE,
+          guid_column: :service_instance_guid
+        }
+      ].freeze
+    end
   end
 end

app/repositories/app_usage_event_repository.rb

@@ -4,6 +4,16 @@
 module VCAP::CloudController
   module Repositories
     class AppUsageEventRepository
+      WAS_RUNNING_EVENT_STATE = 'WAS_RUNNING'.freeze
+      TASK_STARTED_EVENT_STATE = 'TASK_STARTED'.freeze
+      TASK_STOPPED_EVENT_STATE = 'TASK_STOPPED'.freeze
+      # Task baselines get their own state (rather than reusing WAS_RUNNING)
+      # because task events share the app_usage_events table with app events but
+      # carry an empty app_guid: a WAS_RUNNING row keyed by app_guid '' would be
+      # correlated with every other task's baseline by the app lifecycle's
+      # cleanup and swept by the app backfill's stale-row sweep.
+      TASK_WAS_RUNNING_EVENT_STATE = 'TASK_WAS_RUNNING'.freeze
+
       def find(guid)
         AppUsageEvent.find(guid:)
       end
@@ -152,7 +162,7 @@ def purge_and_reseed_started_apps!
       end
 
       def delete_events_older_than(cutoff_age_in_days)
-        Database::OldRecordCleanup.new(AppUsageEvent, cutoff_age_in_days, keep_at_least_one_record: true).delete
+        Database::OldRecordCleanup.new(AppUsageEvent, cutoff_age_in_days: cutoff_age_in_days, keep_at_least_one_record: true, keep_running_records: true).delete
       end
 
       private

app/repositories/service_usage_event_repository.rb

@@ -7,6 +7,7 @@ class ServiceUsageEventRepository
       DELETED_EVENT_STATE = 'DELETED'.freeze
       CREATED_EVENT_STATE = 'CREATED'.freeze
       UPDATED_EVENT_STATE = 'UPDATED'.freeze
+      WAS_RUNNING_EVENT_STATE = 'WAS_RUNNING'.freeze
 
       def find(guid)
         ServiceUsageEvent.find(guid:)
@@ -92,7 +93,7 @@ def purge_and_reseed_service_instances!
       end
 
       def delete_events_older_than(cutoff_age_in_days)
-        Database::OldRecordCleanup.new(ServiceUsageEvent, cutoff_age_in_days, keep_at_least_one_record: true).delete
+        Database::OldRecordCleanup.new(ServiceUsageEvent, cutoff_age_in_days: cutoff_age_in_days, keep_at_least_one_record: true, keep_running_records: true).delete
       end
     end
   end

db/migrations/20260601120000_add_lifecycle_index_to_usage_events.rb

@@ -0,0 +1,68 @@
+Sequel.migration do
+  no_transaction # to use the 'concurrently' option
+
+  up do
+    if database_type == :postgres
+      VCAP::Migration.with_concurrent_timeout(self) do
+        add_index :app_usage_events, %i[state app_guid id],
+                  name: :app_usage_events_lifecycle_index,
+                  if_not_exists: true,
+                  concurrently: true
+
+        add_index :service_usage_events, %i[state service_instance_guid id],
+                  name: :service_usage_events_lifecycle_index,
+                  if_not_exists: true,
+                  concurrently: true
+      end
+
+    elsif database_type == :mysql
+      alter_table :app_usage_events do
+        # rubocop:disable Sequel/ConcurrentIndex
+        add_index %i[state app_guid id], name: :app_usage_events_lifecycle_index unless @db.indexes(:app_usage_events).include?(:app_usage_events_lifecycle_index)
+        # rubocop:enable Sequel/ConcurrentIndex
+      end
+
+      alter_table :service_usage_events do
+        # rubocop:disable Sequel/ConcurrentIndex
+        unless @db.indexes(:service_usage_events).include?(:service_usage_events_lifecycle_index)
+          add_index %i[state service_instance_guid id],
+                    name: :service_usage_events_lifecycle_index
+        end
+        # rubocop:enable Sequel/ConcurrentIndex
+      end
+    end
+  end
+
+  down do
+    if database_type == :postgres
+      VCAP::Migration.with_concurrent_timeout(self) do
+        drop_index :app_usage_events, %i[state app_guid id],
+                   name: :app_usage_events_lifecycle_index,
+                   if_exists: true,
+                   concurrently: true
+
+        drop_index :service_usage_events, %i[state service_instance_guid id],
+                   name: :service_usage_events_lifecycle_index,
+                   if_exists: true,
+                   concurrently: true
+      end
+    end
+
+    if database_type == :mysql
+      alter_table :app_usage_events do
+        # rubocop:disable Sequel/ConcurrentIndex
+        drop_index %i[state app_guid id], name: :app_usage_events_lifecycle_index if @db.indexes(:app_usage_events).include?(:app_usage_events_lifecycle_index)
+        # rubocop:enable Sequel/ConcurrentIndex
+      end
+
+      alter_table :service_usage_events do
+        # rubocop:disable Sequel/ConcurrentIndex
+        if @db.indexes(:service_usage_events).include?(:service_usage_events_lifecycle_index)
+          drop_index %i[state service_instance_guid id],
+                     name: :service_usage_events_lifecycle_index
+        end
+        # rubocop:enable Sequel/ConcurrentIndex
+      end
+    end
+  end
+end

db/migrations/20260601120100_seed_was_running_app_usage_events.rb

@@ -0,0 +1,18 @@
+require 'database/was_running_backfill'
+
+Sequel.migration do
+  no_transaction # backfill manages its own per-batch transactions
+
+  up do
+    logger = Steno.logger('cc.backfill.was_running')
+    if VCAP::WasRunningBackfill.skip?
+      VCAP::WasRunningBackfill.log_skip(logger, 'app')
+    else
+      VCAP::WasRunningBackfill.seed_app_usage_events(self, logger)
+    end
+  end
+
+  down do
+    VCAP::WasRunningBackfill.delete_app_usage_events(self)
+  end
+end

db/migrations/20260601120200_seed_was_running_service_usage_events.rb

@@ -0,0 +1,18 @@
+require 'database/was_running_backfill'
+
+Sequel.migration do
+  no_transaction # backfill manages its own per-batch transactions
+
+  up do
+    logger = Steno.logger('cc.backfill.was_running')
+    if VCAP::WasRunningBackfill.skip?
+      VCAP::WasRunningBackfill.log_skip(logger, 'service')
+    else
+      VCAP::WasRunningBackfill.seed_service_usage_events(self, logger)
+    end
+  end
+
+  down do
+    VCAP::WasRunningBackfill.delete_service_usage_events(self)
+  end
+end

db/migrations/20260601120300_seed_was_running_task_usage_events.rb

@@ -0,0 +1,18 @@
+require 'database/was_running_backfill'
+
+Sequel.migration do
+  no_transaction # backfill manages its own per-batch transactions
+
+  up do
+    logger = Steno.logger('cc.backfill.was_running')
+    if VCAP::WasRunningBackfill.skip?
+      VCAP::WasRunningBackfill.log_skip(logger, 'task')
+    else
+      VCAP::WasRunningBackfill.seed_task_usage_events(self, logger)
+    end
+  end
+
+  down do
+    VCAP::WasRunningBackfill.delete_task_usage_events(self)
+  end
+end

docs/v2/app_usage_events/list_all_app_usage_events.html

@@ -631,9 +631,11 @@ <h4>Body</h4>
                 <ul class="valid_values">
                     <li>STARTED</li>
                     <li>STOPPED</li>
+                    <li>WAS_RUNNING</li>
                     <li>BUILDPACK_SET</li>
                     <li>TASK_STARTED</li>
                     <li>TASK_STOPPED</li>
+                    <li>TASK_WAS_RUNNING</li>
                 </ul>
               </td>
               <td>

docs/v2/service_usage_events/list_service_usage_events.html

@@ -290,6 +290,7 @@ <h4>Body</h4>
                     <li>CREATED</li>
                     <li>DELETED</li>
                     <li>UPDATED</li>
+                    <li>WAS_RUNNING</li>
                 </ul>
               </td>
               <td>

docs/v3/source/includes/resources/app_usage_events/_object.md.erb

@@ -30,3 +30,14 @@ Name | Type | Description
 **instance_count.current** | _integer_ or `null` | Current instance count of the app that this event pertains to, if applicable
 **instance_count.previous** | _integer_ or `null` | Previous instance count of the app that this event pertains to, if applicable
 **links** | [_links object_](#links) | Links to related resources
+
+#### WAS_RUNNING and TASK_WAS_RUNNING events
+
+`WAS_RUNNING` and `TASK_WAS_RUNNING` are synthetic values for `state.current` recorded once per running process (`WAS_RUNNING`) and once per running task (`TASK_WAS_RUNNING`) by a one-time data migration when the keep-running cleanup feature was introduced. They mark every process and task that was already running at the time of the upgrade so that billing consumers can bootstrap from a complete baseline even if the original `STARTED`/`TASK_STARTED` events have been pruned.
+
+**Consumer interpretation** (read `WAS_RUNNING`/`STARTED` as `TASK_WAS_RUNNING`/`TASK_STARTED` for task events, which are keyed by `task.guid`):
+
+* If you have not previously recorded a `STARTED` event for this resource, treat `WAS_RUNNING` as equivalent to `STARTED`.
+* If you have already recorded `STARTED` (or an earlier `WAS_RUNNING`) for this resource, treat as a redundant baseline confirmation and ignore.
+* `created_at` reflects when the backfill migration ran, **not** when the app or task actually started. Treat `WAS_RUNNING` as a baseline marker that the resource was already running as of that timestamp, not as the true start of the running interval.
+* `state.previous` on a `WAS_RUNNING` event is always `null`. Subsequent real events for the same resource will continue to report their actual prior process state in `state.previous` (typically `STARTED`). If you perform chain validation, treat `WAS_RUNNING` as equivalent to `STARTED` for the purpose of validating the next event's `state.previous`.

docs/v3/source/includes/resources/service_usage_events/_object.md.erb

@@ -26,3 +26,13 @@ Name | Type | Description
 **service_broker.guid** | _string_ or `null` | Unique identifier of the service broker that this event pertains to, if applicable
 **service_broker.name** | _string_ or `null` | Name of the service broker that this event pertains to, if applicable
 **links** | [_links object_](#links) | Links to related resources
+
+#### WAS_RUNNING events
+
+`WAS_RUNNING` is a synthetic value for `state` recorded once per existing service instance by a one-time data migration when the keep-running cleanup feature was introduced. It marks every service instance that existed at the time of the upgrade so that billing consumers can bootstrap from a complete baseline of service instances even if the original `CREATED` events have been pruned.
+
+**Consumer interpretation:**
+
+* If you have not previously recorded a `CREATED` event for this service instance, treat `WAS_RUNNING` as equivalent to `CREATED`.
+* If you have already recorded `CREATED` (or an earlier `WAS_RUNNING`) for this instance, treat as a redundant baseline confirmation and ignore.
+* `created_at` reflects when the backfill migration ran, **not** when the service instance was created. Treat `WAS_RUNNING` as a baseline marker that the instance already existed as of that timestamp.

lib/cloud_controller/config_schemas/api_schema.rb

@@ -109,6 +109,7 @@ class ApiSchema < VCAP::Config
           optional(:migration_psql_concurrent_statement_timeout_in_seconds) => Integer,
           optional(:migration_psql_worker_memory_kb) => Integer,
           optional(:skip_bigint_id_migration) => bool,
+          optional(:skip_was_running_backfill) => bool,
           db: {
             optional(:database) => Hash, # db connection hash for sequel
             max_connections: Integer, # max connections in the connection pool

lib/cloud_controller/config_schemas/migrate_schema.rb

@@ -10,6 +10,7 @@ class MigrateSchema < VCAP::Config
           optional(:migration_psql_concurrent_statement_timeout_in_seconds) => Integer,
           optional(:migration_psql_worker_memory_kb) => Integer,
           optional(:skip_bigint_id_migration) => bool,
+          optional(:skip_was_running_backfill) => bool,
 
           db: {
             optional(:database) => Hash, # db connection hash for sequel

lib/database/batch_delete.rb

@@ -11,19 +11,21 @@ def delete
       total_count = 0
 
       loop do
-        set = dataset.limit(amount)
-        break if set.empty?
+        # Fetch the batch's ids in the same query that checks for emptiness, so the
+        # (potentially expensive) filtered dataset is evaluated once per batch.
+        ids = dataset.limit(amount).select_map(:id)
+        break if ids.empty?
 
-        total_count += delete_batch(set)
+        total_count += delete_batch(ids)
       end
 
       total_count
     end
 
     private
 
-    def delete_batch(set)
-      dataset.model.where(id: set.select_map(:id)).delete
+    def delete_batch(ids)
+      dataset.model.where(id: ids).delete
     end
   end
 end