Backend set scheduler lag to zero if no pending work. (#25331)

GitOrigin-RevId: 73b0322d65636789f53a3c712b58909352d704f2

Author: Preslav Le

crates/application/src/scheduled_jobs/metrics.rs

@@ -45,3 +45,11 @@ register_convex_gauge!(
 pub fn log_scheduled_job_execution_lag(lag: Duration) {
     log_gauge(&SCHEDULED_JOB_EXECUTION_LAG_SECONDS, lag.as_secs_f64());
 }
+
+register_convex_gauge!(
+    SCHEDULED_JOB_NUM_RUNNING_TOTAL,
+    "Number of currently executing scheduled jobs"
+);
+pub fn log_num_running_jobs(num_running: usize) {
+    log_gauge(&SCHEDULED_JOB_NUM_RUNNING_TOTAL, num_running as f64);
+}

crates/application/src/scheduled_jobs/mod.rs

@@ -257,6 +257,7 @@ impl<RT: Runtime> ScheduledJobExecutor<RT> {
                     .await?
             };
 
+            metrics::log_num_running_jobs(running_job_ids.len());
             let next_job_future = if let Some(next_job_ts) = next_job_ready_time {
                 let now = self.rt.generate_timestamp()?;
                 Either::Left(if next_job_ts < now {
@@ -265,9 +266,11 @@ impl<RT: Runtime> ScheduledJobExecutor<RT> {
                     // track how far we're behind in our metrics.
                     self.rt.wait(Duration::from_secs(5))
                 } else {
+                    metrics::log_scheduled_job_execution_lag(Duration::from_secs(0));
                     self.rt.wait(next_job_ts - now)
                 })
             } else {
+                metrics::log_scheduled_job_execution_lag(Duration::from_secs(0));
                 Either::Right(std::future::pending())
             };