Skip to content

Commit fd2185a

Browse files
nipunn1313Convex, Inc.
authored and
Convex, Inc.
committed
[1/2] Clean up env var overrides in backend.nomad (#28498)
GitOrigin-RevId: 95b434f972cd51e793b7e1efebff7a8fa3736af4
1 parent d6f4617 commit fd2185a

File tree

1 file changed

+40
-7
lines changed

1 file changed

+40
-7
lines changed

crates/common/src/knobs.rs

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,11 @@ pub static LEASE_LOST_COOL_DOWN: LazyLock<Duration> = LazyLock::new(|| {
141141
/// lost its lease. Duration of 0 means preemption is disabled, all past serving
142142
/// records will be assumed shut down and deleted without attempting to preempt
143143
/// them. We default to 0 in dev as no preemption is necessary.
144+
///
145+
/// Try to proactively preempt past backends for up to 2 minutes. This can
146+
/// be much lower, but we use higher timeout to protect against transiently
147+
/// unreachable backends. By the time elapses, a network partitioning backend,
148+
/// should attempt to write, discover it has lost its lease and self-preempt.
144149
pub static BACKEND_PREEMPTION_TIMEOUT: LazyLock<Duration> = LazyLock::new(|| {
145150
Duration::from_secs(env_config(
146151
"BACKEND_PREEMPTION_TIMEOUT_SECS",
@@ -360,8 +365,15 @@ pub static RESET_DOCUMENT_RETENTION: LazyLock<bool> =
360365
/// The time backend should wait before it acquires the lease. This wait allows
361366
/// for the backend to be added to service discovery, before it renders the
362367
/// previous backends unusable.
363-
pub static BACKEND_STARTUP_DELAY: LazyLock<Duration> =
364-
LazyLock::new(|| Duration::from_secs(env_config("BACKEND_STARTUP_DELAY_SECS", 0)));
368+
///
369+
/// Wait > 5 seconds before acquiring the backend lease, so we are added to
370+
/// traefik before we make the old backend unusable.
371+
pub static BACKEND_STARTUP_DELAY: LazyLock<Duration> = LazyLock::new(|| {
372+
Duration::from_secs(env_config(
373+
"BACKEND_STARTUP_DELAY_SECS",
374+
prod_override(0, 6),
375+
))
376+
});
365377

366378
/// When to start rejecting new additions to the search memory index.
367379
pub static TEXT_INDEX_SIZE_HARD_LIMIT: LazyLock<usize> =
@@ -915,8 +927,16 @@ pub static FUNRUN_ISOLATE_ACTIVE_THREADS: LazyLock<usize> =
915927

916928
/// What percentage of the physical CPU cores can be actively used by the
917929
/// isolate.
918-
pub static BACKEND_ISOLATE_ACTIVE_THREADS_PERCENT: LazyLock<usize> =
919-
LazyLock::new(|| env_config("BACKEND_ISOLATE_ACTIVE_THREADS_PERCENT", 100));
930+
///
931+
/// Give 50% of physical cores to v8. Note that we are still oversubscribing
932+
/// the CPU since we run multiple backends per server. This is fine since we
933+
/// are moving js execution to Funrun.
934+
pub static BACKEND_ISOLATE_ACTIVE_THREADS_PERCENT: LazyLock<usize> = LazyLock::new(|| {
935+
env_config(
936+
"BACKEND_ISOLATE_ACTIVE_THREADS_PERCENT",
937+
prod_override(100, 50),
938+
)
939+
});
920940

921941
/// How long to splay deploying AWS Lambdas due to changes in the backend. This
922942
/// know doesn't delay deploys that are required due to user backends.
@@ -937,7 +957,10 @@ pub static BACKEND_REQUEST_DRAIN_TIMEOUT: LazyLock<Duration> =
937957
/// The kinesis firehose name for streaming usage metrics to the data
938958
// large body of water.
939959
pub static BACKEND_USAGE_FIREHOSE_NAME: LazyLock<Option<String>> = LazyLock::new(|| {
940-
let result = env_config("BACKEND_USAGE_FIREHOSE_NAME", "".to_string());
960+
let result = env_config(
961+
"BACKEND_USAGE_FIREHOSE_NAME",
962+
prod_override("", "cvx-firehose-usage-prod").to_string(),
963+
);
941964
if !result.is_empty() {
942965
Some(result.to_string())
943966
} else {
@@ -1046,13 +1069,23 @@ pub static MAX_PUSH_BYTES: LazyLock<usize> =
10461069
///
10471070
/// Note that the regexes can't contain commas.
10481071
///
1072+
/// Enable sampling for 10% of /api/push_config and 0.001% of all requests
1073+
/// Use knobs to enable to higher limits for individual instances.
1074+
///
10491075
/// Examples:
10501076
/// REQUEST_TRACE_SAMPLE_CONFIG=0.01
10511077
/// REQUEST_TRACE_SAMPLE_CONFIG=/route1=0.50,0.01
10521078
/// REQUEST_TRACE_SAMPLE_CONFIG=/route1=0.50,route2=0.50,0.01
10531079
/// REQUEST_TRACE_SAMPLE_CONFIG=/http/.*=0.50
1054-
pub static REQUEST_TRACE_SAMPLE_CONFIG: LazyLock<SamplingConfig> =
1055-
LazyLock::new(|| env_config("REQUEST_TRACE_SAMPLE_CONFIG", SamplingConfig::default()));
1080+
pub static REQUEST_TRACE_SAMPLE_CONFIG: LazyLock<SamplingConfig> = LazyLock::new(|| {
1081+
env_config(
1082+
"REQUEST_TRACE_SAMPLE_CONFIG",
1083+
prod_override(
1084+
SamplingConfig::default(),
1085+
"/api/push_config=0.1,0.00001".parse().unwrap(),
1086+
),
1087+
)
1088+
});
10561089

10571090
/// If true, the backend will check the rate limiter service for capacity under
10581091
/// the "backend_startup" domain keyed by db cluster name.

0 commit comments

Comments
 (0)