Merge #154939 #155739 #155795 #155798

154939: stress: rename Stress to DevStress r=herkolategan,jeffswenson a=stevendanna

This renames the current implementation of Stress to DevStress and then adds a new Stress function that covers both local and nightly stress runs.

Epic: none
Release note: None

155739: storage: add storage.tombstone_dense_compaction_threshold setting r=RaduBerinde a=RaduBerinde

Add a cluster setting to control the tombstone-dense compaction
threshold.

Fixes: #155715
Release note: None

155795: workload/schemachange: remove FK violation prediction for INSERT operations r=rafiss a=rafiss


Previously, the random schema workload attempted to predict whether INSERT operations would violate foreign key constraints by analyzing the database state and constraint definitions. This prediction logic involved:
- Querying the database to check if FK violations would occur
- Checking for ongoing FK constraint mutations
- Distinguishing between expected and potential FK violations

This commit removes all FK violation prediction logic and makes ForeignKeyViolation unconditionally a potential execution and commit error for INSERT operations. This aligns with the existing pattern used for adding FK constraints, where FK violations are always treated as potential errors due to the asynchronous validation job.

This change simplifies the code by removing ~200 lines of complex prediction logic and deflakes tests that depend on accurate error handling during concurrent schema changes.

Resolves: #155686
Resolves: #154715 
Resolves: #152402

Epic: None

Release note: None

🤖 Generated with [Claude Code](https://claude.com/claude-code)

155798: kvnemesis: log a metrics report r=stevendanna a=miraradeva

This commit adds logging of some key metrics at the end of each kvnemesis run.

Fixes: #153793

Release note: None

----

This is what the report looks like for a run of `TestKVNemesisMultiNode`:
```
Metric                              | Node 1               | Node 2               | Node 3               | Node 4
------------------------------------+----------------------+----------------------+----------------------+---------------------
raft.commands.proposed              | 2906                 | 117                  | 273                  | 118
raft.commands.reproposed.new-lai    | 0                    | 1                    | 12                   | 0
raft.commands.reproposed.unchanged  | 78                   | 0                    | 0                    | 0
txn.server_side.1PC.success         | 84                   | 0                    | 1                    | 0
txnrecovery.successes.committed     | 0                    | 0                    | 0                    | 0
txnwaitqueue.deadlocks_total        | 0                    | 0                    | 4                    | 0
txn.aborts                          | 13                   | 10                   | 8                    | 12
txn.durations                       | μ=25ms p99=260ms     | μ=82ms p99=671ms     | μ=81ms p99=1.476s    | μ=224ms p99=4.832s
txn.restarts.writetooold            | 8                    | 1                    | 2                    | 1
txn.restarts.serializable           | 1                    | 0                    | 0                    | 0
txn.restarts.readwithinuncertainty  | 0                    | 0                    | 0                    | 0
```

Co-authored-by: Steven Danna <[email protected]>
Co-authored-by: Radu Berinde <[email protected]>
Co-authored-by: Rafi Shamim <[email protected]>
Co-authored-by: Mira Radeva <[email protected]>

Author: 5 people

pkg/backup/restore_test.go

@@ -107,7 +107,7 @@ func TestRestoreRetryFastFails(t *testing.T) {
 	// runtime does not exceed the max duration of the retry policy, or
 	// else very few attempts will be made.
 	maxDuration := 500 * time.Millisecond
-	if skip.Stress() {
+	if skip.DevStress() {
 		// Under stress, the restore will take longer to complete, so we need to
 		// increase max duration accordingly.
 		maxDuration = 1500 * time.Millisecond

pkg/ccl/kvccl/kvtenantccl/upgradeinterlockccl/local_test_util_test.go

@@ -107,7 +107,7 @@ func runTest(t *testing.T, variant sharedtestutil.TestVariant, test sharedtestut
 		// extend the TTL by 10x to handle cases where the system is too
 		// overloaded to heartbeat at sub-second intervals.
 		ttlOverride := 250 * time.Millisecond
-		if skip.Stress() {
+		if skip.DevStress() {
 			ttlOverride *= 10
 		}
 		heartbeatOverride := ttlOverride / 10

pkg/crosscluster/replicationtestutils/testutils.go

@@ -582,7 +582,7 @@ func (c *TenantStreamingClusters) SrcExec(exec srcInitExecFunc) {
 
 func WaitUntilStartTimeReached(t *testing.T, db *sqlutils.SQLRunner, ingestionJobID jobspb.JobID) {
 	timeout := 45 * time.Second
-	if skip.Stress() || util.RaceEnabled {
+	if skip.DevStress() || util.RaceEnabled {
 		timeout *= 5
 	}
 	testutils.SucceedsWithin(t, func() error {

pkg/kv/kvnemesis/BUILD.bazel

@@ -113,6 +113,7 @@ go_test(
         "//pkg/util/hlc",
         "//pkg/util/leaktest",
         "//pkg/util/log",
+        "//pkg/util/metric",
         "//pkg/util/randutil",
         "//pkg/util/stop",
         "//pkg/util/syncutil",

pkg/kv/kvnemesis/kvnemesis_test.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -46,6 +47,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
@@ -698,19 +700,91 @@ func testKVNemesisImpl(t testing.TB, cfg kvnemesisTestCfg) {
 	env := &Env{SQLDBs: sqlDBs, Tracker: tr, L: logger, Partitioner: &partitioner, Restarter: tc}
 	failures, err := RunNemesis(ctx, rng, env, config, cfg.concurrency, cfg.numSteps, cfg.mode, dbs...)
 
-	for i := 0; i < cfg.numNodes; i++ {
-		t.Logf("[%d] proposed: %d", i,
-			tc.GetFirstStoreFromServer(t, i).Metrics().RaftCommandsProposed.Count())
-		t.Logf("[%d] reproposed unchanged: %d", i,
-			tc.GetFirstStoreFromServer(t, i).Metrics().RaftCommandsReproposed.Count())
-		t.Logf("[%d] reproposed with new LAI: %d", i,
-			tc.GetFirstStoreFromServer(t, i).Metrics().RaftCommandsReproposedLAI.Count())
-	}
+	logMetricsReport(t, tc)
 
 	require.NoError(t, err, `%+v`, err)
 	require.Zero(t, len(failures), "kvnemesis detected failures") // they've been logged already
 }
 
+func logMetricsReport(t testing.TB, tc *testcluster.TestCluster) {
+	metricsOfInterest := []string{
+		// Raft command metrics
+		"raft.commands.proposed",
+		"raft.commands.reproposed.new-lai",
+		"raft.commands.reproposed.unchanged",
+		// Transaction metrics
+		"txn.aborts",
+		"txn.durations",
+		"txn.restarts.writetooold",
+		"txn.restarts.serializable",
+		"txn.restarts.readwithinuncertainty",
+		"txn.server_side.1PC.success",
+		"txnrecovery.failures",
+		"txnrecovery.successes.aborted",
+		"txnrecovery.successes.committed",
+		"txnwaitqueue.deadlocks_total",
+	}
+
+	numNodes := tc.NumServers()
+	nodeMetrics := make([]map[string]string, numNodes)
+	metricsMap := make(map[string]struct{}, len(metricsOfInterest))
+	for _, m := range metricsOfInterest {
+		metricsMap[m] = struct{}{}
+	}
+	for i := 0; i < numNodes; i++ {
+		nodeMetrics[i] = make(map[string]string)
+		processMetric := func(name string, v interface{}) {
+			switch val := v.(type) {
+			case *metric.Counter:
+				nodeMetrics[i][name] = fmt.Sprintf("%d", val.Count())
+			case *metric.Gauge:
+				nodeMetrics[i][name] = fmt.Sprintf("%d", val.Value())
+			case metric.IHistogram:
+				// TODO(mira): Currently, histograms are assumed to store values of type
+				// time duration.
+				snapshot := val.CumulativeSnapshot()
+				makePretty := func(nanos float64) time.Duration {
+					return time.Duration(nanos).Round(time.Millisecond)
+				}
+				meanDur := makePretty(snapshot.Mean())
+				p50Dur := makePretty(snapshot.ValueAtQuantile(50))
+				p99Dur := makePretty(snapshot.ValueAtQuantile(99))
+				nodeMetrics[i][name] = fmt.Sprintf("μ=%v p50=%v p99=%v", meanDur, p50Dur, p99Dur)
+			default:
+				nodeMetrics[i][name] = fmt.Sprintf("unknown:%T", v)
+			}
+		}
+
+		clientRegistry := tc.Server(i).StorageLayer().MetricsRecorder().AppRegistry()
+		nodeRegistry := tc.Server(i).StorageLayer().MetricsRecorder().NodeRegistry()
+		storeID := tc.GetFirstStoreFromServer(t, i).StoreID()
+		storeRegistry := tc.Server(i).StorageLayer().MetricsRecorder().StoreRegistry(storeID)
+		clientRegistry.Select(metricsMap, func(name string, v interface{}) { processMetric(name, v) })
+		nodeRegistry.Select(metricsMap, func(name string, v interface{}) { processMetric(name, v) })
+		storeRegistry.Select(metricsMap, func(name string, v interface{}) { processMetric(name, v) })
+	}
+
+	header := fmt.Sprintf("%-35s", "Metric")
+	for i := 0; i < numNodes; i++ {
+		header += fmt.Sprintf(" | %-30s", fmt.Sprintf("Node %d", i+1))
+	}
+	t.Log(header)
+
+	separator := strings.Repeat("-", 35)
+	for i := 0; i < numNodes; i++ {
+		separator += "-+-" + strings.Repeat("-", 30)
+	}
+	t.Log(separator)
+
+	for _, metricName := range metricsOfInterest {
+		row := fmt.Sprintf("%-35s", metricName)
+		for i := 0; i < numNodes; i++ {
+			row += fmt.Sprintf(" | %-30s", nodeMetrics[i][metricName])
+		}
+		t.Log(row)
+	}
+}
+
 // TestRunReproductionSteps is a helper that allows quickly running a kvnemesis
 // history.
 func TestRunReproductionSteps(t *testing.T) {

pkg/server/status/recorder.go

@@ -244,6 +244,21 @@ func (mr *MetricsRecorder) AppRegistry() *metric.Registry {
 	return mr.mu.appRegistry
 }
 
+// NodeRegistry returns the metric registry for node-level metrics.
+func (mr *MetricsRecorder) NodeRegistry() *metric.Registry {
+	mr.mu.Lock()
+	defer mr.mu.Unlock()
+	return mr.mu.logRegistry
+}
+
+// StoreRegistry returns the metric registry for store-level metrics
+// corresponding to the provided store ID.
+func (mr *MetricsRecorder) StoreRegistry(id roachpb.StoreID) *metric.Registry {
+	mr.mu.Lock()
+	defer mr.mu.Unlock()
+	return mr.mu.storeRegistries[id]
+}
+
 // AddNode adds various metric registries an initialized server, along
 // with its descriptor and start time.
 // The registries are:

pkg/sql/mvcc_statistics_update_job_test.go

@@ -203,7 +203,7 @@ func TestTenantGlobalAggregatedLivebytes(t *testing.T) {
 		// Exact match for non stress tests, and allow values to differ by up to
 		// 5% in stress situations.
 		confidenceLevel := 0.0
-		if skip.Stress() {
+		if skip.DevStress() {
 			confidenceLevel = 0.05
 		}
 		testutils.SucceedsSoon(t, func() error {

pkg/storage/pebble.go

@@ -524,6 +524,14 @@ var baselineDeletionRate = settings.RegisterIntSetting(
 	settings.NonNegativeInt,
 )
 
+var tombstoneDenseCompactionThreshold = settings.RegisterIntSetting(
+	settings.ApplicationLevel,
+	"storage.tombstone_dense_compaction_threshold",
+	"percentage of tombstone-dense data blocks that trigger a compaction (0 = disabled)",
+	10, // 10%
+	settings.IntInRange(0, 100),
+)
+
 // EngineComparer is a pebble.Comparer object that implements MVCC-specific
 // comparator settings for use with Pebble.
 var EngineComparer = func() pebble.Comparer {
@@ -962,6 +970,9 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) {
 	cfg.opts.TargetByteDeletionRate = func() int {
 		return int(baselineDeletionRate.Get(&cfg.settings.SV))
 	}
+	cfg.opts.Experimental.TombstoneDenseCompactionThreshold = func() float64 {
+		return 0.01 * float64(tombstoneDenseCompactionThreshold.Get(&cfg.settings.SV))
+	}
 	if cfg.opts.Experimental.UseDeprecatedCompensatedScore == nil {
 		cfg.opts.Experimental.UseDeprecatedCompensatedScore = func() bool {
 			return useDeprecatedCompensatedScore.Get(&cfg.settings.SV)

pkg/testutils/skip/skip.go

@@ -188,8 +188,8 @@ func UnderDuressWithIssue(t SkippableTest, githubIssueID int, args ...interface{
 	}
 }
 
-// Duress catures the conditions that currently lead us to
-// believe that tests may be slower than normal.
+// Duress captures the conditions that currently lead us to believe that tests
+// may be slower than normal.
 func Duress() bool {
 	return util.RaceEnabled || Stress() || syncutil.DeadlockEnabled
 }

pkg/testutils/skip/stress.go

@@ -7,17 +7,24 @@ package skip
 
 import "github.com/cockroachdb/cockroach/pkg/util/envutil"
 
-var nightlyStress = envutil.EnvOrDefaultBool("COCKROACH_NIGHTLY_STRESS", false)
+var (
+	nightlyStress = envutil.EnvOrDefaultBool("COCKROACH_NIGHTLY_STRESS", false)
+	stress        = envutil.EnvOrDefaultBool("COCKROACH_STRESS", false)
+)
 
-var stress = envutil.EnvOrDefaultBool("COCKROACH_STRESS", false)
-
-// NightlyStress returns true iff the process is running as part of CockroachDB's
-// nightly stress tests.
+// NightlyStress returns true iff the process is running as part of
+// CockroachDB's nightly stress tests.
 func NightlyStress() bool {
 	return nightlyStress
 }
 
-// Stress returns true iff the process is running under a local _dev_ instance of the stress, i.e., ./dev test ... --stress
-func Stress() bool {
+// DevStress returns true iff the process is running under a local _dev_ instance
+// of the stress, i.e., ./dev test ... --stress
+func DevStress() bool {
 	return stress
 }
+
+// Stress returns true iff the process is running under local or nightly stress.
+func Stress() bool {
+	return DevStress() || NightlyStress()
+}