db: support store-relative paths for WAL dirs

metamorphic/meta.go

@@ -241,7 +241,7 @@ func RunAndCompare(t *testing.T, rootDir string, rOpts ...RunOption) {
 		cmd := exec.Command(binary, args...)
 		out, err := cmd.CombinedOutput()
 		if err != nil {
-			t.Fatalf(`
+			t.Fatalf(`error running %v
 ===== SEED =====
 %d
 ===== ERR =====
@@ -255,6 +255,7 @@ func RunAndCompare(t *testing.T, rootDir string, rOpts ...RunOption) {
 ===== HISTORY =====
 %s
 To reduce:  go test ./internal/metamorphic -tags invariants -run '%s$' --run-dir %s --try-to-reduce -v`,
+				cmd.String(),
 				runOpts.seed,
 				err,
 				out,
@@ -523,13 +524,13 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts .
 		testOpts.Threads = runOpts.maxThreads
 	}
 
-	dir := opts.FS.PathJoin(runDir, "data")
+	dataDir := opts.FS.PathJoin(runDir, "data")
 	// Set up the initial database state if configured to start from a non-empty
 	// database. By default tests start from an empty database, but split
 	// version testing may configure a previous metamorphic tests's database
 	// state as the initial state.
 	if testOpts.initialStatePath != "" {
-		require.NoError(t, setupInitialState(dir, testOpts))
+		require.NoError(t, setupInitialState(dataDir, testOpts))
 	}
 
 	if testOpts.Opts.WALFailover != nil {
@@ -540,19 +541,13 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts .
 			testOpts.Opts.WALFailover = nil
 		} else {
 			testOpts.Opts.WALFailover.Secondary.FS = opts.FS
-			testOpts.Opts.WALFailover.Secondary.Dirname = opts.FS.PathJoin(
-				runDir, testOpts.Opts.WALFailover.Secondary.Dirname)
 		}
 	}
 
-	if opts.WALDir != "" {
-		if runOpts.numInstances > 1 {
-			// TODO(bilal): Allow opts to diverge on a per-instance basis, and use
-			// that to set unique WAL dirs for all instances in multi-instance mode.
-			opts.WALDir = ""
-		} else {
-			opts.WALDir = opts.FS.PathJoin(runDir, opts.WALDir)
-		}
+	if runOpts.numInstances > 1 {
+		// TODO(bilal): Allow opts to diverge on a per-instance basis, and use
+		// that to set unique WAL dirs for all instances in multi-instance mode.
+		opts.WALDir = ""
 	}
 
 	historyFile, err := os.Create(historyPath)
@@ -567,7 +562,7 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts .
 	defer h.Close()
 
 	m := newTest(ops)
-	require.NoError(t, m.init(h, dir, testOpts, runOpts.numInstances, runOpts.opTimeout))
+	require.NoError(t, m.init(h, dataDir, testOpts, runOpts.numInstances, runOpts.opTimeout))
 
 	if err := Execute(m); err != nil {
 		fmt.Fprintf(os.Stderr, "Seed: %d\n", seed)

metamorphic/options.go

@@ -558,7 +558,7 @@ func standardOptions(kf KeyFormat) []*TestOptions {
 `,
 		10: `
 [Options]
-  wal_dir=data/wal
+  wal_dir={store_path}/wal
 `,
 		11: `
 [Level "0"]
@@ -738,7 +738,7 @@ func RandomOptions(
 	opts.MemTableSize = 2 << (10 + uint(rng.IntN(16))) // 2KB - 256MB
 	opts.MemTableStopWritesThreshold = 2 + rng.IntN(5) // 2 - 5
 	if rng.IntN(2) == 0 {
-		opts.WALDir = "data/wal"
+		opts.WALDir = pebble.MakeStoreRelativePath(opts.FS, "wal")
 	}
 
 	// Half the time enable WAL failover.
@@ -761,7 +761,7 @@ func RandomOptions(
 		// must not exceed 119x the probe interval.
 		healthyInterval := scaleDuration(probeInterval, 1.0, 119.0)
 		opts.WALFailover = &pebble.WALFailoverOptions{
-			Secondary: wal.Dir{FS: vfs.Default, Dirname: "data/wal_secondary"},
+			Secondary: wal.Dir{FS: vfs.Default, Dirname: pebble.MakeStoreRelativePath(vfs.Default, "wal_secondary")},
 			FailoverOptions: wal.FailoverOptions{
 				PrimaryDirProbeInterval:      probeInterval,
 				HealthyProbeLatencyThreshold: healthyThreshold,
@@ -975,27 +975,50 @@ func setupInitialState(dataDir string, testOpts *TestOptions) error {
 	// If the test opts are not configured to use a WAL dir, we add the WAL dir
 	// as a 'WAL recovery dir' so that we'll read any WALs in the directory in
 	// Open.
-	walRecoveryPath := testOpts.Opts.FS.PathJoin(dataDir, "wal")
-	if testOpts.Opts.WALDir != "" {
-		// If the test opts are configured to use a WAL dir, we add the data
-		// directory itself as a 'WAL recovery dir' so that we'll read any WALs if
-		// the previous test was writing them to the data directory.
-		walRecoveryPath = dataDir
-	}
-	testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
-		FS:      testOpts.Opts.FS,
-		Dirname: walRecoveryPath,
-	})
+	fs := testOpts.Opts.FS
+	walRecoveryPath := fs.PathJoin(dataDir, "wal")
+	if _, err := fs.Stat(walRecoveryPath); err == nil {
+		// Previous test used a WAL dir.
+		if testOpts.Opts.WALDir == "" {
+			// This test is not using a WAL dir. Add the previous WAL dir as a
+			// recovery dir.
+			testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
+				FS:      fs,
+				Dirname: pebble.MakeStoreRelativePath(fs, "wal"),
+			})
+		} else {
+			// Both the previous test and the current test are using a WAL dir. We
+			// assume that they are the same.
+			if testOpts.Opts.WALDir != pebble.MakeStoreRelativePath(fs, "wal") {
+				return errors.Errorf("unsupported wal dir value %q", testOpts.Opts.WALDir)
+			}
+		}
+	} else {
+		// Previous test did not use a WAL dir.
+		if testOpts.Opts.WALDir != "" {
+			// The current test is using a WAL dir; we add the data directory itself
+			// as a 'WAL recovery dir' so that we'll read any WALs if the previous
+			// test was writing them to the data directory.
+			testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
+				FS:      fs,
+				Dirname: pebble.MakeStoreRelativePath(fs, ""),
+			})
+		}
+	}
 
-	// If the failover dir exists and the test opts are not configured to use
-	// WAL failover, add the failover directory as a 'WAL recovery dir' in case
-	// the previous test was configured to use failover.
+	// If the previous test used WAL failover and this test does not use failover,
+	// add the failover directory as a 'WAL recovery dir' in case the previous
+	// test was configured to use failover.
 	failoverDir := testOpts.Opts.FS.PathJoin(dataDir, "wal_secondary")
-	if _, err := testOpts.Opts.FS.Stat(failoverDir); err == nil && testOpts.Opts.WALFailover == nil {
-		testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
-			FS:      testOpts.Opts.FS,
-			Dirname: failoverDir,
-		})
+	if _, err := testOpts.Opts.FS.Stat(failoverDir); err == nil {
+		if testOpts.Opts.WALFailover == nil {
+			testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
+				FS:      testOpts.Opts.FS,
+				Dirname: pebble.MakeStoreRelativePath(testOpts.Opts.FS, "wal_secondary"),
+			})
+		} else if testOpts.Opts.WALFailover.Secondary.Dirname != pebble.MakeStoreRelativePath(testOpts.Opts.FS, "wal_secondary") {
+			return errors.Errorf("unsupported wal failover dir value %q", testOpts.Opts.WALFailover.Secondary.Dirname)
+		}
 	}
 	return nil
 }

open.go

@@ -377,12 +377,17 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	}
 	if opts.WALFailover != nil {
 		walOpts.Secondary = opts.WALFailover.Secondary
+		walOpts.Secondary.Dirname = resolveStorePath(dirname, walOpts.Secondary.Dirname)
 		walOpts.FailoverOptions = opts.WALFailover.FailoverOptions
 		walOpts.FailoverWriteAndSyncLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
 			Buckets: FsyncLatencyBuckets,
 		})
 	}
-	walDirs := append(walOpts.Dirs(), opts.WALRecoveryDirs...)
+	walDirs := walOpts.Dirs()
+	for _, dir := range opts.WALRecoveryDirs {
+		dir.Dirname = resolveStorePath(dirname, dir.Dirname)
+		walDirs = append(walDirs, dir)
+	}
 	wals, err := wal.Scan(walDirs...)
 	if err != nil {
 		return nil, err
@@ -663,9 +668,9 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 func prepareAndOpenDirs(
 	dirname string, opts *Options,
 ) (walDirname string, dataDir vfs.File, err error) {
-	walDirname = opts.WALDir
-	if opts.WALDir == "" {
-		walDirname = dirname
+	walDirname = dirname
+	if opts.WALDir != "" {
+		walDirname = resolveStorePath(dirname, opts.WALDir)
 	}
 
 	// Create directories if needed.
@@ -684,7 +689,7 @@ func prepareAndOpenDirs(
 		}
 		if opts.WALFailover != nil {
 			secondary := opts.WALFailover.Secondary
-			f, err := mkdirAllAndSyncParents(secondary.FS, secondary.Dirname)
+			f, err := mkdirAllAndSyncParents(secondary.FS, resolveStorePath(dirname, secondary.Dirname))
 			if err != nil {
 				return "", nil, err
 			}

open_test.go

@@ -369,6 +369,7 @@ func TestNewDBFilenames(t *testing.T) {
 func testOpenCloseOpenClose(t *testing.T, fs vfs.FS, root string) {
 	opts := testingRandomized(t, &Options{FS: fs})
 
+	useStoreRelativeWALPath := rand.IntN(2) == 0
 	for _, startFromEmpty := range []bool{false, true} {
 		for _, walDirname := range []string{"", "wal"} {
 			for _, length := range []int{-1, 0, 1, 1000, 10000, 100000} {
@@ -380,7 +381,11 @@ func testOpenCloseOpenClose(t *testing.T, fs vfs.FS, root string) {
 				if walDirname == "" {
 					opts.WALDir = ""
 				} else {
-					opts.WALDir = fs.PathJoin(dirname, walDirname)
+					if useStoreRelativeWALPath {
+						opts.WALDir = MakeStoreRelativePath(fs, walDirname)
+					} else {
+						opts.WALDir = fs.PathJoin(dirname, walDirname)
+					}
 				}
 
 				got, xxx := []byte(nil), ""

options.go

@@ -2159,12 +2159,13 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 // opened without supplying a Options.WALRecoveryDir entry for a directory that
 // may contain WALs required to recover a consistent database state.
 type ErrMissingWALRecoveryDir struct {
-	Dir string
+	Dir       string
+	ExtraInfo string
 }
 
 // Error implements error.
 func (e ErrMissingWALRecoveryDir) Error() string {
-	return fmt.Sprintf("directory %q may contain relevant WALs", e.Dir)
+	return fmt.Sprintf("directory %q may contain relevant WALs%s", e.Dir, e.ExtraInfo)
 }
 
 // CheckCompatibility verifies the options are compatible with the previous options
@@ -2190,6 +2191,8 @@ func (o *Options) CheckCompatibility(previousOptions string) error {
 			}
 		case "Options.wal_dir", "WAL Failover.secondary_dir":
 			switch {
+			case value == "":
+				return nil
 			case o.WALDir == value:
 				return nil
 			case o.WALFailover != nil && o.WALFailover.Secondary.Dirname == value:
@@ -2200,7 +2203,18 @@ func (o *Options) CheckCompatibility(previousOptions string) error {
 						return nil
 					}
 				}
-				return ErrMissingWALRecoveryDir{Dir: value}
+				var buf bytes.Buffer
+				fmt.Fprintf(&buf, "\n  OPTIONS key: %s\n", section+"."+key)
+				if o.WALDir != "" {
+					fmt.Fprintf(&buf, "  o.WALDir: %s\n", o.WALDir)
+				}
+				if o.WALFailover != nil {
+					fmt.Fprintf(&buf, "  o.WALFailover.Secondary.Dirname: %s\n", o.WALFailover.Secondary.Dirname)
+				}
+				for _, d := range o.WALRecoveryDirs {
+					fmt.Fprintf(&buf, "  WALRecoveryDir: %s\n", d)
+				}
+				return ErrMissingWALRecoveryDir{Dir: value, ExtraInfo: buf.String()}
 			}
 		}
 		return nil
@@ -2426,3 +2440,26 @@ func (kc *UserKeyCategories) CategorizeKeyRange(startUserKey, endUserKey []byte)
 	})
 	return kc.rangeNames[p][q]
 }
+
+const storePathIdentifier = "{store_path}"
+
+// MakeStoreRelativePath takes a path that is relative to the store directory
+// and creates a path that can be used for Options.WALDir and wal.Dir.Dirname.
+//
+// This is used in metamorphic tests, so that the test run directory can be
+// copied or moved.
+func MakeStoreRelativePath(fs vfs.FS, relativePath string) string {
+	if relativePath == "" {
+		return storePathIdentifier
+	}
+	return fs.PathJoin(storePathIdentifier, relativePath)
+}
+
+// resolveStorePath is the inverse of MakeStoreRelativePath(). It replaces any
+// storePathIdentifier prefix with the store dir.
+func resolveStorePath(storeDir, path string) string {
+	if remainder, ok := strings.CutPrefix(path, storePathIdentifier); ok {
+		return storeDir + remainder
+	}
+	return path
+}

options_test.go

@@ -192,11 +192,15 @@ func TestOptionsCheckCompatibility(t *testing.T) {
 
 	// Check that an OPTIONS file that configured an explicit WALDir that will
 	// no longer be used errors if it's not also present in WALRecoveryDirs.
-	require.Equal(t, ErrMissingWALRecoveryDir{Dir: "external-wal-dir"},
-		DefaultOptions().CheckCompatibility(`
+	//require.Equal(t, ErrMissingWALRecoveryDir{Dir: "external-wal-dir"},
+	err := DefaultOptions().CheckCompatibility(`
 [Options]
   wal_dir=external-wal-dir
-`))
+`)
+	var missingWALRecoveryDirErr ErrMissingWALRecoveryDir
+	require.True(t, errors.As(err, &missingWALRecoveryDirErr))
+	require.Equal(t, "external-wal-dir", missingWALRecoveryDirErr.Dir)
+
 	// But not if it's configured as a WALRecoveryDir or current WALDir.
 	opts = &Options{WALRecoveryDirs: []wal.Dir{{Dirname: "external-wal-dir"}}}
 	opts.EnsureDefaults()
@@ -214,13 +218,15 @@ func TestOptionsCheckCompatibility(t *testing.T) {
 	// Check that an OPTIONS file that configured a secondary failover WAL dir
 	// that will no longer be used errors if it's not also present in
 	// WALRecoveryDirs.
-	require.Equal(t, ErrMissingWALRecoveryDir{Dir: "failover-wal-dir"},
-		DefaultOptions().CheckCompatibility(`
+	err = DefaultOptions().CheckCompatibility(`
 [Options]
 
 [WAL Failover]
   secondary_dir=failover-wal-dir
-`))
+`)
+	require.True(t, errors.As(err, &missingWALRecoveryDirErr))
+	require.Equal(t, "failover-wal-dir", missingWALRecoveryDirErr.Dir)
+
 	// But not if it's configured as a WALRecoveryDir or current failover
 	// secondary dir.
 	opts = &Options{WALRecoveryDirs: []wal.Dir{{Dirname: "failover-wal-dir"}}}

testdata/open_wal_failover

@@ -59,6 +59,7 @@ grep-between path=(a,data/OPTIONS-000007) start=(\[WAL Failover\]) end=^$
 open path=(a,data)
 ----
 directory "secondary-wals" may contain relevant WALs
+  OPTIONS key: WAL Failover.secondary_dir
 
 # But opening the same directory while providing the secondary path as a WAL
 # recovery dir should succeed.