db: support store-relative paths for WAL dirs

Relative WAL paths (including the actual WAL, the failover path, and
the recovery paths) are (unfortunately) interpreted as relative to the
current working directory.

The cross-version metamorphic test copies a store from a previous run
as the initial state for a new test. The options will fail the
compatibility check since the path changes.

This change adds support for using a special `{store_path}` prefix
inside the path. Any such prefix is replaced with the store directory.

We also improve the missing WAL recovery dir error to show what
directories are actually configured.

Author: RaduBerinde

open.go

@@ -377,12 +377,17 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	}
 	if opts.WALFailover != nil {
 		walOpts.Secondary = opts.WALFailover.Secondary
+		walOpts.Secondary.Dirname = resolveStorePath(dirname, walOpts.Secondary.Dirname)
 		walOpts.FailoverOptions = opts.WALFailover.FailoverOptions
 		walOpts.FailoverWriteAndSyncLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
 			Buckets: FsyncLatencyBuckets,
 		})
 	}
-	walDirs := append(walOpts.Dirs(), opts.WALRecoveryDirs...)
+	walDirs := walOpts.Dirs()
+	for _, dir := range opts.WALRecoveryDirs {
+		dir.Dirname = resolveStorePath(dirname, dir.Dirname)
+		walDirs = append(walDirs, dir)
+	}
 	wals, err := wal.Scan(walDirs...)
 	if err != nil {
 		return nil, err
@@ -663,9 +668,9 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 func prepareAndOpenDirs(
 	dirname string, opts *Options,
 ) (walDirname string, dataDir vfs.File, err error) {
-	walDirname = opts.WALDir
-	if opts.WALDir == "" {
-		walDirname = dirname
+	walDirname = dirname
+	if opts.WALDir != "" {
+		walDirname = resolveStorePath(dirname, opts.WALDir)
 	}
 
 	// Create directories if needed.
@@ -684,7 +689,7 @@ func prepareAndOpenDirs(
 		}
 		if opts.WALFailover != nil {
 			secondary := opts.WALFailover.Secondary
-			f, err := mkdirAllAndSyncParents(secondary.FS, secondary.Dirname)
+			f, err := mkdirAllAndSyncParents(secondary.FS, resolveStorePath(dirname, secondary.Dirname))
 			if err != nil {
 				return "", nil, err
 			}

open_test.go

@@ -369,6 +369,7 @@ func TestNewDBFilenames(t *testing.T) {
 func testOpenCloseOpenClose(t *testing.T, fs vfs.FS, root string) {
 	opts := testingRandomized(t, &Options{FS: fs})
 
+	useStoreRelativeWALPath := rand.IntN(2) == 0
 	for _, startFromEmpty := range []bool{false, true} {
 		for _, walDirname := range []string{"", "wal"} {
 			for _, length := range []int{-1, 0, 1, 1000, 10000, 100000} {
@@ -380,7 +381,11 @@ func testOpenCloseOpenClose(t *testing.T, fs vfs.FS, root string) {
 				if walDirname == "" {
 					opts.WALDir = ""
 				} else {
-					opts.WALDir = fs.PathJoin(dirname, walDirname)
+					if useStoreRelativeWALPath {
+						opts.WALDir = MakeStoreRelativePath(fs, walDirname)
+					} else {
+						opts.WALDir = fs.PathJoin(dirname, walDirname)
+					}
 				}
 
 				got, xxx := []byte(nil), ""

options.go

@@ -2159,12 +2159,13 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error {
 // opened without supplying a Options.WALRecoveryDir entry for a directory that
 // may contain WALs required to recover a consistent database state.
 type ErrMissingWALRecoveryDir struct {
-	Dir string
+	Dir       string
+	ExtraInfo string
 }
 
 // Error implements error.
 func (e ErrMissingWALRecoveryDir) Error() string {
-	return fmt.Sprintf("directory %q may contain relevant WALs", e.Dir)
+	return fmt.Sprintf("directory %q may contain relevant WALs%s", e.Dir, e.ExtraInfo)
 }
 
 // CheckCompatibility verifies the options are compatible with the previous options
@@ -2200,7 +2201,18 @@ func (o *Options) CheckCompatibility(previousOptions string) error {
 						return nil
 					}
 				}
-				return ErrMissingWALRecoveryDir{Dir: value}
+				var buf bytes.Buffer
+				fmt.Fprintf(&buf, "\n  OPTIONS key: %s\n", section+"."+key)
+				if o.WALDir != "" {
+					fmt.Fprintf(&buf, "  o.WALDir: %s\n", o.WALDir)
+				}
+				if o.WALFailover != nil {
+					fmt.Fprintf(&buf, "  o.WALFailover.Secondary.Dirname: %s\n", o.WALFailover.Secondary.Dirname)
+				}
+				for _, d := range o.WALRecoveryDirs {
+					fmt.Fprintf(&buf, "  WALRecoveryDir: %s\n", d)
+				}
+				return ErrMissingWALRecoveryDir{Dir: value, ExtraInfo: buf.String()}
 			}
 		}
 		return nil
@@ -2426,3 +2438,26 @@ func (kc *UserKeyCategories) CategorizeKeyRange(startUserKey, endUserKey []byte)
 	})
 	return kc.rangeNames[p][q]
 }
+
+const storePathIdentifier = "{store_path}"
+
+// MakeStoreRelativePath takes a path that is relative to the store directory
+// and creates a path that can be used for Options.WALDir and wal.Dir.Dirname.
+//
+// This is used in metamorphic tests, so that the test run directory can be
+// copied or moved.
+func MakeStoreRelativePath(fs vfs.FS, relativePath string) string {
+	if relativePath == "" {
+		return storePathIdentifier
+	}
+	return fs.PathJoin(storePathIdentifier, relativePath)
+}
+
+// resolveStorePath is the inverse of MakeStoreRelativePath(). It replaces any
+// storePathIdentifier prefix with the store dir.
+func resolveStorePath(storeDir, path string) string {
+	if remainder, ok := strings.CutPrefix(path, storePathIdentifier); ok {
+		return storeDir + remainder
+	}
+	return path
+}

options_test.go

@@ -192,11 +192,15 @@ func TestOptionsCheckCompatibility(t *testing.T) {
 
 	// Check that an OPTIONS file that configured an explicit WALDir that will
 	// no longer be used errors if it's not also present in WALRecoveryDirs.
-	require.Equal(t, ErrMissingWALRecoveryDir{Dir: "external-wal-dir"},
-		DefaultOptions().CheckCompatibility(`
+	//require.Equal(t, ErrMissingWALRecoveryDir{Dir: "external-wal-dir"},
+	err := DefaultOptions().CheckCompatibility(`
 [Options]
   wal_dir=external-wal-dir
-`))
+`)
+	var missingWALRecoveryDirErr ErrMissingWALRecoveryDir
+	require.True(t, errors.As(err, &missingWALRecoveryDirErr))
+	require.Equal(t, "external-wal-dir", missingWALRecoveryDirErr.Dir)
+
 	// But not if it's configured as a WALRecoveryDir or current WALDir.
 	opts = &Options{WALRecoveryDirs: []wal.Dir{{Dirname: "external-wal-dir"}}}
 	opts.EnsureDefaults()
@@ -214,13 +218,15 @@ func TestOptionsCheckCompatibility(t *testing.T) {
 	// Check that an OPTIONS file that configured a secondary failover WAL dir
 	// that will no longer be used errors if it's not also present in
 	// WALRecoveryDirs.
-	require.Equal(t, ErrMissingWALRecoveryDir{Dir: "failover-wal-dir"},
-		DefaultOptions().CheckCompatibility(`
+	err = DefaultOptions().CheckCompatibility(`
 [Options]
 
 [WAL Failover]
   secondary_dir=failover-wal-dir
-`))
+`)
+	require.True(t, errors.As(err, &missingWALRecoveryDirErr))
+	require.Equal(t, "failover-wal-dir", missingWALRecoveryDirErr.Dir)
+
 	// But not if it's configured as a WALRecoveryDir or current failover
 	// secondary dir.
 	opts = &Options{WALRecoveryDirs: []wal.Dir{{Dirname: "failover-wal-dir"}}}

testdata/open_wal_failover

@@ -59,6 +59,7 @@ grep-between path=(a,data/OPTIONS-000007) start=(\[WAL Failover\]) end=^$
 open path=(a,data)
 ----
 directory "secondary-wals" may contain relevant WALs
+  OPTIONS key: WAL Failover.secondary_dir
 
 # But opening the same directory while providing the secondary path as a WAL
 # recovery dir should succeed.