feat(search): hierarchical path and geohash terms for bleve

services/search/pkg/bleve/backend.go

@@ -74,15 +74,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
 				),
 			},
 		)
-		// Scope below the space root: restrict at query level so totals and
-		// paging respect the path too. Path is a case-preserving keyword
-		// (paths act as references, /Foo and /foo are distinct), so the exact
-		// folder or the folder prefix matches all of, and only, the scope.
+		// scope at query level so totals and paging respect it; the folder
+		// term matches the folder and its descendants (see PathAnalyzer)
 		if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." {
-			q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{
-				&query.TermQuery{FieldVal: "Path", Term: requestedPath},
-				&query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"},
-			}))
+			q.Conjuncts = append(q.Conjuncts, &query.TermQuery{FieldVal: "Path", Term: requestedPath})
 		}
 	}
 

services/search/pkg/bleve/batch.go

@@ -54,131 +54,84 @@ func (b *Batch) indexResource(id string, r search.Resource) error {
 
 func (b *Batch) Move(id, parentID, location string) error {
 	return b.withSizeLimit(func() error {
-		rootResource, err := searchResourceByID(id, b.index)
-		if err != nil {
-			return err
-		}
-		currentPath := rootResource.Path
 		nextPath := utils.MakeRelativePath(location)
-
-		rootResource.Path = nextPath
-		rootResource.Name = path.Base(nextPath)
-		rootResource.ParentID = parentID
-
-		resources := []*search.Resource{rootResource}
-
-		if rootResource.Type == uint64(storageProvider.ResourceType_RESOURCE_TYPE_CONTAINER) {
-			descendantResources, err := searchResourcesByPath(rootResource.RootID, currentPath, b.index)
-			if err != nil {
-				return err
-			}
-
-			for _, descendantResource := range descendantResources {
-				descendantResource.Path = strings.Replace(descendantResource.Path, currentPath, nextPath, 1)
-				resources = append(resources, descendantResource)
+		var currentPath string
+		return b.forSelfAndDescendants(id, func(resource *search.Resource) error {
+			if resource.ID == id {
+				currentPath = resource.Path
+				resource.Path = nextPath
+				resource.Name = path.Base(nextPath)
+				resource.ParentID = parentID
+			} else {
+				resource.Path = strings.Replace(resource.Path, currentPath, nextPath, 1)
 			}
-		}
-
-		for _, resource := range resources {
 			resource.Hidden = search.IsHidden(resource.Path)
-
-			if err := b.indexResource(resource.ID, *resource); err != nil {
-				return err
-			}
-			if b.batch.Size() >= b.size {
-				if err := b.Push(); err != nil {
-					return err
-				}
-			}
-		}
-
-		return nil
+			return b.indexResource(resource.ID, *resource)
+		})
 	})
 }
 
 func (b *Batch) Delete(id string) error {
 	return b.withSizeLimit(func() error {
-		affectedResources, err := searchAndUpdateResourcesDeletionState(id, true, b.index)
-		if err != nil {
-			return err
-		}
-
-		for _, resource := range affectedResources {
-			if err := b.indexResource(resource.ID, *resource); err != nil {
-				return err
-			}
-			if b.batch.Size() >= b.size {
-				if err := b.Push(); err != nil {
-					return err
-				}
-			}
-		}
-
-		return nil
+		return b.setDeleted(id, true)
 	})
 }
 
 func (b *Batch) Restore(id string) error {
 	return b.withSizeLimit(func() error {
-		affectedResources, err := searchAndUpdateResourcesDeletionState(id, false, b.index)
-		if err != nil {
-			return err
-		}
-
-		for _, resource := range affectedResources {
-			if err := b.indexResource(resource.ID, *resource); err != nil {
-				return err
-			}
-			if b.batch.Size() >= b.size {
-				if err := b.Push(); err != nil {
-					return err
-				}
-			}
-		}
+		return b.setDeleted(id, false)
+	})
+}
 
-		return nil
+func (b *Batch) setDeleted(id string, deleted bool) error {
+	return b.forSelfAndDescendants(id, func(resource *search.Resource) error {
+		resource.Deleted = deleted
+		return b.indexResource(resource.ID, *resource)
 	})
 }
 
 func (b *Batch) Purge(id string, onlyDeleted bool) error {
 	return b.withSizeLimit(func() error {
-		rootResource, err := searchResourceByID(id, b.index)
-		if err != nil {
-			return err
-		}
-
-		var affectResources []*search.Resource
-		add := func(resource *search.Resource) {
+		return b.forSelfAndDescendants(id, func(resource *search.Resource) error {
 			if onlyDeleted && !resource.Deleted {
-				return
+				return nil
 			}
+			b.batch.Delete(resource.ID)
+			return nil
+		})
+	})
+}
 
-			affectResources = append(affectResources, resource)
-		}
-
-		add(rootResource)
-
-		if rootResource.Type == uint64(storageProvider.ResourceType_RESOURCE_TYPE_CONTAINER) {
-			descendantResources, err := searchResourcesByPath(rootResource.RootID, rootResource.Path, b.index)
-			if err != nil {
-				return err
-			}
+// fn sees the root first; the root's original path drives the descendant lookup
+func (b *Batch) forSelfAndDescendants(id string, fn func(*search.Resource) error) error {
+	root, err := searchResourceByID(id, b.index)
+	if err != nil {
+		return err
+	}
+	rootID, rootPath := root.RootID, root.Path
+	isContainer := root.Type == uint64(storageProvider.ResourceType_RESOURCE_TYPE_CONTAINER)
 
-			for _, descendantResource := range descendantResources {
-				add(descendantResource)
-			}
+	apply := func(resource *search.Resource) error {
+		if err := fn(resource); err != nil {
+			return err
 		}
-
-		for _, resource := range affectResources {
-			b.batch.Delete(resource.ID)
-			if b.batch.Size() >= b.size {
-				if err := b.Push(); err != nil {
-					return err
-				}
-			}
+		if b.batch.Size() >= b.size {
+			return b.Push()
 		}
+		return nil
+	}
 
+	if err := apply(root); err != nil {
+		return err
+	}
+	if !isContainer {
 		return nil
+	}
+	return forEachResourceByPath(rootID, rootPath, b.index, func(resource *search.Resource) error {
+		if resource.ID == id {
+			return nil
+		}
+		return apply(resource)
 	})
 }
 

services/search/pkg/bleve/bleve.go

@@ -1,8 +1,6 @@
 package bleve
 
 import (
-	"regexp"
-
 	bleveSearch "github.com/blevesearch/bleve/v2/search"
 	storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 
@@ -11,8 +9,6 @@ import (
 	"github.com/opencloud-eu/opencloud/services/search/pkg/search"
 )
 
-var queryEscape = regexp.MustCompile(`([` + regexp.QuoteMeta(`+=&|><!(){}[]^\"~*?:\/`) + `\-\s])`)
-
 func getFieldValue[T any](m map[string]any, key string) (out T) {
 	val, ok := m[key]
 	if !ok {
@@ -84,7 +80,3 @@ func hitToFacet[T any](fields map[string]any, prefix string) *T {
 func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
 	return mapping.Deserialize[search.Resource](match.Fields)
 }
-
-func escapeQuery(s string) string {
-	return queryEscape.ReplaceAllString(s, "\\$1")
-}

services/search/pkg/bleve/descendants_bench_test.go

@@ -0,0 +1,66 @@
+package bleve
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/opencloud-eu/opencloud/pkg/log"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/search"
+)
+
+// reports peak live heap as peak-MB, the number that OOMs servers
+func BenchmarkSearchResourcesByPath(b *testing.B) {
+	for _, n := range []int{20_000, 100_000} {
+		b.Run(fmt.Sprintf("docs=%d", n), func(b *testing.B) {
+			idx, _, err := NewIndex(b.TempDir(), log.NopLogger())
+			if err != nil {
+				b.Fatal(err)
+			}
+			defer idx.Close()
+
+			const rootID = "storage$space!root"
+			batch := idx.NewBatch()
+			for i := 0; i < n; i++ {
+				id := fmt.Sprintf("storage$space!f%06d", i)
+				err := batch.Index(id, search.Resource{
+					ID:       id,
+					RootID:   rootID,
+					ParentID: "storage$space!big",
+					Path:     fmt.Sprintf("./big/dir%02d/file-%06d-%032x.txt", i%50, i, uint64(i)*2654435761),
+					Type:     1,
+				})
+				if err != nil {
+					b.Fatal(err)
+				}
+				if batch.Size() >= 1000 {
+					if err := idx.Batch(batch); err != nil {
+						b.Fatal(err)
+					}
+					batch.Reset()
+				}
+			}
+			if err := idx.Batch(batch); err != nil {
+				b.Fatal(err)
+			}
+
+			b.ResetTimer()
+			peak := peakHeapDuring(func() {
+				for b.Loop() {
+					seen := 0
+					err := forEachResourceByPath(rootID, "./big", idx, func(*search.Resource) error {
+						seen++
+						return nil
+					})
+					if err != nil {
+						b.Fatal(err)
+					}
+					if seen != n {
+						b.Fatalf("expected %d descendants, got %d", n, seen)
+					}
+				}
+			})
+			b.StopTimer()
+			b.ReportMetric(float64(peak)/1e6, "peak-MB")
+		})
+	}
+}