Add BlockCursor (postings a block at a time, with block-max bounds) and optional mergeable/releasable interfaces

block_iterator.go

@@ -0,0 +1,127 @@
+//  Copyright (c) 2026 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package segment
+
+import "github.com/blevesearch/freeway/bitpack"
+
+// PostingsBlockLen is the maximum number of postings a PostingsBlock holds: the
+// length of a block of the bit-packed postings, which is that of freeway's codec.
+const PostingsBlockLen = bitpack.BlockLen
+
+// PostingsBlock is a batch of postings handed out by a BlockPostingsIterator. Only the
+// scalars a scorer needs are carried: doc number, term frequency and field
+// norm. There are no locations.
+//
+// Only the first n entries (n being what the BlockPostingsIterator call returned) are
+// meaningful; the remainder is unspecified. Docs are strictly ascending and
+// are local to the segment. The array is meant to be allocated once by the
+// caller and refilled by every call.
+type PostingsBlock struct {
+	Docs  [PostingsBlockLen]uint32
+	Freqs [PostingsBlockLen]uint32
+	// Norms holds the field norm factor (1/sqrt(fieldLength)) of every doc.
+	// It is filled only when the iterator was opened with norms.
+	Norms [PostingsBlockLen]float32
+}
+
+// BlockPostingsList is an optional capability a PostingsList may satisfy to
+// let a caller consume its postings a block at a time, rather than one
+// posting at a time through a PostingsIterator.
+//
+// Deletions are taken care of by the iterator: deleted docs never appear in a
+// block.
+type BlockPostingsList interface {
+	// BlockIterator returns an iterator positioned before the first posting.
+	// withFreqs / withNorms select which of Freqs / Norms get filled; an
+	// unselected array is left unspecified. prealloc may be a iterator
+	// previously returned by this method, to be reset and reused (nil is
+	// fine).
+	BlockIterator(withFreqs, withNorms bool, prealloc BlockPostingsIterator) (BlockPostingsIterator, error)
+}
+
+// BlockPostingsIterator iterates over a postings list a block at a time, so that
+// whatever sits on top of it pays for an interface call once per batch of
+// postings, not once per posting.
+type BlockPostingsIterator interface {
+	// NextBlock fills b with the next batch of live postings and returns how
+	// many it holds. A return of 0 with a nil error means the iterator is
+	// exhausted, and it stays exhausted.
+	NextBlock(b *PostingsBlock) (n int, err error)
+
+	// SeekBlock fills b with the first batch of live postings whose doc
+	// number is >= target, with the entries below target dropped, and
+	// returns how many it holds; the entries start at index 0. A return of 0
+	// means that nothing at or after target is left. The iterator never moves
+	// backwards, so a target below the last returned doc number behaves like
+	// NextBlock.
+	SeekBlock(target uint64, b *PostingsBlock) (n int, err error)
+
+	// Count is the number of postings of the list, not discounting any
+	// deletions. It's cheap.
+	Count() uint64
+
+	// LiveCount is the exact number of postings of the list that haven't
+	// been deleted. When nothing is deleted it's as cheap as Count; otherwise
+	// the list has to be walked, doc numbers only, so use it when the count is
+	// all that's needed. It's independent of the iterator's position, and
+	// doesn't move it.
+	LiveCount() (uint64, error)
+
+	// BytesRead is the number of bytes read from disk so far by this iterator,
+	// cumulatively.
+	BytesRead() uint64
+}
+
+// BlockBounds describes a block of postings by what bounds the score of
+// every posting in it: the highest term frequency, and the highest norm factor
+// (the shortest field). The score of a posting rises with both, so the score
+// of the pair is an upper bound on the scores of the block. The two need not
+// come from the same posting.
+type BlockBounds struct {
+	// LastDoc is the highest doc number of the block (math.MaxUint32 if there
+	// is no such block).
+	LastDoc uint32
+
+	// MaxFreq is the highest term frequency of the block. It is only
+	// meaningful if FreqBounded.
+	MaxFreq uint32
+	// FreqBounded is false if the frequencies of the block aren't bounded: a
+	// format may only record them up to a cap.
+	FreqBounded bool
+
+	// MaxNorm is the highest norm factor (1/sqrt(field length)) of the block;
+	// +Inf if it isn't known.
+	MaxNorm float32
+}
+
+// BlockMaxPostingsIterator is an optional capability of a BlockPostingsIterator: bounds on the
+// scores of the blocks of the list. It is what block-max pruning (WAND) is made
+// of.
+type BlockMaxPostingsIterator interface {
+	// BoundsAt are the bounds of the first block whose last doc is >= target,
+	// and whether there is such a block. It only looks at the list's skip
+	// data: nothing is decoded and the iterator doesn't move, so it can be
+	// asked about any target, in any order, whenever a pruner wants a bound
+	// without committing to read from there.
+	BoundsAt(target uint32) (BlockBounds, bool)
+
+	// DecodedBounds are the bounds of the block that the last NextBlock or
+	// SeekBlock decoded -- the block the postings it returned are from.
+	DecodedBounds() BlockBounds
+
+	// TermBounds are the bounds of the whole list: the highest frequency and
+	// norm factor over its blocks. LastDoc is that of the last block.
+	TermBounds() BlockBounds
+}

go.mod

@@ -1,10 +1,11 @@
 module github.com/blevesearch/scorch_segment_api/v2
 
-go 1.24.0
+go 1.25.0
 
 require (
 	github.com/RoaringBitmap/roaring/v2 v2.14.5
 	github.com/blevesearch/bleve_index_api v1.4.1-0.20260729060817-8e56340f2a7e
+	github.com/blevesearch/freeway v0.0.0-20260910051309-0770e11c32cd
 )
 
 require (

go.sum

@@ -4,6 +4,8 @@ github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD
 github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/blevesearch/bleve_index_api v1.4.1-0.20260729060817-8e56340f2a7e h1:WJ3JnFiBuZ0m1mXaniDHpbZxKU8hjp7SIhqHopSnaM4=
 github.com/blevesearch/bleve_index_api v1.4.1-0.20260729060817-8e56340f2a7e/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
+github.com/blevesearch/freeway v0.0.0-20260910051309-0770e11c32cd h1:LOQjtgGUFbLlHbw/Ue+SE4nFSWUPMkyTZcCiRLWlF6E=
+github.com/blevesearch/freeway v0.0.0-20260910051309-0770e11c32cd/go.mod h1:KguWkuJ1aPhrMS4svPyA9rz2zrgkNybj546cXgxGrCI=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=

segment.go

@@ -142,6 +142,39 @@ type OptimizablePostingsIterator interface {
 	ReplaceActual(*roaring.Bitmap)
 }
 
+// MergeablePostingsList is an optional capability a PostingsList
+// implementation may satisfy to participate in segment merging without
+// exposing its full on-disk representation: an upper bound on doc frequency
+// and whether the term records locations, both knowable before any iterator
+// over it is constructed.
+type MergeablePostingsList interface {
+	// RawDocFreq returns the number of postings recorded on disk for this
+	// term, before any deletions are taken into account.
+	RawDocFreq() uint64
+
+	// HasLocs reports whether this term records locations.
+	HasLocs() bool
+}
+
+// MergeablePostingsIterator is an optional capability a PostingsIterator
+// implementation may satisfy to participate in segment merging: access to a
+// posting's raw, still-encoded location bytes, so a merge can move them to
+// another segment without decoding and re-encoding them.
+type MergeablePostingsIterator interface {
+	// NextWithLocBytes returns the next posting along with the raw encoded
+	// bytes of its location group (nil if it has none), or exists == false
+	// once the iterator is spent.
+	NextWithLocBytes() (docNum uint64, freq uint64, bytesLoc []byte, exists bool, err error)
+}
+
+// Releasable is an optional capability a PostingsList or PostingsIterator
+// implementation may satisfy to support pooled reuse: Release returns the
+// value to its implementation's internal pool. The caller must not use the
+// value again after calling Release.
+type Releasable interface {
+	Release()
+}
+
 type Posting interface {
 	Number() uint64