package qdf

import "github.com/alex60217101990/qdf/internal/bumparena"

// Arena is a bump allocator for decoded string (and []byte-as-string) bodies.
// Pass it to a decode via WithArena (or Decoder.SetArena) and every copied
// string is packed into the arena's dense, contiguous blocks instead of one
// heap allocation per string. Across an epoch of many decodes this amortizes
// the block allocation to near zero and keeps the strings cache-local; the GC
// scans each block as a single object instead of one object per string.
//
// Lifetime / safety: strings returned by an arena decode ALIAS the arena's
// memory. They stay valid as long as the Arena (or any string from it) is
// reachable — the GC keeps a block alive from an interior pointer, so you need
// not track the buffer manually. The SAFE pattern is one Arena per epoch
// (request / batch / stream window), then drop it:
//
//        a := qdf.NewArena()
//        for _, msg := range batch {
//            var v Event
//            _ = qdf.Unmarshal(msg, &v, qdf.WithArena(a))
//            use(v)          // v's strings live in a
//        }
//        // drop a; the GC frees every block once the decoded values die.
//
// Reset reuses the arena across epochs for zero allocation, but is UNSAFE while
// any string from the previous epoch is still live (see Reset).
//
// An Arena is NOT safe for concurrent use; give each goroutine its own.
type Arena struct {
        b bumparena.Bump
}

// NewArena returns an empty arena. The first decode allocates the first block.
func NewArena() *Arena { return &Arena{b: bumparena.New()} }

// appendStr copies b (non-empty) into the arena and returns an aliasing string.
// Trivial forwarder so the decoder's hot path resolves directly to the bump
// core (this wrapper inlines).
func (a *Arena) appendStr(b []byte) string { return a.b.AppendStr(b) }

// Reset rewinds the arena to reuse its current block, so the next epoch of
// decodes allocates nothing.
//
// UNSAFE: Reset invalidates every string previously returned from this arena —
// the next decode overwrites the same memory. Call it ONLY once every value
// decoded into the arena since the last Reset (or since NewArena) is dead. This
// is a manual use-after-free contract the race detector cannot catch; if unsure,
// drop the Arena and make a new one instead (the GC then frees it safely).
func (a *Arena) Reset() { a.b.Reset() }

package qdf

import (
        "reflect"
        "time"
        "unsafe"
)

// Str is a pointer-free string handle: an (offset, length) pair into the
// owning Batch's slab. 8 bytes, zero pointers — a struct made of Str/Bytes/
// Time and scalars is GC-noscan. Resolve with (*Batch[T]).Str. A handle is
// only meaningful against the Batch that produced it; after Release the
// bytes it points at may be overwritten (debug/race builds panic instead).
type Str struct{ off, len uint32 }

// Bytes is the []byte analogue of Str.
type Bytes struct{ off, len uint32 }

// Time is a pointer-free time value (time.Time carries *Location, which
// would re-introduce GC scanning). Wire-compatible with the timestamp tag.
type Time struct {
        Sec  int64
        Nsec uint32
}

// Batch is a pointer-free decode result: Rows carries handle/scalar structs
// (GC never scans them), the slab owns every byte the handles reference.
type Batch[T any] struct {
        // Field order is GC-scan-tuned (fieldalignment): pointer-bearing fields
        // lead so the scanned prefix is 16 bytes, not 32. Access is by name;
        // declaration order is not part of the API.
        slab *batchSlab
        Rows []T
        // epoch is the slab's generation at decode time, captured so debug/race
        // builds can detect a handle resolved after Release (which bumps
        // slab.epoch and returns the slab to the pool for reuse by an unrelated
        // decode). Always present so Batch's layout is build-tag independent;
        // the production resolve path never reads it (see batch_check_prod.go),
        // so it costs one word and one store, never a branch.
        epoch uint32
}

// UnmarshalBatch decodes data into a pointer-free Batch. T must contain only
// scalars, qdf.Str, qdf.Bytes and qdf.Time fields (see batchPlanOf); string
// bodies land in one pooled slab. Call Release when done to recycle the slab
// and the Rows backing — afterwards Rows and every handle are invalid.
//
// Batch is returned BY VALUE (4 words: a slice header + a slab pointer): the
// generic Batch[T] header itself cannot be pooled (its type is instantiated
// per T), so the only way to avoid an allocation for it on every decode is to
// never heap-allocate it in the first place. Returned by value it is built on
// the caller's stack (or inlined into their struct); (*Batch[T]).Release
// stays a pointer receiver so it can still nil out the fields of an
// addressable local (`b, _ := UnmarshalBatch[T](...); defer b.Release()`).
func UnmarshalBatch[T any](data []byte, opts ...QueryOption) (Batch[T], error) {
        plan, err := batchPlanOf(reflect.TypeFor[T]())
        if err != nil {
                return Batch[T]{}, err
        }
        slab := newBatchSlab()
        var rowsPtr unsafe.Pointer
        takeRows := func(n int) unsafe.Pointer {
                rowsPtr = slab.takeRows(n * int(plan.stride))
                return rowsPtr
        }
        n, err := unmarshalBatchCore(data, plan, slab, takeRows, opts...)
        if err != nil {
                slab.release()
                return Batch[T]{}, err
        }
        var rows []T
        if n > 0 {
                rows = unsafe.Slice((*T)(rowsPtr), n)
        }
        return Batch[T]{Rows: rows, slab: slab, epoch: slab.epoch}, nil
}

// Str resolves a string handle produced by this batch's decode. Debug/race
// builds verify the slab is still the one that produced h before resolving
// (see batch_check_debug.go); production builds inline checkEpoch to nothing.
func (b *Batch[T]) Str(h Str) string {
        b.slab.checkEpoch(b.epoch)
        return b.slab.str(h)
}

// BytesOf resolves a bytes handle. The view aliases the slab: valid until
// Release. An empty wire value resolves to nil (not a non-nil empty slice) —
// the usual qdf empty-slice normalization.
func (b *Batch[T]) BytesOf(h Bytes) []byte {
        b.slab.checkEpoch(b.epoch)
        return b.slab.bytes(h)
}

// TimeOf converts a pointer-free Time to time.Time (UTC).
func (b *Batch[T]) TimeOf(h Time) time.Time {
        return time.Unix(h.Sec, int64(h.Nsec)).UTC()
}

// Release recycles the slab. Rows and all handles become invalid.
func (b *Batch[T]) Release() {
        if b.slab != nil {
                b.slab.release()
                b.slab = nil
        }
        b.Rows = nil
}

//go:build !qdfdebug && !race

package qdf

import "unsafe"

// checkEpoch is a no-op in production builds: resolving a handle after
// Release is documented UB (the slab is pooled and may already be reused by
// an unrelated decode), not a checked error. The empty body on a
// pointer-receiver method with an unused parameter compiles to nothing and
// is inlined away — no branch, no load, zero cost. See batch_check_debug.go
// for the panicking counterpart.
func (s *batchSlab) checkEpoch(want uint32) {}

// str resolves a handle with no checks: 2 loads and an unsafe.String.
func (s *batchSlab) str(h Str) string {
        if h.len == 0 {
                return ""
        }
        return unsafe.String((*byte)(unsafe.Add(s.base(), h.off)), int(h.len))
}

func (s *batchSlab) bytes(h Bytes) []byte {
        if h.len == 0 {
                return nil
        }
        return unsafe.Slice((*byte)(unsafe.Add(s.base(), h.off)), int(h.len))
}

package qdf

import (
        "encoding/binary"
        "errors"
        "math"
        "reflect"
        "sync"
        "time"
        "unsafe"
)

// errBatchNeedFallback is an internal sentinel: decodeBatchColumnar returns it
// when the wire carries something the v1 columnar fast path does not handle
// (a nullable column). unmarshalBatchCore catches it and re-decodes the
// ORIGINAL data through the reflect mirror fallback, which handles every wire.
var errBatchNeedFallback = errors.New("qdf: batch columnar fallback")

// unmarshalBatchCore decodes data into a T-rows region obtained from rows(n):
// the core never names T, all row writes go through unsafe offsets computed
// from plan.stride/plan.fields. rows is a closure over
// (*batchSlab).takeRows(n*plan.stride) supplied by the generic wrapper —
// the core learns n internally (from the columnar header, or from the
// mirror slice length) and calls rows(n) exactly once to get the backing
// pointer, which the wrapper then wraps into []T via unsafe.Slice. This
// keeps the pooled-rows-backing decision (takeRows) out of the core (which
// cannot name T) while letting both decode paths hand it their own
// early-known n.
//
// Fast path: a pure columnar container (tagColStruct) is decoded straight
// into the T rows + slab by decodeBatchColumnar — no mirror, no per-row
// string alloc. Anything else (row-major, hybrid, batched-vector, or a
// columnar payload with a nullable column) falls back to the reflect-driven
// mirror strategy below.
//
// Fallback strategy (correct-first, handles every wire): decode the whole
// payload through the EXISTING reflect-driven Unmarshal into a pooled
// []plan.mirror slice — a runtime struct type with the same wire field names
// as T but with handle types swapped back to string/[]byte/time.Time, so the
// normal decoder needs no new wire logic. Then one copy pass per row scatters
// each mirror row into a T row (memmove of scalar bytes) plus the slab
// (string/bytes bodies, rewritten as Str/Bytes handles) and converts qdf.Time.
// The opts parameter is reserved: arena/noCopy are deliberately inert here —
// the slab supersedes both (it owns every byte a handle points into), and no
// current QueryOption applies. Kept in the signature so the public
// UnmarshalBatch contract does not change when a batch-relevant option lands.
func unmarshalBatchCore(data []byte, plan *batchPlan, slab *batchSlab, rows func(n int) unsafe.Pointer, _ ...QueryOption) (int, error) {

        // --- Columnar fast path -------------------------------------------------
        // Attempt a pure-columnar decode on a pooled decoder. On success the T
        // rows and the slab are fully populated. On the fallback sentinel (or any
        // non-columnar wire) drop through to the mirror path, which re-decodes the
        // ORIGINAL data from scratch (so a rANS-wrapped or hybrid payload is parsed
        // cleanly regardless of how far the attempt advanced).
        if n, ok, err := tryDecodeBatchColumnar(data, plan, slab, rows); ok {
                return n, err
        }

        // Row-major-direct fast path: a plain row-major struct-slice wire (an array
        // header, as opposed to the tagColStruct/tagHybridColStruct container tags)
        // is decoded straight into the T rows + slab — no mirror, no per-row owned
        // string. Any other wire (hybrid, batched-vector, tagNil, …) returns
        // ok=false and drops through to the mirror fallback below.
        if n, ok, err := tryDecodeBatchRowMajor(data, plan, slab, rows); ok {
                return n, err
        }

        return unmarshalBatchMirror(data, plan, slab, rows)
}

// unmarshalBatchMirror is unmarshalBatchCore's fallback strategy, factored out
// so it can also be driven directly — bypassing both fast-path attempts above —
// by a benchmark control that measures the reflect-mirror cost of a wire the
// direct paths would otherwise handle. unmarshalBatchCore's normal call path
// reaches it only after both tryDecodeBatchColumnar and tryDecodeBatchRowMajor
// return ok=false (hybrid, batched-vector, tagNil, a nullable columnar column,
// or — for the benchmark control — a row-major wire decoded on purpose without
// the direct path).
func unmarshalBatchMirror(data []byte, plan *batchPlan, slab *batchSlab, rows func(n int) unsafe.Pointer) (int, error) {
        slot := plan.mirrorSlicePtr.Get().(*mirrorSlot)
        defer plan.mirrorSlicePtr.Put(slot)

        // Direct slice-header access instead of reflect.ValueOf/Elem/Index — the
        // slot caches the raw pointer, so the hot path is reflection-free (and
        // therefore identical under both reflect and qdf_reflect2 builds).
        hdr := (*sliceHeader)(slot.ptr)
        hdr.Len = 0

        if err := Unmarshal(data, slot.box); err != nil {
                return 0, err
        }

        n := hdr.Len
        if n == 0 {
                rows(0)
                return 0, nil
        }

        mirrorBase := hdr.Data
        mirrorStride := plan.mirror.Size()

        // Sum string/bytes body lengths up front so the slab grows exactly once
        // instead of on every append.
        var need int
        for i := range n {
                rowPtr := unsafe.Add(mirrorBase, uintptr(i)*mirrorStride)
                for fi, f := range plan.fields {
                        switch f.kind {
                        case bfStr:
                                s := *(*string)(unsafe.Add(rowPtr, plan.mirrorOff[fi]))
                                need += len(s)
                        case bfBytes:
                                b := *(*[]byte)(unsafe.Add(rowPtr, plan.mirrorOff[fi]))
                                need += len(b)
                        }
                }
        }
        slab.grow(need)

        rowsBase := rows(n)

        if err := batchCopyRows(plan, slab, mirrorBase, rowsBase, n); err != nil {
                return 0, err
        }

        return n, nil
}

// batchCopyRows scatters n mirror rows (starting at mirrorPtr, each
// plan.mirror.Size() bytes apart) into rowsBase (n rows of plan.stride
// bytes each), writing scalars via memmove, strings/bytes into slab (as
// Str/Bytes handles), and qdf.Time from time.Time.
func batchCopyRows(plan *batchPlan, slab *batchSlab, mirrorPtr, rowsBase unsafe.Pointer, n int) error {
        mirrorStride := plan.mirror.Size()
        for i := range n {
                src := unsafe.Add(mirrorPtr, uintptr(i)*mirrorStride)
                dst := unsafe.Add(rowsBase, uintptr(i)*plan.stride)
                for fi, f := range plan.fields {
                        sf := unsafe.Add(src, plan.mirrorOff[fi])
                        df := unsafe.Add(dst, f.off)
                        switch f.kind {
                        case bfStr:
                                s := *(*string)(sf)
                                off, ln, err := slab.append(unsafe.Slice(unsafe.StringData(s), len(s)))
                                if err != nil {
                                        return err
                                }
                                *(*Str)(df) = Str{off: off, len: ln}
                        case bfBytes:
                                b := *(*[]byte)(sf)
                                off, ln, err := slab.append(b)
                                if err != nil {
                                        return err
                                }
                                *(*Bytes)(df) = Bytes{off: off, len: ln}
                        case bfTime:
                                t := *(*time.Time)(sf)
                                // A schema-absent time field decodes to Go's zero time.Time;
                                // converting it would write a large-negative Sec, diverging from
                                // the columnar fast path (which leaves the zeroed rows region as
                                // Time{0,0}). Guard the zero case so both paths agree.
                                if !t.IsZero() {
                                        *(*Time)(df) = Time{Sec: t.Unix(), Nsec: uint32(t.Nanosecond())}
                                }
                        default: // bfScalar
                                copy(unsafe.Slice((*byte)(df), f.width), unsafe.Slice((*byte)(sf), f.width))
                        }
                }
        }
        return nil
}

// scalarKindSize returns the byte width of a scalar batchField, matching
// scalarKindType's type set.
func scalarKindSize(k reflect.Kind) uintptr {
        switch k {
        case reflect.Bool, reflect.Int8, reflect.Uint8:
                return 1
        case reflect.Int16, reflect.Uint16:
                return 2
        case reflect.Int32, reflect.Uint32, reflect.Float32:
                return 4
        case reflect.Int64, reflect.Uint64, reflect.Float64:
                return 8
        case reflect.Int, reflect.Uint, reflect.Uintptr:
                return unsafe.Sizeof(uintptr(0))
        default:
                return 0
        }
}

// tryDecodeBatchColumnar sets up a pooled decoder, reads the header, and — iff
// the top tag is a plain tagColStruct — runs decodeBatchColumnar. It returns
// ok=false to signal "not the fast path, use the mirror fallback": for a
// non-columnar tag, a header/peek error, or the errBatchNeedFallback sentinel.
// A real decode error (ok=true, err!=nil) is surfaced to the caller.
func tryDecodeBatchColumnar(data []byte, plan *batchPlan, slab *batchSlab, rows func(n int) unsafe.Pointer) (n int, ok bool, err error) {
        d := decPool.Get().(*Decoder)
        d.buf = data
        d.i = 0
        d.depth = 0
        d.headerRead = false
        d.mode = Fast
        d.colIndex = false
        d.colMaxLen = 0
        // noCopy makes the string readers ALIAS the input buffer instead of
        // allocating an owned string per distinct value (materializeStr). Every
        // aliased view is immediately copied into the slab (which owns the bytes a
        // handle points into), so the classic noCopy use-after-free hazard never
        // reaches the caller: the alias lives only for the duration of the slab
        // copy. This drops the dict-table / const / raw / inline per-string allocs
        // to zero — the slab copy is the sole owner.
        d.noCopy = true
        d.arena = nil
        d.selectFields = nil
        d.query = nil
        if d.state != nil {
                d.state.reset()
        }
        defer func() {
                d.buf = nil
                d.colMaxLen = 0
                d.noCopy = false
                decPool.Put(d)
        }()

        tag, perr := d.peekTag()
        if perr != nil || tag != tagColStruct {
                return 0, false, nil // non-columnar wire → mirror fallback
        }
        n, derr := decodeBatchColumnar(d, plan, slab, rows)
        if derr != nil {
                if errors.Is(derr, errBatchNeedFallback) {
                        return 0, false, nil // nullable column → mirror fallback re-decodes original
                }
                return 0, true, derr
        }
        return n, true, nil
}

// tryDecodeBatchRowMajor sets up a pooled decoder exactly like
// tryDecodeBatchColumnar and peeks the top tag to DETECT a pure row-major
// struct-slice wire: a plain array header (the tagFixarr range, tagArr16, or
// tagArr32 — the same tag set ReadArrayHeader accepts) whose elements are
// struct-headers, as opposed to the tagColStruct (columnar) or
// tagHybridColStruct (hybrid) container tags. Those two container tags are
// never array-header tags, so checking "is this an array-header tag" alone
// is sufficient to rule them both out — no need to peek past the outer tag.
//
// On detection it decodes the array of per-row structs directly into the T
// rows (obtained from rows(n)) plus the slab, with NO reflect mirror and NO
// per-row owned-string allocation: each row is read via ReadStructHeader
// (shape-interned or plain map) and every wire field is matched by name
// against plan.fields and scattered through unsafe offsets, mirroring the
// generated DecodeQDF discipline. String/bytes bodies alias the input buffer
// (noCopy) and are copied once into the slab as Str/Bytes handles.
//
// Contract (same as tryDecodeBatchColumnar's real-decode branch):
//   - ok=false, nil → not this fast path (non-array-header tag, or a peek
//     error before any byte is consumed) → the caller's mirror fallback
//     re-decodes the ORIGINAL data.
//   - ok=true, nil → decoded successfully.
//   - ok=true, err → a hard decode error AFTER the header was consumed. The
//     cursor is spent, so the caller must NOT fall back to the mirror (that
//     would double-decode); the error is surfaced. Every d.Read* error is
//     propagated and a per-field wire/plan type mismatch surfaces as the
//     value reader's ErrTypeMismatch — the outer array tag proves nothing
//     about the element types, so nothing is scattered without validation.
func tryDecodeBatchRowMajor(data []byte, plan *batchPlan, slab *batchSlab, rows func(n int) unsafe.Pointer) (n int, ok bool, err error) {
        d := decPool.Get().(*Decoder)
        d.buf = data
        d.i = 0
        d.depth = 0
        d.headerRead = false
        d.mode = Fast
        d.colIndex = false
        d.colMaxLen = 0
        // noCopy mirrors tryDecodeBatchColumnar's setup: any string reads this
        // path eventually does must alias-then-copy into the slab, never
        // materialize an owned per-value string.
        d.noCopy = true
        d.arena = nil
        d.selectFields = nil
        d.query = nil
        if d.state != nil {
                d.state.reset()
        }
        defer func() {
                d.buf = nil
                d.colMaxLen = 0
                d.noCopy = false
                decPool.Put(d)
        }()

        tag, perr := d.peekTag()
        if perr != nil {
                return 0, false, nil // header/peek error: let the mirror re-decode and surface it
        }

        // isRowMajor is true only for a plain array-header tag. tagColStruct and
        // tagHybridColStruct are distinct, non-array-header tag values, so this
        // single check is enough to exclude both.
        isRowMajor := tag >= tagFixarr && tag <= tagFixarr|tagFixarrMask || tag == tagArr16 || tag == tagArr32
        if !isRowMajor {
                return 0, false, nil // columnar / hybrid / tagNil / anything else → mirror fallback
        }

        // Detected a row-major struct-slice wire. From here every error is HANDLED
        // (ok=true): ReadArrayHeader consumes the peeked array tag, so the cursor is
        // spent and a mid-stream fall back to the mirror would double-decode.
        n, err = d.ReadArrayHeader()
        if err != nil {
                return 0, true, err
        }
        // Bound n by the INPUT before it drives any allocation: ReadArrayHeader's
        // tagArr32 arm already rejects a count over the remaining bytes, but its
        // tagArr16 arm does not (only 2 header bytes are checked, not the claimed
        // count) — a 3-byte hostile wire can claim up to 65535 rows with zero
        // bytes left to decode them from. Each row is a struct header of >= 1 wire
        // byte, so CheckLength(n, 1) rejects that up front, matching the row-major
        // reflect decoder's identical guard (emitDecodeSliceRowMajorBody,
        // cmd/qdfgen/gen/gen.go) and go-fuzz-oom-triage's "bound every decode
        // allocation by the input" rule.
        if err := d.CheckLength(n, 1); err != nil {
                return 0, true, err
        }
        // Bound the OUTPUT allocation like the columnar path: a wide-struct T with
        // a still-plausible (input-bounded) n could otherwise amplify a modest
        // wire into a very large rows(n) region; this caps n*stride at
        // maxColumnarBytes on top of the input-proportional guard above.
        if err = checkColumnarBytes(n, plan.stride); err != nil {
                return 0, true, err
        }
        if n == 0 {
                rows(0)
                return 0, true, nil
        }
        base := rows(n)

        // Plan fields absent from a row keep their zero value: takeRows hands back
        // a zeroed region, so a field never scattered stays Time{0,0}/0/"" — the
        // same schema-evolution semantics as the columnar and mirror paths.
        for i := range n {
                rowDst := unsafe.Add(base, uintptr(i)*plan.stride)
                names, plainN, shaped, herr := d.ReadStructHeader()
                if herr != nil {
                        return 0, true, herr
                }
                if shaped {
                        // Shape-interned struct: values follow positionally in names order,
                        // no per-value key on the wire.
                        for _, name := range names {
                                nb := unsafe.Slice(unsafe.StringData(name), len(name))
                                if ferr := scatterBatchRowMajorField(d, plan, slab, rowDst, nb); ferr != nil {
                                        return 0, true, ferr
                                }
                        }
                        continue
                }
                // Plain map: plainN (key, value) pairs. The key bytes alias the input;
                // comparing them against a plan-field name (string(b) == name) does not
                // allocate.
                for range plainN {
                        kb, kerr := d.ReadStringBytes()
                        if kerr != nil {
                                return 0, true, kerr
                        }
                        if ferr := scatterBatchRowMajorField(d, plan, slab, rowDst, kb); ferr != nil {
                                return 0, true, ferr
                        }
                }
        }
        return n, true, nil
}

// batchPlanFieldByName returns the plan field whose wire key equals name, or
// nil when name is not a plan field (a forward-compat wire field to skip). The
// name argument aliases the input buffer / an interned shape name; the
// string(name) == f.name comparison is compiled to a zero-alloc byte compare.
func batchPlanFieldByName(plan *batchPlan, name []byte) *batchField {
        for i := range plan.fields {
                if plan.fields[i].name == string(name) {
                        return &plan.fields[i]
                }
        }
        return nil
}

// scatterBatchRowMajorField reads one wire field value for the row at rowDst,
// matched by name against plan.fields, and scatters it through the field's
// unsafe offset. An unmatched name is skipped (forward compat). Type safety is
// enforced by the value reader: ReadInt/ReadUint/ReadFloat*/ReadBool/
// ReadTimestamp/ReadString each return ErrTypeMismatch when the wire tag does
// not match the field's expected kind, so a mismatched element cannot scatter
// garbage — the outer array-header tag never implies the element types.
func scatterBatchRowMajorField(d *Decoder, plan *batchPlan, slab *batchSlab, rowDst unsafe.Pointer, name []byte) error {
        f := batchPlanFieldByName(plan, name)
        if f == nil {
                return d.Skip() // wire field not in plan → skip its single value
        }
        dst := unsafe.Add(rowDst, f.off)
        switch f.kind {
        case bfStr:
                // noCopy → s aliases the input buffer; slab.append copies it (the sole
                // owner), so the alias never escapes.
                s, err := d.ReadString()
                if err != nil {
                        return err
                }
                off, ln, err := slab.append(unsafe.Slice(unsafe.StringData(s), len(s)))
                if err != nil {
                        return err
                }
                *(*Str)(dst) = Str{off: off, len: ln}
        case bfBytes:
                // A nil []byte encodes as tagNil (distinct from an empty bin); the mirror
                // path (decodeBytes → decodeNilSlice) consumes it as a nil slice. Mirror
                // that: consume the tag and leave the zeroed handle (BytesOf resolves
                // Bytes{0,0} to nil), matching batchCopyRows which stores (0,0) for both
                // a nil and an empty []byte.
                t, terr := d.peekTag()
                if terr != nil {
                        return terr
                }
                if t == tagNil {
                        d.i++
                        return nil
                }
                b, err := d.ReadBytes() // noCopy → aliases input; copied into the slab
                if err != nil {
                        return err
                }
                off, ln, err := slab.append(b)
                if err != nil {
                        return err
                }
                *(*Bytes)(dst) = Bytes{off: off, len: ln}
        case bfTime:
                // The wire carries sec/nsec directly (unlike the mirror path, which
                // decodes a Go time.Time and must guard its zero value): a schema-absent
                // time is never read here — it stays the zeroed Time{0,0} takeRows left,
                // matching the columnar path.
                sec, nsec, err := d.ReadTimestamp()
                if err != nil {
                        return err
                }
                *(*Time)(dst) = Time{Sec: sec, Nsec: nsec}
        default: // bfScalar
                return scatterBatchRowMajorScalar(d, dst, f.scalarKind)
        }
        return nil
}

// scatterBatchRowMajorScalar reads one scalar value per the field's Go kind and
// stores it width-narrowed at dst, reusing scatterBatchScalar's width-store
// logic for a single value. The chosen reader validates the wire tag: a wire
// value whose type does not match sk yields ErrTypeMismatch instead of a
// silent reinterpretation of the bits.
func scatterBatchRowMajorScalar(d *Decoder, dst unsafe.Pointer, sk reflect.Kind) error {
        switch sk {
        case reflect.Bool:
                v, err := d.ReadBool()
                if err != nil {
                        return err
                }
                *(*bool)(dst) = v
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                switch scalarKindSize(sk) {
                case 1: // int8
                        *(*int8)(dst) = int8(v)
                case 2: // int16
                        *(*int16)(dst) = int16(v)
                case 4: // int32
                        *(*int32)(dst) = int32(v)
                default: // 8: int, int64
                        *(*int64)(dst) = v
                }
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                v, err := d.ReadUint()
                if err != nil {
                        return err
                }
                switch scalarKindSize(sk) {
                case 1: // uint8/byte
                        *(*uint8)(dst) = uint8(v)
                case 2: // uint16
                        *(*uint16)(dst) = uint16(v)
                case 4: // uint32
                        *(*uint32)(dst) = uint32(v)
                default: // 8: uint, uint64, uintptr
                        *(*uint64)(dst) = v
                }
        case reflect.Float32:
                v, err := d.ReadFloat32()
                if err != nil {
                        return err
                }
                *(*float32)(dst) = v
        case reflect.Float64:
                v, err := d.ReadFloat64()
                if err != nil {
                        return err
                }
                *(*float64)(dst) = v
        default:
                return ErrTypeMismatch
        }
        return nil
}

// decodeBatchColumnar decodes a pure columnar (tagColStruct) payload straight
// into the T rows (obtained from rows(n), see unmarshalBatchCore) plus the
// slab, with no reflect mirror and no per-row string allocation. It mirrors
// decodeColumnar's reading discipline (readColShape bounds, colMaxLen,
// checkColumnarBytes, per-kind dispatch) but scatters into batch fields via
// unsafe offsets and materializes string bodies into the slab as Str/Bytes
// handles.
//
// A nullable wire column is out of scope in v1 (a pointer-free T cannot map
// one anyway): the shape is validated up front and, if any column is nullable,
// errBatchNeedFallback is returned BEFORE any column body is consumed so the
// caller can cleanly re-decode via the mirror path.
func decodeBatchColumnar(d *Decoder, plan *batchPlan, slab *batchSlab, rows func(n int) unsafe.Pointer) (int, error) {
        n, nCols, colLens, err := batchReadColShape(d, plan, slab)
        if err != nil {
                return 0, err
        }

        // Validate the shape up front: any nullable column bails to the mirror
        // fallback before we consume a single column body (partial consumption is
        // harmless — the caller re-decodes the original data from scratch).
        kinds := slab.shapeKinds[:nCols]
        fidx := slab.shapeFidx[:nCols]
        for c := range kinds {
                if kinds[c].isNullable() {
                        return 0, errBatchNeedFallback
                }
        }

        d.colMaxLen = n
        defer func() { d.colMaxLen = 0 }()

        if err := checkColumnarBytes(n, plan.stride); err != nil {
                return 0, err
        }

        // Obtain the T rows backing via the wrapper's takeRows closure. T is
        // guaranteed pointer-free (batchPlanOf rejects any pointer/handle-carrying
        // field), so a plain noscan []byte region is a valid backing for []T: the
        // GC never scans it. Steady-state (slab reused across decodes) this is a
        // cap-reuse + clear, not an allocation — the columnar fast path needs to
        // stay inside its handful-of-allocs budget.
        if n == 0 {
                rows(0)
                return 0, nil
        }
        base := rows(n)

        for c := range kinds {
                if fidx[c] < 0 {
                        // Column present on the wire but not in the plan → skip its body
                        // (forward compat). With no colIndex we must fully decode it to
                        // keep the cursor in sync; with an index we can seek past it.
                        if d.colIndex {
                                if d.i+int(colLens[c]) > len(d.buf) {
                                        return 0, ErrShortBuffer
                                }
                                d.i += int(colLens[c])
                                continue
                        }
                        if err := d.skipColumnValue(kinds[c], n); err != nil {
                                return 0, err
                        }
                        continue
                }
                f := &plan.fields[fidx[c]]
                if err := scatterBatchColumn(d, plan, slab, base, f, kinds[c], n); err != nil {
                        return 0, err
                }
        }
        // Plan fields absent from the wire keep their zero value (the make([]byte)
        // backing is zeroed) — the same schema-evolution semantics as normal decode.
        return n, nil
}

// batchReadColShape is the batch fast path's zero-alloc replacement for
// readColShape's inline-declaration branch. The generic reader materializes
// []string names + []colKind and registers the shape on the decoder state —
// per-message allocations a batch decode pays on every independent payload.
// Batch already knows every field name from the plan, so each wire column
// name is matched as a raw []byte view (string compare against a []byte does
// not allocate) and recorded as a plan-field index in slab scratch:
// slab.shapeFidx[c] (or -1 when the column is not in the plan) and
// slab.shapeKinds[c]. Reading discipline (bounds, MaxInt clauses, colIndex
// lens) mirrors readColShape exactly.
//
// A shape REFERENCE (id != 0) cannot resolve here: the pooled decoder's state
// is reset per batch decode and a single top-level columnar payload declares
// its shape inline, so a reference is either hostile or a multi-block wire
// this path does not handle — bail to the mirror fallback, which replays the
// original bytes through the reference decoder.
func batchReadColShape(d *Decoder, plan *batchPlan, slab *batchSlab) (n, nCols int, colLens []uint32, err error) {
        // Init decoder state unconditionally, exactly like readColShape: the
        // downstream scatter helpers dereference d.state scratch. Today a
        // tagColStruct wire implies FlagDense (whose header path inits state), but
        // that is an incidental invariant — keep the explicit init so a future
        // non-Dense columnar wire cannot nil-panic here.
        if d.state == nil {
                d.state = newDecState()
        }
        d.i++ // consume tagColStruct (caller peeked it)
        n64, k := readUvarint(d.buf[d.i:])
        if k <= 0 {
                return 0, 0, nil, ErrInvalidLength
        }
        d.i += k
        n = int(n64)
        if err := checkColumnarN(n); err != nil {
                return 0, 0, nil, err
        }
        idv, k2 := readUvarint(d.buf[d.i:])
        if k2 <= 0 {
                return 0, 0, nil, ErrInvalidLength
        }
        if idv != 0 {
                return 0, 0, nil, errBatchNeedFallback
        }
        d.i += k2
        cnt64, k3 := readUvarint(d.buf[d.i:])
        if k3 <= 0 {
                return 0, 0, nil, ErrInvalidLength
        }
        d.i += k3
        if cnt64 > uint64(math.MaxInt) { // 32-bit: int() would wrap negative
                return 0, 0, nil, ErrInvalidLength
        }
        nCols = int(cnt64)
        if err := d.CheckLength(nCols, 1); err != nil {
                return 0, 0, nil, err
        }
        if cap(slab.shapeFidx) < nCols {
                slab.shapeFidx = make([]int16, nCols)
                slab.shapeKinds = make([]colKind, nCols)
        }
        fidx := slab.shapeFidx[:nCols]
        kinds := slab.shapeKinds[:nCols]
        for c := range nCols {
                s, err := d.readStringBytes() // view into d.buf; matched, never kept
                if err != nil {
                        return 0, 0, nil, err
                }
                if d.i >= len(d.buf) {
                        return 0, 0, nil, ErrShortBuffer
                }
                kinds[c] = colKind(d.buf[d.i])
                d.i++
                fidx[c] = -1
                for i := range plan.fields {
                        if plan.fields[i].name == string(s) { // no alloc: compare only
                                fidx[c] = int16(i)
                                break
                        }
                }
        }
        // colIndex lens: same pooled read as readColShape.
        if d.colIndex {
                if d.i+4*nCols > len(d.buf) {
                        return 0, 0, nil, ErrShortBuffer
                }
                if d.state == nil {
                        d.state = newDecState()
                }
                if cap(d.state.colLenScratch) >= nCols {
                        colLens = d.state.colLenScratch[:nCols]
                } else {
                        colLens = make([]uint32, nCols)
                }
                d.state.colLenScratch = colLens
                var sum uint64
                for c := range nCols {
                        colLens[c] = binary.LittleEndian.Uint32(d.buf[d.i+4*c:])
                        sum += uint64(colLens[c])
                }
                d.i += 4 * nCols
                if sum > uint64(len(d.buf)-d.i) {
                        return 0, 0, nil, ErrShortBuffer
                }
        }
        return n, nCols, colLens, nil
}

// scatterBatchColumn decodes one wire column of the given kind and scatters it
// into the matched batch field across all n rows. Scalars/bool reuse the
// decoder's *Into scratch then a width-switched store; time uses the sec+nsec
// sub-columns; string/bytes materialize into the slab as handles.
func scatterBatchColumn(d *Decoder, plan *batchPlan, slab *batchSlab, base unsafe.Pointer, f *batchField, kind colKind, n int) error {
        stride := plan.stride
        off := f.off
        st := d.state // non-nil: readColShape initialised it

        switch f.kind {
        case bfStr, bfBytes:
                if kind != colKindString {
                        return ErrTypeMismatch
                }
                if cap(st.colStrHandles) < n {
                        st.colStrHandles = make([]Str, n)
                }
                out := st.colStrHandles[:n]
                if err := readStringColumnHandles(d, n, slab, out); err != nil {
                        return err
                }
                // Str and Bytes have identical layout (off,len uint32); one store path.
                for i := range n {
                        dp := unsafe.Add(base, uintptr(i)*stride+off)
                        *(*Str)(dp) = out[i]
                }
                return nil
        case bfTime:
                if kind != colKindTime {
                        return ErrTypeMismatch
                }
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return err
                }
                sec := st.colScratchI64
                if len(sec) != n {
                        return ErrTypeMismatch
                }
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                nsec := st.colScratchU64
                if len(nsec) != n {
                        return ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return err
                }
                for i := range n {
                        dp := unsafe.Add(base, uintptr(i)*stride+off)
                        *(*Time)(dp) = Time{Sec: sec[i], Nsec: uint32(nsec[i])}
                }
                return nil
        default: // bfScalar
                return scatterBatchScalar(d, base, stride, off, f.scalarKind, kind, n)
        }
}

// batchScalarColKind maps a scalar field's Go kind to the columnar kind the
// wire must carry for it. Used to reject a schema-evolution mismatch (e.g. a
// wire int column landing on a float field) BEFORE decoding, mirroring
// decodeColumnar's exact sh.kinds[c] == col.kind guard — without it the width
// dispatch below would reinterpret the bits and silently corrupt the value.
func batchScalarColKind(k reflect.Kind) (colKind, bool) {
        switch k {
        case reflect.Bool:
                return colKindBool, true
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                return colKindInt, true
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                return colKindUint, true
        case reflect.Float32:
                return colKindFloat32, true
        case reflect.Float64:
                return colKindFloat, true
        default:
                return 0, false
        }
}

// scatterBatchScalar decodes a scalar/bool column into scratch then stores each
// value at base+i*stride+off using the field's Go kind width. The wire colKind
// must equal the field's expected columnar kind (per batchScalarColKind).
func scatterBatchScalar(d *Decoder, base unsafe.Pointer, stride, off uintptr, sk reflect.Kind, kind colKind, n int) error {
        if want, ok := batchScalarColKind(sk); !ok || want != kind {
                return ErrTypeMismatch
        }
        st := d.state
        switch kind {
        case colKindInt:
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return err
                }
                s := st.colScratchI64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                // Width-specialized loops: hoisting the switch out of the per-element
                // path removes a branch+call per cell (the column's width never varies).
                switch w := scalarKindSize(sk); w {
                case 1: // int8
                        for i := range n {
                                *(*int8)(unsafe.Add(base, uintptr(i)*stride+off)) = int8(s[i])
                        }
                case 2: // int16
                        for i := range n {
                                *(*int16)(unsafe.Add(base, uintptr(i)*stride+off)) = int16(s[i])
                        }
                case 4: // int32
                        for i := range n {
                                *(*int32)(unsafe.Add(base, uintptr(i)*stride+off)) = int32(s[i])
                        }
                default: // 8: int, int64
                        for i := range n {
                                *(*int64)(unsafe.Add(base, uintptr(i)*stride+off)) = s[i]
                        }
                }
        case colKindUint:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                switch w := scalarKindSize(sk); w {
                case 1: // uint8/byte
                        for i := range n {
                                *(*uint8)(unsafe.Add(base, uintptr(i)*stride+off)) = uint8(s[i])
                        }
                case 2: // uint16
                        for i := range n {
                                *(*uint16)(unsafe.Add(base, uintptr(i)*stride+off)) = uint16(s[i])
                        }
                case 4: // uint32
                        for i := range n {
                                *(*uint32)(unsafe.Add(base, uintptr(i)*stride+off)) = uint32(s[i])
                        }
                default: // 8: uint, uint64, uintptr
                        for i := range n {
                                *(*uint64)(unsafe.Add(base, uintptr(i)*stride+off)) = s[i]
                        }
                }
        case colKindFloat:
                if err := decodeSliceFloat64Into(d, &st.colScratchF64); err != nil {
                        return err
                }
                s := st.colScratchF64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        *(*float64)(unsafe.Add(base, uintptr(i)*stride+off)) = s[i]
                }
        case colKindFloat32:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        *(*uint32)(unsafe.Add(base, uintptr(i)*stride+off)) = uint32(s[i])
                }
        case colKindBool:
                if err := decodeSliceBoolInto(d, &st.colScratchBool); err != nil {
                        return err
                }
                s := st.colScratchBool
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        *(*bool)(unsafe.Add(base, uintptr(i)*stride+off)) = s[i]
                }
        default:
                return ErrTypeMismatch
        }
        return nil
}

// dictHandleScratchPool pools []Str backing arrays for the dict-table handle slice.
var (
        dictHandleScratchPool = sync.Pool{New: func() any { s := make([]Str, 0, 64); return &s }}
)

func getDictHandleScratch(n int) []Str { return getPooledStrScratch(&dictHandleScratchPool, n) }
func putDictHandleScratch(s []Str)     { putPooledStrScratch(&dictHandleScratchPool, s) }

func getPooledStrScratch(pool *sync.Pool, n int) []Str {
        sp := pool.Get().(*[]Str)
        s := *sp
        if cap(s) < n {
                s = make([]Str, n)
        } else {
                s = s[:n]
        }
        return s
}

func putPooledStrScratch(pool *sync.Pool, s []Str) {
        s = s[:0]
        pool.Put(&s)
}

// readStringColumnHandles decodes a string column body (length n) and writes a
// per-row slab handle into out[0:n], materializing each distinct string body
// into the slab exactly once where the codec makes dedup free:
//
//   - tagColStrConst: one slab.append, the same handle for every row.
//   - dict family (tagColStrDict/DictQ/DictFC): each dict ENTRY is materialized
//     once; a row's handle is its entry's handle (free dedup).
//   - everything else (tagColStrRaw / FSST / alpha / per-row inline strings):
//     the existing readStringColumn machinery decodes each string; we copy the
//     bytes out into the slab before that machinery's scratch is recycled.
//
// bfBytes columns share the string-column wire, so this covers them too.
func readStringColumnHandles(d *Decoder, n int, slab *batchSlab, out []Str) error {
        if n == 0 {
                return nil
        }
        if d.i >= len(d.buf) {
                // n > 0 with no bytes left: truncated/hostile input. Without this the
                // bare tag index below panics — the reference decodeColumnInto guards
                // every entry the same way (public API must error, never panic).
                return ErrShortBuffer
        }
        tag := d.buf[d.i]
        switch tag {
        case tagColStrConst:
                strs, err := d.readStringColumnConst(n)
                if err != nil {
                        return err
                }
                v := strs[0]
                off, ln, err := slab.append(unsafe.Slice(unsafe.StringData(v), len(v)))
                if err != nil {
                        return err
                }
                h := Str{off: off, len: ln}
                for i := range n {
                        out[i] = h
                }
                return nil
        case tagColStrDict, tagColStrDictFC, tagColStrDictQ:
                var (
                        table []string
                        idx   []uint32
                        err   error
                )
                switch tag {
                case tagColStrDictFC:
                        table, idx, err = d.readStringColumnDictFC(n)
                case tagColStrDictQ:
                        table, idx, err = d.readStringColumnDictQ(n)
                default:
                        table, idx, err = d.readStringColumnDict(n)
                }
                if err != nil {
                        return err
                }
                // Materialize each distinct table entry into the slab once, then map
                // per-row indices to their entry handles.
                th := getDictHandleScratch(len(table))
                defer putDictHandleScratch(th)
                for k := range table {
                        v := table[k]
                        off, ln, err := slab.append(unsafe.Slice(unsafe.StringData(v), len(v)))
                        if err != nil {
                                return err
                        }
                        th[k] = Str{off: off, len: ln}
                }
                for i := range n {
                        j := idx[i]
                        if int(j) >= len(table) {
                                return ErrInvalidLength
                        }
                        out[i] = th[j]
                }
                return nil
        case tagColStrFSST:
                // FSST decompresses straight into the slab (no temp scratch + no second
                // copy). readStringColumnFSSTInto mirrors readStringColumnFSST's header
                // parse and every bounds guard verbatim.
                return readStringColumnFSSTInto(d, n, slab, out)
        case tagColStrAlpha:
                // Alpha unpacks its characters straight onto the slab's backing (no temp
                // scratch + no second copy), the same shape as the FSST arm above.
                return d.readStringColumnAlphaInto(n, slab, out)
        default:
                // tagColStrRaw / per-row inline: reuse the general materializer, then
                // copy each body into the slab (distinct strings, so there is no dedup
                // to lose). readStringColumn's result aliases decoder scratch — copy
                // before it is recycled by the next column.
                strs, err := d.readStringColumn(n)
                if err != nil {
                        return err
                }
                for i := range n {
                        v := strs[i]
                        off, ln, err := slab.append(unsafe.Slice(unsafe.StringData(v), len(v)))
                        if err != nil {
                                return err
                        }
                        out[i] = Str{off: off, len: ln}
                }
                return nil
        }
}

package qdf

import (
        "fmt"
        "reflect"
        "strings"
        "sync"
        "unsafe"
)

// batchFieldKind classifies how a batchField's bytes are interpreted when
// scattering a struct-of-arrays batch into per-column storage.
type batchFieldKind uint8

const (
        bfScalar batchFieldKind = iota // bool / ints / uints / floats
        bfStr                          // qdf.Str  <- wire string
        bfBytes                        // qdf.Bytes <- wire bin
        bfTime                         // qdf.Time <- wire timestamp
)

// batchField describes one wire-visible field of a batch-eligible struct:
// its wire key, byte offset within the struct, and how to interpret it.
type batchField struct {
        name       string  // wire key
        off        uintptr // byte offset within T
        width      uintptr // scalar byte width (field.Type.Size()); valid when kind == bfScalar
        kind       batchFieldKind
        scalarKind reflect.Kind // valid when kind == bfScalar
}

// batchPlan is the validated, cached layout of a pointer-free struct type
// eligible for batch (struct-of-arrays) handling. Field order follows the
// same wire field order the normal encoder uses for the equivalent struct
// (flattened embedded, tag-named) — see appendBatchFields.
type batchPlan struct {

        // mirrorSlicePtr pools *mirrorSlot values so repeated UnmarshalBatch calls
        // for the same T neither reallocate the *[]mirror box nor touch
        // reflect.Value on the hot path (the slot caches the raw slice-header
        // pointer alongside the any box Unmarshal needs).
        mirrorSlicePtr sync.Pool

        // Pointer-bearing fields lead, scalars trail (fieldalignment: trims the
        // GC-scanned prefix 128 -> 104 bytes). Constructed with named fields only.
        rt reflect.Type

        // mirror is a runtime-built struct type with the same field names/tags
        // as rt, with handle types swapped back to their decodable wire
        // counterparts: Str->string, Bytes->[]byte, Time->time.Time. Scalars are
        // unchanged. The normal reflect-driven Unmarshal decodes into
        // []mirror; the fallback copy pass (batch_decode.go) then scatters each
        // mirror row into a T row plus slab bytes. Since batchPlanOf only
        // accepts flat (non-nested) field sets, mirror's field offsets align
        // 1:1 with plan.fields via mirrorOff.
        mirror reflect.Type
        fields []batchField

        mirrorOff []uintptr // parallel to fields: byte offset within mirror

        // stride is rt.Size() — scalar tail (see the field-order note above).
        stride uintptr
}

// mirrorSlot pairs the pooled *[]mirror box (handed to Unmarshal as any) with
// its raw pointer, so the fallback path reads/writes the slice header directly
// through *sliceHeader instead of reflect.ValueOf(...).Elem()/Index/UnsafeAddr
// per decode. reflect.New runs only in the pool's New (cold, amortized); the
// hot path is reflection-free, which also keeps it neutral across the
// qdf_reflect2 build-tag split (no runtime reflect.Value in either mode).
type mirrorSlot struct {
        box any            // *[]mirror — the Unmarshal target
        ptr unsafe.Pointer // same value as a raw pointer to the slice header
}

// batchPlans caches reflect.Type -> *batchPlan (success) or error (failure),
// so a type that fails validation once doesn't re-walk reflection on every
// call.
var batchPlans sync.Map

var (
        strType   = reflect.TypeFor[Str]()
        bytesType = reflect.TypeFor[Bytes]()
        timeType2 = reflect.TypeFor[Time]()

        mirrorStringType = reflect.TypeFor[string]()
        mirrorBytesType  = reflect.TypeFor[[]byte]()
)

// buildBatchMirror builds p.mirror (a reflect.StructOf mirroring p.fields
// with handle types swapped for their decodable counterparts) and
// p.mirrorOff (each mirror field's byte offset, parallel to p.fields).
//
// Field names cannot reuse the source type's Go names: p.fields may combine
// fields flattened from anonymous-embedded structs (see appendBatchFields),
// so their original Go identifiers can collide or be unexported through the
// embedding path. Instead each mirror field gets a synthetic exported name
// (F0, F1, ...) carrying a `qdf:"<wire-name>"` tag — the same wire-key
// resolution the normal decoder already applies, so the mirror decodes the
// identical wire regardless of how T's fields were declared.
func buildBatchMirror(p *batchPlan) error {
        sf := make([]reflect.StructField, len(p.fields))
        for i, f := range p.fields {
                var ft reflect.Type
                switch f.kind {
                case bfStr:
                        ft = mirrorStringType
                case bfBytes:
                        ft = mirrorBytesType
                case bfTime:
                        ft = timeType
                default: // bfScalar
                        ft = scalarKindType(f.scalarKind)
                        if ft == nil {
                                return fmt.Errorf("qdf: batch type %s: field %s: unsupported scalar kind %s", p.rt, f.name, f.scalarKind)
                        }
                }
                sf[i] = reflect.StructField{
                        Name: fmt.Sprintf("F%d", i),
                        Type: ft,
                        Tag:  reflect.StructTag(fmt.Sprintf(`qdf:%q`, f.name)),
                }
        }
        mirror := reflect.StructOf(sf)
        p.mirror = mirror
        p.mirrorOff = make([]uintptr, len(p.fields))
        for i := range p.fields {
                p.mirrorOff[i] = mirror.Field(i).Offset
        }
        sliceT := reflect.SliceOf(mirror)
        p.mirrorSlicePtr.New = func() any {
                rv := reflect.New(sliceT)
                return &mirrorSlot{box: rv.Interface(), ptr: unsafe.Pointer(rv.Pointer())}
        }
        return nil
}

// scalarKindType maps a reflect.Kind (as recorded in batchField.scalarKind)
// back to its reflect.Type, for building the mirror struct's scalar fields.
func scalarKindType(k reflect.Kind) reflect.Type {
        switch k {
        case reflect.Bool:
                return reflect.TypeFor[bool]()
        case reflect.Int:
                return reflect.TypeFor[int]()
        case reflect.Int8:
                return reflect.TypeFor[int8]()
        case reflect.Int16:
                return reflect.TypeFor[int16]()
        case reflect.Int32:
                return reflect.TypeFor[int32]()
        case reflect.Int64:
                return reflect.TypeFor[int64]()
        case reflect.Uint:
                return reflect.TypeFor[uint]()
        case reflect.Uint8:
                return reflect.TypeFor[uint8]()
        case reflect.Uint16:
                return reflect.TypeFor[uint16]()
        case reflect.Uint32:
                return reflect.TypeFor[uint32]()
        case reflect.Uint64:
                return reflect.TypeFor[uint64]()
        case reflect.Uintptr:
                return reflect.TypeFor[uintptr]()
        case reflect.Float32:
                return reflect.TypeFor[float32]()
        case reflect.Float64:
                return reflect.TypeFor[float64]()
        default:
                return nil
        }
}

// batchPlanOf returns the cached batchPlan for t, validating and building it
// on first use. t must be a struct type that is entirely pointer-free:
// scalars (bool/int*/uint*/float*), qdf.Str, qdf.Bytes, qdf.Time, or an
// exported flattened-embedded struct composed of the same. See the v1 SCOPE
// DECISION in appendBatchFields for what is rejected.
func batchPlanOf(t reflect.Type) (*batchPlan, error) {
        if v, ok := batchPlans.Load(t); ok {
                if p, ok := v.(*batchPlan); ok {
                        return p, nil
                }
                return nil, v.(error)
        }
        p := &batchPlan{rt: t, stride: t.Size()}
        if t.Kind() != reflect.Struct {
                err := fmt.Errorf("qdf: batch type %s: not a struct", t)
                batchPlans.Store(t, err)
                return nil, err
        }
        if err := appendBatchFields(p, t, 0, ""); err != nil {
                batchPlans.Store(t, err)
                return nil, err
        }
        if err := buildBatchMirror(p); err != nil {
                batchPlans.Store(t, err)
                return nil, err
        }
        actual, _ := batchPlans.LoadOrStore(t, p)
        if pp, ok := actual.(*batchPlan); ok {
                return pp, nil
        }
        return p, nil
}

// wireFieldKey extracts the wire key for a struct field using the same
// qdf/json tag rules as appendStructFields (reflect_desc.go): qdf:"-" or
// json:"-" (first comma segment) skips the field; qdf:"name" or json:"name"
// names it; otherwise the Go field name is used.
func wireFieldKey(sf reflect.StructField) (string, bool) {
        if tag, ok := sf.Tag.Lookup("qdf"); ok {
                if tag == "-" {
                        return "", true
                }
                parts := strings.Split(tag, ",")
                if parts[0] != "" {
                        return parts[0], false
                }
                return sf.Name, false
        }
        if tag, ok := sf.Tag.Lookup("json"); ok {
                parts := strings.Split(tag, ",")
                if parts[0] == "-" {
                        return "", true
                }
                if parts[0] != "" {
                        return parts[0], false
                }
                return sf.Name, false
        }
        return sf.Name, false
}

// appendBatchFields walks t's fields (mirroring appendStructFields's
// embedded-flattening and tag rules so wire order matches the normal
// encoder) and appends validated batchFields to p.
//
// v1 SCOPE DECISION: nested named (non-anonymous) structs and [N]arrays
// inside T are validated pointer-free but rejected with a clear error in
// v1 — they complicate the columnar scatter. Flatten them (embed instead of
// name) or drop them; this may be revisited in a later phase.
func appendBatchFields(p *batchPlan, t reflect.Type, base uintptr, path string) error {
        for sf := range t.Fields() {
                fieldPath := sf.Name
                if path != "" {
                        fieldPath = path + "." + sf.Name
                }
                // Mirror the encoder's embedded flattening (appendStructFields,
                // reflect_desc.go): an anonymous value-struct flattens unless the type
                // carries its own value codec — time.Time or a Marshaler/Unmarshaler
                // implementor (the SAME interface rule the encoder applies; Str/Bytes/
                // Time fall out of it via the handle-type checks below when they reach
                // the regular field path). A `qdf:"-"`/`json:"-"` tag on the embedded
                // field itself opts the whole nested block out, exactly like the
                // encoder — without this the plan's field set diverges from the wire.
                if sf.Anonymous && sf.Type.Kind() == reflect.Struct &&
                        sf.Type != strType && sf.Type != bytesType && sf.Type != timeType2 &&
                        sf.Type != timeType &&
                        !reflect.PointerTo(sf.Type).Implements(reflect.TypeFor[Marshaler]()) &&
                        !reflect.PointerTo(sf.Type).Implements(reflect.TypeFor[Unmarshaler]()) {
                        if _, skip := wireFieldKey(sf); skip {
                                continue
                        }
                        if err := appendBatchFields(p, sf.Type, base+sf.Offset, path); err != nil {
                                return err
                        }
                        continue
                }
                if !sf.IsExported() {
                        continue
                }
                key, skip := wireFieldKey(sf)
                if skip {
                        continue
                }
                bf := batchField{name: key, off: base + sf.Offset}
                switch sf.Type {
                case strType:
                        bf.kind = bfStr
                case bytesType:
                        bf.kind = bfBytes
                case timeType2:
                        bf.kind = bfTime
                case timeType:
                        return fmt.Errorf("qdf: batch type: field %s is time.Time — use qdf.Time", fieldPath)
                default:
                        switch k := sf.Type.Kind(); k {
                        case reflect.Bool,
                                reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
                                reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
                                reflect.Float32, reflect.Float64:
                                bf.kind, bf.scalarKind, bf.width = bfScalar, k, sf.Type.Size()
                        case reflect.Struct:
                                // nested pointer-free struct: flatten is NOT applied to named
                                // (non-anonymous) fields on the wire; they are nested values.
                                // v1: recurse for VALIDATION ONLY, decode via fallback path.
                                if err := validateBatchStruct(sf.Type, fieldPath); err != nil {
                                        return err
                                }
                                return fmt.Errorf("qdf: batch type: field %s: nested struct fields decode via the row-major fallback in v1 — flatten it or use scalar/handle fields", fieldPath)
                        case reflect.Array:
                                if err := validateBatchElem(sf.Type.Elem(), fieldPath); err != nil {
                                        return err
                                }
                                return fmt.Errorf("qdf: batch type: field %s: array fields are v1-fallback only", fieldPath)
                        case reflect.String:
                                return fmt.Errorf("qdf: batch type: field %s is string — use qdf.Str", fieldPath)
                        case reflect.Slice:
                                return fmt.Errorf("qdf: batch type: field %s is a slice — use qdf.Bytes for []byte, or drop the field", fieldPath)
                        case reflect.Map, reflect.Pointer, reflect.Interface, reflect.Chan, reflect.Func:
                                return fmt.Errorf("qdf: batch type: field %s (%s) is not pointer-free", fieldPath, k)
                        default:
                                return fmt.Errorf("qdf: batch type: field %s: unsupported kind %s", fieldPath, k)
                        }
                }
                p.fields = append(p.fields, bf)
        }
        return nil
}

// validateBatchStruct recursively checks that a nested struct type (one that
// v1 rejects for columnar scatter regardless) is at least pointer-free, so
// the caller's error names the actual offending nested field
// (e.g. "In.M") rather than the outer struct field.
func validateBatchStruct(t reflect.Type, path string) error {
        for sf := range t.Fields() {
                fieldPath := path + "." + sf.Name
                if !sf.IsExported() {
                        continue
                }
                if err := validateBatchElem(sf.Type, fieldPath); err != nil {
                        return err
                }
        }
        return nil
}

// validateBatchElem checks that a type (a struct field's type, or an array
// element type) is pointer-free: a scalar, one of the handle types, a
// pointer-free nested struct, or a pointer-free array. Anything else
// (string, slice, map, pointer, interface, chan, func) is rejected.
func validateBatchElem(t reflect.Type, path string) error {
        switch t {
        case strType, bytesType, timeType2:
                return nil
        case timeType:
                return fmt.Errorf("qdf: batch type: field %s is time.Time — use qdf.Time", path)
        }
        switch k := t.Kind(); k {
        case reflect.Bool,
                reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
                reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
                reflect.Float32, reflect.Float64:
                return nil
        case reflect.Struct:
                return validateBatchStruct(t, path)
        case reflect.Array:
                return validateBatchElem(t.Elem(), path)
        case reflect.String:
                return fmt.Errorf("qdf: batch type: field %s is string — use qdf.Str", path)
        case reflect.Slice:
                return fmt.Errorf("qdf: batch type: field %s is a slice — use qdf.Bytes for []byte, or drop the field", path)
        case reflect.Map, reflect.Pointer, reflect.Interface, reflect.Chan, reflect.Func:
                return fmt.Errorf("qdf: batch type: field %s (%s) is not pointer-free", path, k)
        default:
                return fmt.Errorf("qdf: batch type: field %s: unsupported kind %s", path, k)
        }
}

package qdf

import (
        "math"
        "sync"
        "unsafe"
)

// batchSlab owns every byte a Batch's handles point into: one contiguous
// grow-copy buffer. Growth copies the buffer; handles store OFFSETS, so they
// survive growth (only base moves). Pooled: Release returns it for reuse and
// bumps epoch so debug builds can catch stale handles.
type batchSlab struct {
        buf     []byte
        rowsBuf []byte // rows backing (see takeRows); NOT part of buf, never grow-copied
        // Shape scratch for the columnar fast path's zero-alloc shape reader
        // (batchReadColShape): per wire column, the matched plan-field index (-1 =
        // not in plan → skip) and the wire colKind. Reused across decodes with the
        // slab; capacity-managed, never aliased past a decode.
        shapeFidx  []int16
        shapeKinds []colKind
        epoch      uint32
}

const batchSlabInitCap = 4096

var batchSlabPool = sync.Pool{New: func() any {
        return &batchSlab{buf: make([]byte, 0, batchSlabInitCap)}
}}

func newBatchSlab() *batchSlab {
        s := batchSlabPool.Get().(*batchSlab)
        s.buf = s.buf[:0]
        return s
}

// maxBatchSlabBytes caps the slab so a Str/Bytes handle's uint32 offset can
// never wrap: past 4 GiB `uint32(len(s.buf))` would silently truncate and
// handles would resolve to the wrong bytes with no error (the debug bounds
// check cannot catch a wrapped-but-in-range offset). Reaching the cap needs
// a >4 GiB input payload — reject it like every other decode bound instead
// of corrupting.
const maxBatchSlabBytes = math.MaxUint32

// append copies b into the slab and returns its (offset, length). The zero
// handle (0,0) is reserved for the empty string: appends never return len 0.
// Errors when the slab would exceed maxBatchSlabBytes (uint32 handle space).
func (s *batchSlab) append(b []byte) (off, ln uint32, err error) {
        if len(b) == 0 {
                return 0, 0, nil
        }
        if uint64(len(s.buf))+uint64(len(b)) > maxBatchSlabBytes {
                return 0, 0, ErrInvalidLength
        }
        off = uint32(len(s.buf))
        s.buf = append(s.buf, b...)
        return off, uint32(len(b)), nil
}

// grow reserves n more bytes of capacity up front (bulk paths).
func (s *batchSlab) grow(n int) {
        if cap(s.buf)-len(s.buf) < n {
                nb := make([]byte, len(s.buf), len(s.buf)+n+len(s.buf)/2)
                copy(nb, s.buf)
                s.buf = nb
        }
}

// takeRows returns a pointer to a ZEROED nbytes-long region backing the
// caller's []T (the generic wrapper builds the view via unsafe.Slice). The
// region is pooled ON THE SLAB across decodes: if the slab's previous rowsBuf
// already has enough capacity it is resliced and cleared in place; otherwise
// a fresh (already-zeroed) []byte is allocated. Zeroing on the reuse branch
// matters for correctness, not just hygiene: wire fields the columnar/mirror
// path never touches (schema evolution — a plan field absent from this
// message) must read back as the zero value, exactly like a fresh make would
// produce.
//
// GC-safety: rowsBuf is a []byte (noscan) reused as the backing store for
// []T. This is only safe because T is validated pointer-free by batchPlanOf
// (scalars, qdf.Str, qdf.Bytes, qdf.Time only) — the GC never needs to scan
// it. The unsafe.Slice view the wrapper builds over this region must not
// outlive Release: once the slab is pooled, the next decode may reslice and
// overwrite rowsBuf in place (same contract as buf/handles).
func (s *batchSlab) takeRows(nbytes int) unsafe.Pointer {
        if nbytes == 0 {
                return nil
        }
        if cap(s.rowsBuf) >= nbytes {
                s.rowsBuf = s.rowsBuf[:nbytes]
                clear(s.rowsBuf)
        } else {
                s.rowsBuf = make([]byte, nbytes)
        }
        return unsafe.Pointer(unsafe.SliceData(s.rowsBuf))
}

func (s *batchSlab) base() unsafe.Pointer {
        if len(s.buf) == 0 {
                return nil
        }
        return unsafe.Pointer(unsafe.SliceData(s.buf))
}

func (s *batchSlab) release() {
        s.epoch++
        batchSlabPool.Put(s)
}

package qdf

import "math"

// Canonical encoding (OptCanonical) helpers. Float normalization makes
// logically-equal float values serialize identically: -0.0 collapses to +0.0
// and any NaN bit pattern maps to a single canonical quiet NaN. The pooled
// map-key sort scratch (canonKeys* on encState) is reused across maps so the
// sorted-key emit allocates nothing in steady state.

// canonicalizeFloat64 maps -0.0 → +0.0 and any NaN → a canonical quiet NaN, so
// logically-equal float values serialize identically under OptCanonical.
func canonicalizeFloat64(v float64) float64 {
        if v == 0 {
                return 0 // collapses -0.0 to +0.0 (Go: -0.0 == 0)
        }
        if math.IsNaN(v) {
                return math.Float64frombits(0x7FF8000000000000)
        }
        return v
}

// canonicalizeFloat32 maps -0.0 → +0.0 and any NaN → a canonical quiet NaN.
func canonicalizeFloat32(v float32) float32 {
        if v == 0 {
                return 0
        }
        if v != v { // NaN
                return math.Float32frombits(0x7FC00000)
        }
        return v
}

// canonicalizeFloat32Bits normalizes raw float32 bits (the columnar/nullable f32
// path stores bits in a uint64 and never re-floats them).
func canonicalizeFloat32Bits(u uint64) uint64 {
        b := uint32(u)
        f := math.Float32frombits(b)
        if f == 0 {
                return 0
        }
        if f != f {
                return uint64(uint32(0x7FC00000))
        }
        return u
}

// canonicalFloat64Slice returns a slice byte-stable under canonical encoding. If
// s holds no -0.0 and no NaN it returns s unchanged (the common case — one scan,
// no copy, no mutation of the caller's slice). Otherwise it copies the normalized
// values into the pooled canonFloat64 scratch and returns that. The slice entry
// must normalize here because the LE unsafe.Slice memcpy fast path inside
// writePackedFloat64Slice bypasses any per-element hook.
func (e *Encoder) canonicalFloat64Slice(s []float64) []float64 {
        dirty := false
        for _, v := range s {
                if (v == 0 && math.Signbit(v)) || v != v {
                        dirty = true
                        break
                }
        }
        if !dirty {
                return s
        }
        var buf []float64
        if e.state != nil {
                buf = e.state.canonFloat64[:0]
        }
        buf = append(buf, s...)
        for i, v := range buf {
                buf[i] = canonicalizeFloat64(v)
        }
        if e.state != nil {
                e.state.canonFloat64 = buf
        }
        return buf
}

// canonicalFloat32Slice is the float32 analogue of canonicalFloat64Slice.
func (e *Encoder) canonicalFloat32Slice(s []float32) []float32 {
        dirty := false
        for _, v := range s {
                if (v == 0 && math.Signbit(float64(v))) || v != v {
                        dirty = true
                        break
                }
        }
        if !dirty {
                return s
        }
        var buf []float32
        if e.state != nil {
                buf = e.state.canonFloat32[:0]
        }
        buf = append(buf, s...)
        for i, v := range buf {
                buf[i] = canonicalizeFloat32(v)
        }
        if e.state != nil {
                e.state.canonFloat32 = buf
        }
        return buf
}

package qdf

import (
        "math"
        "math/bits"
        "unsafe"
)

// This file exposes the no-reflect columnar primitives that qdfgen-generated
// code calls to transpose a []struct field into the tagColStruct wire frame.
// The byte layout is identical to encodeColumnar (columnar.go) so the reflect
// path and generated code interoperate. Generated code never uses reflection.

// WriteColStructHeader writes the columnar container header: tagColStruct,
// the row count n, and the column shape (field names + kind bytes). The shape
// is declared inline the first time this encoder sees it and reused by id
// afterwards, sharing encState's colStruct shape-id space with the reflect
// path. kinds[i] is the colKind byte for column i (see classifyColKind:
// int*->0, uint*->1, float64->2, bool->3, float32->6); it must match what the
// reflect encoder would emit for the same field so cross-decoding works.
//
// names and kinds back the encoder's shape-id cache by reference on the
// declaring call; the caller must NOT mutate or recycle them for a different
// shape afterward (generated code passes immutable package-level vars).
func (e *Encoder) WriteColStructHeader(n int, names []string, kinds []byte) {
        if e.state == nil {
                e.state = newEncState()
        }
        st := e.state
        e.buf = append(e.buf, tagColStruct)
        e.buf = appendUvarint(e.buf, uint64(n))
        // colKind is a uint8; reinterpret the kinds byte slice without copying.
        var ck []colKind
        if len(kinds) > 0 {
                ck = unsafe.Slice((*colKind)(unsafe.Pointer(&kinds[0])), len(kinds))
        }
        if id := st.colShapeFor(names, ck); id != 0 {
                e.buf = appendUvarint(e.buf, uint64(id))
                return
        }
        e.buf = appendUvarint(e.buf, 0)
        e.buf = appendUvarint(e.buf, uint64(len(names)))
        for i := range names {
                e.WriteString(names[i])
                e.buf = append(e.buf, kinds[i])
        }
        st.colShapeDeclare(names, ck)
}

// colState lazily initializes and returns the encoder's columnar state, which
// owns the reusable per-column transpose scratch.
func (e *Encoder) colState() *encState {
        if e.state == nil {
                e.state = newEncState()
        }
        return e.state
}

// The ScratchX getters hand generated code a reusable column buffer of length n,
// grown from the encoder's pooled columnar scratch (shared with the reflect
// path, which never runs concurrently on the same encoder). Generated code
// fills the buffer and passes it straight to the matching WriteXColumn, so a
// whole columnar []struct encodes with zero per-column allocation.

// ScratchInt returns a reusable []int64 of length n for a signed column.
func (e *Encoder) ScratchInt(n int) []int64 {
        st := e.colState()
        if cap(st.colScratchI64) < n {
                st.colScratchI64 = make([]int64, n)
        }
        st.colScratchI64 = st.colScratchI64[:n]
        return st.colScratchI64
}

// ScratchUint returns a reusable []uint64 of length n for an unsigned column.
func (e *Encoder) ScratchUint(n int) []uint64 {
        st := e.colState()
        if cap(st.colScratchU64) < n {
                st.colScratchU64 = make([]uint64, n)
        }
        st.colScratchU64 = st.colScratchU64[:n]
        return st.colScratchU64
}

// ScratchFloat64 returns a reusable []float64 of length n for a float64 column.
func (e *Encoder) ScratchFloat64(n int) []float64 {
        st := e.colState()
        if cap(st.colScratchF64) < n {
                st.colScratchF64 = make([]float64, n)
        }
        st.colScratchF64 = st.colScratchF64[:n]
        return st.colScratchF64
}

// ScratchFloat32 returns a reusable []float32 of length n for a float32 column.
func (e *Encoder) ScratchFloat32(n int) []float32 {
        st := e.colState()
        if cap(st.colScratchF32) < n {
                st.colScratchF32 = make([]float32, n)
        }
        st.colScratchF32 = st.colScratchF32[:n]
        return st.colScratchF32
}

// ScratchBool returns a reusable []bool of length n for a bool column.
func (e *Encoder) ScratchBool(n int) []bool {
        st := e.colState()
        if cap(st.colScratchBool) < n {
                st.colScratchBool = make([]bool, n)
        }
        st.colScratchBool = st.colScratchBool[:n]
        return st.colScratchBool
}

// WriteIntColumn encodes a column gathered from a signed-integer field. The
// adaptive picker chooses raw/FOR/Delta/RLE/Dict/PFOR per value range.
func (e *Encoder) WriteIntColumn(s []int64) error { return encodeSliceInt64(e, unsafe.Pointer(&s)) }

// WriteUintColumn encodes an unsigned-integer column.
func (e *Encoder) WriteUintColumn(s []uint64) error { return encodeSliceUint64(e, unsafe.Pointer(&s)) }

// WriteFloat64Column encodes a SCALAR float64 column losslessly. OptLossyVec
// targets genuine []float64/[]float32 VECTOR fields, not scalar columns, so this
// path never emits a lossy 0xFD block.
func (e *Encoder) WriteFloat64Column(s []float64) error {
        return encodeSliceFloat64Lossless(e, s)
}

// WriteFloat32Column encodes a float32 column as its 32-bit patterns through
// the unsigned codec, bit-exact and matching the reflect colKindFloat32 path.
// The bit-conversion temp reuses colScratchU64 (free once any uint column it
// shares the buffer with has already been encoded), so no allocation.
func (e *Encoder) WriteFloat32Column(s []float32) error {
        st := e.colState()
        if cap(st.colScratchU64) < len(s) {
                st.colScratchU64 = make([]uint64, len(s))
        }
        u := st.colScratchU64[:len(s)]
        if e.opts.Has(OptCanonical) {
                // Mirror the reflect colKindFloat32 path (columnar.go): under OptCanonical
                // normalize -0.0 -> +0.0 and every NaN -> one quiet NaN so semantically
                // equal columns are byte-identical. Without this the codegen wire diverged
                // from reflect for -0.0/NaN columns under OptCanonical.
                for i, v := range s {
                        u[i] = canonicalizeFloat32Bits(uint64(math.Float32bits(v)))
                }
        } else {
                for i, v := range s {
                        u[i] = uint64(math.Float32bits(v))
                }
        }
        st.colScratchU64 = u
        return encodeSliceUint64(e, unsafe.Pointer(&u))
}

// WriteBoolColumn encodes a bool column.
func (e *Encoder) WriteBoolColumn(s []bool) error { return encodeSliceBool(e, unsafe.Pointer(&s)) }

// ScratchMask returns a zeroed reusable presence bitmap covering n rows
// ((n+7)/8 bytes). Generated code sets bit i for a present (non-nil) nullable
// column row, then passes it to WriteColNullMask.
func (e *Encoder) ScratchMask(n int) []byte {
        st := e.colState()
        mb := (n + 7) >> 3
        if cap(st.colMaskScratch) < mb {
                st.colMaskScratch = make([]byte, mb)
        }
        st.colMaskScratch = st.colMaskScratch[:mb]
        clear(st.colMaskScratch)
        return st.colMaskScratch
}

// WriteColNullMask appends a nullable column's presence bitmap (raw bytes, bit i
// set ⇒ row i present), written before the dense column of present values.
// Layout matches encodeNullableColumn so the reflect path can cross-decode.
func (e *Encoder) WriteColNullMask(mask []byte) { e.buf = append(e.buf, mask...) }

// ReadColNullMask reads a nullable column's presence bitmap for n rows and
// returns it (aliased into the input buffer, read-only) plus the present
// (set-bit) count. The dense column that follows has exactly that many values.
func (d *Decoder) ReadColNullMask(n int) ([]byte, int, error) {
        mb := (n + 7) >> 3
        if d.i+mb > len(d.buf) {
                return nil, 0, ErrShortBuffer
        }
        mask := d.buf[d.i : d.i+mb]
        d.i += mb
        present := 0
        for _, b := range mask {
                present += bits.OnesCount8(b)
        }
        // Reject set padding bits (positions [n%8, 8) of the last byte): a hostile
        // mask can set one while under-filling in-range bits, keeping present<=n yet
        // silently dropping trailing dense values in the scatter loop. Mirrors the
        // reflect sibling readNullableMask.
        if n&7 != 0 && mask[mb-1]>>uint(n&7) != 0 {
                return nil, 0, ErrInvalidLength
        }
        if present > n {
                return nil, 0, ErrInvalidLength
        }
        return mask, present, nil
}

// ScratchString returns a reusable []string of length n for a string column.
func (e *Encoder) ScratchString(n int) []string {
        st := e.colState()
        if cap(st.colScratchStr) < n {
                st.colScratchStr = make([]string, n)
        }
        st.colScratchStr = st.colScratchStr[:n]
        return st.colScratchStr
}

// WriteStringColumn encodes a string column, reusing the Balanced string-column
// picker (dictionary / FSST / raw-slab / plain), which is never larger than the
// per-value row-major encoding. Same wire as the reflect colKindString path.
func (e *Encoder) WriteStringColumn(s []string) { e.writeStringColumn(s) }

// WriteTimeColumn encodes a time.Time column as two sub-columns — sec ([]int64,
// Delta+FOR compresses monotonic series) then nsec ([]uint64) — matching the
// reflect colKindTime path. The caller gathers secs/nsec from the time fields
// (t.UTC().Unix() / t.Nanosecond()); reuse ScratchInt/ScratchUint for them.
func (e *Encoder) WriteTimeColumn(secs []int64, nsec []uint64) error {
        if err := encodeSliceInt64(e, unsafe.Pointer(&secs)); err != nil {
                return err
        }
        return encodeSliceUint64(e, unsafe.Pointer(&nsec))
}

// WriteHybridColStructHeader writes the hybrid columnar container header
// (tagHybridColStruct, row count, shape). The shape lists EVERY field in
// declaration order; residual (non-columnar) fields carry kind byte 0xFF
// (residualKind). Shares the hybrid shape-id space with the reflect path.
func (e *Encoder) WriteHybridColStructHeader(n int, names []string, kinds []byte) {
        st := e.colState()
        e.buf = append(e.buf, tagHybridColStruct)
        e.buf = appendUvarint(e.buf, uint64(n))
        var ck []colKind
        if len(kinds) > 0 {
                ck = unsafe.Slice((*colKind)(unsafe.Pointer(&kinds[0])), len(kinds))
        }
        if id := st.hybridShapeFor(names, ck); id != 0 {
                e.buf = appendUvarint(e.buf, uint64(id))
                return
        }
        e.buf = appendUvarint(e.buf, 0)
        e.buf = appendUvarint(e.buf, uint64(len(names)))
        for i := range names {
                e.WriteString(names[i])
                e.buf = append(e.buf, kinds[i])
        }
        st.hybridShapeDeclare(names, ck)
}

// PeekColStruct reports whether the next byte is a columnar container frame,
// without consuming it. Generated decode uses this to pick the columnar path
// vs the row-major fallback (a tiny slice the encoder kept row-major, or a
// reflect-produced row-major encoding of the same field).
func (d *Decoder) PeekColStruct() bool {
        return d.i < len(d.buf) && d.buf[d.i] == tagColStruct
}

// ReadColStructHeader consumes the columnar container header and returns the
// row count and column shape (names + kind bytes). It mirrors readColShape's
// non-index path. n is bounded by maxColumnarElems.
func (d *Decoder) ReadColStructHeader() (int, []string, []byte, error) {
        cs, err := d.readColShape(maxColumnarElems)
        if err != nil {
                return 0, nil, nil, err
        }
        d.colMaxLen = cs.n
        return cs.n, cs.sh.names, colKindsAsBytes(cs.sh.kinds), nil
}

// colKindsAsBytes reinterprets a cached []colKind (a uint8 alias) as []byte with
// no copy. The result is read-only and transient — valid only until the decoder
// reuses its shape cache; generated code consumes it immediately (or discards
// it), avoiding a per-decode allocation of the kinds slice.
func colKindsAsBytes(kinds []colKind) []byte {
        if len(kinds) == 0 {
                return nil
        }
        return unsafe.Slice((*byte)(unsafe.Pointer(&kinds[0])), len(kinds))
}

// colStateDec lazily initializes and returns the decoder's columnar state,
// which owns the reusable per-column scatter scratch. ReadColStructHeader has
// already created it, but guard anyway for direct callers.
func (d *Decoder) colStateDec() *decState {
        if d.state == nil {
                d.state = newDecState()
        }
        return d.state
}

// The ReadXColumn readers decode one column of n values into the decoder's
// pooled columnar scratch and return it. Generated decode scatters the values
// into struct fields immediately, before the next column read reuses the same
// buffer — so a whole columnar []struct decodes with zero per-column transient
// allocation beyond the result slice itself.

// ReadIntColumn decodes one signed-integer column of n values.
func (d *Decoder) ReadIntColumn(n int) ([]int64, error) {
        st := d.colStateDec()
        s := st.colScratchI64[:0]
        if err := decodeSliceInt64Into(d, &s); err != nil {
                return nil, err
        }
        st.colScratchI64 = s
        if len(s) != n {
                return nil, ErrTypeMismatch
        }
        return s, nil
}

// ReadUintColumn decodes one unsigned-integer column of n values.
func (d *Decoder) ReadUintColumn(n int) ([]uint64, error) {
        st := d.colStateDec()
        s := st.colScratchU64[:0]
        if err := decodeSliceUint64Into(d, &s); err != nil {
                return nil, err
        }
        st.colScratchU64 = s
        if len(s) != n {
                return nil, ErrTypeMismatch
        }
        return s, nil
}

// ReadFloat64Column decodes one float64 column of n values.
func (d *Decoder) ReadFloat64Column(n int) ([]float64, error) {
        st := d.colStateDec()
        s := st.colScratchF64[:0]
        if err := decodeSliceFloat64Into(d, &s); err != nil {
                return nil, err
        }
        st.colScratchF64 = s
        if len(s) != n {
                return nil, ErrTypeMismatch
        }
        return s, nil
}

// ReadFloat32Column decodes one float32 column (32-bit patterns via the
// unsigned codec, mirroring WriteFloat32Column). The bits land in colScratchU64
// and are widened into colScratchF32.
func (d *Decoder) ReadFloat32Column(n int) ([]float32, error) {
        st := d.colStateDec()
        u := st.colScratchU64[:0]
        if err := decodeSliceUint64Into(d, &u); err != nil {
                return nil, err
        }
        st.colScratchU64 = u
        if len(u) != n {
                return nil, ErrTypeMismatch
        }
        if cap(st.colScratchF32) < n {
                st.colScratchF32 = make([]float32, n)
        }
        out := st.colScratchF32[:n]
        for i, v := range u {
                out[i] = math.Float32frombits(uint32(v))
        }
        st.colScratchF32 = out
        return out, nil
}

// ReadBoolColumn decodes one bool column of n values.
func (d *Decoder) ReadBoolColumn(n int) ([]bool, error) {
        st := d.colStateDec()
        s := st.colScratchBool[:0]
        if err := decodeSliceBoolInto(d, &s); err != nil {
                return nil, err
        }
        st.colScratchBool = s
        if len(s) != n {
                return nil, ErrTypeMismatch
        }
        return s, nil
}

// ReadStringColumn decodes one string column of n values, mirroring the reflect
// colKindString decode: a dictionary / FSST / raw-slab block (tag present) goes
// through the shared block reader; otherwise the plain per-value path reads via
// ReadString so state-ref / MTF / repeat encodings written by the picker's
// plain fallback resolve correctly (and share the decode intern cache).
//
// The returned slice is decode-state scratch reused across the columns of one
// struct decode (every path now pools it — see colStrScratch); it is valid only
// until the next string-column read, so scatter it into the destination before
// reading the next column. Every internal columnar decoder (codegen + reflect +
// delta + nullable) drains each column this way.
func (d *Decoder) ReadStringColumn(n int) ([]string, error) {
        if n > 0 && d.i < len(d.buf) && isStringColumnBlockTag(d.buf[d.i]) {
                s, err := d.readStringColumn(n)
                if err != nil {
                        return nil, err
                }
                if len(s) != n {
                        return nil, ErrTypeMismatch
                }
                return s, nil
        }
        out := d.colStrScratch(n)
        for i := range n {
                s, err := d.ReadString()
                if err != nil {
                        return nil, err
                }
                out[i] = s
        }
        return out, nil
}

// colStrScratch returns a length-n []string backed by the decode state's pooled
// string-column scratch, reused across the columns of one struct decode so each
// column does not allocate a fresh result slice. The buffer is shared, so the
// returned slice is valid only until the NEXT string-column read; every internal
// caller scatters its column into the destination structs before reading the
// next column (see ReadStringColumn). The argument must only ever be used as the
// column's RESULT slice — never as an internal table/scratch read while filling
// it, or it would self-overwrite.
func (d *Decoder) colStrScratch(n int) []string {
        if d.colStrNoPool {
                return make([]string, n) // caller retains &elem; needs a stable backing array
        }
        st := d.colStateDec()
        if cap(st.colScratchStr) < n {
                st.colScratchStr = make([]string, n)
        }
        st.colScratchStr = st.colScratchStr[:n]
        return st.colScratchStr
}

// ReadTimeColumn decodes a time.Time column's two sub-columns and returns the
// sec / nsec slices; the caller reconstructs each value as
// time.Unix(sec[i], int64(nsec[i])).UTC(). nsec is validated in range.
func (d *Decoder) ReadTimeColumn(n int) ([]int64, []uint64, error) {
        st := d.colStateDec()
        sec := st.colScratchI64[:0]
        if err := decodeSliceInt64Into(d, &sec); err != nil {
                return nil, nil, err
        }
        st.colScratchI64 = sec
        if len(sec) != n {
                return nil, nil, ErrTypeMismatch
        }
        nsec := st.colScratchU64[:0]
        if err := decodeSliceUint64Into(d, &nsec); err != nil {
                return nil, nil, err
        }
        st.colScratchU64 = nsec
        if len(nsec) != n {
                return nil, nil, ErrTypeMismatch
        }
        if err := checkNsecColumn(nsec); err != nil {
                return nil, nil, err
        }
        return sec, nsec, nil
}

// PeekHybridColStruct reports whether the next byte is a hybrid columnar frame
// (some fields columnar, some residual row-major), without consuming it.
func (d *Decoder) PeekHybridColStruct() bool {
        return d.i < len(d.buf) && d.buf[d.i] == tagHybridColStruct
}

// ReadHybridColStructHeader consumes the hybrid columnar header and returns the
// row count and full shape (every field in declaration order; residual fields
// carry kind byte 0xFF). n is bounded by maxColumnarElems.
func (d *Decoder) ReadHybridColStructHeader() (int, []string, []byte, error) {
        n, sh, err := d.readHybridColShape(maxColumnarElems)
        if err != nil {
                return 0, nil, nil, err
        }
        d.colMaxLen = n
        return n, sh.names, colKindsAsBytes(sh.kinds), nil
}

// ClearColMaxLen resets the per-column length bound. Generated hybrid decode
// calls it between the eligible columns (bounded to n) and the residual block
// (whose nested slices/maps may legitimately exceed n), mirroring
// decodeHybridColumnar.
func (d *Decoder) ClearColMaxLen() { d.colMaxLen = 0 }

// stringColumnsBeneficial samples the given string columns and reports whether a
// columnar string column would beat plain per-value row-major encoding by the
// min-gain threshold. Reflection-free; it mirrors the reflect columnarProbe's
// colKindString branch byte-for-byte so the generated code makes the SAME
// columnar-vs-row-major decision as reflect.
//
// internAware selects the model the reflect probe uses for the element kind:
//   - false (a PURE string-only element): per-value / dict estimate, gain 10 —
//     identical to columnarProbe with internAware=false.
//   - true (a HYBRID string-only element, i.e. residual map/slice fields with no
//     numeric column): additionally credits Dense intern dedup in the row-major
//     baseline (a repeat costs 1 byte) AND alpha-packing on a restricted-alphabet
//     high-cardinality column, with the wider gain 30 — identical to
//     columnarProbe with internAware=true. Without this a hybrid hex-ID element
//     would stay row-major in codegen while reflect goes columnar + alpha.
func stringColumnsBeneficial(internAware bool, cols ...[]string) bool {
        if len(cols) == 0 || len(cols[0]) == 0 {
                return false
        }
        sample := min(len(cols[0]), columnarProbeSample)
        var colBytes, rowBytes int
        for _, strs := range cols {
                // Clamp the per-column sample so a caller passing columns shorter than
                // cols[0] (only possible via the exported StringColumns* entry points)
                // cannot index out of range. Internal codegen callers pass equal-length
                // columns, so n == sample and the estimate is byte-identical.
                n := min(sample, len(strs))
                var seen [columnarProbeSample]string
                nseen := 0
                var tableBytes, perValue, sampleChars int
                prev := ""
                first := true
                for i := range n {
                        s := strs[i]
                        fresh := true
                        for j := 0; j < nseen; j++ {
                                if seen[j] == s {
                                        fresh = false
                                        break
                                }
                        }
                        if fresh && nseen < len(seen) {
                                seen[nseen] = s
                                nseen++
                                tableBytes += 2 + len(s)
                        }
                        if !first && s == prev {
                                perValue++
                        } else {
                                perValue += 2 + len(s)
                        }
                        if internAware && !fresh {
                                rowBytes += 1
                        } else {
                                rowBytes += 2 + len(s)
                        }
                        sampleChars += len(s)
                        prev = s
                        first = false
                }
                dictBytes := tableBytes + (n*bitsForDistinct(nseen)+7)/8
                best := min(perValue, dictBytes)
                // Defer the O(chars) per-byte alphabet scan behind the cheap alpha
                // preconditions (cardinality + average length), so low-card / short string
                // columns skip it entirely — byte-identical to the columnar decision (the
                // scan's alphaOK/alphaCount feed only the alphaEst below). Mirrors
                // columnarProbe.
                if internAware &&
                        nseen*100 >= n*alphaMinDistinctPct &&
                        sampleChars >= n*alphaProbeMinAvgLen {
                        var alphaSeen [256]bool
                        alphaCount := 0
                        alphaOK := true
                        for i := range n {
                                s := strs[i]
                                for k := 0; k < len(s); k++ {
                                        if !alphaSeen[s[k]] {
                                                if alphaCount >= qpackStrAlphaMaxAlphabet {
                                                        alphaOK = false
                                                        break
                                                }
                                                alphaSeen[s[k]] = true
                                                alphaCount++
                                        }
                                }
                                if !alphaOK {
                                        break
                                }
                        }
                        if alphaOK && alphaCount >= 2 {
                                alphaEst := alphaCount + n + (sampleChars*bitsForDistinct(alphaCount)+7)/8
                                if alphaEst < best {
                                        best = alphaEst
                                }
                        }
                }
                colBytes += best
        }
        if rowBytes == 0 {
                return false
        }
        gain := columnarMinGainPct
        if internAware {
                gain = columnarMinGainPctInternAware
        }
        return colBytes*100 <= rowBytes*(100-gain)
}

// StringColumnsBeneficial gates a PURE string-only element's columnar encode in
// generated code (mirrors columnarProbe with internAware=false).
func StringColumnsBeneficial(cols ...[]string) bool { return stringColumnsBeneficial(false, cols...) }

// StringColumnsBeneficialHybrid gates a HYBRID (residual-bearing) string-only
// element's columnar encode in generated code (mirrors columnarProbe with
// internAware=true: intern-credited baseline + alpha-packing estimate + the
// wider gain), so codegen flips such an element into the columnar form exactly
// when reflect does.
func StringColumnsBeneficialHybrid(cols ...[]string) bool {
        return stringColumnsBeneficial(true, cols...)
}

package qdf

// Constant (single-distinct) string column codec: when every value in a string
// column is identical, store it once + the row count instead of n copies. See
// tagColStrConst (wire.go) for the format. Wire-minimal and decodes to a single
// allocation regardless of encoder mode, so it fixes the codegen/Fast path where
// the per-value fallback does not intern repeats.

// tryWriteStringColumnConst writes a constant string column if every element of
// strs is identical, returning true. The all-equal scan bails on the first
// mismatch, so a multi-distinct column costs O(1) here.
func (e *Encoder) tryWriteStringColumnConst(strs []string) bool {
        n := len(strs)
        if n < 2 {
                return false // a 0/1-element column is not worth a dedicated frame
        }
        first := strs[0]
        for i := 1; i < n; i++ {
                if strs[i] != first {
                        return false
                }
        }
        e.writeHeader()
        out := e.buf
        out = append(out, tagColStrConst)
        out = appendUvarint(out, uint64(len(first)))
        out = append(out, first...)
        out = appendUvarint(out, uint64(n))
        e.buf = out
        return true
}

// readStringColumnConst decodes a tagColStrConst block (tag at d.i) and returns
// n shares of the single owned string value. The block's row count must equal
// the columnar header's n (which bounds it), so a tiny input cannot drive a
// large allocation.
func (d *Decoder) readStringColumnConst(n int) ([]string, error) {
        d.i++ // consume tagColStrConst
        l64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if l64 > uint64(len(d.buf)-d.i) {
                return nil, ErrShortBuffer
        }
        l := int(l64)
        v := d.materializeStr(d.buf[d.i : d.i+l]) // shared by every row (aliases input under noCopy)
        d.i += l
        cnt64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if int(cnt64) != n {
                return nil, ErrTypeMismatch
        }
        out := d.colStrScratch(n)
        for i := range out {
                out[i] = v
        }
        return out, nil
}

package qdf

import (
        "encoding/binary"
        "math"
        "reflect"
        "runtime"
        "slices"
        "time"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/fsst"
)

// colKind classifies a struct field for columnar encoding. Only these kinds
// are columnar-eligible; any other field kind makes the whole struct fall
// back to row-major.
type colKind uint8

const (
        colKindInt    colKind = iota // int, int8..int64  → []int64 column
        colKindUint                  // uint, uint8..uint64, uintptr → []uint64 column
        colKindFloat                 // float64 → []float64 column (float32 → colKindFloat32)
        colKindBool                  // bool → []bool column
        colKindString                // string, []byte → consecutive WriteString
        colKindTime                  // time.Time → sec []int64 sub-column + nsec []uint64 sub-column
        // colKindFloat32 → []uint32-bits column (math.Float32bits), encoded with the
        // uint codec. Kept SEPARATE from colKindFloat (which is float64-only) because
        // a float32→float64→float32 round trip is not bit-preserving for NaN payloads
        // (signaling NaNs are quieted). Carrying the raw 32 bits is both lossless and
        // narrower on the wire (4 B vs the 8 B a float64 column used). Appended last
        // so the existing kinds keep their wire values.
        colKindFloat32

        // colKindNullable is OR'd onto a base kind in the columnar shape's kind
        // byte to mark an optional (pointer-to-scalar/string) column: a `*T`
        // field. The column is stored as a presence bitmap (1 bit per row) plus
        // a dense column of only the present values, encoded with the base
        // kind's normal codec. Nullability is a static property of the field
        // type, so it travels in the shape declaration — no separate wire tag.
        colKindNullable colKind = 0x80
)

// base returns the kind without the nullable flag.
func (k colKind) base() colKind { return k &^ colKindNullable }

// isNullable reports whether the column is an optional (pointer) column.
func (k colKind) isNullable() bool { return k&colKindNullable != 0 }

// String returns a human-readable name for the kind, used in error messages.
func (k colKind) String() string {
        n := ""
        switch k.base() {
        case colKindInt:
                n = "int"
        case colKindUint:
                n = "uint"
        case colKindFloat:
                n = "float"
        case colKindFloat32:
                n = "float32"
        case colKindBool:
                n = "bool"
        case colKindString:
                n = "string"
        case colKindTime:
                n = "time"
        default:
                n = "unknown"
        }
        if k.isNullable() {
                n = "*" + n
        }
        return n
}

// colColumn is one field's columnar descriptor: where it lives in each struct
// element and how to (de)serialize the column.
// Fields are ordered to pack without padding AND to keep the GC pointer-scan
// range tight: the interface (elemType, two pointer words) leads, then the
// string, then the non-pointer scalars/flags trail; one colColumn per field is
// built once per columnar struct type and scanned per batch in encodeColumnar.
type colColumn struct {
        // elemType is the pointed-to type for a nullable (*T) column, used to
        // allocate the present values on decode. nil for non-nullable columns.
        elemType reflect.Type
        name     string
        offset   uintptr
        width    uintptr // element width in bytes for the scalar load/store
        kind     colKind // base kind, OR'd with colKindNullable for *T columns
        isByte   bool    // true for []byte string columns (vs string)
}

// residualField describes one struct field that is NOT columnar-eligible
// (map, non-[]byte slice, nested struct, interface, nullable []byte). In a
// hybrid columnar payload these fields are kept row-major — encoded/decoded
// per row via the field's existing typeDesc codecs — instead of disqualifying
// the whole struct from columnar transposition.
type residualField struct {
        desc   *typeDesc // the field's encode/decode closures (row-major path)
        name   string
        offset uintptr
}

// residualKind is a shape-byte sentinel marking a residual field in a hybrid
// columnar shape. It is outside the valid colKind range (base kinds 0x00-0x06,
// nullable-OR'd 0x80-0x86), so it can never be mistaken for a real column kind.
const residualKind colKind = 0xFF

// columnarPlan is cached on the slice element's typeDesc. nil means the
// element type is not columnar-eligible.
//
// Three shapes:
//   - residual == nil          : pure columnar (every field eligible) — tagColStruct
//   - residual != nil          : hybrid (some fields eligible, some residual) — tagHybridColStruct
//   - (buildColumnarPlan nil)  : no eligible field at all — full row-major
type columnarPlan struct {
        cols []colColumn
        // colNames / colKinds mirror cols[].name / cols[].kind as standalone slices,
        // precomputed once at build so encodeColumnar hands them to the shape
        // lookup/declare without rebuilding a transient pair every batch. The plan is
        // immutable after build, so the slice the shape table retains stays valid.
        colNames []string
        colKinds []colKind

        // residual holds the non-columnar fields (in struct declaration order) for a
        // hybrid plan; nil for a pure-columnar plan. hybridNames / hybridKinds are the
        // full field list (ALL fields, declaration order) for the hybrid shape, with
        // residualKind marking the residual entries — built once so the hybrid encode
        // path hands them to the shape lookup/declare without rebuilding per batch.
        residual    []residualField
        hybridNames []string
        hybridKinds []colKind

        // hasStringCol is true when at least one eligible column is a string (not
        // []byte). A hybrid plan with a string column opts into the intern-aware
        // columnar probe under Balanced so a restricted-alphabet ID column can pull
        // the struct into columnar form (where alpha-packing fires) — string-free
        // mixed structs stay row-major, byte-identical to before.
        hasStringCol bool

        stride uintptr // struct size, for base + i*stride addressing
}

// columnarMinElems is the smallest slice length worth transposing; below it
// the shape-declaration + probe overhead is not amortized.
const columnarMinElems = 16

// maxColumnarElems caps the struct count a tagColStruct header may claim. The
// per-byte length sanity check used for row-major slices does not apply here:
// a constant column compresses M structs into a few bytes, so M is not bounded
// by the remaining buffer. This fixed ceiling guards the output MakeSlice
// against a hostile element count while staying well above any realistic batch
// (callers with more rows should shard or stream).
const maxColumnarElems = 1 << 24

// maxColumnarAnyElems caps the row count for the reflective map[string]any
// decode, which allocates one map per row up front (much heavier than the
// typed struct path's single backing slice). Kept well above any realistic
// ad-hoc decode; callers with more rows should decode into a typed struct.
const maxColumnarAnyElems = 1 << 16

// checkColumnarN validates a tagColStruct struct count. Unlike row-major
// slices, a columnar count is not byte-bounded (compressed columns), so it is
// checked against a fixed ceiling rather than the remaining buffer length.
func checkColumnarN(n int) error {
        if n < 0 || n > maxColumnarElems {
                return ErrInvalidLength
        }
        return nil
}

// maxColumnarBytes caps the OUTPUT allocation of a columnar struct decode. The
// element-count ceiling (maxColumnarElems) does NOT bound the n*elemSize output
// slice: a constant / RLE / zero-width column compresses many rows into a few
// wire bytes, so a hostile header can claim maxColumnarElems rows of a wide
// struct and amplify a ~1 KB input into a multi-GB allocation. This byte ceiling
// caps that amplification before the slice is made; callers with larger batches
// shard or stream.
const maxColumnarBytes = 256 << 20

// checkColumnarBytes rejects a columnar output whose total size (n elements of
// elemSize bytes) exceeds maxColumnarBytes — the byte-bounded companion to
// checkColumnarN's element-count ceiling. n must already be >= 0 (checkColumnarN).
func checkColumnarBytes(n int, elemSize uintptr) error {
        if uint64(n)*uint64(elemSize) > maxColumnarBytes {
                return ErrInvalidLength
        }
        return nil
}

// CheckColumnarBytes is the exported guard cmd/qdfgen-generated columnar
// decoders call after ReadColStructHeader / ReadHybridColStructHeader, before
// allocating the row slice, to reject the same memory-amplification a hostile
// row count would cause on the reflect path. elemSize is unsafe.Sizeof(row).
func CheckColumnarBytes(n int, elemSize uintptr) error { return checkColumnarBytes(n, elemSize) }

// buildColumnarPlan classifies a struct's fields into columnar-eligible columns
// and (hybrid) residual fields. Called once at fillDesc time; the result is
// cached so the hot path never reflects.
//
// Returns:
//   - nil                      if td is not a struct, has no fields, or has NO
//     columnar-eligible field (nothing to transpose → full row-major).
//   - plan with residual == nil if EVERY field is eligible (pure columnar).
//   - plan with residual != nil if some fields are eligible and some are not
//     (hybrid: transpose the eligible columns, keep the rest row-major).
//
// A field that classifyColKind rejects (map, non-[]byte slice, nested struct,
// interface) — or a nullable []byte — becomes residual instead of
// disqualifying the whole struct, which is what unlocks columnar for the common
// "mostly-scalar struct with one map/slice field" shape (AD/log/RTB records).
func buildColumnarPlan(td *typeDesc) *columnarPlan {
        if td.kind != reflect.Struct || len(td.fields) == 0 {
                return nil
        }
        cols := make([]colColumn, 0, len(td.fields))
        var residual []residualField
        // hybridNames/hybridKinds record EVERY field in declaration order (with
        // residualKind for the non-columnar ones) so a hybrid shape can reconstruct
        // the struct on decode. Only retained on the plan if it turns out hybrid.
        hybridNames := make([]string, 0, len(td.fields))
        hybridKinds := make([]colKind, 0, len(td.fields))

        addResidual := func(f *fieldDesc) {
                residual = append(residual, residualField{name: f.name, offset: f.offset, desc: f.desc})
                hybridNames = append(hybridNames, f.name)
                hybridKinds = append(hybridKinds, residualKind)
        }

        for i := range td.fields {
                f := &td.fields[i]
                fd := f.desc
                // An optional (*T) field becomes a nullable column: classify the
                // pointed-to type and remember its reflect.Type for decode allocation.
                var elemType reflect.Type
                nullable := false
                if fd.kind == reflect.Pointer {
                        if fd.elem == nil {
                                addResidual(f) // pointer to an undescribable type — keep row-major
                                continue
                        }
                        nullable = true
                        elemType = fd.rType.Elem()
                        fd = fd.elem
                }
                ck, w, isByte, ok := classifyColKind(fd)
                // Not columnar-eligible (or a nullable []byte, still unsupported as a
                // column) → residual rather than disqualifying the whole struct.
                if !ok || (nullable && isByte) {
                        addResidual(f)
                        continue
                }
                if nullable {
                        ck |= colKindNullable
                }
                cols = append(cols, colColumn{name: f.name, offset: f.offset, kind: ck, width: w, isByte: isByte, elemType: elemType})
                hybridNames = append(hybridNames, f.name)
                hybridKinds = append(hybridKinds, ck)
        }

        if len(cols) == 0 {
                return nil // no eligible column — nothing to transpose, full row-major
        }
        names := make([]string, len(cols))
        kinds := make([]colKind, len(cols))
        for i := range cols {
                names[i] = cols[i].name
                kinds[i] = cols[i].kind
        }
        plan := &columnarPlan{cols: cols, stride: td.rType.Size(), colNames: names, colKinds: kinds}
        for i := range cols {
                if cols[i].kind == colKindString && !cols[i].isByte {
                        plan.hasStringCol = true
                        break
                }
        }
        if len(residual) > 0 {
                // Hybrid: keep the full ordered field list for the hybrid shape.
                plan.residual = residual
                plan.hybridNames = hybridNames
                plan.hybridKinds = hybridKinds
        }
        return plan
}

// colShapeDeclare registers a new columnar shape (names + kinds) on the encoder
// and returns its 1-based wire ID. Always appends; call colShapeFor first to
// avoid duplicates.
func (e *encState) colShapeDeclare(names []string, kinds []colKind) uint32 {
        e.colShapeNames = append(e.colShapeNames, names)
        e.colShapeKinds = append(e.colShapeKinds, kinds)
        return uint32(len(e.colShapeNames)) // ids start at 1
}

// colShapeFor returns the 1-based wire ID for an already-declared columnar shape
// whose names and kinds match exactly, or 0 if not found.
func (e *encState) colShapeFor(names []string, kinds []colKind) uint32 {
        for i := range e.colShapeNames {
                if colShapeEq(e.colShapeNames[i], e.colShapeKinds[i], names, kinds) {
                        return uint32(i + 1)
                }
        }
        return 0
}

// colShapeEq reports whether two (names, kinds) pairs are structurally identical.
func colShapeEq(an []string, ak []colKind, bn []string, bk []colKind) bool {
        if len(an) != len(bn) || len(ak) != len(bk) {
                return false
        }
        for i := range an {
                if an[i] != bn[i] || ak[i] != bk[i] {
                        return false
                }
        }
        return true
}

// colShapeDeclareDec appends a new columnar shape to the decoder's table and
// returns a pointer to the stored entry (wire ID = len after append).
func (d *decState) colShapeDeclareDec(names []string, kinds []colKind) *decColShape {
        d.colShapes = append(d.colShapes, decColShape{names: names, kinds: kinds})
        return &d.colShapes[len(d.colShapes)-1]
}

// colShapeLookup returns the columnar shape with the given 1-based wire ID,
// or nil if the ID is out of range.
func (d *decState) colShapeLookup(id uint32) *decColShape {
        if id == 0 || id > uint32(len(d.colShapes)) {
                return nil
        }
        return &d.colShapes[id-1]
}

// hybridShapeDeclare / hybridShapeFor / hybridShapeDeclareDec / hybridShapeLookup
// mirror the colShape* helpers but operate on the SEPARATE hybrid shape table
// (hybridShapeNames/hybridShapeKinds on the encoder, hybridShapes on the
// decoder). Keeping a distinct ID space means a stream can interleave
// tagColStruct and tagHybridColStruct payloads without ID aliasing: hybrid
// shape ID 3 and columnar shape ID 3 are independent. The kinds slice carries
// residualKind (0xFF) for residual fields and a real colKind for eligible ones.
func (e *encState) hybridShapeDeclare(names []string, kinds []colKind) uint32 {
        e.hybridShapeNames = append(e.hybridShapeNames, names)
        e.hybridShapeKinds = append(e.hybridShapeKinds, kinds)
        return uint32(len(e.hybridShapeNames)) // ids start at 1
}

func (e *encState) hybridShapeFor(names []string, kinds []colKind) uint32 {
        for i := range e.hybridShapeNames {
                if colShapeEq(e.hybridShapeNames[i], e.hybridShapeKinds[i], names, kinds) {
                        return uint32(i + 1)
                }
        }
        return 0
}

func (d *decState) hybridShapeDeclareDec(names []string, kinds []colKind) *decColShape {
        d.hybridShapes = append(d.hybridShapes, decColShape{names: names, kinds: kinds})
        return &d.hybridShapes[len(d.hybridShapes)-1]
}

func (d *decState) hybridShapeLookup(id uint32) *decColShape {
        if id == 0 || id > uint32(len(d.hybridShapes)) {
                return nil
        }
        return &d.hybridShapes[id-1]
}

const (
        columnarProbeSample = 32
        // columnarMinGainPct is the minimum estimated wire reduction (percent of
        // the row-major estimate) required to commit to columnar.
        columnarMinGainPct = 10
        // columnarMinGainPctInternAware is the (higher) threshold for the Balanced
        // hybrid path, whose intern-aware baseline can be slightly optimistic on a
        // string-heavy mixed struct (it credits per-column dedup but not cross-column
        // global intern sharing). The wider margin keeps borderline flips — measured
        // at ≈+0.14% on real AD — out, while clear wins (LogEvent −51%, Telemetry
        // −45%, OTLP −18%) clear it comfortably.
        columnarMinGainPctInternAware = 30
)

// columnarProbe samples up to columnarProbeSample elements and estimates
// whether column-major beats row-major on those samples. Conservative: any
// uncertainty falls back to row-major (returns false).
//
// internAware tightens the string-column model for the Balanced hybrid path:
// the row-major baseline credits Dense interning (a repeated value costs a
// 1-byte state-ref, not its full bytes — so a low-cardinality string column is
// cheap row-major and will NOT pull a struct into columnar), and the column
// estimate credits alpha-packing on a restricted-alphabet high-cardinality
// column. Left false for the pure-columnar and FSST paths, where the historical
// model is kept byte-for-byte (no decision change, no regression).
func columnarProbe(plan *columnarPlan, base unsafe.Pointer, n int, fsstEnabled bool, fsstDict *fsst.SymbolTable, internAware bool) bool {
        sample := min(n, columnarProbeSample)
        var rowBytes, colBytes int
        for c := range plan.cols {
                col := &plan.cols[c]
                switch col.kind {
                case colKindInt, colKindUint:
                        // Distinct values bounded to 17 (cardinality > 16 disables the dict
                        // estimate) tracked in a stack array with a linear scan — no map
                        // allocation per probed column.
                        var seen [17]uint64
                        ndistinct := 0
                        // Model the column exactly the way the encoder emits it:
                        //   - row-major: WriteInt is sign-aware (uvarint for v>=0, a fixed-width
                        //     tag for negatives), WriteUint is a plain uvarint — neither zigzags.
                        //   - columnar FOR: pickI64Codec/pickU64Codec pack over the SIGNED (resp.
                        //     unsigned) min/max range. The old code zero-extended every field via
                        //     loadScalarU64, so a narrow negative (int8 -1 → 0xFF) looked like a
                        //     huge positive, corrupting both the FOR spread and the row estimate
                        //     for any signed column holding negatives.
                        isInt := col.kind == colKindInt
                        var mnU, mxU uint64 = math.MaxUint64, 0
                        var mnI, mxI int64 = math.MaxInt64, math.MinInt64
                        for i := range sample {
                                p := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                                var key uint64 // dict identity (bijective either way)
                                if isInt {
                                        iv := loadI64At(p, col.width)
                                        if iv < mnI {
                                                mnI = iv
                                        }
                                        if iv > mxI {
                                                mxI = iv
                                        }
                                        key = uint64(iv)
                                        rowBytes += intRowCost(iv)
                                } else {
                                        uv := loadScalarU64(base, plan.stride, col, i)
                                        if uv < mnU {
                                                mnU = uv
                                        }
                                        if uv > mxU {
                                                mxU = uv
                                        }
                                        key = uv
                                        rowBytes += uvarintLen(uv)
                                }
                                if ndistinct <= 16 {
                                        found := false
                                        for j := 0; j < ndistinct; j++ {
                                                if seen[j] == key {
                                                        found = true
                                                        break
                                                }
                                        }
                                        if !found {
                                                seen[ndistinct] = key
                                                ndistinct++
                                        }
                                }
                        }
                        var spread uint64
                        if isInt {
                                spread = uint64(mxI) - uint64(mnI)
                        } else {
                                spread = mxU - mnU
                        }
                        bits := 0
                        for spread > 0 {
                                bits++
                                spread >>= 1
                        }
                        forEst := 3 + (bits*sample+7)/8
                        dictEst := 1 << 30
                        if ndistinct <= 16 {
                                idxBits := 0
                                for (1 << idxBits) < ndistinct {
                                        idxBits++
                                }
                                dictEst = 3 + ndistinct*8 + (idxBits*sample+7)/8
                        }
                        colBytes += min(forEst, dictEst)
                case colKindFloat:
                        rowBytes += sample * 8
                        colBytes += sample * 8 // raw-LE both ways; no probe win (Gorilla is opt-in)
                case colKindFloat32:
                        // float32 column carries raw 32-bit patterns via the uint codec:
                        // 4 B in the dense column vs 5 B row-major (tag + 4). Without this
                        // case a float32-only struct probed to 0 bytes and stayed row-major.
                        rowBytes += sample * 5
                        colBytes += sample * 4
                case colKindBool:
                        rowBytes += sample
                        colBytes += (sample + 7) / 8
                case colKindTime:
                        // A time.Time column encodes as two sub-columns: sec ([]int64) and
                        // nsec ([]uint64). Estimate both as two FOR-packed integer columns.
                        // Monotonic timestamps compress extremely well with Delta+FOR on sec;
                        // nsec is often 0 or small. Conservative estimate: treat like two
                        // int columns over the sample.
                        var mnSec, mxSec uint64 = math.MaxUint64, 0
                        for i := range sample {
                                p := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                                t := (*time.Time)(p).UTC()
                                v := uint64(t.Unix()) + (1 << 62) // shift to unsigned for range calc (add in uint64 to avoid int64 overflow on far-future ts)
                                if v < mnSec {
                                        mnSec = v
                                }
                                if v > mxSec {
                                        mxSec = v
                                }
                                rowBytes += 8 + 4 // row-major: tagTimestamp + 8-byte sec + 4-byte nsec
                        }
                        spreadSec := mxSec - mnSec
                        bitsSec := 0
                        for spreadSec > 0 {
                                bitsSec++
                                spreadSec >>= 1
                        }
                        // sec sub-column (Delta+FOR compresses monotonic runs to near zero):
                        colBytes += 3 + (bitsSec*sample+7)/8
                        // nsec sub-column (small integers, usually 0): a few bytes overhead
                        colBytes += 3 + (30*sample+7)/8 // conservative: up to 30 bits for nsec
                case colKindString:
                        // Estimate the column two ways over the sample and keep the
                        // cheaper, mirroring the encoder: per-value (with consecutive-repeat
                        // collapse) OR a string dictionary (distinct table + ceil(log2
                        // distinct) bits/row). Without the dictionary term the probe would
                        // decline columnar for a low-cardinality string column that never
                        // repeats consecutively, even though the dictionary crushes it.
                        //
                        // Distinct values are tracked in a stack array with a linear scan,
                        // not a map: the sample is <= columnarProbeSample, so the set is
                        // tiny and a map would heap-allocate buckets on every probed column.
                        var seen [columnarProbeSample]string
                        nseen := 0
                        var tableBytes, perValue, sampleChars int
                        prev := ""
                        first := true
                        for i := range sample {
                                s := loadStringField(base, plan.stride, col, i)
                                fresh := true
                                for j := 0; j < nseen; j++ {
                                        if seen[j] == s {
                                                fresh = false
                                                break
                                        }
                                }
                                if fresh && nseen < len(seen) {
                                        seen[nseen] = s
                                        nseen++
                                        tableBytes += 2 + len(s)
                                }
                                if !first && s == prev {
                                        perValue += 1 // tagStateRepeat
                                } else {
                                        perValue += 2 + len(s)
                                }
                                // Row-major baseline. internAware credits Dense interning: a repeated
                                // value (seen earlier in the column) costs a 1-byte state-ref, not its
                                // full bytes — so a low-cardinality string column is cheap row-major
                                // and will not pull the struct into columnar. The historical model
                                // (full per-value bytes) is kept for the pure/FSST paths.
                                if internAware && !fresh {
                                        rowBytes += 1
                                } else {
                                        rowBytes += 2 + len(s)
                                }
                                sampleChars += len(s)
                                prev = s
                                first = false
                        }
                        dictBytes := tableBytes + (sample*bitsForDistinct(nseen)+7)/8
                        best := min(perValue, dictBytes)
                        // Alpha-packed estimate (intern-aware path only), tightly gated so it
                        // credits only the class alpha actually wins and never flips a borderline
                        // struct: restricted alphabet (<= 64), high-cardinality over the sample
                        // (low-card is cheaper as dict/interned), and values long enough that the
                        // packed body dominates the per-row length prefix. The emit picker still
                        // tries dict/FSST first, so a prefix-shared column is front-coded even
                        // when this estimate brought the struct into columnar — the estimate
                        // governs only the columnar-vs-row-major decision, not the per-column
                        // codec.
                        //
                        // The per-byte alphabet scan is DEFERRED behind these cheap gates
                        // (cardinality + average length, both already known): low-card or short
                        // string columns — the common case — skip the O(chars) scan entirely,
                        // and only a high-card long column (where alpha could win) pays it. The
                        // scan's alphaOK/alphaCount feed only this estimate, so deferring it is
                        // byte-identical to the columnar decision. sampleChars is the full sum
                        // here, but it only affects alphaEst when alphaOK holds (restricted
                        // alphabet), where the old partial sum equalled the full sum anyway.
                        if internAware &&
                                nseen*100 >= sample*alphaMinDistinctPct &&
                                sampleChars >= sample*alphaProbeMinAvgLen {
                                var alphaSeen [256]bool
                                alphaCount := 0
                                alphaOK := true
                                for i := range sample {
                                        s := loadStringField(base, plan.stride, col, i)
                                        for k := 0; k < len(s); k++ {
                                                if !alphaSeen[s[k]] {
                                                        if alphaCount >= qpackStrAlphaMaxAlphabet {
                                                                alphaOK = false
                                                                break
                                                        }
                                                        alphaSeen[s[k]] = true
                                                        alphaCount++
                                                }
                                        }
                                        if !alphaOK {
                                                break
                                        }
                                }
                                if alphaOK && alphaCount >= 2 {
                                        alphaEst := alphaCount + sample + (sampleChars*bitsForDistinct(alphaCount)+7)/8
                                        if alphaEst < best {
                                                best = alphaEst
                                        }
                                }
                        }
                        // FSST competes only when enabled (OptFSST). High-cardinality,
                        // substring-sharing columns (URLs, log lines) where dict and
                        // per-value both stay near raw are exactly where FSST wins — without
                        // this term the probe would route them to row-major and FSST, which
                        // only runs inside the columnar string-column picker, would never
                        // fire. The symbol table is a one-time cost over the whole column, so
                        // it is amortized to the sample window (× sample/n) rather than
                        // charged in full against the 32-row probe.
                        if fsstEnabled {
                                best = min(best, estimateFSSTColumnBytes(base, plan.stride, col, sample, n, fsstDict))
                        }
                        colBytes += best
                default:
                        if !col.kind.isNullable() {
                                continue // unknown kind contributes nothing
                        }
                        // Nullable column. Row-major spends ~1 tag byte per row (tagNil for
                        // absent, a value tag for present) plus the present value bytes;
                        // columnar spends a presence bitmap plus a dense column no larger
                        // than those value bytes (FOR-packing only shrinks it further), so
                        // the mask-vs-byte-per-row difference is the conservative win.
                        valBytes := 0
                        for i := range sample {
                                pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*plan.stride+col.offset))
                                if pp == nil {
                                        continue
                                }
                                switch col.kind.base() {
                                case colKindInt:
                                        valBytes += uvarintLen(zigzagEncode64(loadI64At(pp, col.width)))
                                case colKindUint:
                                        valBytes += uvarintLen(loadU64At(pp, col.width))
                                case colKindFloat:
                                        valBytes += 8
                                case colKindFloat32:
                                        valBytes += 4
                                case colKindBool:
                                        valBytes++
                                case colKindTime:
                                        valBytes += 12 // 8-byte sec + up to 4-byte nsec, matches the non-nullable time estimate
                                }
                        }
                        rowBytes += sample + valBytes
                        colBytes += (sample+7)/8 + valBytes
                }
        }
        if rowBytes == 0 {
                return false
        }
        // The intern-aware (Balanced hybrid) path demands a larger predicted gain.
        // Its row-major baseline credits per-column intern dedup but still cannot see
        // CROSS-column / global intern sharing, so it can be slightly optimistic on a
        // string-heavy mixed struct; a wider margin absorbs that model error so only a
        // clear win (a Delta/FOR numeric or an alpha-packable ID column) pulls the
        // struct into columnar, never a borderline case that would lose a few bytes.
        gain := columnarMinGainPct
        if internAware {
                gain = columnarMinGainPctInternAware
        }
        return colBytes*100 <= rowBytes*(100-gain)
}

func (e *Encoder) encodeColumnar(plan *columnarPlan, base unsafe.Pointer, n int) error {
        st := e.state
        e.buf = append(e.buf, tagColStruct)
        e.buf = appendUvarint(e.buf, uint64(n))

        if id := st.colShapeFor(plan.colNames, plan.colKinds); id != 0 {
                e.buf = appendUvarint(e.buf, uint64(id))
        } else {
                e.buf = appendUvarint(e.buf, 0)
                e.buf = appendUvarint(e.buf, uint64(len(plan.cols)))
                for i := range plan.cols {
                        e.WriteString(plan.cols[i].name)
                        e.buf = append(e.buf, byte(plan.cols[i].kind))
                }
                st.colShapeDeclare(plan.colNames, plan.colKinds)
        }

        idxAt := -1
        if e.colIndex {
                // Backpatch the header flag now that an index is actually emitted, so
                // OptColumnIndex on a non-columnar payload stays a no-op.
                e.buf[e.headerFlagAt] |= FlagColIndex
                idxAt = len(e.buf)
                e.buf = append(e.buf, make([]byte, 4*len(plan.cols))...)
        }
        colStart := len(e.buf)
        for c := range plan.cols {
                col := &plan.cols[c]
                if err := e.encodeOneColumn(plan, base, col, n); err != nil {
                        return err
                }
                if e.colIndex {
                        end := len(e.buf)
                        // The column-index entry is a uint32; a single column body exceeding
                        // 4 GiB would truncate here and desync the decoder's skip cursor. Bail
                        // rather than emit a corrupt index (extreme: needs a >4 GiB column).
                        if end-colStart > math.MaxUint32 {
                                return ErrInvalidLength
                        }
                        binary.LittleEndian.PutUint32(e.buf[idxAt+4*c:], uint32(end-colStart))
                        colStart = end
                }
        }
        return nil
}

// encodeOneColumn emits a single eligible column's body using the pooled
// transpose scratch. It does NO colIndex bookkeeping — the caller backpatches
// the per-column length. Shared by encodeColumnar (tagColStruct) and
// encodeHybridColumnar (tagHybridColStruct) so both get the identical
// per-column encoding (constant/FOR/dict/FSST/Gorilla/bitpack).
func (e *Encoder) encodeOneColumn(plan *columnarPlan, base unsafe.Pointer, col *colColumn, n int) error {
        st := e.state
        if col.kind.isNullable() {
                return e.encodeNullableColumn(base, plan, col, n)
        }
        switch col.kind {
        case colKindInt:
                s := gatherColI64(st.colScratchI64, base, plan.stride, col, n)
                st.colScratchI64 = s
                // OptZoneMap: zone-chunk the column with a min/max zonemap for predicate
                // zone-skip (columnar-only, explicit size-for-query-speed opt-in).
                if e.zonemap && n >= zoneChunkMinLen {
                        e.writeZoneChunkInt64(s)
                        return nil
                }
                return encodeSliceInt64(e, unsafe.Pointer(&s))
        case colKindUint:
                s := gatherColU64(st.colScratchU64, base, plan.stride, col, n)
                st.colScratchU64 = s
                if e.zonemap && n >= zoneChunkMinLen {
                        e.writeZoneChunkUint64(s)
                        return nil
                }
                return encodeSliceUint64(e, unsafe.Pointer(&s))
        case colKindFloat:
                s := st.colScratchF64[:0]
                for i := range n {
                        s = append(s, loadFloat64Field(base, plan.stride, col, i))
                }
                st.colScratchF64 = s
                // OptZoneMap: zone-chunk with a finite min/max zonemap for predicate
                // zone-skip (lossless per zone). Lossless regardless: a SCALAR float64
                // column must never become lossy, even under OptLossyVec.
                if e.zonemap && n >= zoneChunkMinLen {
                        e.writeZoneChunkFloat64(s)
                        return nil
                }
                // Lossless: a SCALAR float64 column must never become lossy, even under
                // OptLossyVec (which targets genuine []float64/[]float32 VECTOR fields).
                return encodeSliceFloat64Lossless(e, s)
        case colKindFloat32:
                // float32 column: store raw 32-bit patterns via the uint codec (4 B,
                // bit-exact). Reuses the u64 scratch — the high 32 bits are always zero.
                s := st.colScratchU64[:0]
                if e.opts.Has(OptCanonical) {
                        for i := range n {
                                s = append(s, canonicalizeFloat32Bits(loadFloat32Bits(base, plan.stride, col, i)))
                        }
                } else {
                        for i := range n {
                                s = append(s, loadFloat32Bits(base, plan.stride, col, i))
                        }
                }
                st.colScratchU64 = s
                return encodeSliceUint64(e, unsafe.Pointer(&s))
        case colKindBool:
                s := st.colScratchBool[:0]
                for i := range n {
                        p := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                        s = append(s, *(*bool)(p))
                }
                st.colScratchBool = s
                return encodeSliceBool(e, unsafe.Pointer(&s))
        case colKindString:
                if col.isByte {
                        for i := range n {
                                p := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                                e.WriteBytes(*(*[]byte)(p))
                        }
                        return nil
                }
                s := st.colScratchStr[:0]
                for i := range n {
                        s = append(s, loadStringField(base, plan.stride, col, i))
                }
                st.colScratchStr = s
                e.writeStringColumn(s)
                return nil
        case colKindTime:
                // Two sub-columns: sec ([]int64) + nsec ([]uint64). Delta+FOR on sec
                // compresses monotonic timestamp series efficiently.
                sec := st.colScratchI64[:0]
                nsec := st.colScratchU64[:0]
                for i := range n {
                        p := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                        t := (*time.Time)(p).UTC()
                        sec = append(sec, t.Unix())
                        nsec = append(nsec, uint64(t.Nanosecond()))
                }
                st.colScratchI64 = sec
                st.colScratchU64 = nsec
                if err := encodeSliceInt64(e, unsafe.Pointer(&sec)); err != nil {
                        return err
                }
                return encodeSliceUint64(e, unsafe.Pointer(&nsec))
        }
        return nil
}

// encodeHybridColumnar emits a slice of mixed structs as tagHybridColStruct:
// the eligible columns transposed (identical per-column encoding to
// tagColStruct, via encodeOneColumn) followed by a per-row residual block where
// the non-columnar fields are encoded row-major using their own codecs. The
// never-larger decision is made by the caller's columnarProbe (it measures the
// eligible columns; the residual block is byte-identical to what row-major
// would emit, so it does not affect the columnar-vs-row-major comparison).
// No FlagColIndex is emitted for hybrid in v1.
func (e *Encoder) encodeHybridColumnar(plan *columnarPlan, base unsafe.Pointer, n int) error {
        st := e.state
        e.buf = append(e.buf, tagHybridColStruct)
        e.buf = appendUvarint(e.buf, uint64(n))

        // Shape: ALL fields in declaration order; residualKind marks residual ones.
        if id := st.hybridShapeFor(plan.hybridNames, plan.hybridKinds); id != 0 {
                e.buf = appendUvarint(e.buf, uint64(id))
        } else {
                e.buf = appendUvarint(e.buf, 0)
                e.buf = appendUvarint(e.buf, uint64(len(plan.hybridNames)))
                for i := range plan.hybridNames {
                        e.WriteString(plan.hybridNames[i])
                        e.buf = append(e.buf, byte(plan.hybridKinds[i]))
                }
                st.hybridShapeDeclare(plan.hybridNames, plan.hybridKinds)
        }

        // Eligible columns, transposed.
        for c := range plan.cols {
                if err := e.encodeOneColumn(plan, base, &plan.cols[c], n); err != nil {
                        return err
                }
        }

        // Residual block: for each row, its non-columnar fields in declaration
        // order, via the field's own (row-major) encoder.
        for i := range n {
                rowPtr := unsafe.Add(base, uintptr(i)*plan.stride)
                for r := range plan.residual {
                        rf := &plan.residual[r]
                        if err := rf.desc.encode(e, unsafe.Add(rowPtr, rf.offset)); err != nil {
                                return err
                        }
                }
        }
        return nil
}

//go:nosplit
func loadFloat64Field(base unsafe.Pointer, stride uintptr, col *colColumn, i int) float64 {
        // colKindFloat is float64-only (width 8); float32 uses colKindFloat32 +
        // loadFloat32Bits, so there is no width==4 case here.
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        return *(*float64)(p)
}

// loadFloat32Bits reads a float32 field's raw IEEE-754 bits (zero-extended to
// uint64) for a colKindFloat32 column. Bit-exact: never goes through a numeric
// float64 conversion, so NaN payloads/signaling bits survive.
//
//go:nosplit
func loadFloat32Bits(base unsafe.Pointer, stride uintptr, col *colColumn, i int) uint64 {
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        return uint64(*(*uint32)(p))
}

// storeFloat32Bits writes raw float32 bits (low 32 of v) into a float32 field
// for a colKindFloat32 column. Inverse of loadFloat32Bits.
//
//go:nosplit
func storeFloat32Bits(base unsafe.Pointer, stride uintptr, col *colColumn, i int, v uint64) {
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        *(*uint32)(p) = uint32(v)
}

// intRowCost returns the row-major wire byte cost of WriteInt(v), so the
// columnar probe scores a signed column against the same encoding the encoder
// actually emits (uvarint for non-negatives, a fixed-width tag for negatives).
func intRowCost(v int64) int {
        if v >= 0 {
                return uvarintLen(uint64(v))
        }
        switch {
        case v >= -negfixintMaxAbs:
                return 1
        case v >= math.MinInt8:
                return 2
        case v >= math.MinInt16:
                return 3
        case v >= math.MinInt32:
                return 5
        default:
                return 9
        }
}

//go:nosplit
func loadScalarU64(base unsafe.Pointer, stride uintptr, col *colColumn, i int) uint64 {
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        switch col.width {
        case 1:
                return uint64(*(*uint8)(p))
        case 2:
                return uint64(*(*uint16)(p))
        case 4:
                return uint64(*(*uint32)(p))
        default:
                return *(*uint64)(p)
        }
}

//go:nosplit
func loadStringField(base unsafe.Pointer, stride uintptr, col *colColumn, i int) string {
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        if col.isByte {
                // Zero-copy view of the []byte field, mirroring the string-field case
                // below (which already aliases the caller's backing). The returned value
                // is consumed synchronously during encode while the source struct is live
                // and is never retained, so aliasing is safe and avoids a heap copy per
                // gathered cell (probe, columnar gather, and column-diff gather).
                b := *(*[]byte)(p)
                if len(b) == 0 {
                        return ""
                }
                return unsafe.String(unsafe.SliceData(b), len(b))
        }
        return *(*string)(p)
}

// loadStringFieldBytes returns a zero-copy []byte view of the i-th string field
// (no allocation). The view aliases the caller's live struct, which is stable
// for the duration of the encode; it is read-only (passed to FSST train/probe).
func loadStringFieldBytes(base unsafe.Pointer, stride uintptr, col *colColumn, i int) []byte {
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        if col.isByte {
                return *(*[]byte)(p)
        }
        s := *(*string)(p)
        return unsafe.Slice(unsafe.StringData(s), len(s))
}

// estimateFSSTColumnBytes estimates the FSST-coded byte cost of a string column
// over the probe sample, amortizing the one-time symbol table across the whole
// column (× sample/n). Zero string copies; reuses one compress scratch buffer.
// Called only when FSST is enabled (OptFSST), so its training cost stays off the
// Speed/Balanced hot path.
func estimateFSSTColumnBytes(base unsafe.Pointer, stride uintptr, col *colColumn, sample, n int, dict *fsst.SymbolTable) int {
        var strs [columnarProbeSample][]byte
        for i := range sample {
                strs[i] = loadStringFieldBytes(base, stride, col, i)
        }
        tbl := dict
        var bld *fsst.Builder
        if tbl == nil {
                bld = fsstBuilderPool.Get().(*fsst.Builder)
                tbl = bld.BuildRounds(strs[:sample], fsstProbeRounds) // coarse estimate
        }
        body := 0
        for i := range sample {
                clen := tbl.CompressedLen(strs[i]) // length only; no throwaway buffer
                body += clen + uvarintLen(uint64(clen))
        }
        sz := body + tbl.SerializedSize()*sample/n
        if bld != nil {
                fsstBuilderPool.Put(bld)
        }
        return sz
}

// colShapeRead is the parsed columnar header: the shape (names+kinds), the row
// count n, and — when the column-length index is present (d.colIndex) — the
// per-column byte lengths. It is the common prefix of every columnar decode
// path (typed struct, dynamic map, and predicate query).
type colShapeRead struct {
        sh      *decColShape
        colLens []uint32 // nil when d.colIndex is false
        n       int
}

// readColShape consumes the tagColStruct tag, the row count, the shape
// (declared inline or by id), and the optional column-length index, validating
// bounds exactly as decodeColumnar did. On return d.i points at the first
// column body. maxN bounds the row count (pass 0 for the struct path, which
// uses only checkColumnarN; pass maxColumnarAnyElems for the map path).
func (d *Decoder) readColShape(maxN int) (colShapeRead, error) {
        var out colShapeRead
        d.i++ // consume tagColStruct
        n64, k := readUvarint(d.buf[d.i:])
        if k <= 0 {
                return out, ErrInvalidLength
        }
        d.i += k
        n := int(n64)
        if err := checkColumnarN(n); err != nil {
                return out, err
        }
        if maxN > 0 && n > maxN {
                return out, ErrInvalidLength
        }
        out.n = n
        if d.state == nil {
                d.state = newDecState()
        }
        idv, k2 := readUvarint(d.buf[d.i:])
        if k2 <= 0 {
                return out, ErrInvalidLength
        }
        if idv > uint64(math.MaxUint32) {
                return out, ErrUnknownStateID // would truncate on the uint32 cast below
        }
        d.i += k2
        if idv == 0 {
                cnt64, k3 := readUvarint(d.buf[d.i:])
                if k3 <= 0 {
                        return out, ErrInvalidLength
                }
                d.i += k3
                if cnt64 > uint64(math.MaxInt) {
                        return out, ErrInvalidLength // would wrap int() on a 32-bit build
                }
                cnt := int(cnt64)
                if err := d.CheckLength(cnt, 1); err != nil {
                        return out, err
                }
                names := make([]string, cnt)
                kinds := make([]colKind, cnt)
                for i := range cnt {
                        s, err := d.readStringBytes()
                        if err != nil {
                                return out, err
                        }
                        names[i] = string(s)
                        if d.i >= len(d.buf) {
                                return out, ErrShortBuffer
                        }
                        kinds[i] = colKind(d.buf[d.i])
                        d.i++
                }
                out.sh = d.state.colShapeDeclareDec(names, kinds)
        } else {
                out.sh = d.state.colShapeLookup(uint32(idv))
                if out.sh == nil {
                        return out, ErrUnknownStateID
                }
        }
        if d.colIndex {
                kk := len(out.sh.kinds)
                if d.i+4*kk > len(d.buf) {
                        return out, ErrShortBuffer
                }
                var colLens []uint32
                if cap(d.state.colLenScratch) >= kk {
                        colLens = d.state.colLenScratch[:kk]
                } else {
                        colLens = make([]uint32, kk)
                }
                d.state.colLenScratch = colLens
                var sum uint64
                for c := range kk {
                        colLens[c] = binary.LittleEndian.Uint32(d.buf[d.i+4*c:])
                        sum += uint64(colLens[c])
                }
                d.i += 4 * kk
                if sum > uint64(len(d.buf)-d.i) {
                        return out, ErrShortBuffer
                }
                out.colLens = colLens
        }
        return out, nil
}

// colVals holds one column decoded into its native scratch, retained until the
// row mask is known so matched rows can be compacted into the output. Exactly
// one typed slice is populated, matching kind.base(). For a nullable column,
// present marks which rows carry a value (dense expanded to length n); a row
// whose present bit is 0 is a nil/zero row.
type colVals struct {
        i64     []int64
        u64     []uint64
        f64     []float64
        b       []bool
        s       []string
        bs      [][]byte
        ts      []time.Time // colKindTime only
        present []uint64    // nullable only; nil otherwise
        kind    colKind     // 1-byte tail: kept last so it adds no padding before the slices
}

// decodeColumnVals decodes a column body (length n) of the given kind into a
// fresh colVals, retaining the decoded slice instead of scattering. isByte
// selects the []byte representation for a string-kind column.
func (d *Decoder) decodeColumnVals(kind colKind, n int, isByte bool) (colVals, error) {
        var cv colVals
        cv.kind = kind
        if kind.isNullable() {
                return d.decodeNullableColumnVals(kind, n)
        }
        switch kind {
        case colKindInt:
                var s []int64
                if err := decodeSliceInt64(d, unsafe.Pointer(&s)); err != nil {
                        return cv, err
                }
                if len(s) != n {
                        return cv, ErrTypeMismatch
                }
                cv.i64 = s
        case colKindUint:
                var s []uint64
                if err := decodeSliceUint64(d, unsafe.Pointer(&s)); err != nil {
                        return cv, err
                }
                if len(s) != n {
                        return cv, ErrTypeMismatch
                }
                cv.u64 = s
        case colKindFloat:
                var s []float64
                if err := decodeSliceFloat64(d, unsafe.Pointer(&s)); err != nil {
                        return cv, err
                }
                if len(s) != n {
                        return cv, ErrTypeMismatch
                }
                cv.f64 = s
        case colKindFloat32:
                // float32 bits live in u64 (high 32 zero); cv.kind tags the f32 meaning.
                var s []uint64
                if err := decodeSliceUint64(d, unsafe.Pointer(&s)); err != nil {
                        return cv, err
                }
                if len(s) != n {
                        return cv, ErrTypeMismatch
                }
                cv.u64 = s
        case colKindBool:
                var s []bool
                if err := decodeSliceBool(d, unsafe.Pointer(&s)); err != nil {
                        return cv, err
                }
                if len(s) != n {
                        return cv, ErrTypeMismatch
                }
                cv.b = s
        case colKindString:
                if d.i < len(d.buf) && isStringColumnBlockTag(d.buf[d.i]) {
                        s, err := d.readStringColumn(n)
                        if err != nil {
                                return cv, err
                        }
                        if isByte {
                                // A string column decodes into a []byte target field too (the
                                // row-major path supports this interchange). The block readers
                                // return strings, so copy each into an owned []byte.
                                bs := make([][]byte, n)
                                for i := range n {
                                        bs[i] = append([]byte(nil), s[i]...)
                                }
                                cv.bs = bs
                        } else {
                                cv.s = s
                        }
                        break
                }
                if isByte {
                        bs := make([][]byte, n)
                        for i := range n {
                                sb, err := d.readStringBytes()
                                if err != nil {
                                        return cv, err
                                }
                                bs[i] = append([]byte(nil), sb...)
                        }
                        cv.bs = bs
                        break
                }
                s := make([]string, n)
                for i := range n {
                        // ReadString shares repeated values via the decode-side intern
                        // cache (and aliases under noCopy), so a low-cardinality column that
                        // fell below the dict gate decodes to ~distinct allocs, not n.
                        str, err := d.ReadString()
                        if err != nil {
                                return cv, err
                        }
                        s[i] = str
                }
                cv.s = s
        case colKindTime:
                var sec []int64
                if err := decodeSliceInt64(d, unsafe.Pointer(&sec)); err != nil {
                        return cv, err
                }
                if len(sec) != n {
                        return cv, ErrTypeMismatch
                }
                var nsec []uint64
                if err := decodeSliceUint64(d, unsafe.Pointer(&nsec)); err != nil {
                        return cv, err
                }
                if len(nsec) != n {
                        return cv, ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return cv, err
                }
                ts := make([]time.Time, n)
                for i := range n {
                        ts[i] = time.Unix(sec[i], int64(nsec[i])).UTC()
                }
                cv.ts = ts
        default:
                return cv, ErrBadTag
        }
        return cv, nil
}

// eval runs term against cv, setting mask bits for matching rows. A nullable
// row whose present bit is 0 never matches. The common non-nullable column
// takes a tight loop with no per-row presence test (split out so the predictable
// presence branch is hoisted out of the hot path entirely).
func (cv *colVals) eval(term *predTerm, n int, mask []uint64) {
        if cv.present == nil {
                cv.evalDense(term, n, mask)
                return
        }
        cv.evalNullable(term, n, mask)
}

// evalDense is eval for a non-nullable column: no presence test per row.
func (cv *colVals) evalDense(term *predTerm, n int, mask []uint64) {
        switch term.want {
        case colKindInt:
                for i := range n {
                        if term.pI64(cv.i64[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindUint:
                for i := range n {
                        if term.pU64(cv.u64[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindFloat:
                for i := range n {
                        if term.pF64(cv.f64[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindFloat32:
                for i := range n {
                        if term.pF64(float64(math.Float32frombits(uint32(cv.u64[i])))) {
                                setBit(mask, i)
                        }
                }
        case colKindBool:
                for i := range n {
                        if term.pBool(cv.b[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindString:
                for i := range n {
                        if term.pStr(cv.strAt(i)) {
                                setBit(mask, i)
                        }
                }
        }
}

// strAt returns row i's string for predicate evaluation, sourcing it from the
// []byte materialization (cv.bs) when the column was projected into a []byte
// target field (cv.s left nil). The string is transient — handed to the
// predicate only — so aliasing the owned cv.bs[i] copy is safe and alloc-free.
func (cv *colVals) strAt(i int) string {
        if cv.s != nil {
                return cv.s[i]
        }
        b := cv.bs[i]
        if len(b) == 0 {
                return ""
        }
        return unsafe.String(unsafe.SliceData(b), len(b))
}

// evalNullable is eval for a nullable column: an absent row never matches.
func (cv *colVals) evalNullable(term *predTerm, n int, mask []uint64) {
        switch term.want {
        case colKindInt:
                for i := range n {
                        if getBit(cv.present, i) && term.pI64(cv.i64[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindUint:
                for i := range n {
                        if getBit(cv.present, i) && term.pU64(cv.u64[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindFloat:
                for i := range n {
                        if getBit(cv.present, i) && term.pF64(cv.f64[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindFloat32:
                for i := range n {
                        if getBit(cv.present, i) && term.pF64(float64(math.Float32frombits(uint32(cv.u64[i])))) {
                                setBit(mask, i)
                        }
                }
        case colKindBool:
                for i := range n {
                        if getBit(cv.present, i) && term.pBool(cv.b[i]) {
                                setBit(mask, i)
                        }
                }
        case colKindString:
                for i := range n {
                        if getBit(cv.present, i) && term.pStr(cv.strAt(i)) {
                                setBit(mask, i)
                        }
                }
        }
}

// evalMasks evaluates term against cv into a fresh T mask (rows that are TRUE).
// For a nullable column it also returns an explicit F mask (rows that are
// FALSE); nil F means the column has no nil rows, so F is the complement of T.
// Built on eval, which already gates on the presence bitmap, so nil rows are
// set in neither T nor F (SQL UNKNOWN).
func (cv *colVals) evalMasks(term *predTerm, n int) (t, f []uint64) {
        t = newBitset(n)
        cv.eval(term, n, t)
        if cv.present == nil {
                return t, nil
        }
        f = slices.Clone(cv.present)
        bitsetAndNot(f, t) // f = present &^ t  (present rows that failed the predicate)
        return t, f
}

// scatterRow writes a non-nullable column's value at source row src into the
// output struct slice at compacted row dst. Mirrors decodeColumnInto's store
// half. Nullable columns are scattered via scatterNullableRowInto (one shared
// backing slab instead of a per-row alloc).
func (cv *colVals) scatterRow(base unsafe.Pointer, plan *columnarPlan, col *colColumn, src, dst int) {
        switch cv.kind {
        case colKindInt:
                storeScalarFromI64(base, plan.stride, col, dst, cv.i64[src])
        case colKindUint:
                storeScalarFromU64(base, plan.stride, col, dst, cv.u64[src])
        case colKindFloat:
                storeFloat64(base, plan.stride, col, dst, cv.f64[src])
        case colKindFloat32:
                storeFloat32Bits(base, plan.stride, col, dst, cv.u64[src])
        case colKindBool:
                *(*bool)(unsafe.Add(base, uintptr(dst)*plan.stride+col.offset)) = cv.b[src]
        case colKindString:
                dp := unsafe.Add(base, uintptr(dst)*plan.stride+col.offset)
                if col.isByte {
                        *(*[]byte)(dp) = append([]byte(nil), cv.bs[src]...)
                } else {
                        *(*string)(dp) = cv.s[src]
                }
        case colKindTime:
                dp := unsafe.Add(base, uintptr(dst)*plan.stride+col.offset)
                *(*time.Time)(dp) = cv.ts[src]
        }
}

// anyAt returns cv's value at row i as an any, mirroring decodeColumnarAny's
// per-kind boxing (int64/uint64/float64/bool, string for string-kind). A
// nullable nil row returns a nil any.
func (cv *colVals) anyAt(i int) any {
        if cv.present != nil && !getBit(cv.present, i) {
                return nil
        }
        switch cv.kind.base() {
        case colKindInt:
                return cv.i64[i]
        case colKindUint:
                return cv.u64[i]
        case colKindFloat:
                return cv.f64[i]
        case colKindFloat32:
                return math.Float32frombits(uint32(cv.u64[i]))
        case colKindBool:
                return cv.b[i]
        case colKindString:
                if cv.bs != nil {
                        return string(cv.bs[i])
                }
                return cv.s[i]
        case colKindTime:
                return cv.ts[i]
        }
        return nil
}

// runQueryColumns resolves d.query's predicate tree against the shape, then
// makes a single forward pass over the wire columns: predicate and projected
// columns are decoded into retained colVals (predicates evaluated via the tree),
// and the rest are skipped (via the column-length index when present, else
// decode-and-discard). It returns the retained column values (indexed by wire
// column, nil where not retained) and the surviving row indices.
//
// isProj reports whether wire column c should be retained for the caller to
// materialise. isByte is forwarded to decodeColumnVals for column c (the typed
// path passes the target field's isByte; the map path passes false).
func (d *Decoder) runQueryColumns(
        sh *decColShape, colLens []uint32, n int,
        isProj func(c int) bool, isByte func(c int) bool,
) (retained []*colVals, matched []int, err error) {
        // Flatten the predicate tree into a local, int-indexed slice (no per-node
        // maps; never mutates the shared QueryOption tree).
        flat := flattenCond(d.query.root)

        // Resolve each leaf to a wire column and validate its kind.
        referenced := make([]bool, len(sh.kinds))
        for i := range flat {
                if flat[i].op != condLeaf {
                        continue
                }
                field := flat[i].term.field
                wi := -1
                for c, name := range sh.names {
                        if name == field {
                                wi = c
                                break
                        }
                }
                if wi < 0 {
                        return nil, nil, &QueryError{Op: "predicate pushdown", Field: field, Err: ErrFieldNotFound}
                }
                if sh.kinds[wi].base() != flat[i].term.want {
                        return nil, nil, &QueryError{Op: "predicate pushdown", Field: field, Want: flat[i].term.want, Got: sh.kinds[wi], Err: ErrTypeMismatch}
                }
                flat[i].col = wi
                referenced[wi] = true
        }

        // Single forward pass: decode projected or referenced columns once, skip rest.
        retained = make([]*colVals, len(sh.kinds))
        colCV := make([]*colVals, len(sh.kinds))
        // Projected-only block columns are deferred: a predicate over OTHER columns
        // narrows the row set first, then these are decoded block-selectively so only
        // the blocks covering matched rows are materialized. Requires the column-length
        // index (colLens) to seek past the deferred column cheaply in this pass.
        type deferredBlock struct{ c, start int }
        var deferred []deferredBlock
        // Projected zone-chunked columns are deferred too: their values must follow the
        // FINAL matched set (a row matched via an OR/NOT sibling can live in a zone this
        // column's predicate bounds skipped), so they are filled selectively after the
        // match is computed.
        var deferredZones []deferredBlock
        for c := range sh.kinds {
                proj := isProj(c)
                ref := referenced[c]
                if !proj && !ref {
                        if colLens != nil {
                                if d.i+int(colLens[c]) > len(d.buf) {
                                        return nil, nil, ErrShortBuffer
                                }
                                d.i += int(colLens[c])
                        } else if e := d.skipColumnValue(sh.kinds[c], n); e != nil {
                                return nil, nil, e
                        }
                        continue
                }
                k := sh.kinds[c]
                // Zone-skip: a referenced zone-chunked int/uint column whose every
                // referencing leaf carries comparison bounds is decoded zone-selectively —
                // zones whose [min,max] cannot match any leaf are skipped, and each leaf's T
                // mask is produced directly (precompT). Needs colLens to reposition past the
                // column afterwards.
                if leaf := singleBoundedLeafForCol(flat, c); leaf != nil && colLens != nil &&
                        !k.isNullable() && (k == colKindInt || k == colKindUint || k == colKindFloat) &&
                        d.i < len(d.buf) && d.buf[d.i] == tagZoneChunk {
                        if d.i+int(colLens[c]) > len(d.buf) {
                                return nil, nil, ErrShortBuffer
                        }
                        start := d.i
                        // Filter only: produce the leaf's precompT mask via zone-skip.
                        if e := d.decodeZoneChunkQuery(start, []*cnode{leaf}, n); e != nil {
                                return nil, nil, e
                        }
                        if proj {
                                deferredZones = append(deferredZones, deferredBlock{c, start})
                        }
                        d.i = start + int(colLens[c]) // leaf uses precompT; colCV[c] stays nil
                        continue
                }
                if proj && !ref && colLens != nil && !k.isNullable() &&
                        (k == colKindInt || k == colKindUint) &&
                        d.i < len(d.buf) && d.buf[d.i] == tagPackBlock {
                        if d.i+int(colLens[c]) > len(d.buf) {
                                return nil, nil, ErrShortBuffer
                        }
                        deferred = append(deferred, deferredBlock{c, d.i})
                        d.i += int(colLens[c])
                        continue
                }
                // Projected-only zone-chunked column: defer it too — a predicate over OTHER
                // columns narrows the rows first, then only the zones covering matched rows
                // are decoded (decodeZoneChunkSelective), instead of full-decoding every zone.
                if proj && !ref && colLens != nil && !k.isNullable() &&
                        (k == colKindInt || k == colKindUint || k == colKindFloat) &&
                        d.i < len(d.buf) && d.buf[d.i] == tagZoneChunk {
                        if d.i+int(colLens[c]) > len(d.buf) {
                                return nil, nil, ErrShortBuffer
                        }
                        deferredZones = append(deferredZones, deferredBlock{c, d.i})
                        d.i += int(colLens[c])
                        continue
                }
                cv, e := d.decodeColumnVals(k, n, isByte(c))
                if e != nil {
                        return nil, nil, e
                }
                if proj {
                        retained[c] = &cv
                }
                if ref {
                        colCV[c] = &cv
                }
        }

        // Bind each leaf to its decoded column values.
        for i := range flat {
                if flat[i].op == condLeaf {
                        flat[i].cv = colCV[flat[i].col]
                }
        }

        var combined []uint64
        if len(flat) == 0 {
                combined = fullBitset(n) // no filter: every row matches
        } else {
                markUnknown(flat)
                combined, _ = evalCond(flat, 0, n)
        }
        matched = matchedIndices(combined, n, nil)

        // Decode the deferred block columns for matched rows only. d.i is irrelevant
        // here (each call seeks via the column's recorded start and restores), so the
        // forward-pass cursor left at the end of all columns is untouched.
        for _, db := range deferred {
                cv, e := d.decodeBlockColumnSelective(db.start, sh.kinds[db.c], n, matched)
                if e != nil {
                        return nil, nil, e
                }
                retained[db.c] = &cv
        }
        // Fill projected zone-chunked columns from the matched set (see deferredZones).
        for _, dz := range deferredZones {
                cv, e := d.decodeZoneChunkSelective(dz.start, n, matched)
                if e != nil {
                        return nil, nil, e
                }
                retained[dz.c] = cv
        }
        return retained, matched, nil
}

// decodeColumnarQuery decodes a columnar struct slice applying d.query: it runs
// the AND of the plan's predicates to select rows, then materialises only the
// matched rows of the projected columns into out (*[]Struct). Filter columns
// need not be projected. Wire order is preserved.
func decodeColumnarQuery(d *Decoder, t reflect.Type, plan *columnarPlan, p unsafe.Pointer) error {
        cs, err := d.readColShape(0)
        if err != nil {
                return err
        }
        n, sh, colLens := cs.n, cs.sh, cs.colLens
        // Bound by bytes before runQueryColumns materialises n-element column
        // scratch (memory amplification from a compressed column count).
        if err := checkColumnarBytes(n, t.Elem().Size()); err != nil {
                return err
        }
        d.colMaxLen = n
        defer func() { d.colMaxLen = 0 }()

        // Projected output columns: wire index -> target plan column (or nil).
        want := wantedColumns(plan, sh.names)
        if d.query.selectFields != nil {
                for c, name := range sh.names {
                        if !slices.Contains(d.query.selectFields, name) {
                                want[c] = nil
                        }
                }
        }

        retained, matched, err := d.runQueryColumns(sh, colLens, n,
                func(c int) bool { return want[c] != nil },
                func(c int) bool { return want[c] != nil && want[c].isByte },
        )
        if err != nil {
                return err
        }

        // Allocate the compacted output slice and scatter matched rows — reuse the
        // caller backing when pointer-free + cap suffices, else fresh.
        base := reuseOrMakeSlice(t, len(matched), p, t.Elem().Size(), noPointers(t.Elem()))
        for c := range sh.kinds {
                cv := retained[c]
                if cv == nil || want[c] == nil {
                        continue
                }
                col := want[c]
                // Full-kind compare (incl. the nullable flag), matching the full-decode
                // path. The scatter below branches on cv.present (the WIRE nullability),
                // so a wire/plan nullability mismatch must be rejected here: otherwise a
                // nullable wire column scattered into a non-nullable plan column hits
                // reflect.SliceOf(nil) (col.elemType==nil) → panic, and the reverse
                // scatters a raw value into a *T field slot → corruption.
                if sh.kinds[c] != col.kind {
                        return ErrTypeMismatch
                }
                if cv.present != nil {
                        // Nullable column: one backing slab holds all present values; each
                        // *T field points into it. Replaces a per-row reflect.New.
                        slab := reflect.MakeSlice(reflect.SliceOf(col.elemType), len(matched), len(matched))
                        slabBase := slab.UnsafePointer()
                        elemSize := col.elemType.Size()
                        for dst, src := range matched {
                                cv.scatterNullableRowInto(base, plan, col, src, dst, slabBase, elemSize)
                        }
                        runtime.KeepAlive(slab)
                        continue
                }
                for dst, src := range matched {
                        cv.scatterRow(base, plan, col, src, dst)
                }
        }
        return nil
}

// decodeColumnarQueryAny is the *[]map[string]any (or *[]any) form of predicate
// pushdown. It runs the AND of the plan's predicates to select rows, then
// returns one map[string]any per matched row containing only the projected
// columns (Select fields, or all columns when no Select was given). Filter
// columns need not be projected. The result is a []any of map[string]any so the
// dynamic slice routing can box it like decodeColumnarAny's. Boxing into any is
// intrinsic to the map form; predicate evaluation stays unboxed.
func decodeColumnarQueryAny(d *Decoder) (any, error) {
        cs, err := d.readColShape(maxColumnarAnyElems)
        if err != nil {
                return nil, err
        }
        n, sh, colLens := cs.n, cs.sh, cs.colLens
        d.colMaxLen = n
        defer func() { d.colMaxLen = 0 }()

        // Projection: selectFields, or all columns when none given.
        projected := make([]bool, len(sh.kinds))
        for c, name := range sh.names {
                projected[c] = d.query.selectFields == nil || slices.Contains(d.query.selectFields, name)
        }

        retained, matched, err := d.runQueryColumns(sh, colLens, n,
                func(c int) bool { return projected[c] },
                func(c int) bool { return false },
        )
        if err != nil {
                return nil, err
        }

        out := make([]any, len(matched))
        for dst, src := range matched {
                row := make(map[string]any, len(sh.kinds))
                for c := range sh.kinds {
                        cv := retained[c]
                        if cv == nil {
                                continue
                        }
                        row[sh.names[c]] = cv.anyAt(src)
                }
                out[dst] = row
        }
        return out, nil
}

func decodeColumnar(d *Decoder, t reflect.Type, plan *columnarPlan, p unsafe.Pointer) error {
        cs, err := d.readColShape(0)
        if err != nil {
                return err
        }
        n := cs.n
        sh := cs.sh
        colLens := cs.colLens
        // Every column holds exactly n elements; bound each column codec's
        // claimed length so a constant/zero-width codec cannot allocate past n.
        d.colMaxLen = n
        defer func() { d.colMaxLen = 0 }()

        // Bound the output by bytes, not just element count: a compressed column
        // can claim maxColumnarElems rows from a tiny input (memory amplification).
        if err := checkColumnarBytes(n, t.Elem().Size()); err != nil {
                return err
        }

        // Reuse the caller's backing when the row struct is pointer-free and the
        // pre-sized slice has cap >= n (decode into a pooled slice), else fresh.
        base := reuseOrMakeSlice(t, n, p, t.Elem().Size(), noPointers(t.Elem()))

        want := wantedColumns(plan, sh.names)
        for c := range sh.kinds {
                col := want[c]
                if col == nil {
                        // unwanted column → skip its body via the index
                        if d.colIndex {
                                if d.i+int(colLens[c]) > len(d.buf) {
                                        return ErrShortBuffer
                                }
                                d.i += int(colLens[c])
                                continue
                        }
                        // No index: there is no column-length to seek past, so we must fully
                        // decode the column body to keep the cursor in sync — its values are
                        // simply discarded rather than scattered into the struct slice.
                        if err := d.skipColumnValue(sh.kinds[c], n); err != nil {
                                return err
                        }
                        continue
                }
                if sh.kinds[c] != col.kind {
                        return ErrTypeMismatch
                }
                if err := d.decodeColumnInto(base, plan, col, n); err != nil {
                        return err
                }
        }
        return nil
}

// readHybridColShape consumes tagHybridColStruct + N + the hybrid shape
// (declare-inline or reuse-by-ID against the SEPARATE hybrid shape table). No
// colIndex (hybrid v1 does not emit one). Mirrors readColShape otherwise.
func (d *Decoder) readHybridColShape(maxN int) (int, *decColShape, error) {
        d.i++ // consume tagHybridColStruct
        n64, k := readUvarint(d.buf[d.i:])
        if k <= 0 {
                return 0, nil, ErrInvalidLength
        }
        d.i += k
        n := int(n64)
        if err := checkColumnarN(n); err != nil {
                return 0, nil, err
        }
        if maxN > 0 && n > maxN {
                return 0, nil, ErrInvalidLength
        }
        if d.state == nil {
                d.state = newDecState()
        }
        idv, k2 := readUvarint(d.buf[d.i:])
        if k2 <= 0 {
                return 0, nil, ErrInvalidLength
        }
        if idv > uint64(math.MaxUint32) {
                return 0, nil, ErrUnknownStateID
        }
        d.i += k2
        if idv == 0 {
                cnt64, k3 := readUvarint(d.buf[d.i:])
                if k3 <= 0 {
                        return 0, nil, ErrInvalidLength
                }
                d.i += k3
                if cnt64 > uint64(math.MaxInt) {
                        return 0, nil, ErrInvalidLength // would wrap int() on a 32-bit build
                }
                cnt := int(cnt64)
                if err := d.CheckLength(cnt, 1); err != nil {
                        return 0, nil, err
                }
                names := make([]string, cnt)
                kinds := make([]colKind, cnt)
                for i := range cnt {
                        s, err := d.readStringBytes()
                        if err != nil {
                                return 0, nil, err
                        }
                        names[i] = string(s)
                        if d.i >= len(d.buf) {
                                return 0, nil, ErrShortBuffer
                        }
                        kinds[i] = colKind(d.buf[d.i])
                        d.i++
                }
                return n, d.state.hybridShapeDeclareDec(names, kinds), nil
        }
        sh := d.state.hybridShapeLookup(uint32(idv))
        if sh == nil {
                return 0, nil, ErrUnknownStateID
        }
        return n, sh, nil
}

func findCol(plan *columnarPlan, name string) *colColumn {
        for c := range plan.cols {
                if plan.cols[c].name == name {
                        return &plan.cols[c]
                }
        }
        return nil
}

func findResidual(plan *columnarPlan, name string) *residualField {
        for r := range plan.residual {
                if plan.residual[r].name == name {
                        return &plan.residual[r]
                }
        }
        return nil
}

// decodeHybridColumnar decodes a tagHybridColStruct payload: the eligible
// columns (transposed) scatter into the result structs exactly as
// decodeColumnar does, then the per-row residual block decodes each
// non-columnar field row-major via its own codec. Schema evolution is handled
// like decodeColumnar: an eligible wire column the target struct lacks is
// skipped; a residual wire field with no target is consumed via d.Skip.
func decodeHybridColumnar(d *Decoder, t reflect.Type, plan *columnarPlan, p unsafe.Pointer) error {
        n, sh, err := d.readHybridColShape(0)
        if err != nil {
                return err
        }
        // Bound every eligible column codec's claimed length to n.
        d.colMaxLen = n
        defer func() { d.colMaxLen = 0 }()

        // Bound the output by bytes, not just element count (memory amplification).
        if err := checkColumnarBytes(n, t.Elem().Size()); err != nil {
                return err
        }

        base := reuseOrMakeSlice(t, n, p, t.Elem().Size(), noPointers(t.Elem()))

        // Eligible columns: shape entries with a real colKind, in wire order
        // (which equals struct declaration order on the encode side). Matched by
        // name to the target plan column.
        for c := range sh.kinds {
                if sh.kinds[c] == residualKind {
                        continue
                }
                col := findCol(plan, sh.names[c])
                if col == nil {
                        // Wire has an eligible column the target struct lacks → skip its body.
                        if err := d.skipColumnValue(sh.kinds[c], n); err != nil {
                                return err
                        }
                        continue
                }
                if sh.kinds[c] != col.kind {
                        return ErrTypeMismatch
                }
                if err := d.decodeColumnInto(base, plan, col, n); err != nil {
                        return err
                }
        }

        // Residual block. Precompute the wire-residual → target mapping once
        // (nil target = a residual field the struct lacks, consumed via Skip).
        var targets []*residualField
        for c := range sh.kinds {
                if sh.kinds[c] == residualKind {
                        targets = append(targets, findResidual(plan, sh.names[c]))
                }
        }
        // Residual fields are row-major, not columnar columns: their slice/map/
        // nested values may legitimately hold any number of elements, unrelated to
        // the row count n. Clear the per-column length bound (set to n above for the
        // eligible columns) before decoding them, or a residual collection longer
        // than n is wrongly rejected by colLenOK as ErrInvalidLength. (A nested
        // columnar residual sets and restores its own colMaxLen.)
        d.colMaxLen = 0
        for i := range n {
                rowPtr := unsafe.Add(base, uintptr(i)*plan.stride)
                for _, rf := range targets {
                        if rf == nil {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                                continue
                        }
                        if err := rf.desc.decode(d, unsafe.Add(rowPtr, rf.offset)); err != nil {
                                return err
                        }
                }
        }
        return nil
}

// decodeColumnInto decodes a single column body from the wire into the matched
// target plan column. Behavior is identical to the per-column body of the
// former positional decode loop.
//
// The int64/uint64/float64/bool cases reuse decoder-held scratch slices
// (d.state.colScratchI64 etc.) so that the transient column buffer is not
// allocated fresh on every call — it is grown once and reused across columns
// and across decode calls (the Decoder is pooled). The scratch slice is only
// valid until the scatter loop below, which copies each element into the
// output struct; it is never aliased by the output.
func (d *Decoder) decodeColumnInto(base unsafe.Pointer, plan *columnarPlan, col *colColumn, n int) error {
        if col.kind.isNullable() {
                return d.decodeNullableColumn(base, plan, col, n)
        }
        st := d.state // always non-nil: readColShape initialises it before the loop
        switch col.kind {
        case colKindInt:
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return err
                }
                s := st.colScratchI64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                scatterColI64(base, plan.stride, col, s)
        case colKindUint:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                scatterColU64(base, plan.stride, col, s)
        case colKindFloat:
                if err := decodeSliceFloat64Into(d, &st.colScratchF64); err != nil {
                        return err
                }
                s := st.colScratchF64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        storeFloat64(base, plan.stride, col, i, s[i])
                }
        case colKindFloat32:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        storeFloat32Bits(base, plan.stride, col, i, s[i])
                }
        case colKindBool:
                if err := decodeSliceBoolInto(d, &st.colScratchBool); err != nil {
                        return err
                }
                s := st.colScratchBool
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        *(*bool)(unsafe.Add(base, uintptr(i)*plan.stride+col.offset)) = s[i]
                }
        case colKindString:
                if d.i < len(d.buf) && isStringColumnBlockTag(d.buf[d.i]) {
                        strs, err := d.readStringColumn(n)
                        if err != nil {
                                return err
                        }
                        for i := range n {
                                dp := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                                if col.isByte {
                                        // A string column scatters into a []byte target field too
                                        // (row-major supports this interchange); copy to owned bytes.
                                        *(*[]byte)(dp) = append([]byte(nil), strs[i]...)
                                } else {
                                        *(*string)(dp) = strs[i]
                                }
                        }
                        break
                }
                for i := range n {
                        dp := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                        if col.isByte {
                                sb, err := d.readStringBytes()
                                if err != nil {
                                        return err
                                }
                                *(*[]byte)(dp) = append([]byte(nil), sb...)
                                continue
                        }
                        // ReadString shares repeated values via the decode-side intern
                        // cache, so a low-cardinality string column that fell below the dict
                        // gate scatters to ~distinct allocs, not one per row.
                        str, err := d.ReadString()
                        if err != nil {
                                return err
                        }
                        *(*string)(dp) = str
                }
        case colKindTime:
                // Decode sec sub-column then nsec sub-column, reusing scratch buffers.
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return err
                }
                sec := st.colScratchI64
                if len(sec) != n {
                        return ErrTypeMismatch
                }
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                nsec := st.colScratchU64
                if len(nsec) != n {
                        return ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return err
                }
                for i := range n {
                        dp := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                        *(*time.Time)(dp) = time.Unix(sec[i], int64(nsec[i])).UTC()
                }
        }
        return nil
}

// skipColumnValue fully decodes a column body of the given kind but discards
// the result. It is the slow correctness fallback for selective decode when the
// producer did not emit a column-length index: with no length to seek past, the
// cursor can only be advanced by actually decoding the body. It mirrors the
// cursor-advancing half of decodeColumnInto without the store/scatter step.
func (d *Decoder) skipColumnValue(kind colKind, n int) error {
        if kind.isNullable() {
                // Skip = mask + dense sub-column of the present values. Reuse the
                // non-nullable arms below for the dense part instead of
                // decodeNullableColumnAny, which boxes every present value into a
                // []any just to be discarded (n allocs per skipped nullable column).
                _, present, err := d.readNullableMask(n)
                if err != nil {
                        return err
                }
                return d.skipColumnValue(kind.base(), present)
        }
        switch kind {
        case colKindInt:
                var s []int64
                return decodeSliceInt64(d, unsafe.Pointer(&s))
        case colKindUint:
                var s []uint64
                return decodeSliceUint64(d, unsafe.Pointer(&s))
        case colKindFloat:
                var s []float64
                return decodeSliceFloat64(d, unsafe.Pointer(&s))
        case colKindFloat32:
                var s []uint64
                return decodeSliceUint64(d, unsafe.Pointer(&s))
        case colKindBool:
                var s []bool
                return decodeSliceBool(d, unsafe.Pointer(&s))
        case colKindString:
                if d.i < len(d.buf) && isStringColumnBlockTag(d.buf[d.i]) {
                        _, err := d.readStringColumn(n)
                        return err
                }
                for range n {
                        if _, err := d.readStringBytes(); err != nil {
                                return err
                        }
                }
                return nil
        case colKindTime:
                // Skip sec sub-column then nsec sub-column.
                var sec []int64
                if err := decodeSliceInt64(d, unsafe.Pointer(&sec)); err != nil {
                        return err
                }
                var nsec []uint64
                return decodeSliceUint64(d, unsafe.Pointer(&nsec))
        default:
                return ErrBadTag
        }
}

//go:nosplit
func storeScalarFromI64(base unsafe.Pointer, stride uintptr, col *colColumn, i int, v int64) {
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        switch col.width {
        case 1:
                *(*int8)(p) = int8(v)
        case 2:
                *(*int16)(p) = int16(v)
        case 4:
                *(*int32)(p) = int32(v)
        default:
                *(*int64)(p) = v
        }
}

//go:nosplit
func storeScalarFromU64(base unsafe.Pointer, stride uintptr, col *colColumn, i int, v uint64) {
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        switch col.width {
        case 1:
                *(*uint8)(p) = uint8(v)
        case 2:
                *(*uint16)(p) = uint16(v)
        case 4:
                *(*uint32)(p) = uint32(v)
        default:
                *(*uint64)(p) = v
        }
}

// gatherColI64 / gatherColU64 / scatterColI64 / scatterColU64 are the bulk
// columnar gather/scatter loops. They hoist the loop-invariant col.width switch
// OUT of the per-element loop (one branch per column instead of per element) and
// presize the destination once. Measured ~13% faster encode / ~16% faster decode
// on narrow-int (int8/16/32, uint8/16/32) columnar structs vs the per-element
// loadScalarU64*/storeScalarFrom* calls; byte-identical wire. The per-element
// helpers above are kept for the probe-sample and query-scatter paths that store
// one value at a time.

func gatherColI64(dst []int64, base unsafe.Pointer, stride uintptr, col *colColumn, n int) []int64 {
        dst = slices.Grow(dst[:0], n)[:n]
        off := col.offset
        switch col.width {
        case 1:
                for i := range n {
                        dst[i] = int64(*(*int8)(unsafe.Add(base, uintptr(i)*stride+off)))
                }
        case 2:
                for i := range n {
                        dst[i] = int64(*(*int16)(unsafe.Add(base, uintptr(i)*stride+off)))
                }
        case 4:
                for i := range n {
                        dst[i] = int64(*(*int32)(unsafe.Add(base, uintptr(i)*stride+off)))
                }
        default:
                for i := range n {
                        dst[i] = *(*int64)(unsafe.Add(base, uintptr(i)*stride+off))
                }
        }
        return dst
}

func gatherColU64(dst []uint64, base unsafe.Pointer, stride uintptr, col *colColumn, n int) []uint64 {
        dst = slices.Grow(dst[:0], n)[:n]
        off := col.offset
        switch col.width {
        case 1:
                for i := range n {
                        dst[i] = uint64(*(*uint8)(unsafe.Add(base, uintptr(i)*stride+off)))
                }
        case 2:
                for i := range n {
                        dst[i] = uint64(*(*uint16)(unsafe.Add(base, uintptr(i)*stride+off)))
                }
        case 4:
                for i := range n {
                        dst[i] = uint64(*(*uint32)(unsafe.Add(base, uintptr(i)*stride+off)))
                }
        default:
                for i := range n {
                        dst[i] = *(*uint64)(unsafe.Add(base, uintptr(i)*stride+off))
                }
        }
        return dst
}

func scatterColI64(base unsafe.Pointer, stride uintptr, col *colColumn, s []int64) {
        off := col.offset
        switch col.width {
        case 1:
                for i := range s {
                        *(*int8)(unsafe.Add(base, uintptr(i)*stride+off)) = int8(s[i])
                }
        case 2:
                for i := range s {
                        *(*int16)(unsafe.Add(base, uintptr(i)*stride+off)) = int16(s[i])
                }
        case 4:
                for i := range s {
                        *(*int32)(unsafe.Add(base, uintptr(i)*stride+off)) = int32(s[i])
                }
        default:
                for i := range s {
                        *(*int64)(unsafe.Add(base, uintptr(i)*stride+off)) = s[i]
                }
        }
}

func scatterColU64(base unsafe.Pointer, stride uintptr, col *colColumn, s []uint64) {
        off := col.offset
        switch col.width {
        case 1:
                for i := range s {
                        *(*uint8)(unsafe.Add(base, uintptr(i)*stride+off)) = uint8(s[i])
                }
        case 2:
                for i := range s {
                        *(*uint16)(unsafe.Add(base, uintptr(i)*stride+off)) = uint16(s[i])
                }
        case 4:
                for i := range s {
                        *(*uint32)(unsafe.Add(base, uintptr(i)*stride+off)) = uint32(s[i])
                }
        default:
                for i := range s {
                        *(*uint64)(unsafe.Add(base, uintptr(i)*stride+off)) = s[i]
                }
        }
}

// checkNsecColumn rejects a time column whose nanosecond sub-column carries a
// value outside [0, 999_999_999], matching Decoder.ReadTimestamp. Without it the
// columnar/nullable time paths would silently normalize a hostile out-of-range
// nsec into a different valid instant instead of erroring.
func checkNsecColumn(nsec []uint64) error {
        for _, v := range nsec {
                if v > 999_999_999 {
                        return ErrInvalidLength
                }
        }
        return nil
}

//go:nosplit
func storeFloat64(base unsafe.Pointer, stride uintptr, col *colColumn, i int, v float64) {
        // colKindFloat is float64-only (width 8); float32 uses colKindFloat32 +
        // storeFloat32Bits, so there is no width==4 case here.
        p := unsafe.Add(base, uintptr(i)*stride+col.offset)
        *(*float64)(p) = v
}

// decodeColumnarAny decodes a tagColStruct payload into a []any of
// map[string]any keyed by column name. Mirrors decodeColumnar's header
// and shape parse exactly; each column is decoded into a temp slice and
// the per-element value boxed into its row's map.
func decodeColumnarAny(d *Decoder) (any, error) {
        // The map-per-row reflection decode allocates n maps up front, far heavier
        // than the struct path's single backing slice, so it gets a tighter
        // element ceiling. A constant column can claim a huge n from a tiny body;
        // without this a small hostile input would drive a multi-gigabyte map
        // allocation. Callers decoding millions of rows should use a typed struct.
        cs, err := d.readColShape(maxColumnarAnyElems)
        if err != nil {
                return nil, err
        }
        n := cs.n
        sh := cs.sh
        colLens := cs.colLens
        d.colMaxLen = n
        defer func() { d.colMaxLen = 0 }()

        out := make([]any, n)
        for i := range out {
                out[i] = make(map[string]any, len(sh.names))
        }
        for c := range sh.kinds {
                name := sh.names[c]
                // When a field filter is active, skip unrequested columns. With the
                // column-length index present we advance past the whole column body
                // without decoding (the perf path); without it we still must decode to
                // stay in sync, but the value is simply not stored below.
                want := d.wantField(name)
                if !want && colLens != nil {
                        if d.i+int(colLens[c]) > len(d.buf) {
                                return nil, ErrShortBuffer
                        }
                        d.i += int(colLens[c])
                        continue
                }
                // store is false only on the no-index skip path: the column body still
                // has to be decoded to keep the cursor in sync, but its values are
                // dropped rather than boxed into the row maps.
                store := want
                if sh.kinds[c].isNullable() {
                        if !store {
                                // Skip path: consume the mask + dense body to keep the cursor in
                                // sync without boxing every present value into []any just to drop
                                // it (mirrors the no-box skip in skipColumnValue).
                                _, present, err := d.readNullableMask(n)
                                if err != nil {
                                        return nil, err
                                }
                                if err := d.skipColumnValue(sh.kinds[c].base(), present); err != nil {
                                        return nil, err
                                }
                                continue
                        }
                        vals, err := d.decodeNullableColumnAny(sh.kinds[c], n)
                        if err != nil {
                                return nil, err
                        }
                        for i := range n {
                                out[i].(map[string]any)[name] = vals[i]
                        }
                        continue
                }
                switch sh.kinds[c] {
                case colKindInt:
                        var s []int64
                        if err := decodeSliceInt64(d, unsafe.Pointer(&s)); err != nil {
                                return nil, err
                        }
                        if len(s) != n {
                                return nil, ErrTypeMismatch
                        }
                        if store {
                                for i := range n {
                                        out[i].(map[string]any)[name] = s[i]
                                }
                        }
                case colKindUint:
                        var s []uint64
                        if err := decodeSliceUint64(d, unsafe.Pointer(&s)); err != nil {
                                return nil, err
                        }
                        if len(s) != n {
                                return nil, ErrTypeMismatch
                        }
                        if store {
                                for i := range n {
                                        out[i].(map[string]any)[name] = s[i]
                                }
                        }
                case colKindFloat:
                        var s []float64
                        if err := decodeSliceFloat64(d, unsafe.Pointer(&s)); err != nil {
                                return nil, err
                        }
                        if len(s) != n {
                                return nil, ErrTypeMismatch
                        }
                        if store {
                                for i := range n {
                                        out[i].(map[string]any)[name] = s[i]
                                }
                        }
                case colKindFloat32:
                        var s []uint64
                        if err := decodeSliceUint64(d, unsafe.Pointer(&s)); err != nil {
                                return nil, err
                        }
                        if len(s) != n {
                                return nil, ErrTypeMismatch
                        }
                        if store {
                                for i := range n {
                                        out[i].(map[string]any)[name] = math.Float32frombits(uint32(s[i]))
                                }
                        }
                case colKindBool:
                        var s []bool
                        if err := decodeSliceBool(d, unsafe.Pointer(&s)); err != nil {
                                return nil, err
                        }
                        if len(s) != n {
                                return nil, ErrTypeMismatch
                        }
                        if store {
                                for i := range n {
                                        out[i].(map[string]any)[name] = s[i]
                                }
                        }
                case colKindString:
                        if d.i < len(d.buf) && isStringColumnBlockTag(d.buf[d.i]) {
                                strs, err := d.readStringColumn(n)
                                if err != nil {
                                        return nil, err
                                }
                                if store {
                                        for i := range n {
                                                out[i].(map[string]any)[name] = strs[i]
                                        }
                                }
                                break
                        }
                        for i := range n {
                                sb, err := d.readStringBytes()
                                if err != nil {
                                        return nil, err
                                }
                                if store {
                                        out[i].(map[string]any)[name] = string(sb)
                                }
                        }
                case colKindTime:
                        var sec []int64
                        if err := decodeSliceInt64(d, unsafe.Pointer(&sec)); err != nil {
                                return nil, err
                        }
                        if len(sec) != n {
                                return nil, ErrTypeMismatch
                        }
                        var nsec []uint64
                        if err := decodeSliceUint64(d, unsafe.Pointer(&nsec)); err != nil {
                                return nil, err
                        }
                        if len(nsec) != n {
                                return nil, ErrTypeMismatch
                        }
                        if err := checkNsecColumn(nsec); err != nil {
                                return nil, err
                        }
                        if store {
                                for i := range n {
                                        out[i].(map[string]any)[name] = time.Unix(sec[i], int64(nsec[i])).UTC()
                                }
                        }
                default:
                        return nil, ErrBadTag
                }
        }
        return out, nil
}

// decodeColumnBodyAny decodes one eligible column body into the per-row maps
// (or discards it when store is false). Mirrors the per-kind switch in
// decodeColumnarAny; used by the hybrid any/skip path (decodeHybridColumnarAny).
func (d *Decoder) decodeColumnBodyAny(kind colKind, name string, n int, store bool, out []any) error {
        if kind.isNullable() {
                vals, err := d.decodeNullableColumnAny(kind, n)
                if err != nil {
                        return err
                }
                if store {
                        for i := range n {
                                out[i].(map[string]any)[name] = vals[i]
                        }
                }
                return nil
        }
        switch kind {
        case colKindInt:
                var s []int64
                if err := decodeSliceInt64(d, unsafe.Pointer(&s)); err != nil {
                        return err
                }
                if len(s) != n {
                        return ErrTypeMismatch
                }
                if store {
                        for i := range n {
                                out[i].(map[string]any)[name] = s[i]
                        }
                }
        case colKindUint:
                var s []uint64
                if err := decodeSliceUint64(d, unsafe.Pointer(&s)); err != nil {
                        return err
                }
                if len(s) != n {
                        return ErrTypeMismatch
                }
                if store {
                        for i := range n {
                                out[i].(map[string]any)[name] = s[i]
                        }
                }
        case colKindFloat:
                var s []float64
                if err := decodeSliceFloat64(d, unsafe.Pointer(&s)); err != nil {
                        return err
                }
                if len(s) != n {
                        return ErrTypeMismatch
                }
                if store {
                        for i := range n {
                                out[i].(map[string]any)[name] = s[i]
                        }
                }
        case colKindFloat32:
                var s []uint64
                if err := decodeSliceUint64(d, unsafe.Pointer(&s)); err != nil {
                        return err
                }
                if len(s) != n {
                        return ErrTypeMismatch
                }
                if store {
                        for i := range n {
                                out[i].(map[string]any)[name] = math.Float32frombits(uint32(s[i]))
                        }
                }
        case colKindBool:
                var s []bool
                if err := decodeSliceBool(d, unsafe.Pointer(&s)); err != nil {
                        return err
                }
                if len(s) != n {
                        return ErrTypeMismatch
                }
                if store {
                        for i := range n {
                                out[i].(map[string]any)[name] = s[i]
                        }
                }
        case colKindString:
                if d.i < len(d.buf) && isStringColumnBlockTag(d.buf[d.i]) {
                        strs, err := d.readStringColumn(n)
                        if err != nil {
                                return err
                        }
                        if store {
                                for i := range n {
                                        out[i].(map[string]any)[name] = strs[i]
                                }
                        }
                        return nil
                }
                for i := range n {
                        sb, err := d.readStringBytes()
                        if err != nil {
                                return err
                        }
                        if store {
                                out[i].(map[string]any)[name] = string(sb)
                        }
                }
        case colKindTime:
                var sec []int64
                if err := decodeSliceInt64(d, unsafe.Pointer(&sec)); err != nil {
                        return err
                }
                if len(sec) != n {
                        return ErrTypeMismatch
                }
                var nsec []uint64
                if err := decodeSliceUint64(d, unsafe.Pointer(&nsec)); err != nil {
                        return err
                }
                if len(nsec) != n {
                        return ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return err
                }
                if store {
                        for i := range n {
                                out[i].(map[string]any)[name] = time.Unix(sec[i], int64(nsec[i])).UTC()
                        }
                }
        default:
                return ErrBadTag
        }
        return nil
}

// decodeHybridColumnarAny decodes a tagHybridColStruct payload into a []any of
// map[string]any rows — the dynamic / any / Skip path (parallel to
// decodeColumnarAny). It fully decodes (not a byte-skip) so the intern + shape
// tables stay in sync for later state-refs. Eligible columns scatter into the
// row maps; each residual field decodes per row via the generic any decoder.
func decodeHybridColumnarAny(d *Decoder) (any, error) {
        n, sh, err := d.readHybridColShape(maxColumnarAnyElems)
        if err != nil {
                return nil, err
        }
        d.colMaxLen = n
        defer func() { d.colMaxLen = 0 }()

        out := make([]any, n)
        for i := range out {
                out[i] = make(map[string]any, len(sh.names))
        }
        // Eligible columns (residualKind entries have no column body — handled below).
        for c := range sh.kinds {
                if sh.kinds[c] == residualKind {
                        continue
                }
                if err := d.decodeColumnBodyAny(sh.kinds[c], sh.names[c], n, true, out); err != nil {
                        return nil, err
                }
        }
        // Residual block: per row, each residual field in shape order via decodeAny.
        // Residual fields are row-major; clear the columnar length bound so a
        // residual collection longer than n is not rejected by colLenOK (see
        // decodeHybridColumnar for the detailed rationale).
        d.colMaxLen = 0
        for i := range n {
                row := out[i].(map[string]any)
                for c := range sh.kinds {
                        if sh.kinds[c] != residualKind {
                                continue
                        }
                        v, err := decodeAny(d)
                        if err != nil {
                                return nil, err
                        }
                        row[sh.names[c]] = v
                }
        }
        return out, nil
}

func classifyColKind(fd *typeDesc) (ck colKind, width uintptr, isByte bool, ok bool) {
        if fd.marshalerKind != 0 {
                return 0, 0, false, false // custom marshaler → row-major
        }
        if fd.hasCustomDecode {
                // Decode-half-only Unmarshaler (the documented asymmetric case): the
                // columnar scatter writes field memory directly, silently skipping the
                // user's UnmarshalQDF — row-major calls it. Keep such fields row-major
                // so both paths agree.
                return 0, 0, false, false
        }
        // time.Time is a struct that has its own scalar codec (encodeTime/decodeTime).
        // It must be detected before the generic struct fall-through so it gets
        // colKindTime instead of being rejected as ineligible.
        if fd.rType == reflect.TypeFor[time.Time]() {
                return colKindTime, unsafe.Sizeof(time.Time{}), false, true
        }
        switch fd.kind {
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                return colKindInt, fd.rType.Size(), false, true
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                return colKindUint, fd.rType.Size(), false, true
        case reflect.Float32:
                return colKindFloat32, fd.rType.Size(), false, true
        case reflect.Float64:
                return colKindFloat, fd.rType.Size(), false, true
        case reflect.Bool:
                return colKindBool, 1, false, true
        case reflect.String:
                return colKindString, 0, false, true
        case reflect.Slice:
                if fd.rType.Elem().Kind() == reflect.Uint8 { // []byte
                        // Known limitation: a []byte column is a plain string column (no
                        // per-row presence bit), kept that way so it stays wire-compatible
                        // with a string column for the string<->[]byte schema interchange
                        // (TestColumnarStringIntoByteField). The consequence is that the
                        // reflect columnar path (n >= columnarMinElems) cannot distinguish a
                        // nil []byte from an empty []byte{} — both decode to nil. The
                        // row-major path (n < columnarMinElems) preserves the distinction.
                        // Same inherent columnar-lossiness class as int->int64 widening and
                        // time monotonic-clock stripping. (The qdfgen codegen path, which has
                        // no string<->[]byte interchange to preserve, routes []byte through a
                        // nullable column and keeps nil vs empty distinct.)
                        return colKindString, 0, true, true
                }
                return 0, 0, false, false
        default:
                return 0, 0, false, false
        }
}

package qdf

// columnar_decode_scratch.go — scratch-buffer-aware column decoders used
// exclusively by decodeColumnInto (the full-decode, non-query path).
//
// The decoder's decState carries colScratchI64/U64/F64/Bool: per-kind
// reusable buffers that survive across columns within a decode call and
// across decode calls (the Decoder is pooled). Columns are processed
// sequentially inside decodeColumnInto; each column's values are scattered
// into the output struct slice immediately after decoding, so the scratch
// buffer is safe to overwrite for the next column.
//
// These Into variants are NOT used by:
//   - decodeColumnVals (query/pushdown path) — that path retains colVals
//     across columns before scatter, so buffers must not alias each other.
//   - decodeNullableColumn — the present-count varies per column; sharing
//     a single scratch there is safe but the int64/uint64 buffers are
//     already narrower (only present values), so nullable columns get the
//     benefit too via decodeSliceInt64Into below.
//
// The helper growI64 / growU64 / growF64 / growBool grow the slice to n
// elements without allocating when the existing cap is sufficient.

import (
        "math"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/bitpack"
        "github.com/alex60217101990/qdf/internal/endian"
)

// ---- grow helpers ----------------------------------------------------------

func growI64(dst *[]int64, n int) {
        if cap(*dst) >= n {
                *dst = (*dst)[:n]
        } else {
                *dst = make([]int64, n)
        }
}

func growU64(dst *[]uint64, n int) {
        if cap(*dst) >= n {
                *dst = (*dst)[:n]
        } else {
                *dst = make([]uint64, n)
        }
}

func growF64(dst *[]float64, n int) {
        if cap(*dst) >= n {
                *dst = (*dst)[:n]
        } else {
                *dst = make([]float64, n)
        }
}

func growBool(dst *[]bool, n int) {
        if cap(*dst) >= n {
                *dst = (*dst)[:n]
        } else {
                *dst = make([]bool, n)
        }
}

// ---- int64 Into variants ---------------------------------------------------

func (d *Decoder) readPackedInt64SliceInto(dst *[]int64) error {
        n, body, err := d.readPackedRawHeader(qpackKindInt64)
        if err != nil {
                return err
        }
        growI64(dst, n)
        if n == 0 {
                return nil
        }
        out := *dst
        if endian.NativeIsLittle {
                bs := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*8)
                copy(bs, body)
                return nil
        }
        for i := range n {
                out[i] = int64(readU64(body[i*8:]))
        }
        return nil
}

func (d *Decoder) readPackedForInt64SliceInto(dst *[]int64) error {
        bitsPer, _, mn, n, body, err := d.readPackedForHeader(qpackKindInt64)
        if err != nil {
                return err
        }
        growI64(dst, n)
        out := *dst
        if bitsPer == 0 {
                for i := range out {
                        out[i] = mn
                }
                return nil
        }
        u := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(out))), n)
        bitpack.Unpack(u, body, bitsPer)
        if mnU := uint64(mn); mnU != 0 {
                for i := range u {
                        u[i] += mnU
                }
        }
        return nil
}

func (d *Decoder) readPackedDeltaForInt64SliceInto(dst *[]int64) error {
        bitsPer, _, first, minDelta, n, body, err := d.readPackedDeltaForHeader(qpackKindInt64)
        if err != nil {
                return err
        }
        growI64(dst, n)
        out := *dst
        if n == 0 {
                return nil
        }
        out[0] = first
        if n == 1 {
                return nil
        }
        if bitsPer == 0 {
                v := first
                for i := 1; i < n; i++ {
                        v += minDelta
                        out[i] = v
                }
                return nil
        }
        if cap(d.deltaScratch) < n-1 {
                d.deltaScratch = make([]uint64, n-1)
        }
        tmp := d.deltaScratch[:n-1]
        bitpack.Unpack(tmp, body, bitsPer)
        minU := uint64(minDelta)
        v := uint64(first)
        for i, dv := range tmp {
                v += dv + minU
                out[i+1] = int64(v)
        }
        return nil
}

func (d *Decoder) readPackedRLEInt64SliceInto(dst *[]int64) error {
        n, err := d.readPackedRLEHeader(qpackKindInt64)
        if err != nil {
                return err
        }
        growI64(dst, n)
        out := *dst
        idx := 0
        for idx < n {
                v64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                runLen, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if runLen == 0 || runLen > uint64(n-idx) {
                        return ErrInvalidLength
                }
                v := zigzagDecode64(v64)
                end := idx + int(runLen)
                out[idx] = v
                for size := 1; idx+size < end; size <<= 1 {
                        copy(out[idx+size:end], out[idx:idx+size])
                }
                idx = end
        }
        return nil
}

func (d *Decoder) readPackedDictInt64SliceInto(dst *[]int64) error {
        count, bitsPer, err := d.readPackedDictHeader(qpackKindInt64)
        if err != nil {
                return err
        }
        var table [qpackDictMaxDistinct]int64
        for i := range count {
                v, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                table[i] = zigzagDecode64(v)
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: int(n64) would wrap negative
                return ErrInvalidLength
        }
        if bitsPer == 0 && n64 > qpackMaxStandaloneCount {
                // count == 1 ⇒ bitsPer == 0: empty index body, no per-element bound.
                // Cap before grow (matches the non-Into sibling readPackedDictInt64Slice).
                return ErrInvalidLength
        }
        n := int(n64)
        growI64(dst, n)
        out := *dst
        if bitsPer == 0 {
                v := table[0]
                for i := range out {
                        out[i] = v
                }
                return nil
        }
        rem := uint64(len(d.buf) - d.i)
        if n64 > rem*8/uint64(bitsPer) {
                return ErrShortBuffer
        }
        bodyBytes := (n*bitsPer + 7) >> 3
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        // Reuse the shared transient unpack scratch (mirrors the non-Into sibling
        // readPackedDictInt64Slice); idx is fully written then mapped into out.
        if cap(d.deltaScratch) < n {
                d.deltaScratch = make([]uint64, n)
        }
        idx := d.deltaScratch[:n]
        bitpack.Unpack(idx, body, bitsPer)
        for i, k := range idx {
                if k >= uint64(count) {
                        return ErrBadTag
                }
                out[i] = table[k]
        }
        return nil
}

func (d *Decoder) readPackedPForInt64SliceInto(dst *[]int64) error {
        if d.i+1 > len(d.buf) || d.buf[d.i] != qpackKindInt64 {
                return ErrTypeMismatch
        }
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return ErrInvalidLength
        }
        if d.i >= len(d.buf) {
                return ErrShortBuffer
        }
        b := int(d.buf[d.i])
        d.i++
        if b > qpackForMaxBits {
                return ErrBadTag
        }
        mz, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        mnU := uint64(zigzagDecode64(mz))
        rem := uint64(len(d.buf) - d.i)
        if b > 0 {
                if n64 > rem*8/uint64(b) {
                        return ErrShortBuffer
                }
        } else if n64 > qpackMaxStandaloneCount {
                // b == 0 (constant base): empty packed body, no per-element buffer bound.
                // colLenOK already caps n64 in a columnar context, but this Into helper is
                // reachable via generated code, so mirror the standalone reader's ceiling
                // defensively (no reliance on colMaxLen being set).
                return ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8/b lets n64 exceed MaxInt -> int(n64) wraps negative
                return ErrInvalidLength
        }
        n := int(n64)
        bodyBytes := (n*b + 7) >> 3
        if d.i+bodyBytes > len(d.buf) {
                return ErrShortBuffer
        }
        growI64(dst, n)
        out := *dst
        u := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(out))), n)
        if b == 0 {
                // All base values equal mnU. Must fully overwrite the reused scratch
                // buffer here — bitpack.Unpack is skipped, so nothing else writes u[:n].
                for k := range u {
                        u[k] = mnU
                }
        } else {
                bitpack.Unpack(u, d.buf[d.i:d.i+bodyBytes], b)
                if mnU != 0 {
                        for k := range u {
                                u[k] += mnU
                        }
                }
        }
        d.i += bodyBytes
        excN64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if excN64 > n64 {
                return ErrInvalidLength
        }
        pos := 0
        for range excN64 {
                dp, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                delta, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                pos += int(dp)
                if pos < 0 || pos >= n {
                        return ErrInvalidLength
                }
                out[pos] = int64(mnU + delta)
        }
        return nil
}

// decodeSliceInt64Into is the scratch-aware equivalent of decodeSliceInt64.
// On return *dst holds the decoded values; the backing array is reused
// across calls when cap is sufficient (steady-state: zero allocation).
func decodeSliceInt64Into(d *Decoder, dst *[]int64) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch t {
        case tagPackRaw:
                d.i++
                return d.readPackedInt64SliceInto(dst)
        case tagPackFor:
                d.i++
                return d.readPackedForInt64SliceInto(dst)
        case tagPackDeltaFor:
                d.i++
                return d.readPackedDeltaForInt64SliceInto(dst)
        case tagPackRLE:
                d.i++
                return d.readPackedRLEInt64SliceInto(dst)
        case tagPackDict:
                d.i++
                return d.readPackedDictInt64SliceInto(dst)
        case tagPackPFor:
                d.i++
                return d.readPackedPForInt64SliceInto(dst)
        case tagPackBlock:
                d.i++
                return d.readBlockInt64Into(dst)
        case tagZoneChunk:
                d.i++
                return d.readZoneChunkInt64Into(dst)
        }
        // Fallback: element-by-element array decode.
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        growI64(dst, n)
        out := *dst
        for i := range n {
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        return nil
}

// ---- uint64 Into variants --------------------------------------------------

func (d *Decoder) readPackedUint64SliceInto(dst *[]uint64) error {
        n, body, err := d.readPackedRawHeader(qpackKindUint64)
        if err != nil {
                return err
        }
        growU64(dst, n)
        if n == 0 {
                return nil
        }
        out := *dst
        if endian.NativeIsLittle {
                bs := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*8)
                copy(bs, body)
                return nil
        }
        for i := range n {
                out[i] = readU64(body[i*8:])
        }
        return nil
}

func (d *Decoder) readPackedForUint64SliceInto(dst *[]uint64) error {
        bitsPer, mn, _, n, body, err := d.readPackedForHeader(qpackKindUint64)
        if err != nil {
                return err
        }
        growU64(dst, n)
        out := *dst
        if bitsPer == 0 {
                for i := range out {
                        out[i] = mn
                }
                return nil
        }
        bitpack.Unpack(out, body, bitsPer)
        if mn != 0 {
                for i := range out {
                        out[i] += mn
                }
        }
        return nil
}

func (d *Decoder) readPackedDeltaForUint64SliceInto(dst *[]uint64) error {
        bitsPer, first, _, minDelta, n, body, err := d.readPackedDeltaForHeader(qpackKindUint64)
        if err != nil {
                return err
        }
        growU64(dst, n)
        out := *dst
        if n == 0 {
                return nil
        }
        out[0] = first
        if n == 1 {
                return nil
        }
        if bitsPer == 0 {
                step := uint64(minDelta)
                v := first
                for i := 1; i < n; i++ {
                        v += step
                        out[i] = v
                }
                return nil
        }
        if cap(d.deltaScratch) < n-1 {
                d.deltaScratch = make([]uint64, n-1)
        }
        tmp := d.deltaScratch[:n-1]
        bitpack.Unpack(tmp, body, bitsPer)
        minU := uint64(minDelta)
        v := first
        for i, dv := range tmp {
                v += dv + minU
                out[i+1] = v
        }
        return nil
}

func (d *Decoder) readPackedRLEUint64SliceInto(dst *[]uint64) error {
        n, err := d.readPackedRLEHeader(qpackKindUint64)
        if err != nil {
                return err
        }
        growU64(dst, n)
        out := *dst
        idx := 0
        for idx < n {
                v, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                runLen, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if runLen == 0 || runLen > uint64(n-idx) {
                        return ErrInvalidLength
                }
                end := idx + int(runLen)
                out[idx] = v
                for size := 1; idx+size < end; size <<= 1 {
                        copy(out[idx+size:end], out[idx:idx+size])
                }
                idx = end
        }
        return nil
}

func (d *Decoder) readPackedDictUint64SliceInto(dst *[]uint64) error {
        count, bitsPer, err := d.readPackedDictHeader(qpackKindUint64)
        if err != nil {
                return err
        }
        var table [qpackDictMaxDistinct]uint64
        for i := range count {
                v, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                table[i] = v
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: int(n64) would wrap negative
                return ErrInvalidLength
        }
        if bitsPer == 0 && n64 > qpackMaxStandaloneCount {
                // count == 1 ⇒ bitsPer == 0: empty index body, no per-element bound.
                // Cap before grow (matches the non-Into sibling readPackedDictUint64Slice).
                return ErrInvalidLength
        }
        n := int(n64)
        growU64(dst, n)
        out := *dst
        if bitsPer == 0 {
                v := table[0]
                for i := range out {
                        out[i] = v
                }
                return nil
        }
        rem := uint64(len(d.buf) - d.i)
        if n64 > rem*8/uint64(bitsPer) {
                return ErrShortBuffer
        }
        bodyBytes := (n*bitsPer + 7) >> 3
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        // Reuse the shared transient unpack scratch (mirrors the non-Into sibling
        // readPackedDictUint64Slice); idx is fully written then mapped into out.
        if cap(d.deltaScratch) < n {
                d.deltaScratch = make([]uint64, n)
        }
        idx := d.deltaScratch[:n]
        bitpack.Unpack(idx, body, bitsPer)
        for i, k := range idx {
                if k >= uint64(count) {
                        return ErrBadTag
                }
                out[i] = table[k]
        }
        return nil
}

func (d *Decoder) readPackedPForUint64SliceInto(dst *[]uint64) error {
        if d.i+1 > len(d.buf) {
                return ErrShortBuffer
        }
        if d.buf[d.i] != qpackKindUint64 {
                return ErrTypeMismatch
        }
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return ErrInvalidLength
        }
        if d.i >= len(d.buf) {
                return ErrShortBuffer
        }
        b := int(d.buf[d.i])
        d.i++
        if b > qpackForMaxBits {
                return ErrBadTag
        }
        mn, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        rem := uint64(len(d.buf) - d.i)
        if b > 0 {
                if n64 > rem*8/uint64(b) {
                        return ErrShortBuffer
                }
        } else if n64 > qpackMaxStandaloneCount {
                // b == 0 constant base: mirror the standalone reader's ceiling defensively
                // (this Into helper is reachable via generated code without colMaxLen set).
                return ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8/b lets n64 exceed MaxInt -> int(n64) wraps negative
                return ErrInvalidLength
        }
        n := int(n64)
        bodyBytes := (n*b + 7) >> 3
        if d.i+bodyBytes > len(d.buf) {
                return ErrShortBuffer
        }
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        growU64(dst, n)
        out := *dst
        if b == 0 {
                // All base values equal mn. Must fully overwrite the reused scratch
                // buffer here — bitpack.Unpack is skipped, so nothing else writes out[:n].
                for k := range out {
                        out[k] = mn
                }
        } else {
                bitpack.Unpack(out, body, b)
                if mn != 0 {
                        for k := range out {
                                out[k] += mn
                        }
                }
        }
        excN64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if excN64 > n64 {
                return ErrInvalidLength
        }
        pos := 0
        for range excN64 {
                dp, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                delta, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                pos += int(dp)
                if pos < 0 || pos >= n {
                        return ErrInvalidLength
                }
                out[pos] = mn + delta
        }
        return nil
}

// decodeSliceUint64Into is the scratch-aware equivalent of decodeSliceUint64.
func decodeSliceUint64Into(d *Decoder, dst *[]uint64) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch t {
        case tagPackRaw:
                d.i++
                return d.readPackedUint64SliceInto(dst)
        case tagPackFor:
                d.i++
                return d.readPackedForUint64SliceInto(dst)
        case tagPackDeltaFor:
                d.i++
                return d.readPackedDeltaForUint64SliceInto(dst)
        case tagPackRLE:
                d.i++
                return d.readPackedRLEUint64SliceInto(dst)
        case tagPackDict:
                d.i++
                return d.readPackedDictUint64SliceInto(dst)
        case tagPackPFor:
                d.i++
                return d.readPackedPForUint64SliceInto(dst)
        case tagPackBlock:
                d.i++
                return d.readBlockUint64Into(dst)
        case tagZoneChunk:
                d.i++
                return d.readZoneChunkUint64Into(dst)
        }
        // Fallback: element-by-element array decode.
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        growU64(dst, n)
        out := *dst
        for i := range n {
                v, err := d.ReadUint()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        return nil
}

// ---- float64 Into variant --------------------------------------------------

// decodeSliceFloat64Into is the scratch-aware equivalent of decodeSliceFloat64.
// Gorilla and ALP are streaming decoders without an easy in-place form;
// they write into a freshly grown slice (still benefits if cap is sufficient).
func decodeSliceFloat64Into(d *Decoder, dst *[]float64) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagColVecLossy {
                vecs, elemF32, used, err2 := readLossyVec(d.buf[d.i:])
                if err2 != nil {
                        return err2
                }
                // A scalar float64 column is one vector (the whole column); reject a
                // multi-vector block rather than silently dropping vecs[1:].
                if len(vecs) != 1 {
                        return ErrInvalidLength
                }
                if elemF32 {
                        return ErrTypeMismatch
                }
                d.i += used
                *dst = vecs[0]
                return nil
        }
        if t == tagZoneChunk {
                d.i++
                return d.readZoneChunkFloat64Into(dst)
        }
        if t == tagPackRaw {
                d.i++
                n, body, err2 := d.readPackedRawHeader(qpackKindFloat64)
                if err2 != nil {
                        return err2
                }
                growF64(dst, n)
                if n == 0 {
                        return nil
                }
                out := *dst
                if endian.NativeIsLittle {
                        bs := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*8)
                        copy(bs, body)
                        return nil
                }
                for i := range n {
                        out[i] = math.Float64frombits(readU64(body[i*8:]))
                }
                return nil
        }
        if t == tagPackGorilla {
                d.i++
                v, err2 := d.readPackedGorillaFloat64Slice()
                if err2 != nil {
                        return err2
                }
                *dst = v
                return nil
        }
        if t == tagPackALP {
                d.i++
                v, err2 := d.readPackedALPFloat64Slice()
                if err2 != nil {
                        return err2
                }
                *dst = v
                return nil
        }
        // Fallback: element-by-element array decode.
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        growF64(dst, n)
        out := *dst
        for i := range n {
                v, err := d.ReadFloat64()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        return nil
}

// ---- bool Into variant -----------------------------------------------------

// decodeSliceBoolInto is the scratch-aware equivalent of decodeSliceBool.
func decodeSliceBoolInto(d *Decoder, dst *[]bool) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagPackBool {
                d.i++
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                // Bound the claimed element count by the columnar row count (colMaxLen)
                // before allocating, mirroring every other columnar codec reader. The
                // body-byte bound below alone would still admit up to 8× the remaining
                // buffer in bool scratch (8 bits/byte); colLenOK ties it to the actual
                // row count so a bool column claiming n >> rows is rejected, not allocated.
                if !d.colLenOK(n64) {
                        return ErrInvalidLength
                }
                if n64 > uint64(len(d.buf)-d.i)*8 {
                        return ErrShortBuffer
                }
                if n64 > uint64(math.MaxInt) { // 32-bit: rem*8 lets n64 exceed MaxInt -> int(n64) wraps negative
                        return ErrInvalidLength
                }
                n := int(n64)
                nBytes := (n + 7) >> 3
                if d.i+nBytes > len(d.buf) {
                        return ErrShortBuffer
                }
                growBool(dst, n)
                out := *dst
                base := d.i
                i := 0
                for ; i+8 <= n; i += 8 {
                        b := d.buf[base+(i>>3)]
                        out[i] = b&(1<<0) != 0
                        out[i+1] = b&(1<<1) != 0
                        out[i+2] = b&(1<<2) != 0
                        out[i+3] = b&(1<<3) != 0
                        out[i+4] = b&(1<<4) != 0
                        out[i+5] = b&(1<<5) != 0
                        out[i+6] = b&(1<<6) != 0
                        out[i+7] = b&(1<<7) != 0
                }
                for ; i < n; i++ {
                        out[i] = d.buf[base+(i>>3)]&(1<<uint(i&7)) != 0
                }
                d.i += nBytes
                return nil
        }
        // Fallback: element-by-element array decode.
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        growBool(dst, n)
        out := *dst
        for i := range n {
                v, err := d.ReadBool()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        return nil
}

package qdf

import "slices"

// wantedColumns maps each WIRE column index to the target plan column to
// decode it into, or nil to skip. Matched by field name. Wire columns whose
// name is absent from the target are skipped; target fields absent from the
// wire are left zero.
func wantedColumns(plan *columnarPlan, names []string) []*colColumn {
        want := make([]*colColumn, len(names))
        for wi, wn := range names {
                for ti := range plan.cols {
                        if plan.cols[ti].name == wn {
                                want[wi] = &plan.cols[ti]
                                break
                        }
                }
        }
        return want
}

// UnmarshalColumns decodes only the named columns of a columnar []struct
// payload into out, leaving the rest undecoded. out may point at a typed
// subset slice (e.g. *[]Subset whose fields are the wanted columns, matched
// by name like Unmarshal) or at the dynamic *[]map[string]any form, where
// only the named columns are stored. When the payload carries the
// column-length index (OptColumnIndex), unrequested columns are skipped
// without decoding; otherwise they are decoded but dropped.
//
// fields names the wire columns to keep. With no fields it behaves like
// Unmarshal.
func UnmarshalColumns(data []byte, out any, fields ...string) error {
        return unmarshal(data, out, fields, false, nil)
}

// wantField reports whether name is in the active selectFields filter. A nil
// filter wants every column (no filtering).
func (d *Decoder) wantField(name string) bool {
        if d.selectFields == nil {
                return true
        }
        return slices.Contains(d.selectFields, name)
}

package qdf

import (
        "math"
        "reflect"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/intern"
        "github.com/alex60217101990/qdf/internal/rans"
)

// Decoder reads QDF wire data from a single input buffer. Call SetInput
// to bind a buffer and the typed Read* methods to walk it. Behaviour is
// undefined if the input is mutated while the decoder holds it.
// Field order groups the pointer-bearing and 8-byte fields first, then the
// int counters, then the 1-byte flags last so the interspersed bools do not
// each force their own padding word (176 bytes vs 208 for the source order).
type Decoder struct {
        buf   []byte
        state *decState

        // arena, when non-nil, receives copied inline string bodies (bump-packed)
        // instead of one heap allocation per string. Caller-owned (see Arena); the
        // decoder only borrows the pointer for one decode and clears it on return
        // to the pool so a pooled decoder never pins a caller's arena. Ignored when
        // noCopy is set (aliasing already avoids the copy).
        arena *Arena

        // keyCache dedupes map keys and other short repeated strings across
        // Unmarshal calls on the same pooled decoder.
        keyCache intern.Cache

        // selectFields, when non-nil, restricts the columnar map (any) decode to
        // the named columns: unrequested columns are skipped via the column-length
        // index when present, or simply not stored when it is absent. Set by
        // UnmarshalColumns for the duration of one decode; must be cleared on
        // reset / return-to-pool / SetInput so it never leaks across decodes.
        selectFields []string

        // query, when non-nil, makes a columnar decode filter rows by the plan's
        // predicates (AND) and project the plan's columns. Set by Unmarshal when
        // QueryOptions are passed; cleared on reset / SetInput so it never leaks.
        query *queryPlan

        // mapFreeList holds maps harvested from a reused []struct{map} (or []map)
        // decode target whose per-element maps decode-slice-reuse is about to zero.
        // Keyed by the map's reflect.Type; reuseOrMakeMap pops a recycled map
        // instead of allocating a fresh one. Lazily initialised; cleared (entries
        // dropped, backing kept) on SetInput so recycled maps never cross into a
        // different decode target.
        mapFreeList map[reflect.Type][]unsafe.Pointer

        // keyIdx is a reused base-key→index map for keyed slice apply. It is cleared
        // (entries dropped, backing kept) and rebuilt per keyed slice; the spike-sized
        // backing is dropped on the apply reset path so a one-off huge slice never
        // pins a large map across pooled reuse.
        keyIdx map[string]int

        // deltaScratch is a reused unpack buffer for the Delta+FOR readers: the
        // bit-unpacked deltas are a transient intermediate (the prefix sum writes
        // the retained out slice), so a per-call make is pure garbage. Grows to the
        // largest column seen; bounded on return to pool.
        deltaScratch []uint64

        i int

        // depth / maxDepth bound recursive decode nesting. The wire dictates
        // nesting for `any`, recursive pointer/slice/map types, so without a
        // guard a crafted deeply-nested payload overflows the goroutine stack —
        // an UNRECOVERABLE fatal error, i.e. a remote DoS. descend/ascend
        // (defer-balanced) cap it at maxDepth (lazily DefaultMaxDepth), returning
        // ErrCycleDetected, symmetric to the encoder's pointer-cycle guard.
        depth    int
        maxDepth int

        // colMaxLen bounds a slice codec's claimed element count while decoding
        // a columnar column, where every column must hold exactly the struct
        // count M. 0 means unbounded (standalone slice decode). It blocks a
        // hostile column whose constant/zero-width codec claims a huge n from a
        // tiny body before the per-element allocation.
        colMaxLen int

        mode Mode

        // noCopy returns aliased string / []byte values instead of copies.
        // Faster, but the caller may not retain the result past the input's
        // lifetime.
        noCopy bool

        // colStrNoPool forces colStrScratch to return a fresh owned []string instead
        // of the reused decode-state scratch. Set only around the nullable string
        // column read, which stores &strs[k] pointers into *string fields — those
        // must reference a stable backing array, not the buffer the next column reuses.
        colStrNoPool bool

        headerRead bool

        // colIndex records whether the header set FlagColIndex. When true a
        // columnar (tagColStruct) payload carries a column-length index right
        // after the shape declaration; decodeColumnar / decodeColumnarAny consume
        // and validate it. Set fresh by readHeader on every decode.
        colIndex bool

        // keyIdxBusy marks keyIdx as borrowed by an in-progress keyed-slice apply so a
        // nested keyed slice routes to a fresh local map instead of clobbering it.
        keyIdxBusy bool
}

// colLenOK reports whether a slice length is acceptable in the current
// decode context. Outside a columnar column (colMaxLen == 0) any length is
// allowed; the row-major and standalone paths bound length by the buffer.
func (d *Decoder) colLenOK(n uint64) bool {
        return d.colMaxLen == 0 || n <= uint64(d.colMaxLen)
}

// InternKey returns a string equal to b, sharing storage with prior
// identical keys when the cache has them.
//
//go:nosplit
func (d *Decoder) InternKey(b []byte) string { return d.keyCache.Make(b) }

// NewDecoder constructs a Decoder. Bind input via SetInput.
func NewDecoder() *Decoder { return &Decoder{} }

// NewDecoderOnBuf binds a Decoder to buf. Equivalent to NewDecoder followed
// by SetInput(buf), without the SetInput branch checks.
func NewDecoderOnBuf(buf []byte) *Decoder {
        return &Decoder{buf: buf}
}

// ReadStringBytes returns the next string or []byte value without copying.
// The returned slice aliases either the input buffer or the decoder's
// state-table storage. Callers that retain it beyond the input's lifetime
// must copy.
func (d *Decoder) ReadStringBytes() ([]byte, error) { return d.readStringBytes() }

// PeekTag returns the next tag byte without advancing the cursor.
func (d *Decoder) PeekTag() (byte, error) { return d.peekTag() }

// ReadStructHeader reads a struct/map header for code-generated DecodeQDF,
// transparently handling both forms a struct takes on the wire:
//   - a shape-interned header (tagMapShape, see Encoder.StructShape) → returns the
//     field names in encoded order with shaped=true; the caller reads len(names)
//     values in that order.
//   - a plain map header (tagMap8/16/32) → returns the entry count as plainN with
//     shaped=false; the caller reads plainN (name, value) pairs inline.
//
// This lets a generated decoder consume both qdfgen shape output and a plain qdf
// map (e.g. from a non-shaped encoder, an older generated type, or the reflect
// path under OptSpeed) without a wire-format negotiation. Exported for
// cmd/qdfgen-generated code.
func (d *Decoder) ReadStructHeader() (names []string, plainN int, shaped bool, err error) {
        tag, err := d.peekTag()
        if err != nil {
                return nil, 0, false, err
        }
        if tag == tagMapShape {
                names, err = decodeMapStringShapeHeader(d)
                return names, 0, true, err
        }
        n, err := d.ReadMapHeader()
        return nil, n, false, err
}

// DecodeValue decodes the next value as a dynamic any — the schemaless form
// (string, bool, int64/uint64/float64, []any, map[string]any, …) that decoding
// into an interface{} produces, mirroring encoding/json. It is the decode
// counterpart of Encoder.EncodeValue and is what qdfgen-generated code calls for
// an interface{} (any) struct field, so a code-generated type can still carry
// fully dynamic data on that field.
func (d *Decoder) DecodeValue(out *any) error {
        v, err := decodeAny(d)
        if err != nil {
                return err
        }
        *out = v
        return nil
}

// IsNil reports whether the next value is the nil tag, consuming it on
// true. Returns (false, nil) for any other tag.
func (d *Decoder) IsNil() (bool, error) {
        t, err := d.peekTag()
        if err != nil {
                return false, err
        }
        if t == tagNil {
                d.i++
                return true, nil
        }
        return false, nil
}

// TagNil exposes the nil-value tag for comparison with PeekTag.
const TagNil = tagNil

// RemainingBytes returns the unread portion of the input buffer. The
// result aliases the buffer and is invalidated by the next SetInput.
func (d *Decoder) RemainingBytes() []byte { return d.buf[d.i:] }

// Advance moves the read cursor forward by n bytes.
func (d *Decoder) Advance(n int) { d.i += n }

// MarkHeaderRead tells the decoder the magic+version header is already
// consumed (e.g. the buffer is a tail slice from a parent decoder). The
// next read will skip the header check.
func (d *Decoder) MarkHeaderRead() { d.headerRead = true }

// descend enters one level of recursive decode, bounding nesting depth so a
// hostile deeply-nested payload cannot overflow the goroutine stack (an
// unrecoverable fatal error). Pair with a deferred ascend. maxDepth is lazily
// initialised so every Decoder construction path is covered.
func (d *Decoder) descend() error {
        if d.maxDepth == 0 {
                d.maxDepth = DefaultMaxDepth
        }
        d.depth++
        if d.depth > d.maxDepth {
                return ErrCycleDetected
        }
        return nil
}

func (d *Decoder) ascend() { d.depth-- }

// SetInput rebinds the decoder to buf, dropping any prior state table.
func (d *Decoder) SetInput(buf []byte) {
        d.buf = buf
        d.i = 0
        d.depth = 0
        d.headerRead = false
        d.mode = Fast
        d.colIndex = false
        d.colMaxLen = 0
        d.selectFields = nil
        d.query = nil
        clear(d.mapFreeList) // drop recycled maps; keep the backing allocated
        if d.state != nil {
                d.state.reset()
        }
}

// SetNoCopy switches the decoder into aliasing mode: string and []byte
// reads return slices that share storage with the input buffer. Faster
// (~2x on string-heavy payloads) with near-zero allocations.
//
// The aliases are valid only while the input is alive and unmodified. This is
// safe — and free — when the input is caller-owned and not reused: an mmap, a
// file fully read into memory, or a buffer allocated fresh per message and
// never pooled (the aliasing headers keep it alive via the GC, so you need not
// track its lifetime). The hazard is buffer REUSE or mutation (a sync.Pool
// buffer, an overwritten scratch slice), which silently corrupts already-decoded
// values — a manual use-after-free the race detector will not catch. See
// WithNoCopy for the full contract.
func (d *Decoder) SetNoCopy(v bool) { d.noCopy = v }

// SetArena directs the decoder to pack copied inline string bodies into a,
// bump-packed, instead of allocating one string per field. Pass nil to disable.
// The decoded strings alias a's memory; see Arena for the lifetime contract.
// Ignored while noCopy is set. Not concurrent-safe: one Arena per goroutine.
func (d *Decoder) SetArena(a *Arena) { d.arena = a }

// Pos returns the current read offset.
func (d *Decoder) Pos() int { return d.i }

// Remaining returns the number of unread bytes.
func (d *Decoder) Remaining() int { return len(d.buf) - d.i }

// CheckLength returns ErrShortBuffer when n claimed elements cannot fit
// in the remaining input. Use before allocating per-element storage.
//
//go:nosplit
func (d *Decoder) CheckLength(n int, perElem int) error {
        if n < 0 {
                return ErrInvalidLength
        }
        if uint64(n)*uint64(perElem) > uint64(len(d.buf)-d.i) {
                return ErrShortBuffer
        }
        return nil
}

// readHeader consumes the 5-byte header once per buffer. The hot path is the
// already-read check; the actual parse is outlined into readHeaderSlow so this
// (and its callers, peekTag/next) stay within the inliner budget.
func (d *Decoder) readHeader() error {
        if d.headerRead {
                return nil
        }
        return d.readHeaderSlow()
}

// readHeaderSlow parses the 5-byte header (and an optional rANS body). Kept out
// of line: it runs once per decode, so inlining its cost into the per-tag hot
// path would only bloat callers.
//
//go:noinline
func (d *Decoder) readHeaderSlow() error {
        if len(d.buf)-d.i < 5 {
                return ErrShortBuffer
        }
        if d.buf[d.i] != Magic0 || d.buf[d.i+1] != Magic1 || d.buf[d.i+2] != Magic2 {
                return ErrBadMagic
        }
        if d.buf[d.i+3] != Version1 {
                return ErrBadVersion
        }
        flags := d.buf[d.i+4]
        d.colIndex = flags&FlagColIndex != 0
        if flags&FlagDense != 0 {
                d.mode = Dense
                if d.state == nil {
                        d.state = newDecState()
                }
        }
        d.i += 5
        d.headerRead = true
        if flags&FlagRANS != 0 {
                rest := d.buf[d.i:]
                origLen, k := readUvarint(rest)
                if k <= 0 {
                        return ErrInvalidLength
                }
                // Bound the allocation: reject a hostile origLen that dwarfs the
                // input. rANS shrinks at best modestly, so a real origLen stays
                // within a small factor of the compressed size. The second clause
                // rejects anything past the int range so the int(origLen) narrowing
                // below cannot truncate a multi-GiB length on a 32-bit build and decode
                // a silently short/wrong body.
                if origLen > uint64(len(d.buf))*64+(1<<20) || origLen > uint64(math.MaxInt) {
                        return ErrInvalidLength
                }
                body, err := rans.Decode(rest[k:], int(origLen))
                if err != nil {
                        return err
                }
                // The reconstructed body is a plain (post-decompression) tag stream;
                // read it from offset 0, exactly as a non-rANS buffer's body.
                d.buf = body
                d.i = 0
        }
        return nil
}

// peekTag returns the next tag without consuming it. The header check is
// inlined here so the common (header-already-read) path has no call; the parse
// is taken only on the first tag of a decode.
func (d *Decoder) peekTag() (byte, error) {
        if !d.headerRead {
                if err := d.readHeaderSlow(); err != nil {
                        return 0, err
                }
        }
        if d.i >= len(d.buf) {
                return 0, ErrShortBuffer
        }
        return d.buf[d.i], nil
}

// next returns the next tag and advances past it.
func (d *Decoder) next() (byte, error) {
        t, err := d.peekTag()
        if err != nil {
                return 0, err
        }
        d.i++
        return t, nil
}

package qdf

import (
        "math"

        "github.com/alex60217101990/qdf/internal/unsafestr"
)

// ----- primitives -----

func (d *Decoder) ReadNil() error {
        t, err := d.next()
        if err != nil {
                return err
        }
        if t != tagNil {
                return ErrTypeMismatch
        }
        return nil
}

func (d *Decoder) ReadBool() (bool, error) {
        t, err := d.next()
        if err != nil {
                return false, err
        }
        switch t {
        case tagTrue:
                return true, nil
        case tagFalse:
                return false, nil
        default:
                return false, ErrTypeMismatch
        }
}

// ReadUint reads a value that was encoded with WriteUint or WriteInt with a
// non-negative argument. Returns an error if the next value is negative.
func (d *Decoder) ReadUint() (uint64, error) {
        t, err := d.next()
        if err != nil {
                return 0, err
        }
        return d.decodeUint(t)
}

func (d *Decoder) decodeUint(t byte) (uint64, error) {
        if t <= tagFixintMax {
                return uint64(t), nil
        }
        switch t {
        case tagUint8:
                if d.i >= len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := uint64(d.buf[d.i])
                d.i++
                return v, nil
        case tagUint16:
                if d.i+2 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := uint64(readU16(d.buf[d.i:]))
                d.i += 2
                return v, nil
        case tagUint32:
                if d.i+4 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := uint64(readU32(d.buf[d.i:]))
                d.i += 4
                return v, nil
        case tagUint64:
                if d.i+8 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := readU64(d.buf[d.i:])
                d.i += 8
                return v, nil
        }
        return 0, ErrTypeMismatch
}

func (d *Decoder) ReadInt() (int64, error) {
        t, err := d.next()
        if err != nil {
                return 0, err
        }
        return d.decodeInt(t)
}

func (d *Decoder) decodeInt(t byte) (int64, error) {
        // fixint positive
        if t <= tagFixintMax {
                return int64(t), nil
        }
        // negfixint range 0xD8..0xDF
        if t >= tagNegfixint && t <= tagNegfixint|tagNegfixintMask {
                return -int64(t&tagNegfixintMask) - 1, nil
        }
        switch t {
        case tagInt8:
                if d.i >= len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := int64(int8(d.buf[d.i]))
                d.i++
                return v, nil
        case tagInt16:
                if d.i+2 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := int64(int16(readU16(d.buf[d.i:])))
                d.i += 2
                return v, nil
        case tagInt32:
                if d.i+4 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := int64(int32(readU32(d.buf[d.i:])))
                d.i += 4
                return v, nil
        case tagInt64:
                if d.i+8 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := int64(readU64(d.buf[d.i:]))
                d.i += 8
                return v, nil
        case tagUint8:
                if d.i >= len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := uint64(d.buf[d.i])
                d.i++
                return int64(v), nil
        case tagUint16:
                if d.i+2 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := uint64(readU16(d.buf[d.i:]))
                d.i += 2
                return int64(v), nil
        case tagUint32:
                if d.i+4 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := uint64(readU32(d.buf[d.i:]))
                d.i += 4
                return int64(v), nil
        case tagUint64:
                if d.i+8 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := readU64(d.buf[d.i:])
                d.i += 8
                if v > math.MaxInt64 {
                        return 0, ErrTypeMismatch
                }
                return int64(v), nil
        }
        return 0, ErrTypeMismatch
}

func (d *Decoder) ReadFloat32() (float32, error) {
        t, err := d.next()
        if err != nil {
                return 0, err
        }
        if t != tagFloat32 {
                return 0, ErrTypeMismatch
        }
        if d.i+4 > len(d.buf) {
                return 0, ErrShortBuffer
        }
        v := math.Float32frombits(readU32(d.buf[d.i:]))
        d.i += 4
        return v, nil
}

func (d *Decoder) ReadFloat64() (float64, error) {
        t, err := d.next()
        if err != nil {
                return 0, err
        }
        switch t {
        case tagFloat64:
                if d.i+8 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := math.Float64frombits(readU64(d.buf[d.i:]))
                d.i += 8
                return v, nil
        case tagFloat32:
                if d.i+4 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                v := float64(math.Float32frombits(readU32(d.buf[d.i:])))
                d.i += 4
                return v, nil
        }
        return 0, ErrTypeMismatch
}

// ReadString returns the next string. If the decoder is in noCopy mode the
// returned string aliases the input buffer; otherwise a copy is made.
//
// When the tag at the cursor resolves through the intern table —
// tagInternStr, tagStateRef, tagStateMTF, tagStatePair, tagStateRepeat —
// readStringBytes leaves d.state.lastID set to the entry it just
// touched. We return the pre-materialised d.state.stringValues[id]
// from that path instead of allocating a fresh `string(b)` copy on
// every state-ref hit. Inline reads (fixstr, str8/16/32) set
// lastID = lruInvalidID, fall through, and pay the copy as before.
func (d *Decoder) ReadString() (string, error) {
        b, err := d.readStringBytes()
        if err != nil {
                return "", err
        }
        if d.noCopy {
                return unsafestr.String(b), nil
        }
        if d.state != nil && d.state.lastID != lruInvalidID {
                if s, ok := d.state.getString(d.state.lastID, d.arena); ok {
                        return s, nil
                }
        }
        if d.arena != nil && len(b) > 0 {
                return d.arena.appendStr(b), nil
        }
        return string(b), nil
}

// materializeStr turns string bytes that ALIAS the input buffer into a string,
// honoring the decoder's zero-copy / arena mode exactly like ReadString. Use it
// at every columnar string-column materialization site whose bytes are a slice
// of d.buf (const / per-value / dict-table / dictQ-table) so columnar decode
// reaches the same zero-copy / arena parity as the row-major path — otherwise
// each value pays an owned string(b) copy. The argument MUST alias d.buf (never
// a reused scratch buffer): under noCopy the result aliases it.
func (d *Decoder) materializeStr(b []byte) string {
        if d.noCopy {
                return unsafestr.String(b)
        }
        if d.arena != nil && len(b) > 0 {
                return d.arena.appendStr(b)
        }
        return string(b)
}

// readStringBytes returns the raw bytes of a string/bin value without
// allocating; the returned slice aliases the input buffer or the decoder
// state table.
func (d *Decoder) readStringBytes() ([]byte, error) {
        t, err := d.next()
        if err != nil {
                return nil, err
        }
        // Inline (non-intern) string / bin reads break the Markov-0 chain so
        // a subsequent tagStateRepeat cannot resurrect a stale state-ref
        // from before the inline emission. State-ref and intern branches
        // below restore the invariant explicitly.
        invalidateLast := d.state != nil
        // fixstr
        if t >= tagFixstr && t <= tagFixstr|tagFixstrMask {
                n := int(t & tagFixstrMask)
                if d.i+n > len(d.buf) {
                        return nil, ErrShortBuffer
                }
                out := d.buf[d.i : d.i+n]
                d.i += n
                if invalidateLast {
                        d.state.lastID = lruInvalidID
                }
                return out, nil
        }
        switch t {
        case tagStr8, tagBin8:
                if d.i >= len(d.buf) {
                        return nil, ErrShortBuffer
                }
                n := int(d.buf[d.i])
                d.i++
                if d.i+n > len(d.buf) {
                        return nil, ErrShortBuffer
                }
                out := d.buf[d.i : d.i+n]
                d.i += n
                if invalidateLast {
                        d.state.lastID = lruInvalidID
                }
                return out, nil
        case tagStr16, tagBin16:
                if d.i+2 > len(d.buf) {
                        return nil, ErrShortBuffer
                }
                n := int(readU16(d.buf[d.i:]))
                d.i += 2
                if d.i+n > len(d.buf) {
                        return nil, ErrShortBuffer
                }
                out := d.buf[d.i : d.i+n]
                d.i += n
                if invalidateLast {
                        d.state.lastID = lruInvalidID
                }
                return out, nil
        case tagStr32, tagBin32:
                if d.i+4 > len(d.buf) {
                        return nil, ErrShortBuffer
                }
                n64 := uint64(readU32(d.buf[d.i:]))
                d.i += 4
                // Compare in uint64 before narrowing: on a 32-bit int a length near
                // 2^32 would become negative and slip past a `d.i+n > len` check, then
                // panic the slice with high < low. Mirrors the tagInternStr guard below.
                if n64 > uint64(len(d.buf)-d.i) {
                        return nil, ErrShortBuffer
                }
                n := int(n64)
                out := d.buf[d.i : d.i+n]
                d.i += n
                if invalidateLast {
                        d.state.lastID = lruInvalidID
                }
                return out, nil
        case tagInternStr, tagInternBin:
                // Read length-prefixed payload, then register it in the state table.
                n64, n := readUvarint(d.buf[d.i:])
                if n <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += n
                // Length validated in uint64 to avoid the int sign-bit pitfall on a
                // hostile varint. A 10-byte varint can encode values up to 2^64-1;
                // our buffer is always < 2^63, so the uint64 comparison is safe.
                if n64 > uint64(len(d.buf)-d.i) {
                        return nil, ErrShortBuffer
                }
                nn := int(n64)
                out := d.buf[d.i : d.i+nn]
                d.i += nn
                if d.state == nil {
                        d.state = newDecState()
                }
                // A conforming encoder caps the intern table at maxInternEntries so ids
                // stay below the 0xFFFF MRU/LRU sentinel; a hostile stream claiming more
                // records would assign id 0xFFFF and corrupt the side-cache chains.
                // Reject it (mirrors the encoder-side cap).
                if len(d.state.values) >= maxInternEntries {
                        return nil, ErrInvalidLength
                }
                // Register the bytes as-is; they alias the input. If the caller
                // later turns this into a copy, the table still references the alias
                // which is fine for the lifetime of the buffer.
                id := d.state.append(out)
                if d.state.lastID != lruInvalidID {
                        d.state.pairRecord(d.state.lastID, id)
                }
                d.state.lastID = id
                return out, nil
        case tagStateRef:
                id64, n := readUvarint(d.buf[d.i:])
                if n <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += n
                if d.state == nil {
                        return nil, ErrUnknownStateID
                }
                // Reject before the uint32 narrowing: a hostile 10-byte varint above
                // 2^32 would truncate to a small, possibly-valid id and silently
                // resolve the WRONG interned string (and desync the LRU/pair chains).
                if id64 > math.MaxUint32 {
                        return nil, ErrUnknownStateID
                }
                out, ok := d.state.get(uint32(id64))
                if !ok {
                        return nil, ErrUnknownStateID
                }
                // Mirror encoder's LRU update so a subsequent tagStateMTF
                // resolves to the same ID position.
                d.state.lruMoveToFront(uint32(id64))
                if d.state.lastID != lruInvalidID {
                        d.state.pairRecord(d.state.lastID, uint32(id64))
                }
                d.state.lastID = uint32(id64)
                return out, nil
        case tagStateMTF:
                rank64, n := readUvarint(d.buf[d.i:])
                if n <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += n
                if d.state == nil {
                        return nil, ErrUnknownStateID
                }
                // Reject before narrowing: a >2^32 varint would truncate to a small
                // rank and resolve a wrong entry instead of failing.
                if rank64 > math.MaxUint32 {
                        return nil, ErrUnknownStateID
                }
                // Side-cache lookup: the encoder emits tagStateMTF only when
                // rank fits in fewer varuint bytes than the raw id, which
                // caps rank at mruRingSize-1 for the common 2-byte id case.
                // The MRU ring resolves that range in O(1) without walking
                // the LRU chain. Older/larger ranks fall back to the chain
                // walk for forward-compatibility.
                rank := uint32(rank64)
                id, ok := d.state.mruIDAtRank(rank)
                if !ok {
                        id, ok = d.state.lruIDAtRank(rank)
                }
                if !ok {
                        return nil, ErrUnknownStateID
                }
                out, ok := d.state.get(id)
                if !ok {
                        return nil, ErrUnknownStateID
                }
                d.state.lruMoveToFront(id)
                if d.state.lastID != lruInvalidID {
                        d.state.pairRecord(d.state.lastID, id)
                }
                d.state.lastID = id
                return out, nil
        case tagStatePair:
                if d.state == nil || d.state.lastID == lruInvalidID {
                        return nil, ErrUnknownStateID
                }
                rank64, n := readUvarint(d.buf[d.i:])
                if n <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += n
                if rank64 >= pairPredK {
                        return nil, ErrUnknownStateID
                }
                prev := d.state.lastID
                id, ok := d.state.pairAtRank(prev, uint8(rank64))
                if !ok {
                        return nil, ErrUnknownStateID
                }
                out, ok := d.state.get(id)
                if !ok {
                        return nil, ErrUnknownStateID
                }
                // Same post-emit bookkeeping as tagStateRef so the encoder
                // and decoder mirror chains diverge nowhere.
                d.state.lruMoveToFront(id)
                d.state.pairRecord(prev, id)
                d.state.lastID = id
                return out, nil
        case tagStateRepeat:
                if d.state == nil || d.state.lastID == lruInvalidID {
                        return nil, ErrUnknownStateID
                }
                out, ok := d.state.get(d.state.lastID)
                if !ok {
                        return nil, ErrUnknownStateID
                }
                // Pair predictor: mirror encoder's self-record.
                d.state.pairRecord(d.state.lastID, d.state.lastID)
                return out, nil
        }
        return nil, ErrTypeMismatch
}

// ReadBytes returns a []byte value. With noCopy = true the result aliases
// the input.
func (d *Decoder) ReadBytes() ([]byte, error) {
        b, err := d.readStringBytes()
        if err != nil {
                return nil, err
        }
        if d.noCopy {
                return b, nil
        }
        out := make([]byte, len(b))
        copy(out, b)
        return out, nil
}

// ReadArrayHeader returns the element count.
func (d *Decoder) ReadArrayHeader() (int, error) {
        t, err := d.next()
        if err != nil {
                return 0, err
        }
        if t >= tagFixarr && t <= tagFixarr|tagFixarrMask {
                return int(t & tagFixarrMask), nil
        }
        switch t {
        case tagArr16:
                if d.i+2 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                n := int(readU16(d.buf[d.i:]))
                d.i += 2
                return n, nil
        case tagArr32:
                if d.i+4 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                // uint64 before narrowing: on a 32-bit int a count >= 2^31 wraps negative,
                // breaking the non-negative API contract and slipping past caller bounds
                // checks. Each element is >= 1 byte, so a count over the remaining bytes is
                // impossible. Mirrors the readStringBytes / decoder_skip tagArr32 guard.
                n64 := uint64(readU32(d.buf[d.i:]))
                d.i += 4
                if n64 > uint64(len(d.buf)-d.i) {
                        return 0, ErrShortBuffer
                }
                return int(n64), nil
        }
        return 0, ErrTypeMismatch
}

// ReadMapHeader returns the key/value pair count.
func (d *Decoder) ReadMapHeader() (int, error) {
        t, err := d.next()
        if err != nil {
                return 0, err
        }
        switch t {
        case tagMap8:
                if d.i >= len(d.buf) {
                        return 0, ErrShortBuffer
                }
                n := int(d.buf[d.i])
                d.i++
                return n, nil
        case tagMap16:
                if d.i+2 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                n := int(readU16(d.buf[d.i:]))
                d.i += 2
                return n, nil
        case tagMap32:
                if d.i+4 > len(d.buf) {
                        return 0, ErrShortBuffer
                }
                // Same 32-bit wrap guard as tagArr32; each entry is >= 2 bytes (key+value),
                // so a count over the remaining bytes is impossible.
                n64 := uint64(readU32(d.buf[d.i:]))
                d.i += 4
                if n64 > uint64(len(d.buf)-d.i) {
                        return 0, ErrShortBuffer
                }
                return int(n64), nil
        }
        return 0, ErrTypeMismatch
}

// ReadTimestamp reads a full-range timestamp and returns (sec, nsec, err).
// sec is seconds since Unix epoch (may be negative for pre-1970 instants).
// nsec is nanoseconds in [0, 999_999_999].
// This replaces the old fixed-8-byte UnixNano encoding (clean break).
func (d *Decoder) ReadTimestamp() (sec int64, nsec uint32, err error) {
        t, err := d.next()
        if err != nil {
                return 0, 0, err
        }
        if t != tagTimestamp {
                return 0, 0, ErrTypeMismatch
        }
        // Read zigzag-encoded seconds.
        secZ, n := readUvarint(d.buf[d.i:])
        if n <= 0 {
                return 0, 0, ErrInvalidLength
        }
        d.i += n
        // Read nanoseconds.
        nsecU, n := readUvarint(d.buf[d.i:])
        if n <= 0 {
                return 0, 0, ErrInvalidLength
        }
        d.i += n
        if nsecU > 999_999_999 {
                return 0, 0, ErrInvalidLength
        }
        return zigzagDecode64(secZ), uint32(nsecU), nil
}

package qdf

import "math"

// Skip advances past one value without materializing it.
//
// Skip recurses through nested arrays and maps, so it bounds nesting depth via
// descend/ascend exactly like the reflect decode path: an unknown struct field
// whose value is a deeply-nested array is skipped here, and without this guard a
// hostile payload of N nested arrays would overflow the goroutine stack (an
// unrecoverable fatal error). The recursive d.Skip() calls below re-enter this
// guard, so every level is counted.
func (d *Decoder) Skip() error {
        if err := d.descend(); err != nil {
                return err
        }
        defer d.ascend()
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch {
        case t <= tagFixintMax:
                d.i++
                return nil
        case t >= tagFixstr && t <= tagFixstr|tagFixstrMask:
                n := int(t & tagFixstrMask)
                if d.i+1+n > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += 1 + n
                return nil
        case t >= tagFixarr && t <= tagFixarr|tagFixarrMask:
                n := int(t & tagFixarrMask)
                d.i++
                for range n {
                        if err := d.Skip(); err != nil {
                                return err
                        }
                }
                return nil
        case t >= tagNegfixint && t <= tagNegfixint|tagNegfixintMask:
                d.i++
                return nil
        }
        switch t {
        case tagNil, tagTrue, tagFalse:
                d.i++
                return nil
        case tagUint8, tagInt8:
                if d.i+2 > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += 2
                return nil
        case tagUint16, tagInt16:
                if d.i+3 > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += 3
                return nil
        case tagUint32, tagInt32, tagFloat32:
                if d.i+5 > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += 5
                return nil
        case tagUint64, tagInt64, tagFloat64:
                if d.i+9 > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += 9
                return nil
        case tagTimestamp:
                // New wire format: tag + uvarint(zigzag(sec)) + uvarint(nsec).
                // Skip two uvarints.
                d.i++ // consume tag
                for range 2 {
                        _, n := readUvarint(d.buf[d.i:])
                        if n <= 0 {
                                return ErrShortBuffer
                        }
                        d.i += n
                }
                return nil
        case tagStr8, tagBin8, tagMap8:
                // Map8 is a count, not a byte length; handle separately below.
                if t == tagMap8 {
                        n, err := d.ReadMapHeader()
                        if err != nil {
                                return err
                        }
                        for range n {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                d.i++ // tag
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                n := int(d.buf[d.i])
                d.i++
                if d.i+n > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += n
                return nil
        case tagStr16, tagBin16, tagArr16, tagMap16:
                d.i++
                if t == tagArr16 {
                        if d.i+2 > len(d.buf) {
                                return ErrShortBuffer
                        }
                        n := int(readU16(d.buf[d.i:]))
                        d.i += 2
                        for range n {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                if t == tagMap16 {
                        if d.i+2 > len(d.buf) {
                                return ErrShortBuffer
                        }
                        n := int(readU16(d.buf[d.i:]))
                        d.i += 2
                        for range n {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                if d.i+2 > len(d.buf) {
                        return ErrShortBuffer
                }
                n := int(readU16(d.buf[d.i:]))
                d.i += 2
                if d.i+n > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += n
                return nil
        case tagStr32, tagBin32, tagArr32, tagMap32:
                d.i++
                if t == tagArr32 {
                        if d.i+4 > len(d.buf) {
                                return ErrShortBuffer
                        }
                        // Compare in uint64 before narrowing: on a 32-bit int a count near 2^32
                        // wraps negative, making `for range n` run zero iterations and Skip
                        // silently succeed without consuming the elements (parse desync). Each
                        // element is >= 1 byte, so a count exceeding the remaining bytes is
                        // impossible. Mirrors the tagStr32/tagBin32 guard below.
                        n64 := uint64(readU32(d.buf[d.i:]))
                        d.i += 4
                        if n64 > uint64(len(d.buf)-d.i) {
                                return ErrShortBuffer
                        }
                        for range int(n64) {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                if t == tagMap32 {
                        if d.i+4 > len(d.buf) {
                                return ErrShortBuffer
                        }
                        // Same 32-bit wrap guard as tagArr32; each entry is >= 2 bytes (key +
                        // value, each >= 1), so a count exceeding the remaining bytes is invalid.
                        n64 := uint64(readU32(d.buf[d.i:]))
                        d.i += 4
                        if n64 > uint64(len(d.buf)-d.i) {
                                return ErrShortBuffer
                        }
                        for range int(n64) {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                if d.i+4 > len(d.buf) {
                        return ErrShortBuffer
                }
                n64 := uint64(readU32(d.buf[d.i:]))
                d.i += 4
                // Compare in uint64 before narrowing: on a 32-bit int a length near 2^32
                // becomes negative, slips past `d.i+n > len`, and `d.i += n` rewinds the
                // cursor (parse desync). Mirrors the read-path tagStr32/tagBin32 guard.
                if n64 > uint64(len(d.buf)-d.i) {
                        return ErrShortBuffer
                }
                d.i += int(n64)
                return nil
        case tagInternStr, tagInternBin:
                // Read+register; Skip semantics still need the state table to stay
                // in sync with the stream.
                _, err := d.readStringBytes()
                return err
        case tagStateRef, tagStateRepeat, tagStateMTF, tagStatePair:
                _, err := d.readStringBytes()
                return err
        case tagMapShape:
                // Wire form mirrors the decode path: either a declaration
                // (shapeID==0, varuint(N), N keys, N values) or a reuse
                // (shapeID>0, looked-up N, N values). Skipping must still
                // advance the shape table on declaration so subsequent
                // references stay consistent.
                d.i++
                shapeID, n := readUvarint(d.buf[d.i:])
                if n <= 0 {
                        return ErrInvalidLength
                }
                if shapeID > uint64(math.MaxUint32) {
                        return ErrUnknownStateID // would truncate on the uint32 cast below
                }
                d.i += n
                if d.state == nil {
                        d.state = newDecState()
                }
                var cnt int
                if shapeID == 0 {
                        cnt64, n := readUvarint(d.buf[d.i:])
                        if n <= 0 {
                                return ErrInvalidLength
                        }
                        d.i += n
                        if cnt64 > uint64(math.MaxInt) { // 32-bit: int(cnt64) would wrap before CheckLength
                                return ErrInvalidLength
                        }
                        if err := d.CheckLength(int(cnt64), 1); err != nil {
                                return err
                        }
                        cnt = int(cnt64)
                        sh := d.state.shapeDeclare()
                        sh.names = make([]string, 0, cnt)
                        for i := 0; i < cnt; i++ {
                                kb, err := d.readStringBytes()
                                if err != nil {
                                        return err
                                }
                                sh.names = append(sh.names, d.keyCache.Make(kb))
                        }
                } else {
                        sh := d.state.shapeLookup(uint32(shapeID))
                        if sh == nil {
                                return ErrUnknownStateID
                        }
                        cnt = len(sh.names)
                }
                for i := 0; i < cnt; i++ {
                        if err := d.Skip(); err != nil {
                                return err
                        }
                }
                return nil
        case tagPackBool:
                d.i++
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                // Validate the element count in uint64 BEFORE the signed cast so
                // a hostile varuint cannot drive nBytes negative and corrupt
                // d.i. n64 elements need ceil(n64/8) bytes.
                rem := uint64(len(d.buf) - d.i)
                if n64 > rem*8 {
                        return ErrShortBuffer
                }
                d.i += int((n64 + 7) >> 3)
                return nil
        case tagPackRaw:
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                k := d.buf[d.i]
                d.i++
                w := qpackRawWidthBytes(k)
                if w == 0 {
                        return ErrBadTag
                }
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if n64 > uint64(len(d.buf)-d.i)/uint64(w) {
                        return ErrShortBuffer
                }
                d.i += int(n64) * w
                return nil
        case tagPackFor:
                d.i++
                if d.i+2 > len(d.buf) {
                        return ErrShortBuffer
                }
                // kind, bits
                d.i++ // skip kind
                bitsPer := int(d.buf[d.i])
                d.i++
                if bitsPer > qpackForMaxBits {
                        return ErrBadTag
                }
                // min varuint
                _, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                // Same overflow guard as tagPackBool. n64 elements times
                // bitsPer bits => ceil/8 bytes; validate in uint64 to avoid the
                // sign-bit pitfall on a hostile varuint.
                rem := uint64(len(d.buf) - d.i)
                if bitsPer > 0 && n64 > rem*8/uint64(bitsPer) {
                        return ErrShortBuffer
                }
                d.i += int((n64*uint64(bitsPer) + 7) / 8)
                return nil
        case tagPackGorilla:
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                k := d.buf[d.i]
                d.i++
                w := qpackRawWidthBytes(k)
                if w != 4 && w != 8 {
                        return ErrBadTag
                }
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if n64 == 0 {
                        return nil
                }
                if d.i+w > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i += w
                // numBits varuint
                nb64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                rem := uint64(len(d.buf) - d.i)
                if nb64 > rem*8 {
                        return ErrShortBuffer
                }
                d.i += int((nb64 + 7) >> 3)
                return nil
        case tagPackALP:
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                excValBytes := 8 // float64 exception value width
                switch d.buf[d.i] {
                case qpackKindFloat64:
                case qpackKindFloat32:
                        excValBytes = 4
                default:
                        return ErrTypeMismatch
                }
                d.i++ // kind
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if n64 == 0 {
                        return nil
                }
                if n64 > alpMaxElems {
                        return ErrInvalidLength
                }
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                d.i++                                           // d exponent
                if _, nr := readUvarint(d.buf[d.i:]); nr <= 0 { // forMin
                        return ErrInvalidLength
                } else {
                        d.i += nr
                }
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                width := int(d.buf[d.i])
                d.i++
                if width > qpackForMaxBits {
                        return ErrBadTag
                }
                if width > 0 {
                        rem := uint64(len(d.buf) - d.i)
                        if n64 > rem*8/uint64(width) {
                                return ErrShortBuffer
                        }
                        d.i += int((n64*uint64(width) + 7) / 8)
                }
                excN, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if excN > n64 {
                        return ErrInvalidLength
                }
                for range excN {
                        _, nr := readUvarint(d.buf[d.i:])
                        if nr <= 0 {
                                return ErrInvalidLength
                        }
                        d.i += nr
                        if d.i+excValBytes > len(d.buf) {
                                return ErrShortBuffer
                        }
                        d.i += excValBytes
                }
                return nil
        case tagPackDeltaFor:
                d.i++
                if d.i+2 > len(d.buf) {
                        return ErrShortBuffer
                }
                d.i++ // skip kind
                bitsPer := int(d.buf[d.i])
                d.i++
                if bitsPer > qpackForMaxBits {
                        return ErrBadTag
                }
                // firstVal varuint
                _, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                // minDelta varuint
                _, nr = readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if n64 >= 2 {
                        // Body holds (n-1) elements at bitsPer each. Bound (n-1) by the
                        // remaining bytes via DIVISION before the multiply — n64 is an
                        // unbounded varuint, so `(n64-1)*bitsPer` would overflow uint64 for a
                        // large n64 (wrapping to a small bodyBytes that passes the size check
                        // and desyncs the skip cursor). Mirrors readPackedDeltaForHeader.
                        rem := uint64(len(d.buf) - d.i)
                        if bitsPer > 0 && n64-1 > rem*8/uint64(bitsPer) {
                                return ErrShortBuffer
                        }
                        bodyBytes := ((n64-1)*uint64(bitsPer) + 7) >> 3
                        d.i += int(bodyBytes)
                }
                return nil
        case tagPackRLE:
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                k := d.buf[d.i]
                d.i++
                if k != qpackKindUint64 && k != qpackKindInt64 {
                        return ErrBadTag
                }
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                // Body is a sequence of (value-varuint, runLen-varuint)
                // pairs whose runLen sum equals n. Walk until n elements
                // consumed; the loop also catches truncated bodies.
                var produced uint64
                for produced < n64 {
                        _, nr := readUvarint(d.buf[d.i:])
                        if nr <= 0 {
                                return ErrInvalidLength
                        }
                        d.i += nr
                        runLen, nr := readUvarint(d.buf[d.i:])
                        if nr <= 0 {
                                return ErrInvalidLength
                        }
                        d.i += nr
                        if runLen == 0 || produced+runLen > n64 {
                                return ErrInvalidLength
                        }
                        produced += runLen
                }
                return nil
        case tagPackDict:
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                k := d.buf[d.i]
                d.i++
                if k != qpackKindUint64 && k != qpackKindInt64 {
                        return ErrBadTag
                }
                distinct64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if distinct64 == 0 || distinct64 > qpackDictMaxDistinct {
                        return ErrBadTag
                }
                for range distinct64 {
                        _, nr := readUvarint(d.buf[d.i:])
                        if nr <= 0 {
                                return ErrInvalidLength
                        }
                        d.i += nr
                }
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                bp := bitsForDistinct(int(distinct64))
                if bp == 0 {
                        return nil
                }
                rem := uint64(len(d.buf) - d.i)
                if n64 > rem*8/uint64(bp) {
                        return ErrShortBuffer
                }
                d.i += int((n64*uint64(bp) + 7) / 8)
                return nil
        case tagPackPFor:
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                k := d.buf[d.i]
                d.i++
                if k != qpackKindUint64 && k != qpackKindInt64 {
                        return ErrBadTag
                }
                n64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                b := int(d.buf[d.i])
                d.i++
                if b > qpackForMaxBits {
                        return ErrBadTag
                }
                // min varuint
                if _, nr := readUvarint(d.buf[d.i:]); nr <= 0 {
                        return ErrInvalidLength
                } else {
                        d.i += nr
                }
                // body: n*b bits
                rem := uint64(len(d.buf) - d.i)
                if b > 0 && n64 > rem*8/uint64(b) {
                        return ErrShortBuffer
                }
                d.i += int((n64*uint64(b) + 7) / 8)
                // exception list: excN pairs of (dPos, delta) varuints
                excN64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if excN64 > n64 {
                        return ErrInvalidLength
                }
                for range excN64 {
                        if _, nr := readUvarint(d.buf[d.i:]); nr <= 0 {
                                return ErrInvalidLength
                        } else {
                                d.i += nr
                        }
                        if _, nr := readUvarint(d.buf[d.i:]); nr <= 0 {
                                return ErrInvalidLength
                        } else {
                                d.i += nr
                        }
                }
                return nil
        case tagColStruct:
                // A columnar []struct payload reached Skip — an unknown struct-slice
                // field under OptBalanced/OptCompression (schema evolution). Decode it
                // via the any path and discard the result: that advances the cursor
                // exactly and replays the shape-table (and any per-column) state a real
                // decode would, keeping later state-refs in sync — a byte-only skip
                // could not. Uses the map-path row ceiling (maxColumnarAnyElems); a
                // skipped columnar field with more rows than that is rejected rather
                // than skipped, which is far above any realistic schema-evolution batch.
                if _, err := decodeColumnarAny(d); err != nil {
                        return err
                }
                return nil
        case tagHybridColStruct:
                // Hybrid columnar []struct (unknown mixed-struct-slice field — emitted
                // under OptFSST/OptCompression, or under Balanced when the intern-aware
                // probe predicts a win). Same rationale as tagColStruct: decode-and-
                // discard via the any path to advance the cursor and replay the intern /
                // shape state, keeping later state-refs in sync.
                if _, err := decodeHybridColumnarAny(d); err != nil {
                        return err
                }
                return nil
        case tagPackBlock:
                // Block-coded int/uint column reached Skip (an unknown []int*/[]uint*
                // field under OptBalanced — writeQPackInt64 emits 0xF0 at top level).
                // Decode-and-discard via the real reader to advance the cursor exactly;
                // the kind byte (after the tag) selects int vs uint.
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                switch d.buf[d.i] {
                case blockKindInt:
                        _, err := d.readBlockInt64()
                        return err
                case blockKindUint:
                        _, err := d.readBlockUint64()
                        return err
                }
                return ErrBadTag
        case tagZoneChunk:
                // Zone-chunked column reached Skip. Defensive: today 0xF1 is only emitted
                // inside tagColStruct columns, but the top-level slice decoders accept it,
                // so a future top-level emission must skip cleanly too. Decode-and-discard
                // via the kind-selected reader.
                d.i++
                if d.i >= len(d.buf) {
                        return ErrShortBuffer
                }
                switch d.buf[d.i] {
                case zoneKindInt:
                        _, err := d.readZoneChunkInt64()
                        return err
                case zoneKindUint:
                        _, err := d.readZoneChunkUint64()
                        return err
                case zoneKindFloat:
                        _, err := d.readZoneChunkFloat64()
                        return err
                }
                return ErrBadTag
        case tagColVecLossy:
                // Lossy float-vector column reached Skip (an unknown []float32/[]float64
                // field under OptLossyVec — 0xFD at top level). The block self-describes
                // its element kind and length; advance by the consumed byte count.
                _, _, used, err := readLossyVec(d.buf[d.i:])
                if err != nil {
                        return err
                }
                d.i += used
                return nil
        case tagVecBatchStruct:
                // Batched lossy vector-column ([]struct with vector fields under
                // OptLossyVec) reached Skip. Not byte-skippable: the per-row non-batched
                // fields can carry intern/shape state-refs, so each must be walked via
                // d.Skip() to replay that state (a raw byte advance would desync later
                // state-refs — same rationale as tagColStruct). The header carries the
                // struct field count so the schemaless walk knows how many fields per row.
                d.i++
                n64, k := readUvarint(d.buf[d.i:])
                if k <= 0 {
                        return ErrInvalidLength
                }
                d.i += k
                if n64 > uint64(maxColumnarElems) {
                        return ErrInvalidLength
                }
                n := int(n64)
                if d.i+3 > len(d.buf) {
                        return ErrShortBuffer
                }
                nv := int(d.buf[d.i])
                mask := d.buf[d.i+1]
                polarMask := d.buf[d.i+2]
                d.i += 3
                if nv > 8 || polarMask&^mask != 0 { // nv fits one mask byte; polar ⊆ mask
                        return ErrBadTag
                }
                nf64, kf := readUvarint(d.buf[d.i:])
                if kf <= 0 {
                        return ErrInvalidLength
                }
                d.i += kf
                // Skip each batched column block (each is self-describing in length).
                for vi := range nv {
                        if mask&(1<<uint(vi)) == 0 {
                                continue
                        }
                        if polarMask&(1<<uint(vi)) != 0 {
                                _, used, err := readNormStream(d.buf[d.i:], n)
                                if err != nil {
                                        return err
                                }
                                d.i += used
                        }
                        _, _, used, err := readLossyVec(d.buf[d.i:])
                        if err != nil {
                                return err
                        }
                        d.i += used
                }
                // Skip per-row non-batched fields, replaying their intern/shape state.
                batched := 0
                for m := mask; m != 0; m &= m - 1 {
                        batched++
                }
                perRow := int(nf64) - batched
                if nf64 > uint64(maxColumnarElems) || perRow < 0 {
                        return ErrBadTag
                }
                for range n {
                        for range perRow {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                        }
                }
                return nil
        }
        return ErrBadTag
}

package qdf

import (
        "encoding/binary"
        "errors"
        "hash/maphash"
        "math"
        "reflect"
        "slices"
        "time"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/rans"
)

// timeType is the cached descriptor key for time.Time, used by the fingerprint
// fast path (a time.Time field is common and the reflect fallback allocates).
var timeType = reflect.TypeFor[time.Time]()

// maxDeltaDepth bounds recursion in the diff/apply/fingerprint value walks,
// matching the encoder's cycle guard. A cyclic or pathologically deep value is
// rejected with ErrCycleDetected instead of overflowing the (uncatchable) stack.
const maxDeltaDepth = 10000

// schemaSeed is a process-stable seed. A fingerprint only needs to be stable
// within a single producer→consumer exchange that shares this binary; cross-
// build stability is not required for Phase 1 (both ends import the same type).
var schemaSeed = maphash.MakeSeed()

// schemaFingerprintCompute hashes a type descriptor's shape (kind + field names +
// recursive field/element kinds) so Apply can reject a patch built for a
// different type. Cycles are broken by a visited set. It is called exactly once
// per descriptor at build time (descBuild) and the result is stored on
// td.schemaFP; Diff/Apply read that field directly with no runtime synchronization.
func schemaFingerprintCompute(td *typeDesc) uint64 {
        var h maphash.Hash
        h.SetSeed(schemaSeed)
        visited := map[*typeDesc]bool{}
        hashDesc(&h, td, visited)
        return h.Sum64()
}

func hashDesc(h *maphash.Hash, td *typeDesc, visited map[*typeDesc]bool) {
        if td == nil {
                _ = h.WriteByte(0xFF)
                return
        }
        if visited[td] {
                _ = h.WriteByte(0xFE) // cycle marker
                return
        }
        visited[td] = true
        _ = h.WriteByte(byte(td.kind))
        _ = h.WriteByte(td.marshalerKind)
        // td.elem is only the value/element descriptor; fold in the parts of the
        // shape it omits so the cross-type guard does not collide: a map's KEY type
        // (else map[int]V and map[string]V share a fingerprint) and an array's LENGTH
        // (else [3]T and [4]T collide).
        switch td.kind {
        case reflect.Map:
                _, _ = h.WriteString(td.rType.Key().String())
        case reflect.Array:
                var b [8]byte
                binary.LittleEndian.PutUint64(b[:], uint64(td.rType.Len()))
                _, _ = h.Write(b[:])
        }
        for i := range td.fields {
                f := &td.fields[i]
                _, _ = h.WriteString(f.name)
                hashDesc(h, f.desc, visited)
        }
        if td.elem != nil {
                hashDesc(h, td.elem, visited)
        }
}

var (
        // ErrInvalidPatch is returned when a patch blob is truncated, has a bad
        // magic/version, or is otherwise malformed.
        ErrInvalidPatch = errors.New("qdf: invalid or truncated patch")
        // ErrPatchSchemaMismatch is returned by Apply when the patch was built for
        // a different type than the supplied base.
        ErrPatchSchemaMismatch = errors.New("qdf: patch schema fingerprint mismatch")
        // ErrPatchBaseMismatch is returned by Apply when the patch carries a base
        // fingerprint that does not match the supplied base value.
        ErrPatchBaseMismatch = errors.New("qdf: patch base fingerprint mismatch")
)

// Diff computes a patch carrying only the structural difference (new − old).
func Diff[T any](old, new T, opts Options) ([]byte, error) {
        return AppendDiff(nil, old, new, opts)
}

// AppendDiff appends the patch to dst and returns the extended slice.
func AppendDiff[T any](dst []byte, old, new T, opts Options) ([]byte, error) {
        td, err := descOf(reflect.TypeFor[T]())
        if err != nil {
                return dst, err
        }
        enc := encPool.Get().(*Encoder)
        enc.Reset()
        enc.applyOpts(opts)

        flags := byte(0)
        if enc.mode == Dense {
                flags |= flagPatchDense
        }
        var baseFP uint64
        if !opts.Has(OptDeltaNoBaseFingerprint) {
                flags |= flagPatchBaseFP // baseFP defaults ON
                baseFP = valueFingerprint(td, unsafe.Pointer(&old))
        }
        schemaFP := td.schemaFP

        start := len(dst)
        enc.buf = writePatchHeader(dst, flags, schemaFP, baseFP)
        enc.MarkHeaderWritten() // QDP header, not QDF: suppress value-codec QDF header

        if err := diffValue(enc, td, unsafe.Pointer(&old), unsafe.Pointer(&new), 0); err != nil {
                enc.buf = nil
                putEnc(enc, &encPool)
                return dst, err
        }
        maybeApplyPatchRANS(enc, start)
        out := slices.Clone(enc.buf)
        enc.buf = nil
        putEnc(enc, &encPool)
        return out, nil
}

// Apply merges patch onto base in place, reconstructing new. Unchanged fields
// are untouched.
func Apply[T any](base *T, patch []byte) error {
        return applyImpl(base, patch, nil)
}

// ApplyArena is Apply with a caller-provided decode arena: every string (and
// []byte-as-string) value the patch REPLACES is copied into arena's contiguous
// bump blocks instead of one heap allocation per string. It is otherwise
// byte-for-byte identical to Apply — same result, same errors, same wire — and
// only differs in where replaced string bodies live.
//
// This helps only when a patch replaces MANY strings (a large changed []string
// or []struct{...string} field, a map with many changed string values). A
// typical small patch changes few strings and the arena has little to batch; in
// that case Apply is the simpler choice.
//
// Lifetime / safety (identical to the decode arena, see Arena): strings written
// into base by ApplyArena ALIAS arena's memory and stay valid only as long as
// arena (or any string from it) is reachable. The safe pattern is one Arena per
// epoch, then drop it; Reset reuses it but invalidates every string from the
// prior epoch. Unchanged string fields already present in base are NOT touched
// and do not alias arena.
func ApplyArena[T any](base *T, patch []byte, arena *Arena) error {
        return applyImpl(base, patch, arena)
}

func applyImpl[T any](base *T, patch []byte, arena *Arena) error {
        if base == nil {
                return ErrTypeMismatch
        }
        td, err := descOf(reflect.TypeFor[T]())
        if err != nil {
                return err
        }
        h, n, err := readPatchHeader(patch)
        if err != nil {
                return err
        }
        if h.schemaFP != td.schemaFP {
                return ErrPatchSchemaMismatch
        }
        if h.flags&flagPatchBaseFP != 0 {
                if valueFingerprint(td, unsafe.Pointer(base)) != h.baseFP {
                        return ErrPatchBaseMismatch
                }
        }
        body := patch[n:]
        if h.flags&flagPatchRANS != 0 {
                body, err = decompressPatchBody(body)
                if err != nil {
                        return err
                }
        }
        if len(body) == 0 {
                // Empty body: the root value was unchanged (diffValue wrote no op).
                // Base already equals new — nothing to apply.
                return nil
        }
        dec := decPool.Get().(*Decoder)
        resetPatchDecoder(dec, body, h.flags&flagPatchDense != 0)
        dec.arena = arena // nil for Apply; set for ApplyArena so replaced strings bump-allocate
        err = applyValue(dec, td, unsafe.Pointer(base), 0)
        dec.buf = nil
        dec.arena = nil // never pin the caller's arena across pooled reuse
        if cap(dec.deltaScratch) > maxRetainedDeltaScratch {
                dec.deltaScratch = nil
        }
        // keyIdx keys alias the caller's base key strings / []struct backing
        // (keyTokenAt); clear() on the small-map branch so a pooled decoder does not
        // GC-pin that data across reuse (mirrors d.values / colScratchStr policy).
        if len(dec.keyIdx) > 1<<16 {
                dec.keyIdx = nil
        } else {
                clear(dec.keyIdx)
        }
        dec.keyIdxBusy = false
        decPool.Put(dec)
        return err
}

// resetPatchDecoder prepares a pooled decoder to read patch body bytes. headerRead
// is forced true so value codecs invoked for opReplace skip the QDF header.
func resetPatchDecoder(dec *Decoder, body []byte, dense bool) {
        dec.SetInput(body)
        // SetInput resets buf/i/depth/colIndex/selectFields/query/mapFreeList/state
        // but leaves noCopy and arena sticky from a prior decode. Inheriting noCopy
        // would alias the caller's patch buffer in an opReplace string → corruption
        // after the caller mutates/reuses it; clear both (mirrors UnmarshalT).
        dec.noCopy = false
        dec.arena = nil
        dec.headerRead = true
        if dense {
                dec.mode = Dense
                if dec.state == nil {
                        dec.state = newDecState()
                }
        }
}

// maybeApplyPatchRANS optionally rANS-compresses the patch body in place after
// the QDP header (offset start). Mirrors maybeApplyRANS.
func maybeApplyPatchRANS(enc *Encoder, start int) {
        if !enc.rans {
                return
        }
        hdr := 13
        if enc.buf[start+4]&flagPatchBaseFP != 0 {
                hdr = 21
        }
        if len(enc.buf)-start < hdr+ransMinBytes {
                return
        }
        body := enc.buf[start+hdr:]
        cand := appendUvarint(make([]byte, 0, len(body)/2+512), uint64(len(body)))
        cand = rans.Encode(cand, body)
        if len(cand) >= len(body) {
                return
        }
        // Decline rANS whenever decompressPatchBody would later reject the frame:
        // it strips the QDP header first and bounds origLen by the post-header body
        // length (len(cand)), not hdr+len(cand). Using the same measure here keeps
        // the encoder from ever emitting a patch its own Apply rejects.
        if uint64(len(body)) > uint64(len(cand))*64+(1<<20) {
                return
        }
        enc.buf = append(enc.buf[:start+hdr], cand...)
        enc.buf[start+4] |= flagPatchRANS
}

// decompressPatchBody reverses maybeApplyPatchRANS: varuint(origLen) + rANS stream.
func decompressPatchBody(body []byte) ([]byte, error) {
        origLen, k := readUvarint(body)
        if k <= 0 {
                return nil, ErrInvalidPatch
        }
        if origLen == 0 {
                // The encoder only emits a rANS body for bodies >= ransMinBytes; a
                // decoded origLen of 0 is malformed.
                return nil, ErrInvalidPatch
        }
        // The magnitude clause caps decompression amplification; the second clause
        // rejects anything past the int range so the int(origLen) narrowing below
        // cannot truncate a multi-GiB length on a 32-bit build and decode a silently
        // short/wrong body (mirrors the rANS body bound in decoder.go).
        if origLen > uint64(len(body))*64+(1<<20) || origLen > uint64(math.MaxInt) {
                return nil, ErrInvalidPatch
        }
        out, err := rans.Decode(body[k:], int(origLen))
        if err != nil {
                return nil, ErrInvalidPatch
        }
        return out, nil
}

// valueFingerprint produces an order-independent hash of a value so map-bearing
// types fingerprint deterministically regardless of map iteration order. It is
// computed once over old (Diff) and once over base (Apply); a mismatch means
// the caller's base is not the old the patch was built against.
func valueFingerprint(td *typeDesc, p unsafe.Pointer) uint64 {
        var h maphash.Hash
        h.SetSeed(schemaSeed)
        fpHash(&h, td, p, 0)
        return h.Sum64()
}

// fpHash hashes the value of type td at p into h. For a pointer-free, padding-
// free ("tight POD") type it issues a single maphash.Write over the whole
// contiguous byte span — collapsing N per-field/per-element writes and all
// reflect dispatch into one vectorized hash. Non-tight types (those carrying
// pointers, strings, padding, or maps) recurse structurally, hashing per field /
// per element so padding bytes are never read. nil-vs-empty markers for
// slice/map/pointer are preserved exactly as the prior reflect walk emitted them.
func fpHash(h *maphash.Hash, td *typeDesc, p unsafe.Pointer, depth int) {
        if depth > maxDeltaDepth {
                // A truncated fingerprint is fine here: the diff walk applies its own cap
                // and surfaces the cycle as ErrCycleDetected.
                return
        }
        if td.tightPOD {
                // One write over the whole value: scalar, tight array, or tightly-packed
                // pointer-free struct. The span is all content (no padding) so the hash is
                // determined purely by the logical value.
                _, _ = h.Write(unsafe.Slice((*byte)(p), td.rType.Size()))
                return
        }
        switch td.kind {
        case reflect.String:
                _, _ = h.WriteString(*(*string)(p))
        case reflect.Bool:
                if *(*bool)(p) {
                        _ = h.WriteByte(1)
                } else {
                        _ = h.WriteByte(0)
                }
        case reflect.Slice:
                sh := (*sliceHeader)(p)
                if sh.Data == nil {
                        _ = h.WriteByte(0) // nil-slice marker (distinct from empty-non-nil)
                        return
                }
                _ = h.WriteByte(1)
                var b [8]byte
                binary.LittleEndian.PutUint64(b[:], uint64(sh.Len))
                _, _ = h.Write(b[:])
                if sh.Len == 0 {
                        return
                }
                if td.rType.Elem().Kind() == reflect.Uint8 {
                        _, _ = h.Write(unsafe.Slice((*byte)(sh.Data), sh.Len))
                        return
                }
                elem := td.elem
                if elem == nil {
                        elem = td.fpElem // primitive-slice fast path: resolved at build
                }
                if elem == nil {
                        elem, _ = descOf(td.rType.Elem())
                }
                stride := td.rType.Elem().Size()
                if elem != nil && elem.tightPOD {
                        // Whole backing array is content: one write over len*stride bytes.
                        _, _ = h.Write(unsafe.Slice((*byte)(sh.Data), uintptr(sh.Len)*stride))
                        return
                }
                for i := range sh.Len {
                        fpHash(h, elem, unsafe.Add(sh.Data, uintptr(i)*stride), depth+1)
                }
        case reflect.Array:
                // A tight array was handled by the tightPOD branch; a non-tight array has a
                // non-tight element (strings/pointers/padding), so hash element by element.
                n := td.rType.Len()
                stride := td.rType.Elem().Size()
                elem := td.elem
                if elem == nil {
                        elem, _ = descOf(td.rType.Elem())
                }
                for i := range n {
                        fpHash(h, elem, unsafe.Add(p, uintptr(i)*stride), depth+1)
                }
        case reflect.Struct:
                // A tight struct was handled above; this one has strings/pointers/padding.
                // A fields-less struct (time.Time, a custom Marshaler) carries no td.fields
                // to walk — hash its real fields via reflect so the fingerprint is not blind
                // to them (consistent with equalValue's DeepEqual for the same shape).
                if len(td.fields) == 0 {
                        if td.rType == timeType {
                                // time.Time is the common fields-less struct. Hash the instant the
                                // codec actually round-trips (UTC sec + nsec) — alloc-free and no
                                // reflect, vs fpHashReflect which walks *Location and allocates.
                                t := (*time.Time)(p).UTC()
                                var b [12]byte
                                binary.LittleEndian.PutUint64(b[:8], uint64(t.Unix()))
                                binary.LittleEndian.PutUint32(b[8:], uint32(t.Nanosecond()))
                                _, _ = h.Write(b[:])
                                return
                        }
                        fpHashReflect(h, reflect.NewAt(td.rType, p).Elem(), depth)
                        return
                }
                // Hash per field (skipping any padding between fields).
                for i := range td.fields {
                        f := &td.fields[i]
                        fpHash(h, f.desc, unsafe.Add(p, f.offset), depth+1)
                }
        case reflect.Pointer:
                ep := *(*unsafe.Pointer)(p)
                if ep == nil {
                        _ = h.WriteByte(0)
                        return
                }
                _ = h.WriteByte(1)
                elem := td.elem
                if elem == nil {
                        elem, _ = descOf(td.rType.Elem())
                }
                fpHash(h, elem, ep, depth+1)
        case reflect.Map:
                mp := *(*unsafe.Pointer)(p)
                if mp == nil {
                        _ = h.WriteByte(0) // nil-map marker
                        return
                }
                _ = h.WriteByte(1)
                v := reflect.NewAt(td.rType, p).Elem()
                // Maps need reflect to iterate; hash key+value with the reflect fallback
                // (no per-entry addressable scratch — that allocated two reflect.New values
                // per entry and regressed map-heavy values). Commutative XOR fold keeps the
                // result order-independent.
                var acc uint64
                for it := v.MapRange(); it.Next(); {
                        var e maphash.Hash
                        e.SetSeed(schemaSeed)
                        fpHashReflect(&e, it.Key(), depth+1)
                        fpHashReflect(&e, it.Value(), depth+1)
                        acc ^= e.Sum64()
                }
                var b [8]byte
                binary.LittleEndian.PutUint64(b[:], acc)
                _, _ = h.Write(b[:])
        case reflect.Interface:
                v := reflect.NewAt(td.rType, p).Elem()
                if v.IsNil() {
                        _ = h.WriteByte(0)
                        return
                }
                _ = h.WriteByte(1)
                el := v.Elem()
                _, _ = h.WriteString(el.Type().String())
                fpHashReflect(h, el, depth+1)
        default:
                // Exotic kinds (chan/func/complex via interface, etc.): hash the type name.
                // Rare and off the hot path.
                _, _ = h.WriteString(td.rType.String())
        }
}

// fpHashReflect is the reflect fallback used only for the boxed dynamic value
// inside an interface, where a stable typed pointer is awkward to obtain. It
// mirrors the prior reflect walk for that one rare branch; perf does not matter.
func fpHashReflect(h *maphash.Hash, v reflect.Value, depth int) {
        if depth > maxDeltaDepth {
                return
        }
        switch v.Kind() {
        case reflect.Map:
                var acc uint64
                iter := v.MapRange()
                for iter.Next() {
                        var e maphash.Hash
                        e.SetSeed(schemaSeed)
                        fpHashReflect(&e, iter.Key(), depth+1)
                        fpHashReflect(&e, iter.Value(), depth+1)
                        acc ^= e.Sum64()
                }
                var b [8]byte
                binary.LittleEndian.PutUint64(b[:], acc)
                _, _ = h.Write(b[:])
        case reflect.Struct:
                // NumField/Field, NOT Fields(): the range-over-func iterator allocates a
                // heap closure per call, and this fingerprint hash recurses per struct on
                // the delta Diff path.
                for i := range v.NumField() {
                        fpHashReflect(h, v.Field(i), depth+1)
                }
        case reflect.Slice, reflect.Array:
                for i := range v.Len() {
                        fpHashReflect(h, v.Index(i), depth+1)
                }
        case reflect.Pointer:
                if v.IsNil() {
                        _ = h.WriteByte(0)
                } else {
                        _ = h.WriteByte(1)
                        fpHashReflect(h, v.Elem(), depth+1)
                }
        case reflect.String:
                _, _ = h.WriteString(v.String())
        case reflect.Bool:
                if v.Bool() {
                        _ = h.WriteByte(1)
                } else {
                        _ = h.WriteByte(0)
                }
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                var b [8]byte
                binary.LittleEndian.PutUint64(b[:], uint64(v.Int()))
                _, _ = h.Write(b[:])
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                var b [8]byte
                binary.LittleEndian.PutUint64(b[:], v.Uint())
                _, _ = h.Write(b[:])
        case reflect.Float32, reflect.Float64:
                var b [8]byte
                binary.LittleEndian.PutUint64(b[:], math.Float64bits(v.Float()))
                _, _ = h.Write(b[:])
        case reflect.Interface:
                if v.IsNil() {
                        _ = h.WriteByte(0)
                        return
                }
                _ = h.WriteByte(1)
                el := v.Elem()
                _, _ = h.WriteString(el.Type().String())
                fpHashReflect(h, el, depth+1)
        default:
                _, _ = h.WriteString(v.Type().String())
        }
}

package qdf

import (
        "math"
        "reflect"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/reflectutil"
)

// applyValue reads one op (op byte + payload) and applies it to the value at baseP.
func applyValue(dec *Decoder, td *typeDesc, baseP unsafe.Pointer, depth int) error {
        if depth > maxDeltaDepth {
                return ErrInvalidPatch
        }
        if dec.i >= len(dec.buf) {
                return ErrInvalidPatch
        }
        op := dec.buf[dec.i]
        dec.i++
        switch op {
        case opReplace:
                return td.decode(dec, baseP)
        case opMerge:
                return applyMerge(dec, td, baseP, depth)
        default:
                return ErrInvalidPatch
        }
}

// applyMerge dispatches a recursive sub-patch by container kind.
func applyMerge(dec *Decoder, td *typeDesc, baseP unsafe.Pointer, depth int) error {
        switch td.kind {
        case reflect.Struct:
                return applyStruct(dec, td, baseP, depth)
        case reflect.Pointer:
                // merge into the pointed-at struct. base pointer must be non-nil: the diff
                // side only emits opMerge for a pointer when both old/new were non-nil, and
                // baseFP guarantees base matches old.
                ptr := *(*unsafe.Pointer)(baseP)
                if ptr == nil {
                        return ErrInvalidPatch
                }
                return applyStruct(dec, td.elem, ptr, depth+1)
        case reflect.Slice:
                if dec.i < len(dec.buf) && dec.buf[dec.i] == tagKeyedSlicePatch {
                        return applyKeyedSlice(dec, td, baseP, depth)
                }
                if dec.i < len(dec.buf) && dec.buf[dec.i] == tagColSlicePatch {
                        return applyColSlice(dec, td, baseP)
                }
                return applySlice(dec, td, baseP, depth)
        case reflect.Array:
                return applyArray(dec, td, baseP, depth)
        case reflect.Map:
                return applyMap(dec, td, baseP, depth)
        default:
                return ErrInvalidPatch // unknown/unmergeable kind
        }
}

// applySlice reads a tagSlicePatch and reconciles the base slice in place:
// resize to newLen (preserving overlap), then apply each entry.
func applySlice(dec *Decoder, td *typeDesc, baseP unsafe.Pointer, depth int) error {
        if dec.i >= len(dec.buf) || dec.buf[dec.i] != tagSlicePatch {
                return ErrInvalidPatch
        }
        dec.i++
        newLen64, k := readUvarint(dec.buf[dec.i:])
        if k <= 0 {
                return ErrInvalidPatch
        }
        dec.i += k

        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
        }
        stride := td.rType.Elem().Size()
        bv := reflect.NewAt(td.rType, baseP).Elem()

        if newLen64 > uint64(math.MaxInt) { // 32-bit: int(newLen64) would truncate silently
                return ErrInvalidPatch
        }
        newLen := int(newLen64)
        if newLen < 0 {
                return ErrInvalidPatch
        }
        // Bound hostile growth: every element beyond the current base length must be
        // described by the patch (>=1 byte each), so the number of grown elements
        // cannot exceed the remaining input. This rejects an allocation-amplification
        // patch (huge newLen, zero entries) while still allowing the legitimate delta
        // case of a large base with a tiny no-grow patch (newLen <= base length).
        if newLen > bv.Len() && uint64(newLen-bv.Len()) > uint64(len(dec.buf)-dec.i) {
                return ErrInvalidPatch
        }

        switch {
        case newLen == bv.Len():
                // no resize
        case newLen < bv.Len():
                // Shrink in place: newLen < len <= cap, so reslicing keeps the existing
                // backing array (the overlap elements survive, the patch overwrites the
                // changed ones) — no allocation for a truncating patch.
                bv.SetLen(newLen)
        default: // grow — must allocate; copy the whole old slice into the larger one
                nv := reflect.MakeSlice(td.rType, newLen, newLen)
                reflect.Copy(nv, bv)
                bv.Set(nv)
        }
        sh := (*sliceHeader)(baseP) // re-read after potential Set (Data pointer moved)

        nEntries, k2 := readUvarint(dec.buf[dec.i:])
        if k2 <= 0 || nEntries > uint64(newLen) {
                return ErrInvalidPatch
        }
        dec.i += k2
        for range nEntries {
                idx, k3 := readUvarint(dec.buf[dec.i:])
                if k3 <= 0 || idx >= uint64(newLen) {
                        return ErrInvalidPatch
                }
                dec.i += k3
                if err := applyValue(dec, elem, unsafe.Add(sh.Data, uintptr(idx)*stride), depth+1); err != nil {
                        return err
                }
        }
        return nil
}

func applyArray(dec *Decoder, td *typeDesc, baseP unsafe.Pointer, depth int) error {
        if dec.i >= len(dec.buf) || dec.buf[dec.i] != tagSlicePatch {
                return ErrInvalidPatch
        }
        dec.i++
        newLen, k := readUvarint(dec.buf[dec.i:])
        if k <= 0 || newLen != uint64(td.rType.Len()) {
                return ErrInvalidPatch
        }
        dec.i += k
        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
        }
        stride := td.rType.Elem().Size()
        nEntries, k2 := readUvarint(dec.buf[dec.i:])
        if k2 <= 0 || nEntries > newLen {
                return ErrInvalidPatch
        }
        dec.i += k2
        for range nEntries {
                idx, k3 := readUvarint(dec.buf[dec.i:])
                if k3 <= 0 || idx >= newLen {
                        return ErrInvalidPatch
                }
                dec.i += k3
                if err := applyValue(dec, elem, unsafe.Add(baseP, uintptr(idx)*stride), depth+1); err != nil {
                        return err
                }
        }
        return nil
}

// applyMap reads a tagMapPatch and mutates the base map in place: set/merge the
// updated keys, then delete the tombstoned keys.
func applyMap(dec *Decoder, td *typeDesc, baseP unsafe.Pointer, depth int) error {
        if dec.i >= len(dec.buf) || dec.buf[dec.i] != tagMapPatch {
                return ErrInvalidPatch
        }
        dec.i++
        mv := reflect.NewAt(td.rType, baseP).Elem()
        if mv.IsNil() {
                // Allocate through reflectutil so the qdf_reflect2 build tag swaps in the
                // faster reflect2 map allocator, matching the decode path.
                reflectutil.MakeMap(td.rType, 0, baseP)
        }
        keyType := td.rType.Key()
        keyDesc, err := descOf(keyType)
        if err != nil {
                return err
        }
        valDesc := td.elem
        if valDesc == nil {
                valDesc, err = descOf(td.rType.Elem())
                if err != nil {
                        return err
                }
        }

        nUpd, k := readUvarint(dec.buf[dec.i:])
        if k <= 0 || nUpd > uint64(len(dec.buf)-dec.i) {
                return ErrInvalidPatch
        }
        dec.i += k
        // Reuse one addressable key/value buffer across all entries: SetMapIndex
        // copies both into the map, so the buffers are free to be overwritten next
        // iteration. keyDesc.decode fully overwrites keyBuf each time; valBuf MUST be
        // reset when there is no existing value, else it would merge onto the previous
        // entry's stale value (reused-buffer underfill).
        keyBuf := reflect.New(keyType).Elem()
        valBuf := reflect.New(valDesc.rType).Elem()
        for range nUpd {
                if err := keyDesc.decode(dec, keyBuf.Addr().UnsafePointer()); err != nil {
                        return err
                }
                if existing := mv.MapIndex(keyBuf); existing.IsValid() {
                        valBuf.Set(existing) // merge target starts from the current value
                } else {
                        valBuf.SetZero() // reset stale state from the previous iteration
                }
                if err := applyValue(dec, valDesc, valBuf.Addr().UnsafePointer(), depth+1); err != nil {
                        return err
                }
                mv.SetMapIndex(keyBuf, valBuf)
        }

        nDel, k2 := readUvarint(dec.buf[dec.i:])
        if k2 <= 0 || nDel > uint64(len(dec.buf)-dec.i) {
                return ErrInvalidPatch
        }
        dec.i += k2
        for range nDel {
                if err := keyDesc.decode(dec, keyBuf.Addr().UnsafePointer()); err != nil {
                        return err
                }
                mv.SetMapIndex(keyBuf, reflect.Value{}) // delete
        }
        return nil
}

// applyStruct reads a tagStructPatch body and overwrites only the listed fields.
func applyStruct(dec *Decoder, td *typeDesc, baseP unsafe.Pointer, depth int) error {
        if dec.i >= len(dec.buf) || dec.buf[dec.i] != tagStructPatch {
                return ErrInvalidPatch
        }
        dec.i++
        nChanged, k := readUvarint(dec.buf[dec.i:])
        if k <= 0 || nChanged > uint64(len(dec.buf)-dec.i) {
                return ErrInvalidPatch // one changed field needs ≥1 byte; bound like applyMap
        }
        dec.i += k
        for range nChanged {
                idx, k2 := readUvarint(dec.buf[dec.i:])
                if k2 <= 0 || idx >= uint64(len(td.fields)) {
                        return ErrInvalidPatch
                }
                dec.i += k2
                f := &td.fields[idx]
                if err := applyValue(dec, f.desc, unsafe.Add(baseP, f.offset), depth+1); err != nil {
                        return err
                }
        }
        return nil
}

package qdf

import (
        "errors"
        "reflect"
        "sync"
        "unsafe"
        "weak"

        "github.com/alex60217101990/qdf/internal/reflectutil"
)

var (
        // ErrBaselineRequired is returned by (*BaselineRegistry).Apply when the
        // patch carries no base fingerprint (the producer used
        // OptDeltaNoBaseFingerprint), so its baseline cannot be content-addressed.
        ErrBaselineRequired = errors.New("qdf: patch has no base fingerprint; baseline registry requires one")
        // ErrBaselineEvicted is returned by (*BaselineRegistry).Apply when the
        // patch's baseline id is not resolvable — it was never registered, or the
        // caller dropped its strong reference and the GC reclaimed it.
        ErrBaselineEvicted = errors.New("qdf: patch baseline not found in registry (never registered or GC-reclaimed)")
)

// BaselineRegistry is a consumer-side, content-addressed cache of recent
// baselines of type T, used to apply a chain of patches in a state-sync stream
// without threading the previous value by hand. It maps each baseline's content
// fingerprint (the same value Diff embeds as baseFP) to a weak.Pointer, so the
// GC reclaims any baseline the caller no longer references. The registry never
// pins a baseline alive.
//
// A BaselineRegistry is safe for concurrent use.
type BaselineRegistry[T any] struct {
        m  map[uint64]weak.Pointer[T]
        td *typeDesc
        mu sync.Mutex
}

// NewBaselineRegistry returns an empty registry for baselines of type T.
func NewBaselineRegistry[T any]() *BaselineRegistry[T] {
        td, _ := descOf(reflect.TypeFor[T]())
        return &BaselineRegistry[T]{m: make(map[uint64]weak.Pointer[T]), td: td}
}

// Len reports the number of live (non-evicted) baselines, pruning dead weak
// entries as a side effect.
func (r *BaselineRegistry[T]) Len() int {
        r.mu.Lock()
        defer r.mu.Unlock()
        for id, wp := range r.m {
                if wp.Value() == nil {
                        delete(r.m, id)
                }
        }
        return len(r.m)
}

// deepClone returns a structural deep copy of *v. It walks the value by its
// *typeDesc over unsafe.Pointer — the same idiom as fpHash / applyValue /
// equalValue — rather than reflect.Value, so it shares the package's fast paths
// and honors the qdf_reflect2 build tag (slice/map backing is allocated through
// reflectutil, exactly as the decode path does).
//
// The clone is byte-for-byte equivalent in everything the fingerprint reads, so
// it always fingerprints identically to its source — the invariant Apply's
// baseFP check relies on. It preserves nil-vs-empty (a nil slice/map stays nil,
// an empty non-nil one stays empty non-nil). Containers (slice/map/pointer) are
// freshly allocated so Apply's in-place mutation of the clone never touches the
// registered baseline.
func deepClone[T any](v *T) *T {
        out := new(T)
        td, err := descOf(reflect.TypeFor[T]())
        if err != nil || td == nil {
                // Unsupported type: descOf failing means no patch can target it, so the
                // clone is unreachable through Apply in practice. A plain value copy is the
                // safe best effort.
                *out = *v
                return out
        }
        cloneValue(td, unsafe.Pointer(out), unsafe.Pointer(v), 0)
        return out
}

// cloneValue writes a deep copy of the value of type td at src into dst,
// mirroring fpHash's structural walk so a clone always fingerprints identically
// to its source. A pointer-free (pod) span is bulk-copied in one byte copy — no
// GC write barrier needed, since it holds no pointers. Every pointer-bearing
// write goes through a typed store (string, unsafe.Pointer) or reflect.Set, so
// the GC write barrier fires; a raw byte copy is never used over a span that
// contains pointers.
func cloneValue(td *typeDesc, dst, src unsafe.Pointer, depth int) {
        if depth > maxDeltaDepth {
                // Mirror the diff/apply/fingerprint walks: stop instead of overflowing the
                // (uncatchable) stack on a cyclic or pathologically deep structure. A
                // truncated clone fingerprints differently, so Apply's baseFP check then
                // rejects it with ErrPatchBaseMismatch — a clean error, never a crash.
                return
        }
        if td.pod {
                // Pointer-free: one byte copy reproduces every byte (scalars, tight
                // arrays/structs, padded pointer-free structs) and is GC-safe.
                sz := td.rType.Size()
                copy(unsafe.Slice((*byte)(dst), sz), unsafe.Slice((*byte)(src), sz))
                return
        }
        switch td.kind {
        case reflect.String:
                *(*string)(dst) = *(*string)(src) // typed string store: GC-barriered
        case reflect.Slice:
                cloneSlice(td, dst, src, depth)
        case reflect.Array:
                cloneArray(td, dst, src, depth)
        case reflect.Struct:
                // Mirror fpHash's struct dispatch. A struct the delta layer treats as
                // fields-less (len(td.fields)==0: time.Time, a custom Marshaler, or a
                // degenerate struct with no qdf-visible fields) is hashed as a whole — by
                // its instant, its marshaled form, or via fpHashReflect over its real
                // fields. Cloning such a struct field-by-field would skip its unexported
                // fields (which is ALL of time.Time's wall/ext/loc), zeroing it and
                // producing a clone whose fingerprint differs → ErrPatchBaseMismatch on a
                // legitimate baseline. A typed value copy reproduces every field, is
                // GC-safe, and shares no mutable structure Apply would touch in place
                // (Apply replaces such a value wholesale via the codec).
                if len(td.fields) == 0 {
                        reflect.NewAt(td.rType, dst).Elem().Set(reflect.NewAt(td.rType, src).Elem())
                        return
                }
                for i := range td.fields {
                        f := &td.fields[i]
                        cloneValue(f.desc, unsafe.Add(dst, f.offset), unsafe.Add(src, f.offset), depth+1)
                }
        case reflect.Pointer:
                sp := *(*unsafe.Pointer)(src)
                if sp == nil {
                        return // nil pointer: dst already nil
                }
                elem := td.elem
                if elem == nil {
                        elem, _ = descOf(td.rType.Elem())
                        if elem == nil {
                                return
                        }
                }
                np := reflect.New(elem.rType) // GC-safe allocation of the pointee
                cloneValue(elem, np.UnsafePointer(), sp, depth+1)
                reflect.NewAt(td.rType, dst).Elem().Set(np) // typed pointer store: barriered
        case reflect.Map:
                cloneMap(td, dst, src, depth)
        case reflect.Interface:
                iv := reflect.NewAt(td.rType, src).Elem()
                if iv.IsNil() {
                        return
                }
                ev := iv.Elem() // dynamic value
                edesc, derr := descOf(ev.Type())
                if derr != nil || edesc == nil {
                        reflect.NewAt(td.rType, dst).Elem().Set(iv) // fallback: shallow box copy
                        return
                }
                srcBuf := reflect.New(ev.Type())
                srcBuf.Elem().Set(ev) // addressable copy of the boxed value
                dstBuf := reflect.New(ev.Type())
                cloneValue(edesc, dstBuf.UnsafePointer(), srcBuf.UnsafePointer(), depth+1)
                reflect.NewAt(td.rType, dst).Elem().Set(dstBuf.Elem()) // box the clone
        default:
                // Exotic kinds (chan/func): a typed value copy (barriered, shallow).
                reflect.NewAt(td.rType, dst).Elem().Set(reflect.NewAt(td.rType, src).Elem())
        }
}

// cloneSlice deep-copies the slice at src into a freshly allocated backing at
// dst, preserving nil-vs-empty.
func cloneSlice(td *typeDesc, dst, src unsafe.Pointer, depth int) {
        sh := (*sliceHeader)(src)
        if sh.Data == nil {
                return // nil slice stays nil (dst already zeroed)
        }
        n := sh.Len
        reflectutil.MakeSlice(td.rType, n, dst) // reflect2-aware; non-nil even for n==0
        if n == 0 {
                return // empty-non-nil preserved
        }
        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
                if elem == nil {
                        return
                }
        }
        dsh := (*sliceHeader)(dst)
        stride := td.rType.Elem().Size()
        if elem.pod {
                // Pointer-free elements: one bulk byte copy (GC-safe, holds no pointers).
                copy(unsafe.Slice((*byte)(dsh.Data), uintptr(n)*stride),
                        unsafe.Slice((*byte)(sh.Data), uintptr(n)*stride))
                return
        }
        for i := range n {
                cloneValue(elem, unsafe.Add(dsh.Data, uintptr(i)*stride),
                        unsafe.Add(sh.Data, uintptr(i)*stride), depth+1)
        }
}

// cloneArray deep-copies a non-pod array (a pod array took the byte-copy fast
// path) element by element.
func cloneArray(td *typeDesc, dst, src unsafe.Pointer, depth int) {
        n := td.rType.Len()
        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
                if elem == nil {
                        return
                }
        }
        stride := td.rType.Elem().Size()
        for i := range n {
                cloneValue(elem, unsafe.Add(dst, uintptr(i)*stride),
                        unsafe.Add(src, uintptr(i)*stride), depth+1)
        }
}

// cloneMap deep-copies the map at src into a freshly allocated map at dst,
// preserving nil-vs-empty. Keys are shared (a map key is immutable; Apply never
// mutates one); values are deep-cloned, because Apply can mutate a value's inner
// containers in place and must not reach through to the registered baseline.
func cloneMap(td *typeDesc, dst, src unsafe.Pointer, depth int) {
        mv := reflect.NewAt(td.rType, src).Elem()
        if mv.IsNil() {
                return // nil map stays nil
        }
        // Allocate through reflectutil so qdf_reflect2 swaps in the faster allocator,
        // matching applyMap / the decode path.
        reflectutil.MakeMap(td.rType, mv.Len(), dst)
        dmv := reflect.NewAt(td.rType, dst).Elem()
        valDesc := td.elem
        if valDesc == nil {
                valDesc, _ = descOf(td.rType.Elem())
        }
        if valDesc == nil {
                dmv.Set(mv) // unsupported value type: shallow copy (still a fresh map)
                return
        }
        valType := td.rType.Elem()
        // Hoist the two scratch holders out of the loop: SetMapIndex copies the value
        // into the map, so a single reusable dstBuf is safe (2 allocs total, not 2×N).
        // SetZero before each clone in case cloneValue underfills for this type.
        srcBuf := reflect.New(valType)
        dstBuf := reflect.New(valType)
        for it := mv.MapRange(); it.Next(); {
                srcBuf.Elem().Set(it.Value()) // addressable copy of the value
                dstBuf.Elem().SetZero()
                cloneValue(valDesc, dstBuf.UnsafePointer(), srcBuf.UnsafePointer(), depth+1)
                dmv.SetMapIndex(it.Key(), dstBuf.Elem())
        }
}

// Register stores v as a resolvable baseline and returns its content id — the
// same fingerprint Diff embeds as baseFP. The registry holds v through a
// weak.Pointer; the CALLER must keep v reachable for it to remain resolvable.
func (r *BaselineRegistry[T]) Register(v *T) uint64 {
        id := r.fingerprint(v)
        r.mu.Lock()
        r.m[id] = weak.Make(v)
        r.mu.Unlock()
        return id
}

// fingerprint computes the content id of *v (identical to Diff's baseFP).
func (r *BaselineRegistry[T]) fingerprint(v *T) uint64 {
        if r.td == nil {
                return 0 // unsupported type; such a T can never produce a patch either
        }
        return valueFingerprint(r.td, unsafe.Pointer(v))
}

// Apply resolves the patch's baseline (by its embedded baseFP) to a *T, applies
// the patch onto a fresh deep copy of it, registers the result as a new
// baseline (so it can base the next patch in the stream), and returns the new
// *T. The caller should keep the returned pointer to chain further patches off
// it; dropping it lets the GC reclaim it.
//
// Errors: ErrBaselineRequired (the patch carries no baseFP — produced with
// OptDeltaNoBaseFingerprint), ErrBaselineEvicted (the baseline id is not
// resolvable), plus any error from the underlying Apply.
//
// Because baselines are content-addressed by a 64-bit fingerprint, two distinct
// values that happen to share a fingerprint (probability ~N²/2⁶⁴, negligible
// for thousands of live baselines) collide on the same map key, and the later
// registration overwrites the earlier one; the earlier baseline then resolves
// as ErrBaselineEvicted even while the caller still holds it.
func (r *BaselineRegistry[T]) Apply(patch []byte) (*T, error) {
        h, _, err := readPatchHeader(patch)
        if err != nil {
                return nil, err
        }
        if h.flags&flagPatchBaseFP == 0 {
                return nil, ErrBaselineRequired
        }
        r.mu.Lock()
        wp, ok := r.m[h.baseFP]
        r.mu.Unlock()
        var base *T
        if ok {
                base = wp.Value()
        }
        if base == nil {
                if ok {
                        r.mu.Lock()
                        if cur, still := r.m[h.baseFP]; still && cur.Value() == nil {
                                delete(r.m, h.baseFP)
                        }
                        r.mu.Unlock()
                }
                return nil, ErrBaselineEvicted
        }
        clone := deepClone(base)
        if err := Apply(clone, patch); err != nil {
                return nil, err
        }
        r.mu.Lock()
        r.m[r.fingerprint(clone)] = weak.Make(clone)
        r.mu.Unlock()
        return clone, nil
}

package qdf

import (
        "bytes"
        "math/bits"
        "reflect"
        "time"
        "unsafe"
)

// Columnar column-level diff. For an equal-length diff of a pure-columnar
// []struct, group changes by column (sparse / arithmetic-delta / dense-whole
// per column) and emit a tagColSlicePatch only when it is strictly smaller than
// the positional patch (build-and-compare, never-larger). See
// docs/superpowers/specs/2026-06-16-columnar-column-diff-design.md.

// colDiffMode* are the per-column reconstruction modes in a tagColSlicePatch
// body.
const (
        colDiffModeSparse     byte = 0 // changed-row indices + changed cells
        colDiffModeDelta      byte = 1 // full per-row arithmetic delta (numeric/bool)
        colDiffModeDenseWhole byte = 2 // whole new column re-shipped
)

// colDiffSparseNum/Den: a column with <= n*(Num/Den) changed cells prefers
// sparse mode; otherwise delta/dense-whole. Benchstat-gated knob (Task 5).
const (
        colDiffSparseNum = 1
        colDiffSparseDen = 4
)

// diffColumnarEligible reports whether a slice's columnar plan can use the
// column-level diff: pure columnar (every field is a column, no residual) with
// fewer than 16384 columns. The 16384 cap (2-byte uvarint max) ensures the
// nChangedCols placeholder in diffColumnar always fits in the reserved space.
// For structs with <128 columns nChanged fits in a single byte (backward-
// compatible wire format). For structs with 128..16383 columns diffColumnar
// reserves a 2-byte padded uvarint placeholder. Structs with >=16384 columns
// are pathological and decline to the positional path.
func diffColumnarEligible(colPlan *columnarPlan) bool {
        return colPlan != nil && colPlan.residual == nil && len(colPlan.cols) < 16384
}

// colKindColumnDiffSupported reports whether a column's kind has a column-diff
// encoder/decoder. Nullable columns are dense-whole only (pickColMode forces
// it).
//
// String/[]byte columns are supported via the wire-stateless dict/raw codecs
// (writeStringColumnStateless): those bodies ship their own table/indices and
// touch e.state only for scratch, so they can be built on the shared encoder
// state during diffColumnar's build-and-compare without polluting the intern
// state table. The stateful intern fallback (e.WriteString) and FSST are never
// used on this path.
//
// BLOCKED: nullable with a STRING base. encodeNullableColumn's string case calls
// the stateful e.writeStringColumn (which can fall through to e.WriteString),
// and nullable columns only have a dense-whole path (encodeOneColumn →
// encodeNullableColumn), so there is no stateless route for it yet. It stays
// gated out and falls back to the positional differ. All other nullable bases
// (int/uint/float/float32/bool/time) are stateless and fully supported.
func colKindColumnDiffSupported(col *colColumn) bool {
        if col.kind.isNullable() {
                // Dense-whole only (pickColMode forces it). A string base re-ships through
                // the stateful nullable string codec (encodeNullableColumn → WriteString),
                // for which there is no stateless route → gate it out.
                return col.kind.base() != colKindString
        }
        switch col.kind {
        case colKindInt, colKindUint, colKindFloat, colKindFloat32,
                colKindBool, colKindTime, colKindString:
                return true
        default:
                return false
        }
}

// Benchstat (Intel i7-9750H, OptBalanced, clustered 1000-row batch, ~1/3 of one
// int column changed; -benchmem -count=8): diff ~130 us/op, 22 KB, 23 allocs;
// apply ~58 us/op, 25 KB, 3 allocs. Wire: column body 1007 B vs positional body
// 3276 B (0.31x, a 3.25x shrink). The build-and-compare picker keeps the smaller
// body, so the diff cost (positional build + column build) is paid only on an
// eligible equal-length pure-columnar batch and the wire is never larger.
// Decision: KEEP unconditionally — no pre-gate needed at this scale.
//
// diffColumnar attempts to write a tagColSlicePatch body that is smaller than
// the positional body. It returns (true, nil) when it has placed the winning
// body (column OR positional) into enc.buf, (false, nil) when an unsupported
// column kind means the caller should fall back to the positional path, or
// (false, err) on a real error. n == oldLen == newLen is guaranteed by the
// caller. Exactly one body remains in enc.buf on a true return.
func diffColumnar(enc *Encoder, elem *typeDesc, plan *columnarPlan, stride uintptr,
        oldData, newData unsafe.Pointer, n, depth int) (bool, error) {
        if enc.state == nil {
                enc.state = newEncState()
        }
        st := enc.state

        // Detect which rows changed and, per column, which columns are affected. A
        // column whose kind is not yet column-diff-supported forces a decline of the
        // WHOLE slice only if it actually changed (an unchanged unsupported column
        // emits nothing, so its presence is harmless). Decide before touching enc.buf.
        st.deltaColBitmap = newChangedBitmap(st.deltaColBitmap, n)
        markChangedRows(st.deltaColBitmap, plan, stride, oldData, newData, n, elem.pod)

        // Never-larger trial: build a positional baseline (the size-comparison
        // alternative) and the column body, keep the smaller. Suspend interning for
        // the whole trial so the DISCARDED candidate cannot leak intern ids whose
        // wire definitions are thrown away with it — a later state-ref to such an id
        // dangles (ErrUnknownStateID). This is the same trap diffKeyedSlice guards
        // against; the column string body is already wire-stateless
        // (writeStringColumnStateless), but the positional baseline interned
        // normally and, when the column body won, left orphaned ids in enc.state.
        // Under suspension the only intern substate a body mutates is lastID, which
        // is captured after the positional build and restored if positional wins.
        prevSuspended := enc.stateSuspended
        enc.stateSuspended = true
        defer func() { enc.stateSuspended = prevSuspended }()

        // preTrialLastID is the intern lastID the decoder reaches when the column
        // body wins: that body is wire-stateless (no inline strings, no state-refs),
        // so a decoder reading it never touches lastID and stays at its pre-slice
        // value. The encoder must end there too, or the two diverge and a later
        // pair/repeat state-ref desyncs (the keyed picker keeps the same invariant).
        preTrialLastID := st.lastID
        // shapeStart anchors the shape-id counter the same way lastID is anchored: a
        // suspended StructShape (a codegen marshaler's e.StructShape in the positional
        // trial) still advances shapeCount to keep the decoder aligned, so the
        // discarded candidate's declarations must be rebased off shapeCount when the
        // wire-stateless column body wins — else the next shape id desyncs
        // (ErrUnknownStateID). Mirrors diffKeyedSlice.
        shapeStart := st.shapeCount
        // colShape/hybridShape tables are a SEPARATE id-space (id = table position).
        // A codegen marshaler reached through the positional trial (a nested
        // columnar-eligible []struct field) declares entries via WriteColStructHeader
        // with no stateSuspended gate; when the column body wins, those declaration
        // bytes are truncated with the positional segment, so the entries must be
        // truncated too — a surviving entry would make a LATER encode of the same
        // shape emit a state-ref to an id the decoder never saw (ErrUnknownStateID).
        colShapeStart := len(st.colShapeNames)
        hybridShapeStart := len(st.hybridShapeNames)

        // Build the positional body into enc.buf first (the comparison baseline).
        posStart := len(enc.buf)
        if err := diffElemsPositional(enc, elem, stride, oldData, n, newData, n, depth); err != nil {
                return false, err
        }
        posLen := len(enc.buf) - posStart
        posEndLastID := st.lastID
        shapeAfterPos := st.shapeCount

        body := st.deltaColBuf[:0]
        body = append(body, tagColSlicePatch)
        body = appendUvarint(body, uint64(n))
        colCountPos := len(body)
        // Reserve space for nChangedCols. Structs with <128 columns guarantee
        // nChanged < 128, so a single byte suffices (backward-compatible wire
        // format). Structs with >=128 columns may have nChanged >=128, so reserve
        // two bytes (padded uvarint encoding of 0: 0x80, 0x00) to accommodate up to
        // 16383 changed columns. The apply side always uses readUvarint, so both
        // sizes decode correctly.
        if len(plan.cols) < 128 {
                body = append(body, 0) // 1-byte uvarint placeholder
        } else {
                body = append(body, 0x80, 0) // 2-byte padded uvarint placeholder for 0
        }
        nChanged := 0
        // Column codecs write through enc.buf; swap it to point at our scratch body,
        // restore enc.buf afterward.
        savedBuf := enc.buf
        for ci := range plan.cols {
                col := &plan.cols[ci]
                rows := colChangedRows(st.deltaColRows, st.deltaColBitmap, col, stride, oldData, newData, n)
                st.deltaColRows = rows
                if len(rows) == 0 {
                        continue
                }
                if !colKindColumnDiffSupported(col) {
                        // A changed column we cannot encode → abandon the column patch; the
                        // positional body already in enc.buf handles the whole slice.
                        enc.buf = savedBuf
                        st.deltaColBuf = body[:0]
                        st.lastID = posEndLastID // positional wins; track its kept bytes
                        st.shapeCount = shapeAfterPos
                        return true, nil
                }
                nChanged++
                body = appendUvarint(body, uint64(ci))
                mode := pickColMode(col, len(rows), n)
                body = append(body, mode)
                enc.buf = body
                var err error
                switch mode {
                case colDiffModeSparse:
                        err = encodeSparseColumn(enc, col, stride, oldData, newData, rows)
                case colDiffModeDelta:
                        err = encodeDeltaColumn(enc, col, stride, oldData, newData, n)
                case colDiffModeDenseWhole:
                        if col.kind.base() == colKindString {
                                // encodeOneColumn's string path can hit the stateful intern fallback
                                // (writeStringColumn → WriteString) and pollute enc.state shared with
                                // the discarded positional body. Gather the whole column and emit a
                                // wire-stateless dict/raw body instead. Non-string kinds are stateless
                                // and use encodeOneColumn unchanged.
                                s := st.colScratchStr[:0]
                                for i := range n {
                                        s = append(s, loadStringField(newData, stride, col, i))
                                }
                                st.colScratchStr = s
                                enc.writeStringColumnStateless(s)
                        } else {
                                err = enc.encodeOneColumn(plan, newData, col, n)
                        }
                default:
                        err = ErrInvalidPatch
                }
                if err != nil {
                        enc.buf = savedBuf
                        st.deltaColBuf = body[:0]
                        return false, err
                }
                body = enc.buf
        }
        enc.buf = savedBuf
        // Patch the nChangedCols count into the reserved placeholder bytes.
        // For structs with <128 columns nChanged <128, so single byte is valid
        // (backward-compatible wire format). For structs with >=128 columns two
        // bytes were reserved; write a padded 2-byte uvarint (covers 0..16383).
        if len(plan.cols) < 128 {
                body[colCountPos] = byte(nChanged) // nChanged < 128, single byte valid
        } else {
                body[colCountPos] = byte(nChanged) | 0x80 // low 7 bits + continuation
                body[colCountPos+1] = byte(nChanged >> 7) // high bits
        }
        st.deltaColBuf = body

        // Never-larger: keep the column body only if it strictly beats positional.
        if len(body) < posLen {
                enc.buf = enc.buf[:posStart]
                enc.buf = append(enc.buf, body...)
                // Column body wins. It is wire-stateless, so a decoder leaves lastID at
                // its pre-slice value — restore the encoder to match (the positional
                // build may have moved lastID, e.g. to lruInvalidID on a changed string
                // column). Without this the encoder/decoder lastID + pair predictor
                // diverge after the slice.
                st.lastID = preTrialLastID
                // The kept column body is wire-stateless, so rebase shapeCount off the
                // discarded positional trial's declarations (column body declares none, so
                // this resolves to shapeStart; the general form mirrors diffKeyedSlice).
                st.shapeCount = shapeStart + (st.shapeCount - shapeAfterPos)
                // Truncate colShape/hybridShape entries the discarded positional trial
                // declared: their wire declarations were just truncated with the
                // positional segment, so a surviving table entry would let a later encode
                // of the same shape emit a state-ref the decoder cannot resolve. The kept
                // column body declares none, so everything past the snapshot belongs to
                // the discarded candidate.
                if len(st.colShapeNames) > colShapeStart {
                        st.colShapeNames = st.colShapeNames[:colShapeStart]
                        st.colShapeKinds = st.colShapeKinds[:colShapeStart]
                }
                if len(st.hybridShapeNames) > hybridShapeStart {
                        st.hybridShapeNames = st.hybridShapeNames[:hybridShapeStart]
                        st.hybridShapeKinds = st.hybridShapeKinds[:hybridShapeStart]
                }
        } else {
                // Positional wins; restore lastID to its post-positional value so it
                // tracks the bytes the decoder will actually read.
                st.lastID = posEndLastID
                st.shapeCount = shapeAfterPos
        }
        // Either way exactly one body remains and the slice is handled.
        return true, nil
}

// newChangedBitmap returns a []uint64 with ceil(n/64) words, reusing dst's
// backing when large enough, cleared to zero.
func newChangedBitmap(dst []uint64, n int) []uint64 {
        words := (n + 63) >> 6
        if cap(dst) >= words {
                dst = dst[:words]
                for i := range dst {
                        dst[i] = 0
                }
                return dst
        }
        return make([]uint64, words)
}

// markChangedRows sets bit i for every row whose bytes differ between old and
// new. Returns true if any row changed.
//
// For a pointer-free (POD) element a row changed iff its whole stride span
// differs, so one SIMD bytes.Equal per row replaces a per-column-per-cell sweep
// — the same block-memcmp fast path the positional differ uses (equalSliceEV).
// A padding-only difference can spuriously flag a row, but then every column's
// per-cell compare in colChangedRows finds no real change → the column is
// skipped → no wrong data, only a wasted attribution pass (the documented POD
// trade). A non-POD element (string/[]byte columns) must compare per column.
func markChangedRows(bm []uint64, plan *columnarPlan, stride uintptr,
        oldData, newData unsafe.Pointer, n int, pod bool) bool {
        any := false
        if pod {
                for i := range n {
                        off := uintptr(i) * stride
                        ob := unsafe.Slice((*byte)(unsafe.Add(oldData, off)), stride)
                        nb := unsafe.Slice((*byte)(unsafe.Add(newData, off)), stride)
                        if !bytes.Equal(ob, nb) {
                                bm[i>>6] |= 1 << (uint(i) & 63)
                                any = true
                        }
                }
                return any
        }
        for ci := range plan.cols {
                col := &plan.cols[ci]
                for i := range n {
                        if bm[i>>6]&(1<<(uint(i)&63)) != 0 {
                                continue // already marked by an earlier column
                        }
                        if !colCellEqual(col, stride, oldData, newData, i) {
                                bm[i>>6] |= 1 << (uint(i) & 63)
                                any = true
                        }
                }
        }
        return any
}

// colChangedRows appends, in ascending order, the indices where col differs
// between old and new, restricted to rows already flagged in bm (iterating bm
// via bits.TrailingZeros64 word-skip). dst is reused.
func colChangedRows(dst []int, bm []uint64, col *colColumn,
        stride uintptr, oldData, newData unsafe.Pointer, n int) []int {
        dst = dst[:0]
        for w := range bm {
                word := bm[w]
                base := w << 6
                for word != 0 {
                        b := bits.TrailingZeros64(word)
                        word &= word - 1
                        i := base + b
                        if i >= n {
                                break
                        }
                        if !colCellEqual(col, stride, oldData, newData, i) {
                                dst = append(dst, i)
                        }
                }
        }
        return dst
}

// colCellEqual reports whether column col's cell i is byte/value-equal between
// old and new.
func colCellEqual(col *colColumn, stride uintptr, oldData, newData unsafe.Pointer, i int) bool {
        off := uintptr(i)*stride + col.offset
        op := unsafe.Add(oldData, off)
        np := unsafe.Add(newData, off)
        if col.kind.isNullable() {
                return nullableCellEqual(col, op, np)
        }
        switch col.kind {
        case colKindString:
                if col.isByte {
                        return bytes.Equal(*(*[]byte)(op), *(*[]byte)(np))
                }
                return *(*string)(op) == *(*string)(np)
        case colKindTime:
                ot := (*time.Time)(op).UTC()
                nt := (*time.Time)(np).UTC()
                return ot.Unix() == nt.Unix() && ot.Nanosecond() == nt.Nanosecond()
        default:
                // pointer-free scalar: compare width bytes.
                w := col.width
                return bytes.Equal(unsafe.Slice((*byte)(op), w), unsafe.Slice((*byte)(np), w))
        }
}

// nullableCellEqual compares two *T cells: both nil, or both non-nil with equal
// pointees (string by value, otherwise width-bytes).
func nullableCellEqual(col *colColumn, op, np unsafe.Pointer) bool {
        pa := *(*unsafe.Pointer)(op)
        pb := *(*unsafe.Pointer)(np)
        if pa == nil || pb == nil {
                return pa == pb
        }
        switch col.kind.base() {
        case colKindString:
                return *(*string)(pa) == *(*string)(pb)
        case colKindTime:
                // Mirror colCellEqual's semantic time compare: a raw width-byte memcmp
                // over time.Time would over-report changes because the same instant can
                // have two internal encodings (wall+monotonic vs ext) and a *Location
                // pointer, re-shipping the whole column for an unchanged value.
                ot := (*time.Time)(pa).UTC()
                nt := (*time.Time)(pb).UTC()
                return ot.Unix() == nt.Unix() && ot.Nanosecond() == nt.Nanosecond()
        default:
                return bytes.Equal(unsafe.Slice((*byte)(pa), col.width), unsafe.Slice((*byte)(pb), col.width))
        }
}

// pickColMode chooses the reconstruction mode for a column given how many of
// its n cells changed. Size heuristic only — the top-level build-and-compare is
// the never-larger guarantee.
func pickColMode(col *colColumn, nChanged, n int) byte {
        if col.kind.isNullable() {
                return colDiffModeDenseWhole // nullable: whole-column re-ship only
        }
        if nChanged*colDiffSparseDen <= n*colDiffSparseNum {
                return colDiffModeSparse
        }
        switch col.kind {
        case colKindInt, colKindUint, colKindBool:
                return colDiffModeDelta
        default: // float, float32, string, []byte, time
                return colDiffModeDenseWhole
        }
}

// writeStringColumnStateless emits a string column using only self-contained
// codecs (dict or raw), never the stateful intern fallback (e.WriteString) or
// FSST — safe to build on a shared encoder state during the column-diff
// build-and-compare. The dict/raw bodies ship their own table/indices and read
// e.state only for scratch, so the discarded loser body cannot pollute the
// intern state table and the kept body never emits a state-table ref the decoder
// did not build. Decode is automatic: decodeColumnInto/readStringColumn dispatch
// on the bulk tag we always emit.
func (e *Encoder) writeStringColumnStateless(strs []string) {
        if e.tryWriteStringColumnDict(strs) {
                return
        }
        e.writeStringColumnRawForced(strs)
}

// encodeSparseColumn writes nChanged, the gap-encoded ascending row indices, and
// the changed cells as a length-nChanged subcolumn via the column codec. Only
// int/uint/bool reach this via sparse mode for some kinds; float/float32/string/
// []byte/time can also be sparse (pickColMode returns sparse when changes are
// few), but never delta.
func encodeSparseColumn(enc *Encoder, col *colColumn,
        stride uintptr, oldData, newData unsafe.Pointer, rows []int) error {
        _ = oldData
        enc.buf = appendUvarint(enc.buf, uint64(len(rows)))
        prev := 0
        for _, r := range rows {
                enc.buf = appendUvarint(enc.buf, uint64(r-prev))
                prev = r
        }
        st := enc.state
        switch col.kind {
        case colKindInt:
                s := st.colScratchI64[:0]
                for _, r := range rows {
                        s = append(s, loadIntCell(col, stride, newData, r))
                }
                st.colScratchI64 = s
                return encodeSliceInt64(enc, unsafe.Pointer(&s))
        case colKindUint:
                s := st.colScratchU64[:0]
                for _, r := range rows {
                        s = append(s, loadUintCell(col, stride, newData, r))
                }
                st.colScratchU64 = s
                return encodeSliceUint64(enc, unsafe.Pointer(&s))
        case colKindFloat:
                s := st.colScratchF64[:0]
                for _, r := range rows {
                        s = append(s, loadFloat64Field(newData, stride, col, r))
                }
                st.colScratchF64 = s
                // Lossless: a SCALAR float64 column must never become lossy under
                // OptLossyVec (which targets genuine []float64/[]float32 VECTOR fields).
                return encodeSliceFloat64Lossless(enc, s)
        case colKindFloat32:
                // Float32 cells are gathered as raw bits and emitted via the uint64 codec,
                // which never re-floats them — so the canonical -0.0/NaN normalization that
                // columnar.go applies must be repeated here under OptCanonical.
                canon := enc.opts.Has(OptCanonical)
                s := st.colScratchU64[:0]
                for _, r := range rows {
                        bits := loadFloat32Bits(newData, stride, col, r)
                        if canon {
                                bits = canonicalizeFloat32Bits(bits)
                        }
                        s = append(s, bits)
                }
                st.colScratchU64 = s
                return encodeSliceUint64(enc, unsafe.Pointer(&s))
        case colKindBool:
                s := st.colScratchBool[:0]
                for _, r := range rows {
                        p := unsafe.Add(newData, uintptr(r)*stride+col.offset)
                        s = append(s, *(*bool)(p))
                }
                st.colScratchBool = s
                return encodeSliceBool(enc, unsafe.Pointer(&s))
        case colKindString:
                // Both string and []byte columns gather the changed cells into a []string
                // and emit a wire-stateless dict/raw body (never the intern fallback). A
                // []byte field is viewed as a string via loadStringField (col.isByte), so
                // the same codec handles both; the decoder copies back to owned bytes.
                s := st.colScratchStr[:0]
                for _, r := range rows {
                        s = append(s, loadStringField(newData, stride, col, r))
                }
                st.colScratchStr = s
                enc.writeStringColumnStateless(s)
                return nil
        case colKindTime:
                sec := st.colScratchI64[:0]
                nsec := st.colScratchU64[:0]
                for _, r := range rows {
                        p := unsafe.Add(newData, uintptr(r)*stride+col.offset)
                        tt := (*time.Time)(p).UTC()
                        sec = append(sec, tt.Unix())
                        nsec = append(nsec, uint64(tt.Nanosecond()))
                }
                st.colScratchI64 = sec
                st.colScratchU64 = nsec
                if err := encodeSliceInt64(enc, unsafe.Pointer(&sec)); err != nil {
                        return err
                }
                return encodeSliceUint64(enc, unsafe.Pointer(&nsec))
        }
        return ErrInvalidPatch
}

// encodeDeltaColumn writes the full-length per-row arithmetic delta column
// (new[i] - old[i]). Unchanged cells are 0 → delta/FOR/RLE crush them.
func encodeDeltaColumn(enc *Encoder, col *colColumn,
        stride uintptr, oldData, newData unsafe.Pointer, n int) error {
        st := enc.state
        switch col.kind {
        case colKindInt:
                // Gather new and old contiguously (each gather hoists the width switch
                // once), then a vectorizable subtract — no per-element width branch.
                cur := gatherColI64(st.colScratchI64, newData, stride, col, n)
                prev := gatherColI64(st.deltaColAuxI64, oldData, stride, col, n)
                st.colScratchI64, st.deltaColAuxI64 = cur, prev
                for i := range cur {
                        cur[i] -= prev[i]
                }
                return encodeSliceInt64(enc, unsafe.Pointer(&cur))
        case colKindUint:
                cur := gatherColU64(st.colScratchU64, newData, stride, col, n)
                prev := gatherColU64(st.deltaColAuxU64, oldData, stride, col, n)
                st.colScratchU64, st.deltaColAuxU64 = cur, prev
                for i := range cur {
                        cur[i] -= prev[i]
                }
                return encodeSliceUint64(enc, unsafe.Pointer(&cur))
        case colKindBool:
                // changed-flag column: true where the cell flipped. Apply XORs base where set.
                s := st.colScratchBool[:0]
                for i := range n {
                        po := unsafe.Add(oldData, uintptr(i)*stride+col.offset)
                        pn := unsafe.Add(newData, uintptr(i)*stride+col.offset)
                        s = append(s, *(*bool)(po) != *(*bool)(pn))
                }
                st.colScratchBool = s
                return encodeSliceBool(enc, unsafe.Pointer(&s))
        }
        return ErrInvalidPatch
}

// loadIntCell / loadUintCell / storeIntCell / storeUintCell read or write a
// width-normalized scalar at column cell i. They reuse the loadI64At/loadU64At/
// storeI64At/storeU64At width helpers (nullable_col.go) so the width switch
// lives in one place.
func loadIntCell(col *colColumn, stride uintptr, data unsafe.Pointer, i int) int64 {
        return loadI64At(unsafe.Add(data, uintptr(i)*stride+col.offset), col.width)
}

func loadUintCell(col *colColumn, stride uintptr, data unsafe.Pointer, i int) uint64 {
        return loadU64At(unsafe.Add(data, uintptr(i)*stride+col.offset), col.width)
}

func storeIntCell(col *colColumn, stride uintptr, data unsafe.Pointer, i int, v int64) {
        storeI64At(unsafe.Add(data, uintptr(i)*stride+col.offset), col.width, v)
}

func storeUintCell(col *colColumn, stride uintptr, data unsafe.Pointer, i int, v uint64) {
        storeU64At(unsafe.Add(data, uintptr(i)*stride+col.offset), col.width, v)
}

// applyColSlice applies a tagColSlicePatch onto the base slice at baseP. It does
// not recurse (every column is a flat reconstruction), so it takes no depth.
func applyColSlice(dec *Decoder, td *typeDesc, baseP unsafe.Pointer) error {
        if dec.i >= len(dec.buf) || dec.buf[dec.i] != tagColSlicePatch {
                return ErrInvalidPatch
        }
        dec.i++
        // The columnar plan lives on the slice descriptor (td.colPlan), built by
        // descBuild; it is not mirrored onto the element desc.
        plan := td.colPlan
        if plan == nil {
                return ErrInvalidPatch
        }
        n64, k := readUvarint(dec.buf[dec.i:])
        if k <= 0 {
                return ErrInvalidPatch
        }
        dec.i += k
        bv := reflect.NewAt(td.rType, baseP).Elem()
        if int64(n64) != int64(bv.Len()) {
                return ErrInvalidPatch // equal-length invariant
        }
        n := int(n64)
        base := (*sliceHeader)(baseP).Data
        if dec.state == nil {
                dec.state = newDecState()
        }
        // Bound every column codec's element claim to n (matches decodeColumnInto and
        // the other columnar decode sites); applySparseColumn tightens it to nc for
        // the subcolumn. Defends against an allocation-amplification patch.
        savedColMax := dec.colMaxLen
        dec.colMaxLen = n
        defer func() { dec.colMaxLen = savedColMax }()
        nCols64, k2 := readUvarint(dec.buf[dec.i:])
        if k2 <= 0 || nCols64 > uint64(len(plan.cols)) {
                return ErrInvalidPatch
        }
        dec.i += k2
        prevCol := -1
        for range int(nCols64) {
                colIdx64, k3 := readUvarint(dec.buf[dec.i:])
                if k3 <= 0 || colIdx64 >= uint64(len(plan.cols)) {
                        return ErrInvalidPatch
                }
                dec.i += k3
                if int(colIdx64) <= prevCol {
                        return ErrInvalidPatch // columns must be ascending, no duplicates
                }
                prevCol = int(colIdx64)
                col := &plan.cols[colIdx64]
                if dec.i >= len(dec.buf) {
                        return ErrInvalidPatch
                }
                mode := dec.buf[dec.i]
                dec.i++
                var err error
                switch mode {
                case colDiffModeSparse:
                        err = applySparseColumn(dec, plan, col, base, n)
                case colDiffModeDelta:
                        err = applyDeltaColumn(dec, plan, col, base, n)
                case colDiffModeDenseWhole:
                        err = dec.decodeColumnInto(base, plan, col, n)
                default:
                        err = ErrInvalidPatch
                }
                if err != nil {
                        return err
                }
        }
        return nil
}

// applySparseColumn reads nChanged, gap-decodes ascending row indices, decodes
// the length-nChanged subcolumn, and scatters into base at those rows.
//
// Gap encoding: gap[0] = idx0, gap[j] = idx[j] - idx[j-1] (>= 1 for j>0).
func applySparseColumn(dec *Decoder, plan *columnarPlan, col *colColumn, base unsafe.Pointer, n int) error {
        nc64, k := readUvarint(dec.buf[dec.i:])
        // nc==0 is never emitted by a conforming encoder (a column with no changed
        // rows is skipped); reject it before it sets colMaxLen=0, which colLenOK reads
        // as "unbounded" and would let a constant-codec body force a large make().
        if k <= 0 || nc64 == 0 || nc64 > uint64(n) {
                return ErrInvalidPatch
        }
        dec.i += k
        nc := int(nc64)
        st := dec.state
        rows := st.deltaColRows[:0]
        prev := -1
        for j := range nc {
                gap, kg := readUvarint(dec.buf[dec.i:])
                if kg <= 0 {
                        return ErrInvalidPatch
                }
                dec.i += kg
                // A legitimate gap never exceeds n (j==0: idx=gap<n; j>0: idx=prev+gap<n
                // with prev>=0 → gap<n). Reject larger values BEFORE the int() narrowing:
                // a huge uint64 gap wraps int(gap) negative, and prev+int(gap) can land
                // back inside [0,n) BELOW prev — passing the range check while silently
                // breaking the ascending/no-duplicate row-index invariant.
                if gap > uint64(n) {
                        return ErrInvalidPatch
                }
                var idx int
                if j == 0 {
                        idx = int(gap)
                } else {
                        if gap == 0 {
                                return ErrInvalidPatch // non-ascending
                        }
                        idx = prev + int(gap)
                }
                if idx < 0 || idx >= n {
                        return ErrInvalidPatch
                }
                rows = append(rows, idx)
                prev = idx
        }
        st.deltaColRows = rows
        // The subcolumn has exactly nc elements; tighten the per-column bound so a
        // codec cannot claim more than the changed-cell count before the len check.
        savedColMax := dec.colMaxLen
        dec.colMaxLen = nc
        defer func() { dec.colMaxLen = savedColMax }()
        switch col.kind {
        case colKindInt:
                if err := decodeSliceInt64Into(dec, &st.colScratchI64); err != nil {
                        return err
                }
                s := st.colScratchI64
                if len(s) != nc {
                        return ErrTypeMismatch
                }
                for j, r := range rows {
                        storeIntCell(col, plan.stride, base, r, s[j])
                }
        case colKindUint:
                if err := decodeSliceUint64Into(dec, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != nc {
                        return ErrTypeMismatch
                }
                for j, r := range rows {
                        storeUintCell(col, plan.stride, base, r, s[j])
                }
        case colKindFloat:
                if err := decodeSliceFloat64Into(dec, &st.colScratchF64); err != nil {
                        return err
                }
                s := st.colScratchF64
                if len(s) != nc {
                        return ErrTypeMismatch
                }
                for j, r := range rows {
                        storeFloat64(base, plan.stride, col, r, s[j])
                }
        case colKindFloat32:
                if err := decodeSliceUint64Into(dec, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != nc {
                        return ErrTypeMismatch
                }
                for j, r := range rows {
                        storeFloat32Bits(base, plan.stride, col, r, s[j])
                }
        case colKindBool:
                if err := decodeSliceBoolInto(dec, &st.colScratchBool); err != nil {
                        return err
                }
                s := st.colScratchBool
                if len(s) != nc {
                        return ErrTypeMismatch
                }
                for j, r := range rows {
                        *(*bool)(unsafe.Add(base, uintptr(r)*plan.stride+col.offset)) = s[j]
                }
        case colKindString:
                // Producer emitted a wire-stateless dict/raw body (writeStringColumnStateless)
                // for both string and []byte columns; readStringColumn dispatches on the
                // bulk tag. dec.colMaxLen is already tightened to nc above.
                strs, err := dec.readStringColumn(nc)
                if err != nil {
                        return err
                }
                if len(strs) != nc {
                        return ErrTypeMismatch
                }
                if col.isByte {
                        for j, r := range rows {
                                dp := unsafe.Add(base, uintptr(r)*plan.stride+col.offset)
                                *(*[]byte)(dp) = append([]byte(nil), strs[j]...)
                        }
                } else {
                        for j, r := range rows {
                                *(*string)(unsafe.Add(base, uintptr(r)*plan.stride+col.offset)) = strs[j]
                        }
                }
        case colKindTime:
                if err := decodeSliceInt64Into(dec, &st.colScratchI64); err != nil {
                        return err
                }
                sec := st.colScratchI64 // distinct scratch from nsec (U64) → no copy needed
                if len(sec) != nc {
                        return ErrTypeMismatch
                }
                if err := decodeSliceUint64Into(dec, &st.colScratchU64); err != nil {
                        return err
                }
                nsec := st.colScratchU64
                if len(nsec) != nc {
                        return ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return err
                }
                for j, r := range rows {
                        dp := unsafe.Add(base, uintptr(r)*plan.stride+col.offset)
                        *(*time.Time)(dp) = time.Unix(sec[j], int64(nsec[j])).UTC()
                }
        default:
                return ErrInvalidPatch
        }
        return nil
}

// applyDeltaColumn reads the full-length delta column and adds it onto base.
func applyDeltaColumn(dec *Decoder, plan *columnarPlan, col *colColumn, base unsafe.Pointer, n int) error {
        st := dec.state
        switch col.kind {
        case colKindInt:
                if err := decodeSliceInt64Into(dec, &st.colScratchI64); err != nil {
                        return err
                }
                s := st.colScratchI64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        storeIntCell(col, plan.stride, base, i, loadIntCell(col, plan.stride, base, i)+s[i])
                }
        case colKindUint:
                if err := decodeSliceUint64Into(dec, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        storeUintCell(col, plan.stride, base, i, loadUintCell(col, plan.stride, base, i)+s[i])
                }
        case colKindBool:
                // changed-flag column: flip base where the flag is set.
                if err := decodeSliceBoolInto(dec, &st.colScratchBool); err != nil {
                        return err
                }
                s := st.colScratchBool
                if len(s) != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        if s[i] {
                                p := unsafe.Add(base, uintptr(i)*plan.stride+col.offset)
                                *(*bool)(p) = !*(*bool)(p)
                        }
                }
        default:
                return ErrInvalidPatch
        }
        return nil
}

package qdf

import (
        "bytes"
        "reflect"
        "slices"
        "time"
        "unsafe"
)

// diffValue compares the value at oldP/newP and, if changed, writes one op
// (op byte + payload). The caller has already written any preceding selector
// (field index / slice index / map key).
func diffValue(enc *Encoder, td *typeDesc, oldP, newP unsafe.Pointer, depth int) error {
        if depth > maxDeltaDepth {
                return ErrCycleDetected
        }
        if equalValue(td, oldP, newP, depth) {
                return nil
        }
        switch td.kind {
        case reflect.Struct:
                if len(td.fields) == 0 { // time.Time / marshaler struct: whole replace
                        return writeReplace(enc, td, newP)
                }
                enc.buf = append(enc.buf, opMerge)
                return diffStruct(enc, td, oldP, newP, depth)
        case reflect.Pointer:
                op, np := *(*unsafe.Pointer)(oldP), *(*unsafe.Pointer)(newP)
                if op == nil || np == nil {
                        // presence change (both-nil was handled by equalValue earlier) → replace
                        return writeReplace(enc, td, newP)
                }
                if td.elem != nil && td.elem.kind == reflect.Struct && len(td.elem.fields) > 0 {
                        enc.buf = append(enc.buf, opMerge)
                        return diffStruct(enc, td.elem, op, np, depth+1)
                }
                return writeReplace(enc, td, newP)
        case reflect.Slice:
                if td.rType.Elem().Kind() == reflect.Uint8 {
                        return writeReplace(enc, td, newP) // []byte: whole replace
                }
                // A nil↔non-nil transition is not expressible by a positional slice patch
                // (apply rebuilds via MakeSlice, which is always non-nil). Fall back to a
                // whole-value replace so the field codec preserves nil-vs-empty.
                if ((*sliceHeader)(oldP).Data == nil) != ((*sliceHeader)(newP).Data == nil) {
                        return writeReplace(enc, td, newP)
                }
                elem := td.elem
                if elem == nil {
                        elem, _ = descOf(td.rType.Elem())
                }
                if elem != nil && elem.keyed && keyTokenable(elem.keyDesc) {
                        enc.buf = append(enc.buf, opMerge)
                        return diffKeyedSlice(enc, td, elem, oldP, newP, depth)
                }
                enc.buf = append(enc.buf, opMerge)
                return diffSlice(enc, td, oldP, newP, depth)
        case reflect.Array:
                if td.rType.Elem().Kind() == reflect.Uint8 {
                        return writeReplace(enc, td, newP) // [N]byte: whole replace
                }
                enc.buf = append(enc.buf, opMerge)
                return diffArray(enc, td, oldP, newP, depth)
        case reflect.Map:
                // Same nil↔non-nil concern as slices: applyMap never reconstructs a nil
                // map, so a nilness transition must be a whole-value replace.
                if (*(*unsafe.Pointer)(oldP) == nil) != (*(*unsafe.Pointer)(newP) == nil) {
                        return writeReplace(enc, td, newP)
                }
                enc.buf = append(enc.buf, opMerge)
                return diffMap(enc, td, oldP, newP, depth)
        default:
                // Scalars/string/[]byte and presence/nilness changes are a whole-value
                // replace; structs/slices/arrays/maps/pointers merge in their own cases above.
                return writeReplace(enc, td, newP)
        }
}

// diffSlice writes a tagSlicePatch body for a non-[]byte slice. The whole-slice
// equality short-circuit already happened in diffValue→equalValue (one SIMD
// bytes.Equal over POD backing arrays), so we only get here on a real difference.
func diffSlice(enc *Encoder, td *typeDesc, oldP, newP unsafe.Pointer, depth int) error {
        oh := (*sliceHeader)(oldP)
        nh := (*sliceHeader)(newP)
        stride := td.rType.Elem().Size()
        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
        }
        // td.colPlan (the slice element's columnar plan, built on the slice
        // descriptor) is threaded in so diffColumnar routes off a parameter rather
        // than a field on the shared element typeDesc — no write to a published desc.
        return diffElems(enc, elem, td.colPlan, stride, oh.Data, oh.Len, nh.Data, nh.Len, depth)
}

func diffArray(enc *Encoder, td *typeDesc, oldP, newP unsafe.Pointer, depth int) error {
        n := td.rType.Len()
        stride := td.rType.Elem().Size()
        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
        }
        // Arrays are never columnar-eligible here (colPlan lives on slice element
        // descriptors), so route straight to the positional differ.
        return diffElemsPositional(enc, elem, stride, oldP, n, newP, n, depth)
}

// diffMap writes a tagMapPatch: updated/added keys (each with a recursive op),
// then a tombstone list of deleted keys. Keys are the identity, so there is no
// positional ambiguity. Two passes (count, then emit) avoid buffer surgery.
func diffMap(enc *Encoder, td *typeDesc, oldP, newP unsafe.Pointer, depth int) error {
        ov := reflect.NewAt(td.rType, oldP).Elem()
        nv := reflect.NewAt(td.rType, newP).Elem()
        keyType := td.rType.Key()
        keyDesc, err := descOf(keyType)
        if err != nil {
                return err
        }
        valDesc := td.elem
        if valDesc == nil {
                valDesc, err = descOf(td.rType.Elem())
                if err != nil {
                        return err
                }
        }

        // Compare map values with the SAME comparator diffValue uses (equalValue),
        // not reflect.Value.Equal: the latter compares pointer identity for *T values
        // (inconsistent with diffValue's deref) and panics on non-comparable values
        // (map/slice). Reuse two addressable buffers to avoid per-key allocation.
        oCmp := reflect.New(valDesc.rType).Elem()
        nCmp := reflect.New(valDesc.rType).Elem()
        valEqual := func(oVal, nVal reflect.Value) bool {
                oCmp.Set(oVal)
                nCmp.Set(nVal)
                return equalValue(valDesc, oCmp.Addr().UnsafePointer(), nCmp.Addr().UnsafePointer(), depth)
        }

        enc.buf = append(enc.buf, tagMapPatch)

        // Under OptCanonical both emit passes (updates/adds and deletion tombstones)
        // must visit keys in sorted order so logically-equal patches serialize
        // byte-identically. The count passes are order-independent, so they keep the
        // fast MapRange loop; only the two emit passes route through the sorted
        // iterator, which reuses the SAME kind-specialized sort as the reflect map
        // encoder (typed pooled scratch for string/int/uint, a reflect-comparator
        // fallback for exotic key kinds).
        canon := enc.opts.Has(OptCanonical)

        // Pass 1: count updates/additions.
        nUpd := 0
        for it := nv.MapRange(); it.Next(); {
                oVal := ov.MapIndex(it.Key())
                if oVal.IsValid() && valEqual(oVal, it.Value()) {
                        continue
                }
                nUpd++
        }
        enc.buf = appendUvarint(enc.buf, uint64(nUpd))
        // Pass 2: emit updates/additions.
        keyBuf := reflect.New(keyType).Elem()
        emitUpd := func(k reflect.Value) error {
                nVal := nv.MapIndex(k)
                oVal := ov.MapIndex(k)
                if oVal.IsValid() && valEqual(oVal, nVal) {
                        return nil
                }
                keyBuf.Set(k)
                if err := keyDesc.encode(enc, keyBuf.Addr().UnsafePointer()); err != nil {
                        return err
                }
                if oVal.IsValid() {
                        // The skip check above ran valEqual (oVal is valid), which already set
                        // oCmp/nCmp to (oVal, nVal); reuse those addressable buffers directly.
                        return diffValue(enc, valDesc,
                                oCmp.Addr().UnsafePointer(), nCmp.Addr().UnsafePointer(), depth+1)
                }
                // Addition: oVal is invalid, so valEqual short-circuited and nCmp is
                // stale; set it from nVal and reuse the buffer for the replace.
                nCmp.Set(nVal)
                return writeReplace(enc, valDesc, nCmp.Addr().UnsafePointer())
        }
        if canon {
                if err := enc.canonSortedMapKeys(nv, keyType, emitUpd); err != nil {
                        return err
                }
        } else {
                for it := nv.MapRange(); it.Next(); {
                        if err := emitUpd(it.Key()); err != nil {
                                return err
                        }
                }
        }

        // Deletions: keys in old, absent in new.
        nDel := 0
        for it := ov.MapRange(); it.Next(); {
                if !nv.MapIndex(it.Key()).IsValid() {
                        nDel++
                }
        }
        enc.buf = appendUvarint(enc.buf, uint64(nDel))
        emitDel := func(k reflect.Value) error {
                if nv.MapIndex(k).IsValid() {
                        return nil
                }
                keyBuf.Set(k)
                return keyDesc.encode(enc, keyBuf.Addr().UnsafePointer())
        }
        if canon {
                if err := enc.canonSortedMapKeys(ov, keyType, emitDel); err != nil {
                        return err
                }
        } else {
                for it := ov.MapRange(); it.Next(); {
                        if err := emitDel(it.Key()); err != nil {
                                return err
                        }
                }
        }
        return nil
}

// canonSortedMapKeys iterates the map's keys in canonical sorted order, calling
// fn for each. For string/int/uint key kinds a single reflect.Value holder is
// allocated once and reused across all iterations (1 alloc total instead of N),
// matching the pattern in reflect_encode.go's encodeMapCanonical. fn is called
// synchronously, so the holder is valid for each call. Exotic / float / bool
// key kinds fall back to rv.MapKeys() sorted by canonReflectKeyCompare.
//
// Re-entrancy: canonKeysBusy is held for the full duration of the typed gather
// (set by gatherStringKeys/IntKeys/UintKeys, cleared by canonKeysRelease). Any
// recursive call from within fn for a nested map's value sees canonKeysBusy=true
// and gets a fresh local scratch — exactly the behaviour the gather guards
// provide, with no additional latch needed here.
func (e *Encoder) canonSortedMapKeys(rv reflect.Value, keyType reflect.Type, fn func(reflect.Value) error) error {
        switch keyType.Kind() {
        case reflect.String:
                ks, gp := e.gatherStringKeys(rv)
                kh := reflect.New(keyType).Elem()
                for _, k := range ks {
                        kh.SetString(k)
                        if err := fn(kh); err != nil {
                                e.canonKeysRelease(gp)
                                return err
                        }
                }
                e.canonKeysRelease(gp)
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                ks, gp := e.gatherIntKeys(rv)
                kh := reflect.New(keyType).Elem()
                for _, k := range ks {
                        kh.SetInt(k)
                        if err := fn(kh); err != nil {
                                e.canonKeysRelease(gp)
                                return err
                        }
                }
                e.canonKeysRelease(gp)
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                ks, gp := e.gatherUintKeys(rv)
                kh := reflect.New(keyType).Elem()
                for _, k := range ks {
                        kh.SetUint(k)
                        if err := fn(kh); err != nil {
                                e.canonKeysRelease(gp)
                                return err
                        }
                }
                e.canonKeysRelease(gp)
        default:
                // Float / bool / struct / array / interface keys: gather and sort with the
                // shared stable comparator (also covers the rare exotic kinds).
                keys := rv.MapKeys()
                slices.SortFunc(keys, canonReflectKeyCompare)
                for _, k := range keys {
                        if err := fn(k); err != nil {
                                return err
                        }
                }
        }
        return nil
}

// diffElems is the shared element differ for slices and arrays. For an
// equal-length pure-columnar slice it routes through diffColumnar (column-level
// patch, never-larger picker); every other case falls through to the positional
// differ. Arrays never reach diffColumnar (they call diffElemsPositional
// directly via diffArray).
func diffElems(enc *Encoder, elem *typeDesc, colPlan *columnarPlan, stride uintptr,
        oldData unsafe.Pointer, oldLen int, newData unsafe.Pointer, newLen int, depth int) error {
        if oldLen == newLen && oldLen >= columnarMinElems && diffColumnarEligible(colPlan) {
                handled, err := diffColumnar(enc, elem, colPlan, stride, oldData, newData, oldLen, depth)
                if err != nil {
                        return err
                }
                if handled {
                        return nil
                }
        }
        return diffElemsPositional(enc, elem, stride, oldData, oldLen, newData, newLen, depth)
}

// diffElemsPositional is the positional element differ: one tagSlicePatch body
// listing each changed/appended element index with its recursive op.
func diffElemsPositional(enc *Encoder, elem *typeDesc, stride uintptr,
        oldData unsafe.Pointer, oldLen int, newData unsafe.Pointer, newLen int, depth int) error {
        minLen := min(newLen, oldLen)
        // elem.pod was precomputed at build (noPointersWalk of the element type) —
        // read the field instead of walking per call. elem is always non-nil here
        // (resolved by diffSlice/diffArray); guard defensively against an unresolved
        // descriptor.
        pod := elem != nil && elem.pod
        // The appended tail [minLen,newLen) is always emitted, so reserve at least
        // that many up front; byte-identical, saves a realloc round on growth.
        entries := make([]int, 0, newLen-minLen)
        for i := range minLen {
                oP := unsafe.Add(oldData, uintptr(i)*stride)
                nP := unsafe.Add(newData, uintptr(i)*stride)
                var same bool
                if pod {
                        same = bytes.Equal(unsafe.Slice((*byte)(oP), stride), unsafe.Slice((*byte)(nP), stride))
                } else {
                        same = equalValue(elem, oP, nP, depth)
                }
                if !same {
                        entries = append(entries, i)
                }
        }
        for i := minLen; i < newLen; i++ {
                entries = append(entries, i)
        }

        enc.buf = append(enc.buf, tagSlicePatch)
        enc.buf = appendUvarint(enc.buf, uint64(newLen))
        enc.buf = appendUvarint(enc.buf, uint64(len(entries)))
        for _, i := range entries {
                enc.buf = appendUvarint(enc.buf, uint64(i))
                nP := unsafe.Add(newData, uintptr(i)*stride)
                if i < oldLen {
                        oP := unsafe.Add(oldData, uintptr(i)*stride)
                        if err := diffValue(enc, elem, oP, nP, depth+1); err != nil {
                                return err
                        }
                } else {
                        if err := writeReplace(enc, elem, nP); err != nil { // appended element
                                return err
                        }
                }
        }
        return nil
}

// writeReplace writes opReplace + the whole new value via the normal codec.
func writeReplace(enc *Encoder, td *typeDesc, newP unsafe.Pointer) error {
        enc.buf = append(enc.buf, opReplace)
        return td.encode(enc, newP)
}

// diffStruct writes a tagStructPatch body with only the changed fields.
func diffStruct(enc *Encoder, td *typeDesc, oldP, newP unsafe.Pointer, depth int) error {
        var changed []int
        for i := range td.fields {
                f := &td.fields[i]
                if !equalValue(f.desc, unsafe.Add(oldP, f.offset), unsafe.Add(newP, f.offset), depth) {
                        changed = append(changed, i)
                }
        }
        enc.buf = append(enc.buf, tagStructPatch)
        enc.buf = appendUvarint(enc.buf, uint64(len(changed)))
        for _, i := range changed {
                f := &td.fields[i]
                enc.buf = appendUvarint(enc.buf, uint64(i))
                if err := diffValue(enc, f.desc,
                        unsafe.Add(oldP, f.offset), unsafe.Add(newP, f.offset), depth+1); err != nil {
                        return err
                }
        }
        return nil
}

// equalValue reports whether the value of type described by td at aP equals the
// one at bP. POD scalars reduce to a width compare; strings/[]byte to a SIMD
// memcmp (bytes.Equal); containers recurse. This is the diff walk's unchanged
// fast path — for an unchanged subtree it is the ONLY work done.
func equalValue(td *typeDesc, aP, bP unsafe.Pointer, depth int) bool {
        if depth > maxDeltaDepth {
                // Force the diff path, which hits its own cap and errors out.
                return false
        }
        switch td.kind {
        case reflect.Bool:
                return *(*bool)(aP) == *(*bool)(bP)
        case reflect.Int, reflect.Uint, reflect.Uintptr:
                // Platform-width: 8 bytes on 64-bit, 4 on 32-bit. Reading *(*uint64)
                // here would be an OOB read on 32-bit (see slices_fast.go width gating).
                return *(*uint)(aP) == *(*uint)(bP)
        case reflect.Int64, reflect.Uint64, reflect.Float64:
                return *(*uint64)(aP) == *(*uint64)(bP)
        case reflect.Int8, reflect.Uint8:
                return *(*uint8)(aP) == *(*uint8)(bP)
        case reflect.Int16, reflect.Uint16:
                return *(*uint16)(aP) == *(*uint16)(bP)
        case reflect.Int32, reflect.Uint32, reflect.Float32:
                return *(*uint32)(aP) == *(*uint32)(bP)
        case reflect.String:
                return *(*string)(aP) == *(*string)(bP)
        case reflect.Slice:
                // A nil slice and an empty-non-nil slice both have len 0 but the field
                // codec preserves the distinction; treat them as unequal so diffValue
                // emits an op (a whole-value opReplace via its nilness check).
                if ((*sliceHeader)(aP).Data == nil) != ((*sliceHeader)(bP).Data == nil) {
                        return false
                }
                if td.rType.Elem().Kind() == reflect.Uint8 {
                        return bytes.Equal(*(*[]byte)(aP), *(*[]byte)(bP))
                }
                return equalSliceEV(td, aP, bP, depth)
        case reflect.Array:
                return equalArrayEV(td, aP, bP, depth)
        case reflect.Struct:
                // time.Time and custom-marshaler structs have no td.fields. Use
                // reflect.DeepEqual, NOT reflect.Value.Equal, which panics when the struct
                // holds a non-comparable field (slice/map/func) — a real crash for a
                // Marshaler type with e.g. a []int field. Rare path; correctness over speed.
                if len(td.fields) == 0 {
                        if td.rType == timeType {
                                // time.Time is the common fields-less struct. Compare the instant the
                                // codec actually round-trips (absolute sec + nsec) — no reflect, and it
                                // avoids a spurious change when only the monotonic-clock reading or the
                                // *Location differs (which the UTC sec+nsec wire form drops anyway).
                                at, bt := (*time.Time)(aP), (*time.Time)(bP)
                                return at.Unix() == bt.Unix() && at.Nanosecond() == bt.Nanosecond()
                        }
                        av := reflect.NewAt(td.rType, aP).Elem()
                        bv := reflect.NewAt(td.rType, bP).Elem()
                        // Fast path: a comparable fields-less struct (time.Time — common, on the
                        // hot path for any timestamped value — and comparable Marshaler structs)
                        // compares with reflect.Value.Equal: no .Interface() boxing, no DeepEqual
                        // reflection. Only a NON-comparable Marshaler field (slice/map/func), where
                        // Value.Equal would panic, falls back to DeepEqual.
                        if td.rType.Comparable() {
                                return av.Equal(bv)
                        }
                        return reflect.DeepEqual(av.Interface(), bv.Interface())
                }
                for i := range td.fields {
                        f := &td.fields[i]
                        if !equalValue(f.desc, unsafe.Add(aP, f.offset), unsafe.Add(bP, f.offset), depth+1) {
                                return false
                        }
                }
                return true
        case reflect.Pointer:
                ap, bp := *(*unsafe.Pointer)(aP), *(*unsafe.Pointer)(bP)
                if ap == nil || bp == nil {
                        return ap == bp
                }
                return equalValue(td.elem, ap, bp, depth+1)
        case reflect.Map:
                // nil map vs empty-non-nil map: same nil-vs-empty distinction as slices.
                if (*(*unsafe.Pointer)(aP) == nil) != (*(*unsafe.Pointer)(bP) == nil) {
                        return false
                }
                return equalMapEV(td, aP, bP, depth)
        default:
                // Interface and exotic kinds: reflect.Value.Equal panics on a
                // non-comparable dynamic value (a []int / map / func inside an any), so
                // compare structurally with DeepEqual, which handles them. This path is
                // rare and off the hot path.
                av := reflect.NewAt(td.rType, aP).Elem()
                bv := reflect.NewAt(td.rType, bP).Elem()
                return reflect.DeepEqual(av.Interface(), bv.Interface())
        }
}

// equalSliceEV compares two non-[]byte slices element-by-element. For fixed-size
// pointer-free element types it does one bytes.Equal over the backing arrays
// (the block-memcmp fast path).
//
// Note: uses the existing sliceHeader type from reuse.go (fields Data/Len/Cap).
func equalSliceEV(td *typeDesc, aP, bP unsafe.Pointer, depth int) bool {
        ah := (*sliceHeader)(aP)
        bh := (*sliceHeader)(bP)
        if ah.Len != bh.Len {
                return false
        }
        n := ah.Len
        if n == 0 {
                return true
        }
        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
        }
        stride := td.rType.Elem().Size()
        if elem != nil && elem.pod {
                // POD memcmp compares padding too: a padding-only diff yields a spurious opReplace, never wrong data — acceptable for Phase 1.
                ab := unsafe.Slice((*byte)(ah.Data), uintptr(n)*stride)
                bb := unsafe.Slice((*byte)(bh.Data), uintptr(n)*stride)
                return bytes.Equal(ab, bb)
        }
        for i := range n {
                if !equalValue(elem, unsafe.Add(ah.Data, uintptr(i)*stride),
                        unsafe.Add(bh.Data, uintptr(i)*stride), depth+1) {
                        return false
                }
        }
        return true
}

func equalArrayEV(td *typeDesc, aP, bP unsafe.Pointer, depth int) bool {
        if td.rType.Elem().Kind() == reflect.Uint8 {
                n := uintptr(td.rType.Len())
                return bytes.Equal(unsafe.Slice((*byte)(aP), n), unsafe.Slice((*byte)(bP), n))
        }
        n := td.rType.Len()
        stride := td.rType.Elem().Size()
        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
        }
        if elem != nil && elem.pod {
                // POD memcmp compares padding too: a padding-only diff yields a spurious opReplace, never wrong data — acceptable for Phase 1.
                total := uintptr(n) * stride
                return bytes.Equal(unsafe.Slice((*byte)(aP), total), unsafe.Slice((*byte)(bP), total))
        }
        for i := range n {
                if !equalValue(elem, unsafe.Add(aP, uintptr(i)*stride),
                        unsafe.Add(bP, uintptr(i)*stride), depth+1) {
                        return false
                }
        }
        return true
}

func equalMapEV(td *typeDesc, aP, bP unsafe.Pointer, depth int) bool {
        av := reflect.NewAt(td.rType, aP).Elem()
        bv := reflect.NewAt(td.rType, bP).Elem()
        if av.Len() != bv.Len() {
                return false
        }
        if av.Len() == 0 {
                // Both maps empty (lengths matched) and equalValue's Map case already
                // pre-checked nil-vs-non-nil before calling here, so matching nilness +
                // zero entries ⇒ equal contents. Skip the two reflect.New comparison
                // buffers — the common (nil/empty map) case allocated them for nothing.
                return true
        }
        valDesc := td.elem
        if valDesc == nil {
                var err error
                valDesc, err = descOf(td.rType.Elem())
                if err != nil {
                        return false
                }
        }
        aCmp := reflect.New(valDesc.rType).Elem()
        bCmp := reflect.New(valDesc.rType).Elem()
        iter := av.MapRange()
        for iter.Next() {
                bVal := bv.MapIndex(iter.Key())
                if !bVal.IsValid() {
                        return false
                }
                aCmp.Set(iter.Value())
                bCmp.Set(bVal)
                if !equalValue(valDesc, aCmp.Addr().UnsafePointer(), bCmp.Addr().UnsafePointer(), depth+1) {
                        return false
                }
        }
        return true
}

package qdf

import (
        "math"
        "reflect"
        "unsafe"
)

// keyToken returns a string usable as a Go map key that uniquely identifies the
// element's key field value WITHOUT allocating. elemP points at an element of a
// keyed struct type td; td.keyOff/td.keyDesc locate the key field. Delegates to
// keyTokenAt over the key field pointer.
func keyToken(td *typeDesc, elemP unsafe.Pointer) string {
        return keyTokenAt(td.keyDesc, unsafe.Add(elemP, td.keyOff))
}

// keyTokenAt returns the token for a key value of descriptor kd located at kp.
//   - string key: the key string itself (its content is the identity).
//   - scalar / [N]byte key: unsafe.String over the key's raw bytes. Valid only
//     while the backing value is alive — true for the duration of a single
//     Diff/Apply call. Safe because these kinds are gap-free (no padding within
//     the key value), so the bytes ARE the whole value.
//   - exotic comparable key (a comparable struct): the allocating reflect
//     fallback (rare; the keyed path is gated to the kinds above elsewhere, so
//     this branch is defensive).
func keyTokenAt(kd *typeDesc, kp unsafe.Pointer) string {
        switch kd.kind {
        case reflect.String:
                return *(*string)(kp)
        case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
                reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
                reflect.Uint64, reflect.Uintptr, reflect.Float32, reflect.Float64:
                return unsafe.String((*byte)(kp), int(kd.rType.Size()))
        case reflect.Array:
                if kd.rType.Elem().Kind() == reflect.Uint8 {
                        return unsafe.String((*byte)(kp), kd.rType.Len())
                }
                return keyTokenReflect(kd, kp)
        default:
                return keyTokenReflect(kd, kp)
        }
}

// keyedLinearMax is the element count below which keyed matching uses a linear
// scan (no map allocation). O(n^2) but n small; cheaper than a map's hashing
// plus warm-up allocation, and fully alloc-free.
const keyedLinearMax = 32

// keyTokenable reports whether the key kind is one keyTokenAt handles without
// the allocating reflect fallback (so the keyed path is worth taking).
func keyTokenable(kd *typeDesc) bool {
        switch kd.kind {
        case reflect.String, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16,
                reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16,
                reflect.Uint32, reflect.Uint64, reflect.Uintptr, reflect.Float32, reflect.Float64:
                return true
        case reflect.Array:
                return kd.rType.Elem().Kind() == reflect.Uint8
        default:
                return false
        }
}

func diffKeyedSlice(enc *Encoder, td, elem *typeDesc, oldP, newP unsafe.Pointer, depth int) error {
        oh := (*sliceHeader)(oldP)
        nh := (*sliceHeader)(newP)
        stride := td.rType.Elem().Size()
        oldKeyAt := func(i int) string { return keyToken(elem, unsafe.Add(oh.Data, uintptr(i)*stride)) }
        newKeyAt := func(i int) string { return keyToken(elem, unsafe.Add(nh.Data, uintptr(i)*stride)) }

        lookup, dup, release := buildKeyLookup(&enc.keyIdx, &enc.keyIdxBusy, oldKeyAt, oh.Len)
        defer release()
        // Thread a reusable scratch map into hasDupNewKeys so calls with
        // n > keyedLinearMax reuse existing bucket storage instead of
        // heap-allocating per call. Fall back to nil (fresh alloc inside
        // hasDupNewKeys) when busy due to a nested keyed-slice diff.
        var newKeyScratch map[string]struct{}
        if !enc.newKeyIdxBusy && nh.Len > keyedLinearMax {
                enc.newKeyIdxBusy = true
                if enc.newKeyIdx == nil {
                        enc.newKeyIdx = make(map[string]struct{}, nh.Len)
                }
                newKeyScratch = enc.newKeyIdx
                defer func() {
                        clear(newKeyScratch)
                        enc.newKeyIdxBusy = false
                }()
        }
        if dup || hasDupNewKeys(newKeyAt, nh.Len, newKeyScratch) {
                // Ambiguous identity → positional fallback. diffValue already wrote opMerge;
                // diffSlice writes its own tagSlicePatch body, so apply dispatches correctly.
                return diffSlice(enc, td, oldP, newP, depth)
        }

        // Never-larger picker (docs/DELTA.md:86). The keyed patch wins big on a
        // reorder (preserved elements cost nothing) but LOSES on high key turnover:
        // the new-order key list plus the full per-op replaces expand past — and on
        // full rotation even past a full re-encode — the positional alternative. So
        // build both bodies and keep the smaller, mirroring diffColumnar.
        //
        // Building a discarded candidate would normally leak intern definitions
        // (ids whose wire defs are thrown away → a later state-ref dangles), the
        // trap diffColumnar dodges via a wire-stateless string column. A keyed
        // element diff recurses through arbitrary value codecs, so that trick does
        // not generalise; instead suspend interning for the whole trial. Both
        // candidates and the kept winner are then wire-stateless, the size compare
        // is exact, and the only intern substate a suspended body mutates is lastID
        // (captured after the positional build, restored if positional wins).
        prevSuspended := enc.stateSuspended
        enc.stateSuspended = true
        defer func() { enc.stateSuspended = prevSuspended }()

        // shapeStart anchors the shape-id counter: a suspended StructShape (a
        // qdfgen EncoderMarshaler element field) emits a fresh declaration and
        // advances shapeCount per emit, so the discarded candidate's declarations
        // must be re-based off the counter or a later shape ref desyncs.
        shapeStart := uint32(0)
        if enc.state != nil {
                shapeStart = enc.state.shapeCount
        }

        posStart := len(enc.buf)
        if err := diffSlice(enc, td, oldP, newP, depth); err != nil {
                return err
        }
        posLen := len(enc.buf) - posStart
        posEndLastID, shapeAfterPos, haveLast := uint32(0), shapeStart, enc.state != nil
        if haveLast {
                posEndLastID = enc.state.lastID
                shapeAfterPos = enc.state.shapeCount
        }

        // Build the keyed body APPENDED after the positional one, using enc.buf
        // itself as scratch (no extra allocation): keep whichever is smaller.
        keyedStart := len(enc.buf)
        if err := encodeKeyedSlicePatch(enc, elem, oh, nh, stride, lookup, oldKeyAt, newKeyAt, depth); err != nil {
                return err
        }
        keyedLen := len(enc.buf) - keyedStart

        if keyedLen < posLen {
                // Shift the keyed body down over the positional one. lastID already
                // reflects the keyed build (emitted last), matching the kept body.
                copy(enc.buf[posStart:], enc.buf[keyedStart:])
                enc.buf = enc.buf[:posStart+keyedLen]
                // Keyed wins: only its declarations reach the decoder, so drop the
                // discarded positional candidate's from the shape counter.
                if haveLast {
                        enc.state.shapeCount = shapeStart + (enc.state.shapeCount - shapeAfterPos)
                }
        } else {
                // Positional wins; drop the trailing keyed body and restore lastID and
                // the shape counter to their post-positional values so they track the
                // bytes the decoder will read.
                enc.buf = enc.buf[:keyedStart]
                if haveLast {
                        enc.state.lastID = posEndLastID
                        enc.state.shapeCount = shapeAfterPos
                }
        }
        return nil
}

// encodeKeyedSlicePatch writes a tagKeyedSlicePatch body: an optional new-order
// key list (when the key sequence changed) followed by the per-key ops. Factored
// out of diffKeyedSlice so the never-larger picker can build it as one candidate.
func encodeKeyedSlicePatch(enc *Encoder, elem *typeDesc, oh, nh *sliceHeader, stride uintptr,
        lookup keyLookup, oldKeyAt, newKeyAt func(int) string, depth int) error {
        orderChanged := oh.Len != nh.Len
        if !orderChanged {
                for i := range nh.Len {
                        if oldKeyAt(i) != newKeyAt(i) {
                                orderChanged = true
                                break
                        }
                }
        }

        enc.buf = append(enc.buf, tagKeyedSlicePatch)
        flags := byte(0)
        if orderChanged {
                flags |= flagKeyedOrderChanged
        }
        enc.buf = append(enc.buf, flags)

        if orderChanged {
                enc.buf = appendUvarint(enc.buf, uint64(nh.Len))
                for i := range nh.Len {
                        nP := unsafe.Add(nh.Data, uintptr(i)*stride)
                        if err := elem.keyDesc.encode(enc, unsafe.Add(nP, elem.keyOff)); err != nil {
                                return err
                        }
                }
        }

        // Count ops, then emit (so nOps is known before the entries).
        nOps := 0
        for i := range nh.Len {
                nP := unsafe.Add(nh.Data, uintptr(i)*stride)
                oi, ok := lookupGet(lookup, oldKeyAt, oh.Len, newKeyAt(i))
                if ok && equalValue(elem, unsafe.Add(oh.Data, uintptr(oi)*stride), nP, depth) {
                        continue
                }
                nOps++
        }
        enc.buf = appendUvarint(enc.buf, uint64(nOps))
        for i := range nh.Len {
                nP := unsafe.Add(nh.Data, uintptr(i)*stride)
                k := newKeyAt(i)
                oi, ok := lookupGet(lookup, oldKeyAt, oh.Len, k)
                if ok {
                        oP := unsafe.Add(oh.Data, uintptr(oi)*stride)
                        if equalValue(elem, oP, nP, depth) {
                                continue
                        }
                        if err := elem.keyDesc.encode(enc, unsafe.Add(nP, elem.keyOff)); err != nil {
                                return err
                        }
                        if err := diffValue(enc, elem, oP, nP, depth+1); err != nil {
                                return err
                        }
                } else {
                        if err := elem.keyDesc.encode(enc, unsafe.Add(nP, elem.keyOff)); err != nil {
                                return err
                        }
                        if err := writeReplace(enc, elem, nP); err != nil {
                                return err
                        }
                }
        }
        return nil
}

// applyKeyedSlice reads a tagKeyedSlicePatch body and reconciles the base slice
// by element identity (the key field), mirroring diffKeyedSlice on the encode
// side. Element copies use typed reflect Set so the compiler emits GC write
// barriers (raw memmove would not); the new slice is built with reflect.MakeSlice.
func applyKeyedSlice(dec *Decoder, td *typeDesc, baseP unsafe.Pointer, depth int) error {
        if depth > maxDeltaDepth {
                return ErrInvalidPatch
        }
        if dec.i >= len(dec.buf) || dec.buf[dec.i] != tagKeyedSlicePatch {
                return ErrInvalidPatch
        }
        dec.i++
        if dec.i >= len(dec.buf) {
                return ErrInvalidPatch
        }
        flags := dec.buf[dec.i]
        dec.i++

        elem := td.elem
        if elem == nil {
                elem, _ = descOf(td.rType.Elem())
        }
        if elem == nil || !elem.keyed || elem.keyDesc == nil {
                return ErrInvalidPatch
        }
        elemType := td.rType.Elem()
        stride := elemType.Size()
        bh := (*sliceHeader)(baseP)
        baseKeyAt := func(i int) string { return keyToken(elem, unsafe.Add(bh.Data, uintptr(i)*stride)) }

        if flags&flagKeyedOrderChanged == 0 {
                // Value-only updates on the same key sequence; apply each op in place.
                lookup, _, release := buildKeyLookup(&dec.keyIdx, &dec.keyIdxBusy, baseKeyAt, bh.Len)
                defer release()
                nOps, k := readUvarint(dec.buf[dec.i:])
                if k <= 0 || nOps > uint64(len(dec.buf)-dec.i) {
                        return ErrInvalidPatch
                }
                dec.i += k
                keyHold := reflect.New(elem.keyDesc.rType).Elem()
                for range nOps {
                        if err := elem.keyDesc.decode(dec, keyHold.Addr().UnsafePointer()); err != nil {
                                return err
                        }
                        tok := keyTokenAt(elem.keyDesc, keyHold.Addr().UnsafePointer())
                        oi, ok := lookupGet(lookup, baseKeyAt, bh.Len, tok)
                        if !ok {
                                return ErrInvalidPatch // op for a key not in base (divergent base)
                        }
                        if err := applyValue(dec, elem, unsafe.Add(bh.Data, uintptr(oi)*stride), depth+1); err != nil {
                                return err
                        }
                }
                return nil
        }

        // orderChanged: read new key order, build base lookup, construct the new slice.
        newLen64, k := readUvarint(dec.buf[dec.i:])
        if k <= 0 {
                return ErrInvalidPatch
        }
        dec.i += k
        if newLen64 > uint64(math.MaxInt) { // 32-bit: int(newLen64) would truncate silently
                return ErrInvalidPatch
        }
        newLen := int(newLen64)
        if newLen < 0 || uint64(newLen) > uint64(len(dec.buf)-dec.i) {
                return ErrInvalidPatch
        }
        // Decode all order keys into ONE contiguous typed buffer so each token can
        // alias its stable element (no per-key string copy). keysV stays live for the
        // function, keeping the key bytes (and any string-key heap content) reachable.
        order := make([]string, newLen)
        keysV := reflect.MakeSlice(reflect.SliceOf(elem.keyDesc.rType), newLen, newLen)
        keysBase := keysV.UnsafePointer()
        ksize := elem.keyDesc.rType.Size()
        for i := range newLen {
                kp := unsafe.Add(keysBase, uintptr(i)*ksize)
                if err := elem.keyDesc.decode(dec, kp); err != nil {
                        return err
                }
                order[i] = keyTokenAt(elem.keyDesc, kp) // aliases keysV (stable) — no copy
        }

        lookup, _, release := buildKeyLookup(&dec.keyIdx, &dec.keyIdxBusy, baseKeyAt, bh.Len)
        defer release()

        nv := reflect.MakeSlice(td.rType, newLen, newLen)
        nb := nv.UnsafePointer()
        orderIdx := make(map[string]int, newLen)
        for i, tok := range order {
                if _, dup := orderIdx[tok]; dup {
                        // Duplicate identity in the decoded order is ambiguous; the diff side
                        // never emits one (hasDupNewKeys routes to positional), so reject a
                        // hostile/divergent patch rather than silently mis-assigning slots.
                        return ErrInvalidPatch
                }
                orderIdx[tok] = i
        }
        filled := make([]bool, newLen)
        // Copy unchanged base elements into their new slots (GC-safe typed Set).
        for i := range newLen {
                oi, ok := lookupGet(lookup, baseKeyAt, bh.Len, order[i])
                if ok {
                        dst := reflect.NewAt(elemType, unsafe.Add(nb, uintptr(i)*stride)).Elem()
                        src := reflect.NewAt(elemType, unsafe.Add(bh.Data, uintptr(oi)*stride)).Elem()
                        dst.Set(src)
                        filled[i] = true
                }
        }
        // Apply ops into their slots by key.
        nOps, k2 := readUvarint(dec.buf[dec.i:])
        if k2 <= 0 || nOps > uint64(len(dec.buf)-dec.i) {
                return ErrInvalidPatch
        }
        dec.i += k2
        keyHold2 := reflect.New(elem.keyDesc.rType).Elem()
        for range nOps {
                if err := elem.keyDesc.decode(dec, keyHold2.Addr().UnsafePointer()); err != nil {
                        return err
                }
                tok := keyTokenAt(elem.keyDesc, keyHold2.Addr().UnsafePointer())
                ni, ok := orderIdx[tok]
                if !ok {
                        return ErrInvalidPatch
                }
                slot := unsafe.Add(nb, uintptr(ni)*stride)
                if err := applyValue(dec, elem, slot, depth+1); err != nil {
                        return err
                }
                filled[ni] = true
        }
        for i := range newLen {
                if !filled[i] {
                        return ErrInvalidPatch // a key with neither a base match nor an op
                }
        }
        reflect.NewAt(td.rType, baseP).Elem().Set(nv)
        return nil
}

// keyLookup chooses linear (small n, no map) vs a built key→index map. When a
// map is used it is carried in m so lookupGet reads the exact map this build
// produced, immune to a nested keyed-slice frame reusing the shared scratch.
type keyLookup struct {
        m      map[string]int
        useMap bool
}

// buildKeyLookup builds an old/base key→index lookup. For n > keyedLinearMax it
// borrows the caller's reusable scratch map (enc.keyIdx / dec.keyIdx) when it is
// free; if a parent keyed-slice frame already holds it (re-entrancy via a nested
// keyed slice), it allocates a fresh local map so the parent's lookup is never
// clobbered. release() returns the borrow (no-op for the linear / nested cases).
func buildKeyLookup(m *map[string]int, busy *bool, keyAt func(int) string, n int) (lk keyLookup, dup bool, release func()) {
        noop := func() {}
        if n <= keyedLinearMax {
                for i := range n {
                        ki := keyAt(i)
                        for j := range i {
                                if keyAt(j) == ki {
                                        return keyLookup{}, true, noop
                                }
                        }
                }
                return keyLookup{useMap: false}, false, noop
        }
        var lm map[string]int
        release = noop
        if !*busy {
                *busy = true
                if *m == nil {
                        *m = make(map[string]int, n)
                } else {
                        clear(*m)
                }
                lm = *m
                release = func() { *busy = false }
        } else {
                lm = make(map[string]int, n)
        }
        for i := range n {
                ki := keyAt(i)
                if _, exists := lm[ki]; exists {
                        return keyLookup{useMap: true, m: lm}, true, release
                }
                lm[ki] = i
        }
        return keyLookup{useMap: true, m: lm}, false, release
}

func lookupGet(l keyLookup, keyAt func(int) string, n int, key string) (int, bool) {
        if l.useMap {
                i, ok := l.m[key]
                return i, ok
        }
        for i := range n {
                if keyAt(i) == key {
                        return i, true
                }
        }
        return 0, false
}

func hasDupNewKeys(keyAt func(int) string, n int, scratch map[string]struct{}) bool {
        if n <= keyedLinearMax {
                for i := range n {
                        ki := keyAt(i)
                        for j := range i {
                                if keyAt(j) == ki {
                                        return true
                                }
                        }
                }
                return false
        }
        seen := scratch
        if seen == nil {
                seen = make(map[string]struct{}, n)
        }
        for i := range n {
                k := keyAt(i)
                if _, ok := seen[k]; ok {
                        return true
                }
                seen[k] = struct{}{}
        }
        return false
}

// keyTokenReflect is the allocating fallback for exotic comparable key types
// (a comparable struct). Rare and off the hot path; the keyed diff/apply path is
// gated (keyTokenable, a later task) to the non-fallback kinds, so a struct key
// falls back to positional diff rather than relying on this token.
func keyTokenReflect(kd *typeDesc, kp unsafe.Pointer) string {
        return reflect.NewAt(kd.rType, kp).Elem().String()
}

package qdf

import "encoding/binary"

// Patch container wire format. A patch is NOT a normal qdf blob: it carries
// its own magic ('QDP') so a patch can never be mistaken for a full value and
// vice versa. Header layout:
//
//        'Q' 'D' 'P'  patchVersion1            4 bytes magic+version
//        flags        (1 byte)                 patch flag bits
//        schemaFP     (8 bytes, little-endian) hash of typeDesc(T)
//        baseFP       (8 bytes, little-endian) present iff flags&flagPatchBaseFP
//        body...                               root patch (rANS-framed iff flagPatchRANS)
const (
        patchMagic0   = 'Q'
        patchMagic1   = 'D'
        patchMagic2   = 'P'
        patchVersion1 = 0x01
)

// Patch flag bits (header byte 4).
const (
        flagPatchDense  byte = 1 << 0 // field names interned in the patch body
        flagPatchRANS   byte = 1 << 1 // body after the header is rANS-compressed
        flagPatchBaseFP byte = 1 << 2 // baseFP field present (8 bytes)
)

// Patch body tags. Disjoint numbering from the value tag space (wire.go) — a
// patch body is parsed by the delta reader, never by the value tag dispatcher.
const (
        tagStructPatch     = 0x01 // varuint(nChanged), nChanged×(varuint(fieldIdx), op)
        tagSlicePatch      = 0x02 // varuint(newLen), varuint(nEntries), nEntries×(varuint(idx), op)
        tagMapPatch        = 0x03 // varuint(nUpdate), nUpdate×(key-value, op), varuint(nDelete), nDelete×(key-value)
        tagKeyedSlicePatch = 0x04 // flags byte, [if orderChanged: varuint(newLen)+newLen×key], varuint(nOps), nOps×(key, op)
        tagColSlicePatch   = 0x05 // varuint(n), varuint(nChangedCols), per col: varuint(colIdx), mode byte, body
)

// Keyed-slice-patch flag bits (the flags byte after tagKeyedSlicePatch).
const (
        flagKeyedOrderChanged = 1 << 0 // keys were added/removed/reordered; the new key order follows
)

// Op bytes. Every changed location is one op byte + payload.
const (
        opUnchanged = 0x00 // never written; sparse encoding omits unchanged locations
        opReplace   = 0x01 // payload is the whole new value (normal td.encode codec)
        opMerge     = 0x02 // payload is a recursive patch body (tagStructPatch/Slice/Map)
)

type patchHeader struct {
        schemaFP uint64
        baseFP   uint64
        flags    byte
}

// writePatchHeader appends the header to dst. baseFP is written only when
// flags&flagPatchBaseFP is set.
func writePatchHeader(dst []byte, flags byte, schemaFP, baseFP uint64) []byte {
        dst = append(dst, patchMagic0, patchMagic1, patchMagic2, patchVersion1, flags)
        dst = binary.LittleEndian.AppendUint64(dst, schemaFP)
        if flags&flagPatchBaseFP != 0 {
                dst = binary.LittleEndian.AppendUint64(dst, baseFP)
        }
        return dst
}

// readPatchHeader parses the header and returns it plus the number of header
// bytes consumed. The body starts at buf[n:].
func readPatchHeader(buf []byte) (patchHeader, int, error) {
        var h patchHeader
        if len(buf) < 13 || buf[0] != patchMagic0 || buf[1] != patchMagic1 ||
                buf[2] != patchMagic2 || buf[3] != patchVersion1 {
                return h, 0, ErrInvalidPatch
        }
        h.flags = buf[4]
        h.schemaFP = binary.LittleEndian.Uint64(buf[5:13])
        n := 13
        if h.flags&flagPatchBaseFP != 0 {
                if len(buf) < 21 {
                        return h, 0, ErrInvalidPatch
                }
                h.baseFP = binary.LittleEndian.Uint64(buf[13:21])
                n = 21
        }
        return h, n, nil
}

package qdf

import (
        "encoding/binary"
        "math"
        "slices"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/fsst"
        "github.com/alex60217101990/qdf/internal/rans"
        "github.com/alex60217101990/qdf/internal/unsafestr"
        "github.com/alex60217101990/qdf/internal/vecquant"
)

// ransMinBytes is the smallest message body worth a rANS attempt. Below it
// the 256-entry frequency table dwarfs any entropy saving, so the picker
// would reject it anyway — this threshold just skips the wasted encode.
const ransMinBytes = 512

// maybeApplyRANS rewrites the message at e.buf[start:] in place to a
// rANS-compressed body when that is strictly smaller than the plain body,
// setting FlagRANS in the header. start is the message's header offset (0 for
// Marshal, len(dst) for AppendMarshal). No-op unless OptRANS is set.
func (e *Encoder) maybeApplyRANS(start int) {
        if !e.rans {
                return
        }
        if e.customFramed {
                // A top-level Marshaler forced Fast framing and its bytes are
                // opts-invariant by contract; do not reframe them with FlagRANS.
                return
        }
        const hdr = 5
        if len(e.buf)-start < hdr+ransMinBytes {
                return
        }
        body := e.buf[start+hdr:]
        cand := appendUvarint(make([]byte, 0, len(body)/2+512), uint64(len(body)))
        cand = rans.Encode(cand, body)
        if len(cand) >= len(body) {
                return // no win — keep the plain body
        }
        // Decline rANS if the frame we'd emit would trip the decoder's origLen
        // bound (decoder.go: origLen > len(buf)*64 + 1MiB rejects, to cap the decode
        // allocation). A body that compresses more than ~64x — e.g. a multi-MiB
        // low-entropy/repeated payload — has an origLen the decoder won't accept, so
        // rANS would make a wire its own Unmarshal rejects. Keep the plain body
        // (the decoder bounds it by remaining input as usual). The emitted frame is
        // hdr + len(cand) bytes, matching the decoder's len(d.buf) for this message.
        if uint64(len(body)) > uint64(hdr+len(cand))*64+(1<<20) {
                return
        }
        e.buf = append(e.buf[:start+hdr], cand...)
        e.buf[start+4] |= FlagRANS
}

// preInternEntry caches a caller-registered string by its backing
// pointer and length. id == preInternUnseen marks an entry that
// has not yet been interned on the wire — the first WriteString
// that matches still goes through the normal intern path so the
// decoder sees the tagInternStr record; subsequent matches reuse
// the resolved id and skip the hash + slot probe.
// See Encoder.PreIntern for the contract.
type preInternEntry struct {
        ptr unsafe.Pointer
        n   uintptr
        id  uint32
        // Naturally 24 B: ptr(8)+n(8)+id(4) rounds up to the 8 B struct alignment.
}

const preInternUnseen = ^uint32(0)

// Encoder writes a single QDF value into a growing internal buffer. Reset
// drops the buffer contents and (in Dense mode) the intern table so the
// encoder can be reused.
// Field order groups the pointer-bearing / slice / map fields and the
// 8-byte counters first, then the 1-byte flags (mode + the many bools) last
// so the interspersed bools do not each force a padding word (200 bytes vs
// 232 for the source order).
type Encoder struct { // betteralign:ignore — hand-tuned for SIZE (200 vs 232 bytes); bools packed last
        state *encState

        // fsstDict, when non-nil, is a pre-trained FSST symbol table supplied via
        // FSSTDict.Marshal. The encoder uses it instead of training a table per
        // string column (the dominant FSST encode cost), so train-once-reuse-many
        // collapses per-batch encode to compression only. The table is bounded and
        // immutable; it is never mutated by the encoder.
        fsstDict *fsst.SymbolTable

        // fsstCachedTbl holds the FSST symbol table trained on the most recent
        // batch when no user-supplied fsstDict is set. Reused across consecutive
        // same-encoder batches to eliminate the per-batch Build training cost
        // (~140–311 µs). Retrained every fsstReuseInterval batches so the table
        // adapts when string distributions shift. Cleared by Reset() on pool
        // recycle so a pooled encoder never carries stale data into the next
        // caller; survives resetForReuse() so a streaming caller amortises training.
        fsstCachedTbl  *fsst.SymbolTable
        fsstBatchCount int // consecutive reuses of fsstCachedTbl since last retrain

        // keyIdx is a reused (clear-not-realloc) old-key→index map for keyed slice
        // diff. Lives on the Encoder so a many-element keyed slice builds its match
        // table once per pool acquire; dropped past a spike cap in Reset(). Single
        // pointer word — grouped with state/fsstDict ahead of the slices so the GC
        // pointer-scan range stays tight (the slices' len/cap words trail).
        keyIdx map[string]int

        buf []byte

        // alpScratch is a reused FOR-mantissa staging buffer for the ALP float
        // writer (mirrors the decoder's deltaScratch). Lives on the Encoder, not
        // encState, so the row-major float path reuses it without needing a state.
        alpScratch []uint64

        // pforExcPos / pforExcDelta collect exception (relPos, delta) pairs during
        // the PFOR pack pass so the second O(n) scan of s is eliminated. Storing
        // relative positions pre-computed during the pack loop lets the emit pass
        // loop over the compact scratch instead of re-scanning the full slice.
        // Pointer-free ([]uint64); retained across calls, dropped past the ceiling.
        pforExcPos   []uint64
        pforExcDelta []uint64

        // alpExcScratch collects exception indices during the ALP float pack pass
        // so the second O(n) re-scan of the slice is eliminated (mirrors the PFOR
        // pattern above). Pointer-free ([]int); retained across calls.
        alpExcScratch []int

        // wideF64 reuses the []float32 -> []float64 widening buffer for the lossy
        // vector codec, retained across encodes and dropped past the ceiling in
        // resetForReuse. For f32 fields this avoids one allocation per encode;
        // for f64 fields it holds a defensive copy so appendLossyVec cannot mutate
        // the caller's slice.
        wideF64 []float64

        // vecBatchFlat / vecBatchRows reuse the gather scratch for the batched lossy
        // vector-column codec: vecBatchFlat is the flat [n*dim]float64 backing and
        // vecBatchRows the [n] row views into it. Retained across encodes, dropped
        // past the ceiling in Reset.
        vecBatchFlat []float64
        vecBatchRows [][]float64
        // vecBatchNorms reuses the per-vector L2 norm scratch for the polar-split
        // variant of the batched vector codec.
        vecBatchNorms []float64

        // preIntern is an opt-in identity cache populated by PreIntern.
        // When non-empty WriteString does a linear scan against it
        // before falling to the intern table — a pointer-and-length
        // match means we already know the intern id and can skip the
        // hash + slot probe. The slice is empty (and the check is a
        // single branch-predicted compare) when no caller has opted
        // in, so it does not regress the default Marshal path.
        preIntern []preInternEntry

        // wideI64 / wideU64 are reused widening scratch for the QPack slice
        // encoders: []int32 / []uint32 must be promoted to []int64 / []uint64 so
        // the int64/uint64 codec pickers can score them. The widen → pick → emit
        // sequence is atomic (no nested slice encode runs between fill and the last
        // read), so a single scratch per element type is safely reused across every
        // narrow-int slice field in a message — turning one make per field into
        // zero. Bounded on return to the encoder pool (putEnc).
        wideI64 []int64
        wideU64 []uint64

        // blkPlanI64 / blkPlanU64 cache the per-block codec picks made while costing
        // the per-block adaptive codec, so the emit pass reuses them instead of
        // re-picking every block (pickI64Codec is the dominant encode cost). Reused
        // across columns; bounded on return to the pool.
        blkPlanI64 []blockPlanI64
        blkPlanU64 []blockPlanU64

        // zoneMM caches the per-zone (min,max) pairs — already reduced to their
        // uvarint payload (zigzag for int64, raw for uint64) — computed while
        // costing the min/max zonemap against the linear one in writeZoneChunk*.
        // When the linear map wins it is unused; when it loses the emit pass reuses
        // these instead of a second minMaxI64/minMaxU64 scan. Reused across columns.
        zoneMM []uint64

        // vecScratch reuses the lossy vector codec's rotate/row/coord buffers across
        // encodes, retained between batches and dropped past the ceiling in
        // resetForReuse. Mirrors alpScratch. Placed last among the pointer-bearing
        // fields: its embedded struct ends in a non-pointer bool (Overflowed), so
        // trailing it keeps that byte outside the GC pointer-scan prefix.
        vecScratch vecquant.Scratch

        // minIntern is the minimum string length eligible for interning;
        // shorter values go in line.
        minIntern int

        // maxStateEntries caps the intern table. Past the cap, new strings go
        // in line; existing IDs still resolve.
        maxStateEntries int

        // headerFlagAt is the byte offset of the header flag byte in e.buf,
        // recorded by writeHeader so encodeColumnar can backpatch FlagColIndex
        // only when it actually emits a column index.
        headerFlagAt int

        // depth tracks nested pointer/struct traversal. Pointer cycles do
        // not crash the process; encodePtr increments depth on entry and
        // returns ErrCycleDetected when it exceeds maxDepth. Lightweight
        // alternative to a per-pointer set (no allocation per call).
        depth    int
        maxDepth int

        // ifaceDepth > 0 while encoding a value reached through a dynamic (interface)
        // field/element — a schemaless position that the decoder reads back via
        // decodeAny. The batched lossy-vector []struct block (tagVecBatchStruct) has
        // no decodeAny case, so encodeSlice suppresses it in this context and falls
        // back to the columnar path (tagColStruct), which decodeAny does handle.
        ifaceDepth int

        // vecBudget is the fidelity target used when OptLossyVec is active.
        // No effect unless OptLossyVec is set. Zero value resolves to MinCosine(0.999).
        // Placed among the 8-byte fields (it is pointer-free, 8-byte aligned) so the
        // trailing 1-byte flags pack contiguously without a padding gap on either side.
        vecBudget VectorBudget

        // opts is the bit-mask of feature toggles. mode and qpack are
        // derived from it at configure time so the hot path can stay on
        // fast bool / Mode compares; the rest of the codecs (MTF, Pair,
        // ShapeIntern) check the corresponding bit directly via opts. Placed
        // next to the 1-byte flags so the uint32 packs without a padding gap.
        opts Options

        mode Mode

        headerOut bool
        // customFramed is set when a top-level Marshaler emitted the body and forced
        // a Fast (flag 0) header. maybeApplyRANS must then leave the wire alone — a
        // Marshaler's bytes are opts-invariant by contract, so the entropy pass must
        // not reframe them with FlagRANS (see TestMarshaler_AlwaysFastFraming).
        customFramed bool

        // qpack switches the slice fast paths to QPack codecs (bitpack, FOR,
        // Gorilla, raw-LE bulk). When set, the header's FlagQPack bit is
        // emitted as an early hint so a reader that does not implement the
        // codec tags fails fast on the header rather than mid-stream.
        qpack bool

        // gorillaFloat lets encodeSliceFloat64 / encodeSliceFloat32 probe
        // the input and pick Gorilla XOR for smooth time-series. Costs
        // ~10× more CPU per slice than raw-LE bulk, so it stays off in
        // OptBalanced; flipped on by OptCompression. Implies qpack.
        gorillaFloat bool

        // rans enables the order-0 rANS entropy post-pass over the finished
        // body at the top-level Marshal entry points (never per nested value,
        // never in streaming). Set from OptRANS; applied only when it shrinks.
        rans bool

        // fsst enables the FSST string codec for columnar string columns. Set
        // from OptFSST; applied only when strictly smaller than dict/per-value.
        // Implies qpack (columnar path requires OptQPack).
        fsst bool

        // colIndex makes encodeColumnar emit a fixed-width uint32 column-length
        // table after the shape declaration and before the column bodies, and
        // backpatch FlagColIndex onto the header. Set from OptColumnIndex. Lets a
        // reader skip columns without decoding them.
        colIndex bool

        // zonemap stores eligible columnar int columns as zone-chunked containers
        // (tag 0xF1) with a per-zone [min,max] map for predicate zone-skip. Set from
        // OptZoneMap (requires qpack). See qpack_zonechunk.go.
        zonemap bool

        // pairPred / mtf cache OptPairPred / OptMTF (both on in OptBalanced) so the
        // hot Dense state-ref emit path tests a bool field instead of re-running
        // opts.Has() several times per repeated value. Set in applyOpts, cleared in
        // Reset — same pattern as qpack/rans/fsst/colIndex.
        pairPred bool
        mtf      bool

        // keyIdxBusy marks keyIdx as borrowed by an in-progress keyed-slice diff so a
        // nested keyed slice routes to a fresh local map instead of clobbering it.
        keyIdxBusy bool

        // newKeyIdx is a reused seen-map for hasDupNewKeys (delta_keyed.go), cleared
        // (not reallocated) per diff call so large new-key sets above keyedLinearMax
        // reuse the bucket storage instead of heap-allocating per call. Kept on the
        // Encoder (not encState) next to keyIdx so both keyed-diff scratch fields
        // share the same cache neighbourhood. newKeyIdxBusy guards against
        // re-entrancy: a nested keyed slice sees true and falls back to a fresh map.
        newKeyIdx     map[string]struct{}
        newKeyIdxBusy bool

        // stateSuspended turns off every wire-stateful encoding — string/[]byte
        // interning, struct-shape interning, map-shape interning — for the duration
        // of a keyed-slice delta's never-larger trial (delta_keyed.go). A keyed patch
        // is compared against the positional alternative by building both bodies and
        // keeping the smaller; a stateful DISCARDED body leaks shared-state ids whose
        // wire definitions are thrown away, so a later reference into the kept wire
        // dangles (ErrUnknownStateID). Suspending makes both candidate bodies — and
        // the emitted winner — wire-stateless, so the trial is pollution-free, the
        // size comparison is exact, and the kept body is self-consistent. The only
        // substate a suspended body mutates is lastID (the inline-string reset, which
        // the decoder mirrors). Saved/restored across nested keyed slices so
        // re-entrancy keeps the outer suspension.
        stateSuspended bool
}

// Suspended reports whether wire-stateful encodings (string/shape/map-shape
// interning, the columnar shape table) are suspended — true inside a delta
// never-larger trial (delta_keyed.go / delta_columnar.go). Generated code
// checks it before emitting a columnar frame: WriteColStructHeader declares
// into the shared shape table with ids the discarded trial candidate never
// ships, so a suspended encode must fall back to the row-major body exactly
// like the reflect columnar path (which gates on the same flag).
func (e *Encoder) Suspended() bool { return e.stateSuspended }

// applyOpts mirrors the options bitmask onto the cached mode / qpack
// fields so hot-path checks compile to a single bool / Mode compare
// instead of a bit-test. It is safe to call on a pooled encoder;
// pair with state setup as needed (callers that need OptDense must
// also point state at a non-nil encState).
//
//go:nosplit
func (e *Encoder) applyOpts(opts Options) {
        e.opts = opts
        if opts.Has(OptDense) {
                e.mode = Dense
                // Lazily allocate the Dense intern state. A pooled encoder starts with
                // state == nil (set in the pool's New) and never allocates it while it
                // only serves OptSpeed; the first Dense encode brings it up. A reused
                // encoder keeps its state — Reset() clears it before this runs.
                if e.state == nil {
                        e.state = newEncState()
                }
        } else {
                e.mode = Fast
        }
        e.qpack = opts.Has(OptQPack)
        e.gorillaFloat = e.qpack && opts.Has(OptGorillaFloat)
        e.rans = opts.Has(OptRANS)
        e.colIndex = opts.Has(OptColumnIndex)
        e.zonemap = e.qpack && opts.Has(OptZoneMap)
        e.fsst = e.qpack && opts.Has(OptFSST)
        e.pairPred = opts.Has(OptPairPred)
        e.mtf = opts.Has(OptMTF)
}

// DefaultMaxDepth caps reflect-path pointer/struct recursion. Set
// large enough for any legitimate payload (10 000) while still
// rejecting genuine cycles before the goroutine stack overflows.
const DefaultMaxDepth = 10_000

// Mode selects the wire dialect.
type Mode uint8

const (
        // Fast writes strings and []byte in line. No intern bookkeeping.
        Fast Mode = 0

        // Dense writes repeated strings/bytes once and references them by ID
        // thereafter. Smaller output on repetitive payloads; slightly slower
        // per call.
        Dense Mode = 1
)

// NewEncoder returns an Encoder. The internal buffer is allocated lazily
// on first write.
func NewEncoder(mode Mode) *Encoder {
        e := &Encoder{
                mode:            mode,
                minIntern:       4,
                maxStateEntries: 1 << 14,
                maxDepth:        DefaultMaxDepth,
        }
        if mode == Dense {
                e.state = newEncState()
                e.opts = OptBalanced
                e.qpack = true
                // OptBalanced includes OptPairPred | OptMTF; mirror them onto the cached
                // flags this constructor sets by hand (it bypasses applyOpts).
                e.pairPred = true
                e.mtf = true
        } else {
                e.opts = OptSpeed
        }
        return e
}

// NewEncoderWith returns an Encoder configured by the option bit-mask
// directly. Dense state machinery is allocated only when OptDense is
// set. Defaults for intern threshold / depth match NewEncoder.
func NewEncoderWith(opts Options) *Encoder {
        e := &Encoder{
                minIntern:       4,
                maxStateEntries: 1 << 14,
                maxDepth:        DefaultMaxDepth,
        }
        e.applyOpts(opts) // allocates Dense state when OptDense is set
        return e
}

// NewEncoderOnBuf returns an Encoder that appends to buf at its current
// length. The buffer is NOT truncated; pass an empty slice for a fresh
// encoding. Call SetBuffer afterwards to truncate.
func NewEncoderOnBuf(buf []byte, mode Mode) *Encoder {
        e := NewEncoder(mode)
        e.buf = buf
        return e
}

// Reset truncates the buffer and resets the intern table. Capacities
// are preserved. opts / mode / qpack are also reset to OptSpeed so a
// pooled encoder does not leak its previous configuration into the
// next caller — apply the desired options explicitly via applyOpts /
// SetQPack after Reset.
func (e *Encoder) Reset() {
        e.resetForReuse()
        e.opts = OptSpeed
        e.mode = Fast
        e.qpack = false
        e.gorillaFloat = false
        e.rans = false
        e.colIndex = false
        e.fsst = false
        e.zonemap = false
        e.pairPred = false
        e.mtf = false
        e.fsstDict = nil
        // Drop the streaming FSST table cache so a pooled encoder does not carry
        // stale table data into the next caller's workload.
        e.fsstCachedTbl = nil
        e.fsstBatchCount = 0
}

// resetForReuse clears the per-message / per-stream encoder state — the output
// buffer, frame header flag, dense intern/shape state, and the row-scaled
// scratch — while KEEPING the configured options/mode/codec flags. Encoder.Reset
// layers the option reset on top (a pooled encoder is reconfigured per acquire);
// StreamEncoder.Reset calls this directly so one encoder + its grown intern table
// is reused across independent streams without a fresh newEncState allocation.
func (e *Encoder) resetForReuse() {
        e.buf = e.buf[:0]
        e.customFramed = false
        // Row-scaled ALP staging scratch: retain across batches, drop only past the
        // hard ceiling so a one-off giant float slice can't pin unbounded memory.
        // Pure []uint64 (no pointers) so no clear is needed.
        if cap(e.alpScratch) > maxRetainedColScratch {
                e.alpScratch = nil
        }
        // PFOR exception scratch: pointer-free, drop both when oversized (they grow
        // together on the same workload so a single ceiling check suffices).
        if cap(e.pforExcPos) > maxRetainedColScratch {
                e.pforExcPos = nil
                e.pforExcDelta = nil
        }
        // ALP exception-index scratch: pointer-free, drop when oversized.
        if cap(e.alpExcScratch) > maxRetainedColScratch {
                e.alpExcScratch = nil
        }
        e.vecScratch.Reset()
        if cap(e.wideF64) > maxRetainedColScratch {
                e.wideF64 = nil
        }
        if cap(e.vecBatchFlat) > maxRetainedColScratch {
                e.vecBatchFlat = nil
        }
        // vecBatchRows holds slice headers into vecBatchFlat; drop it independently
        // (many rows × small dim keeps flat under the ceiling) and clear first so the
        // retained headers do not GC-pin the prior flat backing.
        if cap(e.vecBatchRows) > maxRetainedColScratch || e.vecBatchFlat == nil {
                clear(e.vecBatchRows[:cap(e.vecBatchRows)])
                e.vecBatchRows = nil
        }
        if cap(e.vecBatchNorms) > maxRetainedColScratch {
                e.vecBatchNorms = nil
        }
        // Keyed-diff match map: drop a spike-sized backing, else clear() it in place.
        // The keys are unsafe.String / string-header aliases into the caller's prior
        // key strings or []struct element backing (keyTokenAt), so leaving them
        // populated across a pool recycle GC-pins that caller data until the next
        // keyed build — mirror the clear-on-reset policy already applied to the other
        // aliasing string-keyed pooled state (colScratchStr, d.values, strDictMap).
        if len(e.keyIdx) > 1<<16 {
                e.keyIdx = nil
        } else {
                clear(e.keyIdx)
        }
        e.keyIdxBusy = false
        // newKeyIdx: keys are unsafe.String aliases into caller backing; clear to
        // drop GC-pinning references. Drop past spike cap like keyIdx above.
        if len(e.newKeyIdx) > 1<<16 {
                e.newKeyIdx = nil
        } else {
                clear(e.newKeyIdx)
        }
        e.newKeyIdxBusy = false
        e.headerOut = false
        if e.state != nil {
                e.state.reset()
        }
        e.depth = 0
        e.ifaceDepth = 0
        if e.maxDepth == 0 {
                e.maxDepth = DefaultMaxDepth
        }
        // Drop any PreIntern entries — they reference caller-supplied backing
        // pointers that are not safe to assume valid across a pool / stream recycle.
        if e.preIntern != nil {
                e.preIntern = e.preIntern[:0]
        }
}

// ApplyOpts reconfigures the encoder via the options bit-mask.
// Equivalent to recreating the encoder with NewEncoderWith but
// preserves the existing pool buffer and (Dense-mode) intern
// state. The pool-backed Marshal* entry points call this on every
// acquire; callers driving an Encoder directly use it to switch
// between OptSpeed and OptBalanced without re-allocating.
//
//go:nosplit
func (e *Encoder) ApplyOpts(opts Options) { e.applyOpts(opts) }

// Canonical reports whether OptCanonical is set on this encoder. Generated
// EncodeQDF code calls it to decide whether to emit map entries in sorted
// (deterministic) key order, matching the reflect encoder's canonical path.
func (e *Encoder) Canonical() bool { return e.opts.Has(OptCanonical) }

// EncodeValue runs the reflect-driven Marshal pipeline on v
// against this Encoder. Convenience for callers driving an
// Encoder directly (e.g. with PreIntern) — equivalent to what
// the pool-backed Marshal does internally.
func (e *Encoder) EncodeValue(v any) error { return encodeReflect(e, v) }

// EncodeAny encodes v as a value reached through a dynamic (interface/any)
// position — the codegen entry point for any-typed struct fields and container
// elements (qdfgen emits this for `any` fields). Unlike EncodeValue (a
// top-level, typed-on-decode entry point), it marks the encode as schemaless by
// raising e.ifaceDepth, exactly as the reflect encoder's []any / map[K]any path
// does via encodeIface. That suppresses codecs whose blocks only decode on the
// typed path — OptLossyVec's tagColVecLossy (0xFD) and tagVecBatchStruct (0xFE)
// — because a value reached through an any is read back via decodeAny, which
// has no case for them. Without this, a codegen `any` field holding a
// []float32/[]float64 or a batchable []struct emitted an undecodable block.
func (e *Encoder) EncodeAny(v any) error {
        if v == nil {
                e.WriteNil()
                return nil
        }
        // Bound recursion through the dynamic dispatch (mirrors encodeIface): an
        // any-typed field can form a cycle the static *T guard does not see.
        if e.maxDepth != 0 {
                e.depth++
                if e.depth > e.maxDepth {
                        e.depth--
                        return ErrCycleDetected
                }
                defer func() { e.depth-- }()
        }
        e.ifaceDepth++
        defer func() { e.ifaceDepth-- }()
        return encodeReflect(e, v)
}

// PreIntern registers the given strings against the encoder's
// intern table up front. Subsequent WriteString / WriteBytes
// calls that pass the SAME backing string header (i.e. the same
// underlying byte pointer and length) skip the hash + slot probe
// and emit a state-ref directly against the cached intern id.
//
// Intended for power users who know their hot string pool ahead
// of time — service names, region codes, enum-like values drawn
// from a fixed slice. Real telemetry payloads draw 90 %+ of
// their dense intern hits from such pools. Skipping the
// hash/probe on those calls is the main per-emit cost left on
// the encode hot path.
//
// Requires OptDense to be applied first (otherwise the call is a
// no-op). The registered identities are dropped on Reset so a
// pooled encoder does not carry caller-supplied pointers across
// recycles.
//
// Safety: the caller must keep the backing memory of every
// PreIntern'd string alive for the lifetime of the next encode
// call. For literals embedded in a slice / global / struct
// field this is automatic; for short-lived stack strings the
// caller is responsible.
func (e *Encoder) PreIntern(strs ...string) {
        if e.state == nil || !e.opts.Has(OptDense) {
                return
        }
        if e.preIntern == nil {
                e.preIntern = make([]preInternEntry, 0, len(strs))
        }
        for _, s := range strs {
                if len(s) < e.minIntern {
                        continue
                }
                // Record the caller's backing pointer + length without
                // touching the intern table yet. The first WriteString
                // that matches will run the regular intern path
                // (emitting tagInternStr on the wire so the decoder can
                // register the entry) and back-fill the id here.
                e.preIntern = append(e.preIntern, preInternEntry{
                        ptr: unsafe.Pointer(unsafe.StringData(s)),
                        n:   uintptr(len(s)),
                        id:  preInternUnseen,
                })
        }
}

// SetMaxDepth caps reflect-path pointer/struct recursion. The default
// (DefaultMaxDepth = 10000) is sufficient for any normal payload and
// rejects pointer cycles before they stack-overflow the goroutine.
// Set to 0 to disable the check entirely — only safe when the caller
// can prove the input graph is acyclic.
func (e *Encoder) SetMaxDepth(d int) { e.maxDepth = d }

// Bytes returns the encoded payload. It aliases the encoder's buffer and
// is only valid until the next write or Reset.
func (e *Encoder) Bytes() []byte { return e.buf }

// Take returns the encoded payload and detaches it from the encoder. The
// caller takes ownership.
func (e *Encoder) Take() []byte {
        out := e.buf
        e.buf = nil
        e.headerOut = false
        if e.state != nil {
                e.state.reset()
        }
        return out
}

// SetBuffer installs a caller-owned buffer (truncated to length 0).
func (e *Encoder) SetBuffer(b []byte) {
        e.buf = b[:0]
        e.headerOut = false
}

// AdoptBuffer installs b as the working buffer, preserving its current
// length. Used to continue writing into a buffer that already carries
// data — for example, after a nested type returned its extended slice.
// headerOut is left unchanged.
func (e *Encoder) AdoptBuffer(b []byte) {
        e.buf = b
}

// SetIntern overrides the Dense-mode tuning knobs. Zero values keep the
// current setting.
func (e *Encoder) SetIntern(min int, cap int) {
        if min > 0 {
                e.minIntern = min
        }
        if cap > 0 {
                if cap > maxInternEntries {
                        cap = maxInternEntries // keep max id below the 0xFFFF uint16 sentinel
                }
                e.maxStateEntries = cap
        }
}

// streamHeaderLen is the fixed byte length writeHeader emits: the 3 magic bytes,
// the version byte, and the flag byte.
const streamHeaderLen = 5

func (e *Encoder) writeHeader() {
        if e.headerOut {
                return
        }
        flag := byte(0)
        if e.mode == Dense {
                flag |= FlagDense
        }
        if e.qpack {
                flag |= FlagQPack
        }
        // FlagColIndex is NOT set here: encodeColumnar backpatches it only when it
        // actually emits a column index, so OptColumnIndex on a non-columnar
        // payload stays a true no-op (header byte unchanged).
        e.headerFlagAt = len(e.buf) + 4
        e.buf = append(e.buf, Magic0, Magic1, Magic2, Version1, flag)
        e.headerOut = true
}

// SetQPack toggles QPack codec emission. When true, slice fast paths
// produce packed/encoded forms (bitpacked bools, FOR-packed integers,
// Gorilla-encoded floats, raw-LE bulk) instead of per-element tag streams.
// Setting must happen before the first write of the value (the header is
// emitted lazily and carries the FlagQPack hint when this is on).
func (e *Encoder) SetQPack(v bool) { e.qpack = v }

// SetVectorBudget sets the fidelity target used when OptLossyVec is active.
// No effect unless OptLossyVec is in the encoder's options.
func (e *Encoder) SetVectorBudget(b VectorBudget) { e.vecBudget = b }

// QPack reports whether QPack codec emission is enabled.
func (e *Encoder) QPack() bool { return e.qpack }

// EnsureHeader forces a header write if one has not been emitted yet.
func (e *Encoder) EnsureHeader() { e.writeHeader() }

// MarkHeaderWritten tells the encoder the QDF header is already present
// in its buffer (e.g. left there by a parent encoder). The next write
// will skip the header emission.
func (e *Encoder) MarkHeaderWritten() { e.headerOut = true }

// AppendBytes appends raw, already-valid wire bytes. Bypasses tag
// dispatch; used by generated code to emit pre-encoded field-name
// prefixes.
func (e *Encoder) AppendBytes(p []byte) {
        e.writeHeader()
        e.buf = append(e.buf, p...)
}

// ----- primitives -----

func (e *Encoder) WriteNil() {
        e.writeHeader()
        e.buf = append(e.buf, tagNil)
}

func (e *Encoder) WriteBool(v bool) {
        e.writeHeader()
        if v {
                e.buf = append(e.buf, tagTrue)
        } else {
                e.buf = append(e.buf, tagFalse)
        }
}

func (e *Encoder) WriteUint(v uint64) {
        e.writeHeader()
        switch {
        case v <= tagFixintMax:
                e.buf = append(e.buf, byte(v))
        case v <= math.MaxUint8:
                e.buf = append(e.buf, tagUint8, byte(v))
        case v <= math.MaxUint16:
                e.buf = appendU16(append(e.buf, tagUint16), uint16(v))
        case v <= math.MaxUint32:
                e.buf = appendU32(append(e.buf, tagUint32), uint32(v))
        default:
                e.buf = appendU64(append(e.buf, tagUint64), v)
        }
}

func (e *Encoder) WriteInt(v int64) {
        e.writeHeader()
        if v >= 0 {
                e.WriteUint(uint64(v))
                return
        }
        switch {
        case v >= -negfixintMaxAbs:
                // negfixint packs -1..-8 as 0xD8..0xDF; decoder mirrors as
                // -(int8(tag & 0x07) + 1).
                e.buf = append(e.buf, tagNegfixint|byte(-v-1))
        case v >= math.MinInt8:
                e.buf = append(e.buf, tagInt8, byte(int8(v)))
        case v >= math.MinInt16:
                e.buf = appendU16(append(e.buf, tagInt16), uint16(int16(v)))
        case v >= math.MinInt32:
                e.buf = appendU32(append(e.buf, tagInt32), uint32(int32(v)))
        default:
                e.buf = appendU64(append(e.buf, tagInt64), uint64(v))
        }
}

func (e *Encoder) WriteFloat32(v float32) {
        e.writeHeader()
        if e.opts.Has(OptCanonical) {
                v = canonicalizeFloat32(v)
        }
        e.buf = appendU32(append(e.buf, tagFloat32), math.Float32bits(v))
}

func (e *Encoder) WriteFloat64(v float64) {
        e.writeHeader()
        if e.opts.Has(OptCanonical) {
                v = canonicalizeFloat64(v)
        }
        e.buf = appendU64(append(e.buf, tagFloat64), math.Float64bits(v))
}

// WriteString writes s. In Dense mode (OptDense set), eligible strings
// are intern-encoded.
func (e *Encoder) WriteString(s string) {
        e.writeHeader()
        st := e.state
        dense := e.opts.Has(OptDense)
        if dense && !e.stateSuspended && st != nil && len(s) >= e.minIntern && int(st.internLoad) < e.maxStateEntries {
                // PreIntern identity fast path: if the caller registered
                // this exact backing pointer + length via Encoder.PreIntern,
                // the cached id (after first sight on the wire) lets us
                // skip the hash + slot probe and emit a state-ref directly.
                // The slice is empty for default callers, so the gating
                // `len > 0` check is a single branch-predicted compare
                // that does not regress the default path.
                //
                // On the first WriteString of a PreIntern'd string the
                // entry's id is still preInternUnseen — we fall through
                // to the normal intern path (which emits tagInternStr on
                // the wire so the decoder can register the entry) and
                // back-fill the id below.
                preInternIdx := -1
                if len(e.preIntern) > 0 {
                        sp := unsafe.Pointer(unsafe.StringData(s))
                        sn := uintptr(len(s))
                        for i := range e.preIntern {
                                if e.preIntern[i].ptr == sp && e.preIntern[i].n == sn {
                                        if e.preIntern[i].id != preInternUnseen {
                                                id := e.preIntern[i].id
                                                if st.lastID == id {
                                                        e.buf = append(e.buf, tagStateRepeat)
                                                        if e.pairPred {
                                                                st.pairRecord(id, id)
                                                        }
                                                        return
                                                }
                                                e.emitStateRef(id)
                                                return
                                        }
                                        preInternIdx = i
                                        break
                                }
                        }
                }
                id, ok := st.lookupOrAssign(s)
                if preInternIdx >= 0 {
                        e.preIntern[preInternIdx].id = id
                }
                if ok {
                        // Repeat hot path: hand-inlined out of emitStateRef so the
                        // most common Dense hit avoids the non-inlinable call.
                        if st.lastID == id {
                                e.buf = append(e.buf, tagStateRepeat)
                                if e.pairPred {
                                        st.pairRecord(id, id)
                                }
                                return
                        }
                        e.emitStateRef(id)
                        return
                }
                e.buf = append(e.buf, tagInternStr)
                e.buf = appendUvarint(e.buf, uint64(len(s)))
                e.buf = appendString(e.buf, s)
                if st.lastID != lruInvalidID && e.pairPred {
                        st.pairRecord(st.lastID, id)
                }
                st.lastID = id
                return
        }
        if dense && st != nil {
                st.lastID = lruInvalidID
        }
        e.writeStringInline(s)
}

// emitStateRef writes a state-ref to id. Four forms are possible, the
// encoder picks the smallest:
//
//        tagStateRepeat                   1 byte total, when id == lastID
//        tagStatePair  + varuint(pairR)   1 + uvarintLen(pairR) bytes,
//                                         when id is in lastID's predictor ring
//        tagStateMTF   + varuint(mtfR)    1 + uvarintLen(mtfR) bytes
//        tagStateRef   + varuint(id)      1 + uvarintLen(id) bytes
//
// MTF rank comes from the encState LRU. The pair rank comes from the
// per-prev successor ring (Markov-1 predictor). The wire never grows
// over the plain tagStateRef encoding because we only pick the
// alternative when its varuint is strictly shorter than the raw id
// varuint.
//
// Every successful emit moves id to the LRU head AND records the
// (prev, id) transition in the pair predictor so the decoder's mirror
// chain stays in sync.
func (e *Encoder) emitStateRef(id uint32) {
        st := e.state
        pairOn := e.pairPred
        if st.lastID == id {
                e.buf = append(e.buf, tagStateRepeat)
                if pairOn {
                        st.pairRecord(id, id)
                }
                return
        }
        prev := st.lastID
        prevValid := prev != lruInvalidID
        // Small-id fast path: id<128 means the raw state-ref is already a
        // 2-byte literal (tag + 1-byte varuint). Neither MTF (rank varuint
        // ≥1 byte) nor Pair (rank varuint =1 byte) can be strictly shorter,
        // so we write the raw form directly and skip the LRU walk entirely.
        if id < 0x80 {
                st.lruMoveFront(id)
                e.buf = append(e.buf, tagStateRef, byte(id))
                if prevValid && pairOn {
                        st.pairRecord(prev, id)
                }
                st.lastID = id
                return
        }
        // Pair predictor: when (prev, id) is in the predictor ring the
        // payload is always a single byte (ranks 0..3), so the pair form
        // is strictly shorter than the raw state-ref whenever id needs a
        // multi-byte varuint — which the small-id branch above just
        // excluded.
        if prevValid && pairOn {
                if st.pairLookup(prev, id) {
                        st.lruMoveFront(id)
                        // Top-1 predictor: rank is always 0 (see encState.pairLookup
                        // comment), so the rank byte is a hard-coded literal here.
                        e.buf = append(e.buf, tagStatePair, 0)
                        st.pairRecord(prev, id)
                        st.lastID = id
                        return
                }
        }
        // MTF — fall back on raw if the rank varuint is not shorter than
        // the id varuint. When OptMTF is off the LRU is still maintained
        // (the chain must stay in sync for the rest of the codec) but the
        // rank is never emitted; the raw state-ref form is used instead.
        //
        // Rank discovery used to walk the LRU linked list head→tail
        // looking for id (state.go's old lruMoveToFront). Profiling on
        // telemetry workloads showed that pointer-chase walk consumed
        // >50% of encode CPU because each step is a cache-cold random
        // index into lruNext[]. The MRU ring side-cache (mruRank) gives
        // O(1) rank for the recent-128 emits — a contiguous, cache-warm
        // scan that covers exactly the rank range where MTF beats the
        // 2-byte raw state-ref (rank ≤ 127). On a ring miss the chain
        // rank is necessarily ≥ 128 so the raw form would be picked
        // anyway; we skip the walk entirely and emit raw.
        idLen := uvarintLen(uint64(id))
        if e.mtf {
                if rank, ok := st.mruRank(id); ok {
                        st.lruMoveFront(id)
                        if rankLen := uvarintLen(uint64(rank)); rankLen < idLen {
                                e.buf = append(e.buf, tagStateMTF)
                                e.buf = appendUvarint(e.buf, uint64(rank))
                        } else {
                                e.buf = append(e.buf, tagStateRef)
                                e.buf = appendUvarint(e.buf, uint64(id))
                        }
                } else {
                        st.lruMoveFront(id)
                        e.buf = append(e.buf, tagStateRef)
                        e.buf = appendUvarint(e.buf, uint64(id))
                }
        } else {
                st.lruMoveFront(id)
                e.buf = append(e.buf, tagStateRef)
                e.buf = appendUvarint(e.buf, uint64(id))
        }
        if prevValid && pairOn {
                st.pairRecord(prev, id)
        }
        st.lastID = id
}

// WriteStringInline forces an in-line encoding even when Dense intern would
// be eligible. Use for fields known to be unique per message.
func (e *Encoder) WriteStringInline(s string) {
        e.writeHeader()
        // In Dense mode the decoder resets lastID to invalid on every inline string
        // read, so the encoder must do the same here — otherwise a later repeated
        // value emits tagStateRepeat against a lastID the decoder has already
        // dropped, desyncing the state tables (silent wrong value or
        // ErrUnknownStateID). Mirrors WriteString's own inline fallthrough.
        if e.opts.Has(OptDense) && e.state != nil {
                e.state.lastID = lruInvalidID
        }
        e.writeStringInline(s)
}

// stringInlineHeaderLen returns the number of header bytes writeStringInline
// emits for a string of length n (fixstr 1, str8 2, str16 3, str32 5). Kept
// next to writeStringInline so the two stay in lockstep; used by size
// estimators (e.g. the tagColStrRaw never-larger gate).
func stringInlineHeaderLen(n int) int {
        switch {
        case n <= int(tagFixstrMask):
                return 1
        case n <= math.MaxUint8:
                return 2
        case n <= math.MaxUint16:
                return 3
        default:
                return 5
        }
}

func (e *Encoder) writeStringInline(s string) {
        n := len(s)
        // One Grow up front for worst-case header (5 B) + body beats append's
        // amortized growth on the hot path.
        b := slices.Grow(e.buf, 5+n)
        switch {
        case n <= int(tagFixstrMask):
                b = append(b, tagFixstr|byte(n))
        case n <= math.MaxUint8:
                b = append(b, tagStr8, byte(n))
        case n <= math.MaxUint16:
                b = appendU16(append(b, tagStr16), uint16(n))
        default:
                b = appendU32(append(b, tagStr32), uint32(n))
        }
        b = append(b, s...)
        e.buf = b
}

// WriteBytes writes a []byte. In Dense mode, eligible payloads are
// intern-encoded. The intern table is keyed on the byte sequence, so a
// string and a []byte with identical content share an ID.
func (e *Encoder) WriteBytes(b []byte) {
        e.writeHeader()
        st := e.state
        dense := e.opts.Has(OptDense)
        if dense && !e.stateSuspended && st != nil && len(b) >= e.minIntern && int(st.internLoad) < e.maxStateEntries {
                key := unsafestr.String(b)
                id, ok := st.lookupOrAssign(key)
                if ok {
                        if st.lastID == id {
                                e.buf = append(e.buf, tagStateRepeat)
                                if e.pairPred {
                                        st.pairRecord(id, id)
                                }
                                return
                        }
                        e.emitStateRef(id)
                        return
                }
                e.buf = append(e.buf, tagInternBin)
                e.buf = appendUvarint(e.buf, uint64(len(b)))
                e.buf = append(e.buf, b...)
                if st.lastID != lruInvalidID && e.pairPred {
                        st.pairRecord(st.lastID, id)
                }
                st.lastID = id
                return
        }
        if dense && st != nil {
                st.lastID = lruInvalidID
        }
        e.writeBytesInline(b)
}

func (e *Encoder) writeBytesInline(p []byte) {
        n := len(p)
        out := slices.Grow(e.buf, 5+n)
        switch {
        case n <= math.MaxUint8:
                out = append(out, tagBin8, byte(n))
        case n <= math.MaxUint16:
                out = appendU16(append(out, tagBin16), uint16(n))
        default:
                out = appendU32(append(out, tagBin32), uint32(n))
        }
        out = append(out, p...)
        e.buf = out
}

// WriteArrayHeader writes the header for an array of n elements. The
// caller must follow with exactly n element writes. n must not exceed
// math.MaxUint32 — the wire array count is a uint32, so a larger n is
// unrepresentable and panics rather than silently truncating the count
// (which would desync the decoder against the n bodies that follow).
func (e *Encoder) WriteArrayHeader(n int) {
        e.writeHeader()
        if n < 0 {
                // A negative count would narrow to a garbage byte (byte(-1)==0xFF) and
                // desync the decoder. Treat it as the empty header the caller's
                // for-i<n loop would actually emit (zero iterations).
                n = 0
        }
        if uint64(n) > math.MaxUint32 {
                // The wire count is a uint32 (there is no tagArr64). Truncating would
                // emit a count smaller than the n bodies the caller writes next, an
                // undecodable desync; fail loud on this unrepresentable input instead.
                panic("qdf: array length exceeds uint32 wire limit")
        }
        switch {
        case n <= int(tagFixarrMask):
                e.buf = append(e.buf, tagFixarr|byte(n))
        case n <= math.MaxUint16:
                e.buf = appendU16(append(e.buf, tagArr16), uint16(n))
        default:
                e.buf = appendU32(append(e.buf, tagArr32), uint32(n))
        }
}

// WriteMapHeader writes the header for a map of n entries. The caller
// must follow with exactly n key/value pairs. n must not exceed
// math.MaxUint32 — the wire map count is a uint32, so a larger n is
// unrepresentable and panics rather than silently truncating the count.
func (e *Encoder) WriteMapHeader(n int) {
        e.writeHeader()
        if n < 0 {
                // See WriteArrayHeader: a negative count narrows to a garbage byte and
                // desyncs the decoder; emit the empty header instead.
                n = 0
        }
        if uint64(n) > math.MaxUint32 {
                // See WriteArrayHeader: the wire count is a uint32 (no tagMap64), so a
                // larger count is unrepresentable; fail loud rather than truncate.
                panic("qdf: map length exceeds uint32 wire limit")
        }
        switch {
        case n <= math.MaxUint8:
                e.buf = append(e.buf, tagMap8, byte(n))
        case n <= math.MaxUint16:
                e.buf = appendU16(append(e.buf, tagMap16), uint16(n))
        default:
                e.buf = appendU32(append(e.buf, tagMap32), uint32(n))
        }
}

// StructShape begins a shape-interned struct emission for a code-generated type
// (the decode-time counterpart is Decoder.ReadStructHeader). token is a stable
// per-type address — a package-level var the generated EncodeQDF passes;
// fieldHdrs are the pre-encoded fixstr/strN field-name headers in field order.
//
// On the first emit for token on this encoder it DECLARES the shape — tagMapShape,
// id 0, the field count, then the names — so the decoder registers it; every
// later emit writes only tagMapShape + the shape ID. The caller then writes the
// field VALUES in field order (no names). Across a slice of the same struct
// threaded through one encoder, the field names are written once instead of per
// record. Reuses the shared shape-ID space, so a generated buffer stays decodable
// by the reflection path (tagMapShape is standard wire).
func (e *Encoder) StructShape(token *byte, fieldHdrs [][]byte) {
        e.writeHeader()
        if e.state == nil {
                e.state = newEncState()
        }
        st := e.state
        if e.stateSuspended {
                // Inside a never-larger trial (diffKeyedSlice / diffColumnar): emit a
                // self-contained declaration every time and never bind or reference a
                // token. A bound id whose declaration is thrown away with the losing
                // candidate would dangle (ErrUnknownStateID) — the same leak the trial
                // suspends interning to avoid. Still advance the shape counter so it
                // tracks the decoder (which registers a shape per declaration it reads);
                // the trial re-bases the counter for the discarded candidate.
                st.shapeDeclareEnc()
                e.buf = append(e.buf, tagMapShape)
                e.buf = appendUvarint(e.buf, 0) // 0 ⇒ declaration follows
                e.buf = appendUvarint(e.buf, uint64(len(fieldHdrs)))
                for _, h := range fieldHdrs {
                        e.buf = append(e.buf, h...)
                }
                return
        }
        if id := st.shapeForToken(token); id != 0 {
                e.buf = append(e.buf, tagMapShape)
                e.buf = appendUvarint(e.buf, uint64(id))
                return
        }
        id := st.shapeDeclareEnc()
        st.shapeBindToken(token, id)
        e.buf = append(e.buf, tagMapShape)
        e.buf = appendUvarint(e.buf, 0) // 0 ⇒ declaration follows
        e.buf = appendUvarint(e.buf, uint64(len(fieldHdrs)))
        for _, h := range fieldHdrs {
                e.buf = append(e.buf, h...)
        }
}

// WriteTimestamp writes a full-range timestamp as two uvarints:
// sec (zigzag-encoded signed int64 seconds since Unix epoch) and
// nsec (unsigned uint32 nanoseconds in [0, 999_999_999]).
// This replaces the old fixed-8-byte UnixNano encoding (clean break).
func (e *Encoder) WriteTimestamp(sec int64, nsec uint32) {
        e.writeHeader()
        e.buf = append(e.buf, tagTimestamp)
        e.buf = appendUvarint(e.buf, zigzagEncode64(sec))
        e.buf = appendUvarint(e.buf, uint64(nsec))
}

// ----- helpers -----

func appendU16(b []byte, v uint16) []byte { return binary.LittleEndian.AppendUint16(b, v) }
func appendU32(b []byte, v uint32) []byte { return binary.LittleEndian.AppendUint32(b, v) }
func appendU64(b []byte, v uint64) []byte { return binary.LittleEndian.AppendUint64(b, v) }

// appendString uses the runtime's append-string fast path to avoid the
// implicit []byte(s) copy.
func appendString(b []byte, s string) []byte { return append(b, s...) }

func readU16(b []byte) uint16 { return binary.LittleEndian.Uint16(b) }
func readU32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
func readU64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }

package qdf

import (
        "errors"
        "fmt"
)

var (
        ErrShortBuffer    = errors.New("qdf: short buffer")
        ErrBadMagic       = errors.New("qdf: bad magic / not a qdf stream")
        ErrBadVersion     = errors.New("qdf: unsupported wire version")
        ErrBadTag         = errors.New("qdf: unknown tag")
        ErrTypeMismatch   = errors.New("qdf: type mismatch on decode")
        ErrInvalidLength  = errors.New("qdf: invalid length prefix")
        ErrUnknownStateID = errors.New("qdf: unknown state-table id")
        ErrUnsupported    = errors.New("qdf: unsupported type")
        ErrCycleDetected  = errors.New("qdf: pointer cycle detected (max depth exceeded)")
        ErrFieldNotFound  = errors.New("qdf: query predicate field not found")
        // ErrStreamBadFlags is returned by StreamDecoder.Decode when the stream
        // header carries whole-payload flags that do not apply to a frame stream
        // (FlagRANS / FlagColIndex). A conforming StreamEncoder never sets them; a
        // hostile stream claiming them would, if honored, swap the shared window
        // buffer mid-stream and desync framing. The decoder is latched broken.
        ErrStreamBadFlags = errors.New("qdf: stream header has unsupported whole-payload flags")
)

// QueryError describes why a filtering/projecting decode (Unmarshal with
// QueryOptions) could not proceed. It wraps one of ErrUnsupported,
// ErrTypeMismatch, or ErrFieldNotFound, so callers can categorise the failure
// with errors.Is and read the specifics with errors.As.
type QueryError struct {
        Err   error   // wrapped sentinel
        Op    string  // e.g. "predicate pushdown"
        Field string  // filter/projection field involved, if any
        Want  colKind // kind implied by the predicate's T (kind mismatches)
        Got   colKind // kind found on the wire (kind mismatches)
}

func (e *QueryError) Error() string {
        switch {
        case e.Field != "" && errors.Is(e.Err, ErrTypeMismatch):
                return fmt.Sprintf("qdf: %s: field %q kind %s does not match wire kind %s: %v",
                        e.Op, e.Field, e.Want, e.Got, e.Err)
        case e.Field != "":
                return fmt.Sprintf("qdf: %s: field %q: %v", e.Op, e.Field, e.Err)
        default:
                return fmt.Sprintf("qdf: %s: %v", e.Op, e.Err)
        }
}

func (e *QueryError) Unwrap() error { return e.Err }

// batchdecode compares plain Unmarshal against UnmarshalBatch on a held
// decode result.
//
// A decoded []struct{ ...string fields... } that you HOLD (a cache, an
// in-memory index, a streaming pipeline's working set) puts one pointer per
// string field per row into the heap — the garbage collector has to walk
// every one of them on every mark phase for as long as you keep the slice
// alive. UnmarshalBatch[T] decodes into a pointer-free Batch[T] instead:
// string/time.Time fields become qdf.Str/qdf.Time handles (an offset/length
// or two plain integers), so []T carries zero pointers and the collector
// skips scanning it entirely. See docs/BATCH-HANDLES.md for the full design
// and the measured numbers this example's shape is drawn from.
//
//        go run ./examples/batchdecode
package main

import (
        "fmt"
        "time"

        "github.com/alex60217101990/qdf"
)

// Source is the wire shape: everything is a normal Go type, encoded and
// decoded like any other qdf struct.
type Source struct {
        TS     time.Time `qdf:"ts"`
        Region string    `qdf:"region"` // low-cardinality: a handful of repeated values
        Msg    string    `qdf:"msg"`    // high-cardinality: distinct per row
        ID     int64     `qdf:"id"`
        Val    float64   `qdf:"val"`
}

// Row is the SAME wire shape, but every pointer-carrying field is spelled
// as its pointer-free handle type. UnmarshalBatch validates this once per
// type and rejects anything that isn't strictly pointer-free (see
// docs/BATCH-HANDLES.md's type-rules table).
type Row struct {
        ID     int64    `qdf:"id"`
        Region qdf.Str  `qdf:"region"`
        Msg    qdf.Str  `qdf:"msg"`
        TS     qdf.Time `qdf:"ts"`
        Val    float64  `qdf:"val"`
}

// regions is deliberately small: a low-cardinality string column is where
// the dict codec (tagColStrDict family) pays off hardest for UnmarshalBatch
// — each DISTINCT value is copied into the slab once, not once per row.
var regions = []string{"us-east", "us-west", "eu-central", "ap-south"}

func main() {
        const n = 1000 // >= columnarMinElems (16), so the wire is columnar
        src := make([]Source, n)
        base := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
        for i := range src {
                src[i] = Source{
                        ID:     int64(i),
                        Region: regions[i%len(regions)],
                        Msg:    fmt.Sprintf("event %d processed in region shard %d", i, i%len(regions)), // high-cardinality
                        TS:     base.Add(time.Duration(i) * time.Second),
                        Val:    float64(i) * 0.5,
                }
        }

        data, err := qdf.Marshal(src, qdf.OptBalanced|qdf.OptDense|qdf.OptShapeIntern)
        if err != nil {
                panic(err)
        }
        fmt.Printf("encoded: %d rows, %d bytes\n\n", n, len(data))

        // --- Decode A: plain Unmarshal into the string-bearing struct. -----
        // Every Region/Msg field becomes a real Go string: a heap pointer the
        // GC must walk on every mark phase for as long as `plain` is held.
        var plain []Source
        t0 := time.Now()
        if err := qdf.Unmarshal(data, &plain); err != nil {
                panic(err)
        }
        plainTime := time.Since(t0)

        // --- Decode B: UnmarshalBatch into the pointer-free handle struct. -
        // Region/Msg/TS become qdf.Str/qdf.Time: plain integers, no pointers.
        // []Row is GC-noscan regardless of row count. Str resolution is lazy —
        // decode itself does not materialize a Go string until you call b.Str.
        t1 := time.Now()
        b, err := qdf.UnmarshalBatch[Row](data)
        if err != nil {
                panic(err)
        }
        batchTime := time.Since(t1)
        defer b.Release()

        fmt.Println("decode time (single run, NOT a benchmark — see batch_bench_test.go for real numbers):")
        fmt.Printf("  Unmarshal:      %v\n", plainTime)
        fmt.Printf("  UnmarshalBatch: %v\n\n", batchTime)

        // Prove the handles resolve to the same data plain Unmarshal produced.
        fmt.Println("spot-check (handles resolve to real data):")
        for _, i := range []int{0, 500, 999} {
                fmt.Printf("  row %-3d region=%-10s msg=%q val=%v\n",
                        i, b.Str(b.Rows[i].Region), b.Str(b.Rows[i].Msg), b.Rows[i].Val)
        }
        fmt.Println()

        // Low-cardinality columns are free deduplication: every row sharing a
        // Region value shares the SAME slab handle (dict codec, one slab copy
        // per distinct value — not per row). Rows 0 and 4 share "us-east".
        sameHandle := b.Rows[0].Region == b.Rows[4].Region
        fmt.Printf("rows 0 and 4 share one slab entry for %q: %v\n\n", b.Str(b.Rows[0].Region), sameHandle)

        // --- Streaming reuse: decode -> use -> Release -> decode again. -----
        // This is the shape a long-running pipeline actually runs in: each
        // iteration's slab and Rows backing come from a sync.Pool, so once the
        // pool is warm this loop settles to a small, constant allocation floor
        // instead of growing with iteration count (see BatchSteadyState in
        // batch_bench_test.go: 2 allocs/op once warm).
        const iterations = 1000
        var checksum int64
        for range iterations {
                rb, err := qdf.UnmarshalBatch[Row](data)
                if err != nil {
                        panic(err)
                }
                checksum += rb.Rows[0].ID + int64(len(rb.Rows))
                rb.Release() // slab + Rows backing return to the pool for the next iteration
        }
        fmt.Printf("streaming reuse: %d decode+Release iterations, checksum=%d (proves every iteration decoded real data)\n",
                iterations, checksum)
}

// decodealloc shows the decode-side allocation lever. Decode is usually
// allocation/GC-bound, and the dominant cost is copying each decoded string out
// of the input buffer onto the heap. A decode Arena moves those strings into one
// bump-allocated region, collapsing thousands of tiny heap allocations into a
// handful of big ones — freed together when the batch is dropped.
//
//        go run ./examples/decodealloc
package main

import (
        "fmt"
        "runtime"

        "github.com/alex60217101990/qdf"
)

type Record struct {
        ID      string `qdf:"id"`
        Service string `qdf:"service"`
        Message string `qdf:"message"`
}

func main() {
        const n = 2000
        msgs := make([][]byte, n)
        for i := range msgs {
                b, err := qdf.Marshal(Record{
                        ID:      fmt.Sprintf("req-%06d", i),
                        Service: "api",
                        Message: "request handled",
                }, qdf.OptSpeed)
                if err != nil {
                        panic(err)
                }
                msgs[i] = b
        }

        // Default decode: each message's strings are copied to the heap.
        def := mallocs(func() {
                var r Record
                for _, m := range msgs {
                        if err := qdf.Unmarshal(m, &r); err != nil {
                                panic(err)
                        }
                }
        })

        // Arena decode: strings alias one bump-allocated region for the batch.
        arena := mallocs(func() {
                a := qdf.NewArena()
                var r Record
                for _, m := range msgs {
                        if err := qdf.Unmarshal(m, &r, qdf.WithArena(a)); err != nil {
                                panic(err)
                        }
                }
        })

        fmt.Printf("decode %d messages:\n", n)
        fmt.Printf("  default: %6d heap allocations\n", def)
        fmt.Printf("  arena:   %6d heap allocations  (%.0f%% fewer)\n",
                arena, 100*(1-float64(arena)/float64(def)))
}

// mallocs reports the number of heap allocations f performs.
func mallocs(f func()) uint64 {
        var before, after runtime.MemStats
        runtime.GC()
        runtime.ReadMemStats(&before)
        f()
        runtime.ReadMemStats(&after)
        return after.Mallocs - before.Mallocs
}

// embeddings demonstrates the opt-in lossy vector codec for AI embeddings.
//
// Embeddings don't need bit-exact storage — only their nearest-neighbour
// geometry has to survive. Under OptLossyVec, qdf quantizes the []float32 /
// []float64 vector columns of a []struct to a fidelity budget you set
// (min cosine, max relative error, or target SNR) and entropy-codes the result,
// with a never-larger fallback to the lossless body.
//
//        go run ./examples/embeddings
package main

import (
        "fmt"
        "math"

        "github.com/alex60217101990/qdf"
)

type Doc struct {
        ID  string    `qdf:"id"`
        Emb []float32 `qdf:"emb"`
}

func main() {
        const n, dim = 2000, 128
        docs := make([]Doc, n)
        for i := range docs {
                v := make([]float32, dim)
                for j := range v {
                        v[j] = float32(math.Sin(float64(i*dim+j) * 0.017))
                }
                docs[i] = Doc{ID: fmt.Sprintf("doc-%05d", i), Emb: v}
        }

        // Lossless baseline.
        lossless, err := qdf.Marshal(docs, qdf.OptBalanced)
        if err != nil {
                panic(err)
        }

        // Lossy vector codec targeting a minimum cosine similarity of 0.99.
        enc := qdf.NewEncoderWith(qdf.OptBalanced | qdf.OptLossyVec)
        enc.SetVectorBudget(qdf.MinCosine(0.99))
        if err := enc.EncodeValue(docs); err != nil {
                panic(err)
        }
        lossy := enc.Bytes()

        // Decode and measure the worst-case cosine similarity actually achieved.
        var back []Doc
        if err := qdf.Unmarshal(lossy, &back); err != nil {
                panic(err)
        }
        worst := 1.0
        for i := range docs {
                if c := cosine(docs[i].Emb, back[i].Emb); c < worst {
                        worst = c
                }
        }

        fmt.Printf("vectors:        %d × %d dims\n", n, dim)
        fmt.Printf("lossless:       %7d bytes\n", len(lossless))
        fmt.Printf("lossy (cos≥.99):%7d bytes  (%.1f%% smaller)\n",
                len(lossy), 100*(1-float64(len(lossy))/float64(len(lossless))))
        fmt.Printf("worst cosine:   %.4f (target 0.99)\n", worst)
}

func cosine(a, b []float32) float64 {
        var dot, na, nb float64
        for i := range a {
                x, y := float64(a[i]), float64(b[i])
                dot += x * y
                na += x * x
                nb += y * y
        }
        return dot / (math.Sqrt(na)*math.Sqrt(nb) + 1e-30)
}

// query demonstrates predicate pushdown: filtering a columnar batch without
// fully decoding it.
//
// When a []struct is encoded columnar, qdf stores each field as its own column.
// A WHERE predicate (WhereCmp / WhereRange / Where) is evaluated against the
// relevant column(s), and only the matching rows — and only the columns you keep
// — are materialised. You query the bytes instead of decoding everything first.
//
//        go run ./examples/query
package main

import (
        "fmt"

        "github.com/alex60217101990/qdf"
)

type Event struct {
        Level string `qdf:"level"`
        Msg   string `qdf:"msg"`
        Code  int32  `qdf:"code"`
}

func main() {
        const n = 5000
        batch := make([]Event, n)
        for i := range batch {
                level := "INFO"
                switch {
                case i%20 == 0:
                        level = "ERROR"
                case i%5 == 0:
                        level = "WARN"
                }
                batch[i] = Event{Level: level, Code: int32(200 + (i%6)*100), Msg: "..."}
        }

        data, err := qdf.Marshal(batch, qdf.OptBalanced)
        if err != nil {
                panic(err)
        }

        // Typed bound predicate on a numeric column (>= 400).
        var serverErrors []Event
        if err := qdf.Unmarshal(data, &serverErrors, qdf.WhereCmp("code", qdf.GE, int32(400))); err != nil {
                panic(err)
        }

        // Range predicate on the same column (300..399 inclusive).
        var redirects []Event
        if err := qdf.Unmarshal(data, &redirects, qdf.WhereRange("code", int32(300), int32(399))); err != nil {
                panic(err)
        }

        // Arbitrary closure predicate on a string column.
        var onlyErr []Event
        if err := qdf.Unmarshal(data, &onlyErr, qdf.Where("level", func(s string) bool { return s == "ERROR" })); err != nil {
                panic(err)
        }

        fmt.Printf("encoded:       %d rows in %d bytes\n", n, len(data))
        fmt.Printf("code >= 400:   %d rows (selective decode)\n", len(serverErrors))
        fmt.Printf("code in 300-399: %d rows\n", len(redirects))
        fmt.Printf("level == ERROR: %d rows\n", len(onlyErr))
}

// streaming shows the multi-message win: a StreamEncoder keeps ONE intern
// dictionary across every message in the stream, so a string that repeats from
// one message to the next (service names, region codes, status labels) is
// written in full once and referenced by a 1-byte id thereafter.
//
// Encoding each message independently (or with a per-record format) re-emits
// those strings every time.
//
//        go run ./examples/streaming
package main

import (
        "bytes"
        "encoding/json"
        "fmt"

        "github.com/alex60217101990/qdf"
)

type Event struct {
        Service string `qdf:"service" json:"service"`
        Region  string `qdf:"region"  json:"region"`
        Code    int32  `qdf:"code"    json:"code"`
}

func main() {
        services := []string{"auth", "api", "billing"}
        regions := []string{"eu-west-1", "us-east-1"}

        const n = 1000
        events := make([]Event, n)
        for i := range events {
                events[i] = Event{services[i%3], regions[i%2], int32(200 + (i%4)*100)}
        }

        // One stream, one shared intern dictionary across all n messages.
        var sink bytes.Buffer
        enc := qdf.NewStreamEncoderWith(&sink, qdf.OptBalanced)
        for _, e := range events {
                if err := enc.Encode(e); err != nil {
                        panic(err)
                }
        }
        if err := enc.Flush(); err != nil {
                panic(err)
        }
        _ = enc.Close()
        streamed := sink.Len()

        // Baseline: encode each message on its own (no cross-message dictionary).
        independent := 0
        for _, e := range events {
                b, err := qdf.Marshal(e, qdf.OptBalanced)
                if err != nil {
                        panic(err)
                }
                independent += len(b)
        }
        jsonBytes, _ := json.Marshal(events)

        // Decode the whole stream back.
        dec := qdf.NewStreamDecoder(&sink)
        decoded := 0
        for {
                var out Event
                if dec.Decode(&out) != nil {
                        break
                }
                decoded++
        }

        fmt.Printf("messages:            %d\n", n)
        fmt.Printf("encoding/json:       %7d bytes\n", len(jsonBytes))
        fmt.Printf("qdf, per-message:    %7d bytes\n", independent)
        fmt.Printf("qdf, shared stream:  %7d bytes  (%.1fx smaller than per-message)\n",
                streamed, float64(independent)/float64(streamed))
        fmt.Printf("decoded back:        %d messages\n", decoded)
}

// telemetry demonstrates qdf's core win: it compresses *across* records.
//
// A batch of log lines repeats the same handful of service / level / region
// strings thousands of times. Per-record formats (json, msgpack, protobuf)
// re-encode those strings every row; qdf's Dense mode interns them once and
// columnar-compresses the numeric columns, so the wire shrinks with batch size.
//
//        go run ./examples/telemetry
package main

import (
        "encoding/json"
        "fmt"

        "github.com/alex60217101990/qdf"
)

type LogRecord struct {
        Service string `qdf:"service" json:"service"`
        Level   string `qdf:"level"   json:"level"`
        Region  string `qdf:"region"  json:"region"`
        Message string `qdf:"msg"     json:"msg"`
        Code    int32  `qdf:"code"    json:"code"`
}

func main() {
        services := []string{"auth", "api", "billing", "search"}
        levels := []string{"INFO", "WARN", "ERROR"}
        regions := []string{"eu-west-1", "us-east-1", "ap-south-1"}

        const n = 5000
        batch := make([]LogRecord, n)
        for i := range batch {
                batch[i] = LogRecord{
                        Service: services[i%len(services)],
                        Level:   levels[i%len(levels)],
                        Region:  regions[i%len(regions)],
                        Code:    int32(200 + (i%5)*100),
                        Message: "request handled",
                }
        }

        jsonBytes, err := json.Marshal(batch)
        if err != nil {
                panic(err)
        }
        qdfBytes, err := qdf.Marshal(batch, qdf.OptBalanced)
        if err != nil {
                panic(err)
        }

        // Round-trip: decode straight back into the same shape.
        var back []LogRecord
        if err := qdf.Unmarshal(qdfBytes, &back); err != nil {
                panic(err)
        }
        ok := len(back) == len(batch) && back[0] == batch[0] && back[n-1] == batch[n-1]

        fmt.Printf("records:       %d\n", n)
        fmt.Printf("encoding/json: %7d bytes\n", len(jsonBytes))
        fmt.Printf("qdf balanced:  %7d bytes  (%.1fx smaller than json)\n",
                len(qdfBytes), float64(len(jsonBytes))/float64(len(qdfBytes)))
        fmt.Printf("round-trip ok: %v\n", ok)
}

//go:build !qdf_simd

package qdf

// Default bulk float encoders. Element-by-element through WriteFloatN.
// Replaced under -tags qdf_simd by a tighter inlined loop.

func encodeSliceFloat32Impl(e *Encoder, s []float32) error {
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteFloat32(s[i])
        }
        return nil
}

func encodeSliceFloat64Impl(e *Encoder, s []float64) error {
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteFloat64(s[i])
        }
        return nil
}

package qdf

import (
        "slices"

        "github.com/alex60217101990/qdf/internal/fsst"
        "github.com/alex60217101990/qdf/internal/unsafestr"
)

// FSSTDict is a pre-trained, immutable FSST symbol table.
//
// Training the per-column symbol table is the dominant cost of FSST encoding.
// Train one FSSTDict over representative samples and reuse it across many
// FSSTDict.Marshal calls: each encode then skips training and pays only
// compression — the train-once, encode-many pattern (the "Static" in FSST).
//
// An FSSTDict is bounded (≤255 symbols, a few KB) and never mutated after
// training, so it is safe for concurrent use and holds a fixed, small amount of
// memory. It cannot grow, so reusing one across the life of a process does not
// leak. The wire stays self-describing: each FSST column still carries its
// table, so output produced with a dictionary decodes with a plain Unmarshal.
type FSSTDict struct {
        tbl *fsst.SymbolTable
}

// TrainFSSTDict learns an FSST symbol table from representative byte samples
// (typically a slice of the string column you will encode). Deterministic: the
// same samples always yield the same dictionary.
func TrainFSSTDict(samples [][]byte) *FSSTDict {
        return &FSSTDict{tbl: fsst.BuildSymbolTable(samples)}
}

// TrainFSSTDictStrings is TrainFSSTDict over []string, viewing each string's
// bytes without copying (the strings are only read during training).
func TrainFSSTDictStrings(samples []string) *FSSTDict {
        b := make([][]byte, len(samples))
        for i, s := range samples {
                b[i] = unsafestr.Bytes(s)
        }
        return &FSSTDict{tbl: fsst.BuildSymbolTable(b)}
}

// fsstRequired is the option set FSST needs to actually fire: the columnar
// prerequisites (OptDense + OptShapeIntern gate encodeColumnar; OptQPack gates
// e.fsst) plus OptFSST itself. FSSTDict.Marshal ORs this into the caller's opts
// so a bare FSSTDict.Marshal(v, 0) still compresses instead of silently being a
// no-op. OptCompression / OptBalanced already include the columnar bits.
const fsstRequired = OptDense | OptQPack | OptShapeIntern | OptFSST

// Marshal encodes v using this pre-trained dictionary for FSST string columns.
// It enables the FSST codec and its columnar prerequisites (Dense, QPack,
// ShapeIntern), so even FSSTDict.Marshal(v, OptSpeed) compresses; combine with
// OptCompression to add the float codecs and rANS.
func (d *FSSTDict) Marshal(v any, opts Options) ([]byte, error) {
        return marshalDict(v, opts|fsstRequired, d.tbl)
}

// AppendMarshal encodes v with this dictionary and appends to dst (see
// AppendMarshal). Enables FSST and its columnar prerequisites.
func (d *FSSTDict) AppendMarshal(dst []byte, v any, opts Options) ([]byte, error) {
        return appendMarshalDict(dst, v, opts|fsstRequired, d.tbl)
}

// marshalDict is Marshal with an optional pre-trained FSST table (nil = train
// per batch). Marshal delegates here with a nil dict.
func marshalDict(v any, opts Options, dict *fsst.SymbolTable) ([]byte, error) {
        enc := encPool.Get().(*Encoder)
        enc.Reset()
        enc.applyOpts(opts)
        enc.fsstDict = dict
        if err := encodeReflect(enc, v); err != nil {
                putEnc(enc, &encPool)
                return nil, err
        }
        enc.maybeApplyRANS(0)
        var out []byte
        if cap(enc.buf) > marshalDetachThreshold {
                out = enc.buf
                enc.buf = nil
        } else {
                out = slices.Clone(enc.buf)
        }
        putEnc(enc, &encPool) // cap a spike-sized buffer / widening scratch before pooling
        return out, nil
}

// appendMarshalDict is AppendMarshal with an optional pre-trained FSST table.
func appendMarshalDict(dst []byte, v any, opts Options, dict *fsst.SymbolTable) ([]byte, error) {
        enc := encPool.Get().(*Encoder)
        enc.Reset()
        enc.applyOpts(opts)
        enc.fsstDict = dict
        start := len(dst)
        enc.buf = dst
        if err := encodeReflect(enc, v); err != nil {
                // Detach the caller's dst before pooling: putEnc only nils buf past
                // maxPooledBuf, so a normal-sized dst would stay aliased in the pooled
                // encoder and the next encode would overwrite the caller's backing array.
                enc.buf = nil
                putEnc(enc, &encPool)
                return dst, err
        }
        enc.maybeApplyRANS(start)
        out := enc.buf
        enc.buf = nil
        putEnc(enc, &encPool) // cap a spike-sized widening scratch before pooling
        return out, nil
}

// Package bitflag provides tiny generic helpers for bit-flag enums over any
// unsigned integer type. It centralizes the set/test idioms so flag enums (e.g.
// a comparison operator built from less/equal/greater bits) read by intent
// rather than raw `&` / `|` expressions and stay consistent across the codebase.
package bitflag

// Flag is any unsigned integer usable as a bit set.
type Flag interface {
        ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uint
}

// Has reports whether v has ANY of the bits in f set (v & f != 0). For a
// single-bit f this is the usual membership test.
func Has[F Flag](v, f F) bool { return v&f != 0 }

// All reports whether v has ALL of the bits in f set (v & f == f).
func All[F Flag](v, f F) bool { return v&f == f }

// Set returns v with the bits in f set.
func Set[F Flag](v, f F) F { return v | f }

// Clear returns v with the bits in f cleared.
func Clear[F Flag](v, f F) F { return v &^ f }

// Toggle returns v with the bits in f flipped.
func Toggle[F Flag](v, f F) F { return v ^ f }

// Package bitpack implements the pure bit-packing and SIMD kernel layer
// shared by qdf's integer codecs (FOR, Delta+FOR, dict, PFOR, ALP). It
// encodes []uint64 deltas as a tight LSB-first bit-stream and decodes it
// back, with byte-aligned and variable-width fast paths dispatched to
// hand-written AVX2 (amd64) / NEON (arm64) kernels under the qdf_simd
// build tag and a portable scalar fallback otherwise.
//
// The layer is dependency-closed: it imports only the standard library
// and golang.org/x/sys/cpu, and holds no qdf wire-format or codec state.
// All bit layouts are LSB-first within each byte; Pack and Unpack are
// exact inverses for every width in [1, 56].
package bitpack

import "math/bits"

// Pack writes len(vals)*bits bits into out, LSB-first within each byte.
// out must have len >= ceil(len(vals)*bits/8). bits must be in [1, 56].
func Pack(out []byte, vals []uint64, bitsPer int) {
        if bitsPer == 0 || len(vals) == 0 {
                return
        }
        mask := uint64(1)<<uint(bitsPer) - 1
        var acc uint64
        var have uint
        pos := 0
        for _, v := range vals {
                acc |= (v & mask) << have
                have += uint(bitsPer)
                for have >= 8 {
                        out[pos] = byte(acc)
                        acc >>= 8
                        have -= 8
                        pos++
                }
        }
        if have > 0 {
                out[pos] = byte(acc)
        }
}

// Unpack reads len(out)*bits bits from in. bits must be in [1, 56]. in
// must have len >= ceil(len(out)*bits/8). Byte-aligned widths take a
// dedicated zero-extend fast path (asm under qdf_simd, otherwise scalar
// memcpy-with-mask); other widths use the variable-width / sliding-window
// decoders.
func Unpack(out []uint64, in []byte, bitsPer int) {
        switch bitsPer {
        case 8:
                unpackBits8(out, in)
                return
        case 10:
                unpackBits10(out, in)
                return
        case 12:
                unpackBits12(out, in)
                return
        case 14:
                unpackBits14(out, in)
                return
        case 16:
                unpackBits16(out, in)
                return
        case 20:
                unpackBits20(out, in)
                return
        case 32:
                unpackBits32(out, in)
                return
        }
        // Remaining small widths (1-7, 9, 11, 13) go through the general
        // 4-value VPSRLVQ kernel; the wider non-aligned widths (15, 17-19,
        // 21-28) use the 2-value kernel. Both fall back to the scalar window
        // on non-SIMD builds or non-AVX2 CPUs. Widths >= 29 stay scalar.
        if bitsPer <= 14 {
                unpackBitsVar(out, in, bitsPer)
                return
        }
        if bitsPer <= 28 {
                unpackBitsVarWide(out, in, bitsPer)
                return
        }
        bitUnpackU64LEFast(out, in, bitsPer)
}

// UnpackScalar is the original byte-at-a-time decoder, kept as a parity
// reference (oracle) for the fast paths' tests.
func UnpackScalar(out []uint64, in []byte, bitsPer int) {
        if bitsPer == 0 {
                for i := range out {
                        out[i] = 0
                }
                return
        }
        mask := uint64(1)<<uint(bitsPer) - 1
        var acc uint64
        var have uint
        pos := 0
        for i := range out {
                for have < uint(bitsPer) {
                        acc |= uint64(in[pos]) << have
                        have += 8
                        pos++
                }
                out[i] = acc & mask
                acc >>= uint(bitsPer)
                have -= uint(bitsPer)
        }
}

// PackChunk writes a chunk of values starting at element offset elemOff in
// the output bit-stream. It is Pack generalised to write into the middle
// of an existing buffer. bitsPer must be in [0, 56].
func PackChunk(out []byte, vals []uint64, bitsPer int, elemOff int) {
        if bitsPer == 0 || len(vals) == 0 {
                return
        }
        bitOff := elemOff * bitsPer
        // Byte-aligned widths land on whole-byte boundaries (bitsPer multiple
        // of 8 ⇒ bitOff multiple of 8), so each value is an independent LE
        // store with no cross-byte accumulator. These dedicated packers mirror
        // the byte-aligned unpack fast paths and get a SIMD variant under
        // qdf_simd.
        // elemOff is always a multiple of the 64-element chunk size, so bitOff
        // is byte-aligned for every width below (64*b is a multiple of 8). The
        // dedicated packers write whole-byte chunks with a SIMD variant under
        // qdf_simd; 10/12/14/20 use VPSLLVQ + lane-OR, 8/16/32 use VPSHUFB.
        switch bitsPer {
        case 8:
                packBits8(out[bitOff>>3:], vals)
                return
        case 10:
                packBits10(out[bitOff>>3:], vals)
                return
        case 12:
                packBits12(out[bitOff>>3:], vals)
                return
        case 14:
                packBits14(out[bitOff>>3:], vals)
                return
        case 16:
                packBits16(out[bitOff>>3:], vals)
                return
        case 20:
                packBits20(out[bitOff>>3:], vals)
                return
        case 32:
                packBits32(out[bitOff>>3:], vals)
                return
        }
        mask := uint64(1)<<uint(bitsPer) - 1
        pos := bitOff >> 3
        bitInByte := uint(bitOff & 7)
        var acc uint64
        if bitInByte > 0 {
                acc = uint64(out[pos])
        }
        have := bitInByte
        for _, v := range vals {
                acc |= (v & mask) << have
                have += uint(bitsPer)
                for have >= 8 {
                        out[pos] = byte(acc)
                        acc >>= 8
                        have -= 8
                        pos++
                }
        }
        if have > 0 {
                // merge with any existing high bits in out[pos] (zero on a freshly
                // cleared body, but safe in case the caller passed a populated buf)
                out[pos] = byte(acc) | (out[pos] &^ byte((1<<have)-1))
        }
}

// BitsForDelta returns the number of bits required to represent values in
// [0, d], i.e. bits.Len64(d). 0 => no bits needed (all equal).
func BitsForDelta(d uint64) int {
        return bits.Len64(d)
}

// decodeHex4Scalar fills dst from a 4-bit-packed nibble stream src, mapping each
// nibble through lut: dst[2i] = lut[src[i]&0x0f], dst[2i+1] = lut[src[i]>>4].
// len(dst) outputs are produced; an odd len(dst) consumes the low nibble of the
// final src byte only. It is the portable reference for DecodeHex4 and the
// scalar tail of the SIMD kernels — keep it bit-identical to them.
func decodeHex4Scalar(dst, src []byte, lut *[16]byte) {
        n := len(dst)
        full := n &^ 1
        for k := 0; k < full; k += 2 {
                b := src[k>>1]
                dst[k] = lut[b&0x0f]
                dst[k+1] = lut[b>>4]
        }
        if full < n {
                dst[full] = lut[src[full>>1]&0x0f]
        }
}

package bitpack

import "encoding/binary"

// bitUnpackU64LEFast is a wider-window replacement for Unpack.
// It carries a 128-bit sliding window (hi||lo) and refills it with a
// single 64-bit little-endian load from the input rather than one byte
// at a time. For bitsPer in [1, 56] the inner loop reduces to:
//
//        out[i] = lo & mask
//        lo = (lo >> b) | (hi << (64-b))
//        hi >>= b
//
// which is 4 dependent ops per element with no inner refill branch on
// the hot iterations.
//
// Bit layout is unchanged: LSB-first within each byte. Output of
// Pack is consumed by this function bit-for-bit.
func bitUnpackU64LEFast(out []uint64, in []byte, bitsPer int) {
        if bitsPer == 0 {
                clear(out)
                return
        }
        n := len(out)
        if n == 0 {
                return
        }
        mask := uint64(1)<<uint(bitsPer) - 1
        b := uint(bitsPer)

        var lo, hi uint64
        var have uint
        pos := 0
        end := len(in)

        for i := range n {
                // Refill: ensure 128-bit window has at least `b` valid bits.
                // Invariant: when we enter the refill, have < b <= 56, so the
                // remaining bits all live in lo and hi is zero.
                if have < b {
                        if pos+8 <= end {
                                v := binary.LittleEndian.Uint64(in[pos:])
                                pos += 8
                                lo |= v << have
                                if have > 0 {
                                        hi = v >> (64 - have)
                                }
                                have += 64
                        } else {
                                // Tail: byte-by-byte top-up while bytes remain. When
                                // `have` exceeds 56 a byte straddles the lo/hi
                                // boundary, so split the loaded byte across both
                                // halves to avoid losing the high overflow bits.
                                for have < 64 && pos < end {
                                        v := uint64(in[pos])
                                        lo |= v << have
                                        if have > 56 {
                                                hi |= v >> (64 - have)
                                        }
                                        pos++
                                        have += 8
                                }
                                // No second hi-fill loop: this loop exits with have >= 64 (or the
                                // buffer drained), and b <= 64, so any straddling high bits were
                                // already captured by the `have > 56` split above.
                        }
                }
                out[i] = lo & mask
                lo = (lo >> b) | (hi << (64 - b))
                hi >>= b
                have -= b
        }
}

//go:build (!amd64 && !arm64) || !qdf_simd

package bitpack

import "encoding/binary"

// DecodeHex4 fills dst from a 4-bit nibble stream src via the 16-entry LUT
// (dst[2i]=lut[src[i]&0xf], dst[2i+1]=lut[src[i]>>4]). Scalar fallback for
// non-SIMD builds.
func DecodeHex4(dst, src []byte, lut *[16]byte) {
        decodeHex4Scalar(dst, src, lut)
}

// unpackBits32 fallback for non-amd64 builds and amd64 builds without
// the qdf_simd tag.
func unpackBits32(out []uint64, in []byte) {
        for i := range out {
                out[i] = uint64(binary.LittleEndian.Uint32(in[i*4:]))
        }
}

// unpackBits16 fallback.
func unpackBits16(out []uint64, in []byte) {
        for i := range out {
                out[i] = uint64(binary.LittleEndian.Uint16(in[i*2:]))
        }
}

// unpackBits8 fallback.
func unpackBits8(out []uint64, in []byte) {
        for i := range out {
                out[i] = uint64(in[i])
        }
}

// packBits8 fallback: low byte of each value, contiguous.
func packBits8(out []byte, vals []uint64) {
        for i, v := range vals {
                out[i] = byte(v)
        }
}

// packBits16 fallback: low 2 bytes LE per value.
func packBits16(out []byte, vals []uint64) {
        for i, v := range vals {
                binary.LittleEndian.PutUint16(out[i*2:], uint16(v))
        }
}

// packBits32 fallback: low 4 bytes LE per value.
func packBits32(out []byte, vals []uint64) {
        for i, v := range vals {
                binary.LittleEndian.PutUint32(out[i*4:], uint32(v))
        }
}

// packBits10/12/14/20 fallbacks: scalar accumulator packer.
func packBits10(out []byte, vals []uint64) { Pack(out, vals, 10) }
func packBits12(out []byte, vals []uint64) { Pack(out, vals, 12) }
func packBits14(out []byte, vals []uint64) { Pack(out, vals, 14) }
func packBits20(out []byte, vals []uint64) { Pack(out, vals, 20) }

// unpackBitsVar / unpackBitsVarWide fallbacks: scalar sliding window.
func unpackBitsVar(out []uint64, in []byte, bitsPer int) {
        bitUnpackU64LEFast(out, in, bitsPer)
}

func unpackBitsVarWide(out []uint64, in []byte, bitsPer int) {
        bitUnpackU64LEFast(out, in, bitsPer)
}

// unpackBits10/12/14/20 fallbacks: the scalar sliding-window decoder.
func unpackBits10(out []uint64, in []byte) { bitUnpackU64LEFast(out, in, 10) }
func unpackBits12(out []uint64, in []byte) { bitUnpackU64LEFast(out, in, 12) }
func unpackBits14(out []uint64, in []byte) { bitUnpackU64LEFast(out, in, 14) }
func unpackBits20(out []uint64, in []byte) { bitUnpackU64LEFast(out, in, 20) }

// PackBoolsLSB fallback: plain scalar bit-by-bit pack.
func PackBoolsLSB(dst []byte, src []bool, n int) {
        for i := range n {
                if src[i] {
                        dst[i>>3] |= 1 << uint(i&7)
                }
        }
}

// Package bufpool implements size-classed, sharded byte-slice pools.
//
// Buffers are bucketed by capacity, so a small Get does not receive a
// previously-released multi-MiB slice. The pool stores *[]byte to avoid
// the interface-conversion allocation that putting a []byte directly
// into sync.Pool incurs.
package bufpool

import (
        "sync"
        "unsafe"
)

const (
        classSmall  = 1 << 9  // 512 B
        classMedium = 1 << 13 // 8 KiB
        classLarge  = 1 << 16 // 64 KiB
        classHuge   = 1 << 20 // 1 MiB
)

type shard struct {
        small  sync.Pool
        medium sync.Pool
        large  sync.Pool
        huge   sync.Pool
}

// numShards is a power of two so the shard index masks cleanly.
const numShards = 16

var shards [numShards]shard

func init() {
        for i := range shards {
                s := &shards[i]
                s.small.New = func() any { b := make([]byte, 0, classSmall); return &b }
                s.medium.New = func() any { b := make([]byte, 0, classMedium); return &b }
                s.large.New = func() any { b := make([]byte, 0, classLarge); return &b }
                s.huge.New = func() any { b := make([]byte, 0, classHuge); return &b }
        }
}

// shardIndex returns a shard index derived from the caller's goroutine stack
// address. Different goroutines have different stack bases, so this distributes
// across shards without any atomic operation.
func shardIndex() uint32 {
        var hint uintptr
        // Goroutine stacks are allocated in 8 KiB (1<<13) granules; shifting by 13
        // discards the intra-frame bits and uses the goroutine-specific part of the
        // address. The lower bits of the shifted value are then masked to numShards-1.
        return uint32(uintptr(unsafe.Pointer(&hint))>>13) & (numShards - 1)
}

// Get returns a *[]byte with cap >= hint and length 0. Anything larger
// than classHuge is allocated outside the pool.
func Get(hint int) *[]byte {
        s := &shards[shardIndex()]
        var p any
        switch {
        case hint <= classSmall:
                p = s.small.Get()
        case hint <= classMedium:
                p = s.medium.Get()
        case hint <= classLarge:
                p = s.large.Get()
        case hint <= classHuge:
                p = s.huge.Get()
        default:
                b := make([]byte, 0, hint)
                return &b
        }
        b := p.(*[]byte)
        // Put files by cap, so a class pool may hold a buffer whose cap is below
        // this class's hint ceiling (e.g. a caller returned a sub-512 B slice into
        // the small pool). Honor the documented "cap >= hint" contract: if the
        // pooled buffer is too small, drop it and allocate a right-sized one.
        if cap(*b) < hint {
                nb := make([]byte, 0, hint)
                return &nb
        }
        *b = (*b)[:0]
        return b
}

// Put returns b to the pool. Buffers grown past classHuge are dropped so
// the GC can reclaim them.
func Put(b *[]byte) {
        if b == nil || *b == nil {
                return
        }
        c := cap(*b)
        *b = (*b)[:0]
        s := &shards[shardIndex()]
        switch {
        case c <= classSmall:
                s.small.Put(b)
        case c <= classMedium:
                s.medium.Put(b)
        case c <= classLarge:
                s.large.Put(b)
        case c <= classHuge:
                s.huge.Put(b)
        }
}

// Package bumparena is a monotonic bump allocator for decoded string bodies.
// It is the reusable core behind the public qdf.Arena: a value is copied into a
// densely-packed block and returned as a string aliasing that copy, so a decode
// with many string fields costs ~one allocation per block instead of one per
// string. Blocks grow geometrically and are allocated without zero-fill.
//
// It does NOT carry the encoder's interning id-table (see internal/internarena);
// decode returns strings directly, never looks values up by id, so the id-table
// would be pure overhead.
package bumparena

import (
        "unsafe"

        "github.com/alex60217101990/qdf/internal/unsafestr"
)

const (
        // firstBlock is the first block size; subsequent blocks grow geometrically.
        firstBlock = 512
        // blockCap caps geometric growth so one block (and thus the retention of any
        // single returned string) is bounded.
        blockCap = 1 << 16
)

// Bump is a monotonic bump allocator. The zero value is NOT ready; use New.
// Not safe for concurrent use.
type Bump struct {
        buf  []byte // current block; full blocks are abandoned to the GC
        off  int    // bump cursor into buf
        next int    // size of the next fresh block (geometric growth)
}

// New returns an empty bump allocator. The first AppendStr allocates the first
// block.
func New() Bump { return Bump{next: firstBlock} }

// AppendStr copies b (which must be non-empty) into the arena and returns a
// string aliasing the copy. Successive calls pack contiguously. The rare block
// allocation is split into the //go:noinline appendStrGrow so the hot path
// stays small; AppendStr itself does not currently fit the inline budget (the
// call node for the grow branch keeps it over), but the fast path is a bounds
// check plus a copy — cheap relative to the per-decode allocation it avoids.
func (a *Bump) AppendStr(b []byte) string {
        n := len(b)
        off := a.off
        if off+n > len(a.buf) {
                return a.appendStrGrow(b)
        }
        dst := a.buf[off : off+n]
        a.off = off + n
        copy(dst, b)
        return unsafe.String(unsafe.SliceData(dst), n)
}

// appendStrGrow is the cold path: the value does not fit the current block, so
// allocate a fresh one. Out of line so AppendStr stays small.
//
//go:noinline
func (a *Bump) appendStrGrow(b []byte) string {
        n := len(b)
        // dirtmake: no zero-fill — copy overwrites exactly what we expose. Geometric
        // block growth (like a Vec / std::vector): few allocations on a large epoch,
        // little waste on a small one. The old block is abandoned to the GC, still
        // kept alive by any strings that alias it.
        a.buf = unsafestr.DirtBytes(max(a.next, n))
        if a.next < blockCap {
                a.next *= 2
        }
        dst := a.buf[:n]
        a.off = n
        copy(dst, b)
        return unsafe.String(unsafe.SliceData(dst), n)
}

// Reset rewinds the cursor to reuse the current block. It does NOT clear or
// zero memory; the next AppendStr overwrites it. Callers must ensure every
// string previously returned is dead before calling Reset.
func (a *Bump) Reset() { a.off = 0 }

package fsst

import "slices"

const (
        buildRounds = 3 // refinement iterations

        // maxSampleBytes caps the bytes of input scanned per training round. FSST
        // table quality saturates well before a whole large column is consumed, so
        // bounding the sample keeps encode cost flat regardless of column size.
        maxSampleBytes = 1 << 13 // 8 KiB

        // counterLog sizes the flat candidate counter (1<<counterLog slots). Large
        // enough to hold the distinct symbols + adjacent pairs of an 8 KiB sample
        // at a < 0.75 load factor; once full it stops admitting new keys (the
        // frequent ones are already present), bounding memory and time.
        counterLog = 13
)

// symKey is a fixed-size, comparable key for a candidate symbol of up to 8
// bytes — counting it never materializes a string.
type symKey struct {
        lo uint64 // up to 8 bytes, packed little-endian
        n  uint8  // length 1..8
}

func packKey(b []byte) symKey {
        var lo uint64
        for i := range b {
                lo |= uint64(b[i]) << (8 * i)
        }
        return symKey{lo, uint8(len(b))}
}

// counter is a flat open-addressed frequency table for symKeys. It replaces a
// map[symKey]int: no bucket chasing, an integer hash, and a hard capacity that
// drops new keys once full (frequent candidates are already counted). Reused
// across rounds via reset.
type counter struct {
        keys    []symKey
        cnt     []int32
        usedIdx []uint32 // occupied slot indices, so reset/iterate are O(distinct) not O(table)
        mask    uint32
}

func newCounter() *counter {
        n := 1 << counterLog
        return &counter{
                keys:    make([]symKey, n),
                cnt:     make([]int32, n),
                usedIdx: make([]uint32, 0, 2048),
                mask:    uint32(n - 1),
        }
}

func (c *counter) reset() {
        // Clear only the slots we touched — the table is 8192 wide but a single
        // training round fills far fewer, so a full clear(c.cnt) dominated the
        // per-round cost on small (probe-sample) inputs.
        for _, i := range c.usedIdx {
                c.cnt[i] = 0
        }
        c.usedIdx = c.usedIdx[:0]
}

// add increments the count for k, inserting it if absent and capacity remains.
func (c *counter) add(k symKey) {
        h := uint32(k.lo) ^ uint32(k.lo>>32)
        h = h*2654435761 + uint32(k.n)*40503
        i := h & c.mask
        for {
                if c.cnt[i] == 0 {
                        if len(c.usedIdx)<<2 >= len(c.cnt)*3 { // load factor 0.75: stop admitting
                                return
                        }
                        c.keys[i] = k
                        c.cnt[i] = 1
                        c.usedIdx = append(c.usedIdx, i)
                        return
                }
                if c.keys[i] == k {
                        if c.cnt[i] < 0x7fffffff { // saturate: 0 means empty, never wrap to it
                                c.cnt[i]++
                        }
                        return
                }
                i = (i + 1) & c.mask
        }
}

type candidate struct {
        k    symKey
        gain int
}

// Builder holds reusable training scratch — the counter, candidate/keys buffers,
// and the table itself (whose symbol slice and first-byte index buckets retain
// their capacity across calls). The columnar probe estimates FSST per string
// column and the encoder re-trains per column, so without reuse each call
// re-allocated the counter and ~255 first-byte bucket slices (the dominant FSST
// encode allocation). Not safe for concurrent use; pool one per goroutine.
type Builder struct {
        c    *counter
        cand []candidate
        keys []symKey
        scan [][]byte // reused sample-prefix header for the trim case
        t    SymbolTable
}

func NewBuilder() *Builder {
        return &Builder{
                c:    newCounter(),
                cand: make([]candidate, 0, 2048),
                keys: make([]symKey, 0, maxSymbols),
        }
}

// Build trains a table from samples, reusing the Builder's scratch. The returned
// table ALIASES the Builder's internal table and is valid only until the next
// Build on the same Builder — a caller that retains the table (a pre-trained
// dictionary) must use BuildSymbolTable, which returns a fresh table.
//
// Deterministic: identical samples always yield an identical table.
func (b *Builder) Build(samples [][]byte) *SymbolTable {
        return b.BuildRounds(samples, buildRounds)
}

// BuildRounds is Build with an explicit refinement-round count. The columnar
// probe uses fewer rounds for its size ESTIMATE (a rough table is enough to
// decide columnar-vs-row-major), while the encoder uses the full buildRounds
// for the table it actually emits.
func (b *Builder) BuildRounds(samples [][]byte, rounds int) *SymbolTable {
        scan := b.sampleByBytes(samples, maxSampleBytes)
        b.t.reset() // empty: round 0 is all single-byte tokens
        for range rounds {
                b.c.reset()
                empty := len(b.t.symbols) == 0
                for _, s := range scan {
                        tokenizeCount(&b.t, s, b.c, empty)
                }
                b.keys, b.cand = topCandidates(b.c, b.cand[:0], b.keys[:0])
                b.t.fillFromKeys(b.keys)
        }
        // Drop the []byte views into the caller's strings so a pooled Builder does
        // not pin that memory until its next Build (the trim path retains them in
        // b.scan). cand/keys are value types (no pointers) and need no such clear.
        clear(b.scan)
        return &b.t
}

// BuildSymbolTable trains a fresh, independent table (for a retained dictionary
// or one-shot use). The wire stores the table, so table quality affects ratio
// only, never correctness.
func BuildSymbolTable(samples [][]byte) *SymbolTable {
        return NewBuilder().Build(samples)
}

// reset clears the table for reuse, keeping the symbol slice and candidate
// index capacities so a subsequent fillFromKeys does not re-allocate.
func (t *SymbolTable) reset() {
        t.symbols = t.symbols[:0]
        t.cands = t.cands[:0]
}

// fillFromKeys rebuilds the table in place from packed candidate keys — each
// symbol's bytes written straight into its fixed array, no intermediate [][]byte.
func (t *SymbolTable) fillFromKeys(keys []symKey) {
        t.reset()
        for _, k := range keys {
                if k.n == 0 || k.n > maxSymLen || len(t.symbols) >= maxSymbols {
                        continue
                }
                var s symbol
                s.len = k.n
                for i := 0; i < int(k.n); i++ {
                        s.bytes[i] = byte(k.lo >> (8 * i))
                }
                s.val = k.lo // packKey already packed exactly k.n bytes
                if k.n >= maxSymLen {
                        s.mask = ^uint64(0)
                } else {
                        s.mask = (uint64(1) << (8 * k.n)) - 1
                }
                t.symbols = append(t.symbols, s)
        }
        t.buildIndex()
}

// sampleByBytes returns the leading prefix of samples whose total length first
// reaches budget, truncating the final element so the SCANNED bytes are bounded
// by budget regardless of any single element's size (a lone multi-MB string
// must not make training O(that string)). Deterministic; never mutates the
// caller's backing data (it shallow-copies the small header slice only when it
// has to trim).
func (b *Builder) sampleByBytes(samples [][]byte, budget int) [][]byte {
        total := 0
        for i := range samples {
                total += len(samples[i])
                if total >= budget {
                        over := total - budget
                        if over > 0 && over < len(samples[i]) {
                                // Reuse the Builder's scan header instead of allocating a fresh
                                // [][]byte every train; the trimmed prefix is read-only here.
                                b.scan = append(b.scan[:0], samples[:i+1]...)
                                b.scan[i] = samples[i][:len(samples[i])-over]
                                return b.scan
                        }
                        return samples[:i+1]
                }
        }
        return samples
}

// tokenizeCount greedily tokenizes s with the current table and counts each
// emitted symbol and each adjacent-symbol pair. The pair is a contiguous slice
// of s (the previous token immediately precedes the current one), so counting
// never concatenates or allocates. When empty is set (round 0, no symbols yet)
// every token is a single byte, so the per-position match scan is skipped.
func tokenizeCount(t *SymbolTable, s []byte, c *counter, empty bool) {
        i := 0
        prevStart, prevLen := 0, 0
        for i < len(s) {
                n := 1
                if !empty {
                        if _, m := t.match(s[i:]); m != 0 {
                                n = m
                        }
                }
                c.add(packKey(s[i : i+n]))
                if prevLen != 0 && prevLen+n <= maxSymLen {
                        c.add(packKey(s[prevStart : i+n])) // prev+cur, contiguous
                }
                prevStart, prevLen = i, n
                i += n
        }
}

// topCandidates selects the top-255 candidate keys by gain = freq*len, with a
// deterministic total order (gain desc, len desc, packed-bytes asc). The cand
// and keys scratch slices are caller-owned and reused across rounds.
// It returns keys plus the (possibly grown) cand backing so the caller can
// retain it across rounds/Build calls and avoid re-growing the candidate slice.
func topCandidates(c *counter, cand []candidate, keys []symKey) ([]symKey, []candidate) {
        for _, i := range c.usedIdx { // only occupied slots, not the whole 8192-wide table
                k := c.keys[i]
                if k.n == 0 || k.n > maxSymLen {
                        continue
                }
                cand = append(cand, candidate{k, int(c.cnt[i]) * int(k.n)})
        }
        // High-cardinality columns produce thousands of candidates of which only 255
        // are kept; quickselect the top 255 in O(n) and sort just those, instead of
        // fully sorting the whole set (the dominant FSST training cost).
        if len(cand) > maxSymbols {
                selectTopK(cand, maxSymbols)
                cand = cand[:maxSymbols]
        }
        // slices.SortFunc (typed) avoids sort.Slice's reflection-based swapper.
        slices.SortFunc(cand, func(a, b candidate) int {
                if candLess(a, b) {
                        return -1
                }
                if candLess(b, a) {
                        return 1
                }
                return 0
        })
        for i := range cand {
                keys = append(keys, cand[i].k)
        }
        return keys, cand
}

// candLess reports whether a outranks b: higher gain, then longer symbol, then
// smaller packed bytes. A strict total order on distinct keys (no ties).
func candLess(a, b candidate) bool {
        if a.gain != b.gain {
                return a.gain > b.gain
        }
        if a.k.n != b.k.n {
                return a.k.n > b.k.n
        }
        return a.k.lo < b.k.lo
}

// selectTopK partitions cand (Hoare quickselect, O(n) average) so the k
// highest-ranked candidates by candLess occupy cand[:k] (unordered). Used to
// avoid fully sorting a large candidate set when only the top maxSymbols are
// kept. The selected SET is deterministic for a given input.
func selectTopK(cand []candidate, k int) {
        lo, hi := 0, len(cand)-1
search:
        for lo < hi {
                pivot := cand[(lo+hi)/2]
                i, j := lo, hi
                for i <= j {
                        for candLess(cand[i], pivot) {
                                i++
                        }
                        for candLess(pivot, cand[j]) {
                                j--
                        }
                        if i <= j {
                                cand[i], cand[j] = cand[j], cand[i]
                                i++
                                j--
                        }
                }
                switch {
                case k <= j:
                        hi = j
                case k >= i:
                        lo = i
                default:
                        break search
                }
        }
}

// Package fsst implements FSST (Fast Static Symbol Table) string compression:
// a learned table of up to 255 substrings (each 1..8 bytes) replaced by 1-byte
// codes, with code 255 reserved as an escape for a following literal byte.
// Decompression is a table lookup; the table is stored on the wire.
package fsst

import (
        "encoding/binary"
        "errors"
)

const (
        escapeCode = 0xFF // 255: next byte is an emitted literal
        maxSymbols = 255  // codes 0..254
        maxSymLen  = 8
)

// symbol holds up to 8 bytes of a table entry. bytes/len drive decode; val/mask
// are the same bytes packed little-endian (plus a length mask) so the hot
// match path is a single masked uint64 compare instead of a byte loop.
type symbol struct {
        bytes [8]byte
        val   uint64 // bytes packed little-endian
        mask  uint64 // (1<<(8*len))-1
        len   uint8
}

// packVal returns the little-endian uint64 of b and its length mask.
func packVal(b []byte) (val, mask uint64) {
        for i := range b {
                val |= uint64(b[i]) << (8 * i)
        }
        if len(b) >= 8 {
                mask = ^uint64(0)
        } else {
                mask = (uint64(1) << (8 * len(b))) - 1
        }
        return
}

// cand is a compression-index entry: the match-relevant fields of a symbol
// inlined so the hot match loop scans a contiguous, cache-friendly slice instead
// of gathering each symbol out of t.symbols by code. val/mask drive the single
// masked uint64 compare; code/length are the result.
type cand struct {
        val, mask uint64
        code      uint8
        length    uint8
}

// SymbolTable maps codes 0..254 to symbols and indexes them for compression.
type SymbolTable struct {
        symbols []symbol // index == code
        // Compression index: all candidates in one flat slice, grouped by first
        // byte and sorted longest-symbol-first within each group. cands[first[b] :
        // first[b+1]] is byte b's group. One backing allocation (reused across
        // rebuilds) and sequential scan — no 256 sub-slices, no gather into symbols.
        cands []cand
        first [257]int32
}

// newSymbolTable builds a table from raw symbol byte-strings (≤255, each 1..8
// bytes). Symbols longer than maxSymLen or empty are skipped; excess past 255
// is truncated. buildIndex then groups them first-byte, longest-first so
// Compress's first hit is the longest match.
func newSymbolTable(raw [][]byte) *SymbolTable {
        t := &SymbolTable{}
        for _, b := range raw {
                if len(b) == 0 || len(b) > maxSymLen || len(t.symbols) >= maxSymbols {
                        continue
                }
                var s symbol
                s.len = uint8(len(b))
                copy(s.bytes[:], b)
                s.val, s.mask = packVal(b)
                t.symbols = append(t.symbols, s)
        }
        t.buildIndex()
        return t
}

// buildIndex rebuilds the flat first-byte-grouped candidate index from
// t.symbols. Each group is sorted longest-symbol-first (tie-break by code, for
// determinism) so match's first prefix hit is the longest. The cands backing
// array is reused when it already has the capacity (the pooled-builder path
// rebuilds this every training round).
func (t *SymbolTable) buildIndex() {
        var cnt [256]int32
        for i := range t.symbols {
                cnt[t.symbols[i].bytes[0]]++
        }
        var off int32
        for b := range 256 {
                t.first[b] = off
                off += cnt[b]
        }
        t.first[256] = off
        if cap(t.cands) < int(off) {
                t.cands = make([]cand, off)
        } else {
                t.cands = t.cands[:off]
        }
        var cur [256]int32
        copy(cur[:], t.first[:256])
        for i := range t.symbols {
                s := &t.symbols[i]
                b0 := s.bytes[0]
                t.cands[cur[b0]] = cand{val: s.val, mask: s.mask, code: uint8(i), length: s.len}
                cur[b0]++
        }
        for b := range 256 {
                lo, hi := t.first[b], t.first[b+1]
                for i := lo + 1; i < hi; i++ {
                        for j := i; j > lo; j-- {
                                a, c := &t.cands[j-1], &t.cands[j]
                                if c.length > a.length || (c.length == a.length && c.code < a.code) {
                                        *a, *c = *c, *a
                                } else {
                                        break
                                }
                        }
                }
        }
}

// match returns the code and length of the longest symbol that is a prefix of
// s, or (0,0) if none matches. Hot path: load up to 8 input bytes once and test
// each candidate (longest-first) with a single masked uint64 compare over the
// contiguous first-byte group (no per-candidate gather into t.symbols).
func (t *SymbolTable) match(s []byte) (uint8, int) {
        var x uint64
        if len(s) >= 8 {
                x = binary.LittleEndian.Uint64(s)
        } else {
                for i := range s {
                        x |= uint64(s[i]) << (8 * i)
                }
        }
        avail := len(s)
        b0 := int(byte(x))
        cs := t.cands[t.first[b0]:t.first[b0+1]]
        for k := range cs {
                c := &cs[k]
                if int(c.length) <= avail && x&c.mask == c.val {
                        return c.code, int(c.length)
                }
        }
        return 0, 0
}

// Compress appends the FSST encoding of src to dst and returns dst.
func (t *SymbolTable) Compress(src, dst []byte) []byte {
        i := 0
        for i < len(src) {
                code, n := t.match(src[i:])
                if n == 0 {
                        dst = append(dst, escapeCode, src[i])
                        i++
                        continue
                }
                dst = append(dst, code)
                i += n
        }
        return dst
}

// CompressedLen returns the number of bytes Compress would emit for src,
// without materializing the output. The columnar size probe only needs the
// coded length to compare codecs, so this avoids allocating (and growing) a
// throwaway destination buffer per sampled string.
func (t *SymbolTable) CompressedLen(src []byte) int {
        n, i := 0, 0
        for i < len(src) {
                _, m := t.match(src[i:])
                if m == 0 {
                        n += 2 // escapeCode + literal byte
                        i++
                        continue
                }
                n++ // single code byte
                i += m
        }
        return n
}

// Decompress appends the decoding of codes to dst and returns dst. It is
// defensive: a truncated trailing escape stops cleanly rather than panicking,
// so it is safe to call on arbitrary input.
func (t *SymbolTable) Decompress(codes, dst []byte) []byte {
        i := 0
        for i < len(codes) {
                c := codes[i]
                i++
                if c == escapeCode {
                        if i >= len(codes) {
                                break
                        }
                        dst = append(dst, codes[i])
                        i++
                        continue
                }
                if int(c) >= len(t.symbols) {
                        // unknown code on arbitrary input: skip (Decompress never panics).
                        continue
                }
                s := &t.symbols[c]
                dst = append(dst, s.bytes[:s.len]...)
        }
        return dst
}

// DecompressN appends the decoding of codes to dst but never lets len(dst)
// exceed limit, returning ok=false the moment a symbol or literal would
// overflow. The decode path pre-sizes the destination slab to limit and passes
// limit here, so a malformed block cannot drive an append past the slab's
// capacity (no transient over-allocation / heap-exhaustion). Never panics.
func (t *SymbolTable) DecompressN(codes, dst []byte, limit int) ([]byte, bool) {
        i := 0
        for i < len(codes) {
                c := codes[i]
                i++
                if c == escapeCode {
                        if i >= len(codes) {
                                // Dangling escape with no literal byte: a corrupt block. Reject
                                // rather than break-as-success, which would return a string one
                                // byte short reported as ok (matches the unknown-code branch below).
                                return dst, false
                        }
                        if len(dst) >= limit {
                                return dst, false
                        }
                        dst = append(dst, codes[i])
                        i++
                        continue
                }
                if int(c) >= len(t.symbols) {
                        // A code with no symbol is a corrupt block (a conforming encoder only
                        // emits codes < len(symbols) or escapeCode). Reject rather than silently
                        // skipping it, which would return a truncated string as success.
                        return dst, false
                }
                s := &t.symbols[c]
                if len(dst)+int(s.len) > limit {
                        return dst, false
                }
                dst = append(dst, s.bytes[:s.len]...)
        }
        return dst, true
}

// MarshalTo appends the serialized symbol table to dst:
//
//        uvarint(count) then count × [ len:1B(1..8) | bytes ].
func (t *SymbolTable) MarshalTo(dst []byte) []byte {
        dst = binary.AppendUvarint(dst, uint64(len(t.symbols)))
        for i := range t.symbols {
                dst = append(dst, t.symbols[i].len)
                dst = append(dst, t.symbols[i].bytes[:t.symbols[i].len]...)
        }
        return dst
}

// SerializedSize returns the number of bytes MarshalTo would write, without
// allocating. Used by the columnar probe to estimate FSST cost cheaply.
func (t *SymbolTable) SerializedSize() int {
        sz := uvarintSize(uint64(len(t.symbols)))
        for i := range t.symbols {
                sz += 1 + int(t.symbols[i].len)
        }
        return sz
}

// uvarintSize returns the encoded length of x as a uvarint (no allocation).
func uvarintSize(x uint64) int {
        n := 1
        for x >= 0x80 {
                x >>= 7
                n++
        }
        return n
}

var errBadTable = errors.New("fsst: malformed symbol table")

// UnmarshalSymbolTable parses a table from b and returns it plus the number of
// bytes consumed. All lengths are validated against b before any use; never
// panics on malformed input.
func UnmarshalSymbolTable(b []byte) (*SymbolTable, int, error) {
        t := &SymbolTable{}
        n, err := UnmarshalInto(b, t)
        if err != nil {
                return nil, 0, err
        }
        return t, n, nil
}

// Reset clears t's symbol and index data while retaining backing allocations,
// making t safe to reuse via UnmarshalInto from a sync.Pool.
func (t *SymbolTable) Reset() {
        t.symbols = t.symbols[:0]
        t.cands = t.cands[:0]
        t.first = [257]int32{}
}

// Clone returns an independent deep copy of t. Use it to retain a table
// beyond the lifetime of the Builder that trained it: Builder.Build returns a
// table that aliases the Builder's internal SymbolTable and is invalidated the
// next time Build is called on the same Builder (e.g. when the Builder is
// returned to a pool). Clone's copies (symbols and cands slices) contain only
// value types (no pointer fields), so the clone neither pins the original
// Builder nor shares any mutable state with it.
func (t *SymbolTable) Clone() *SymbolTable {
        c := &SymbolTable{first: t.first}
        if len(t.symbols) > 0 {
                c.symbols = make([]symbol, len(t.symbols))
                copy(c.symbols, t.symbols)
        }
        if len(t.cands) > 0 {
                c.cands = make([]cand, len(t.cands))
                copy(c.cands, t.cands)
        }
        return c
}

// UnmarshalInto parses a symbol table from b into t, reusing t's existing
// slice backing arrays. It is equivalent to UnmarshalSymbolTable but avoids
// the intermediate [][]byte allocation, making it suitable for pooled callers.
// Returns the number of bytes consumed from b.
func UnmarshalInto(b []byte, t *SymbolTable) (int, error) {
        cnt, n := binary.Uvarint(b)
        if n <= 0 || cnt > maxSymbols {
                return 0, errBadTable
        }
        off := n
        t.Reset()
        for range cnt {
                if off >= len(b) {
                        return 0, errBadTable
                }
                l := int(b[off])
                off++
                if l < 1 || l > maxSymLen || off+l > len(b) {
                        return 0, errBadTable
                }
                var s symbol
                src := b[off : off+l]
                s.len = uint8(l)
                copy(s.bytes[:], src)
                s.val, s.mask = packVal(src)
                t.symbols = append(t.symbols, s)
                off += l
        }
        t.buildIndex()
        return off, nil
}

// Package hadamard implements a randomized fast Walsh–Hadamard rotation
// R = (1/√n)·H·D, where D is a seed-driven random ±1 diagonal and H is the
// Walsh–Hadamard matrix. R is orthonormal, so its inverse is Rᵀ = D·(1/√n)·H.
// The rotation spreads per-coordinate outliers evenly, concentrating the data
// toward a Gaussian shape that low-bit scalar/lattice quantization handles well.
// Only a uint64 seed is stored on the wire — the matrix is never materialized.
package hadamard

import "math"

// NextPow2 returns the smallest power of two ≥ n (minimum 1).
func NextPow2(n int) int {
        if n <= 1 {
                return 1
        }
        p := 1
        for p < n {
                p <<= 1
        }
        return p
}

// signs derives a deterministic ±1 per index from seed via splitmix64.
func sign(seed uint64, i int) float64 {
        z := seed + uint64(i+1)*0x9e3779b97f4a7c15
        z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9
        z = (z ^ (z >> 27)) * 0x94d049bb133111eb
        z ^= z >> 31
        if z&1 == 0 {
                return 1
        }
        return -1
}

func checkPow2(n int) {
        if n == 0 || n&(n-1) != 0 {
                panic("hadamard: length must be a power of two")
        }
}

// Forward applies R·x = (1/√n)·H·(D·x) in place.
func Forward(x []float64, seed uint64) {
        n := len(x)
        checkPow2(n)
        for i := range x {
                x[i] *= sign(seed, i)
        }
        fwht(x)
        s := 1.0 / math.Sqrt(float64(n))
        for i := range x {
                x[i] *= s
        }
}

// Inverse applies R⁻¹·y = D·(1/√n)·H·y in place.
func Inverse(y []float64, seed uint64) {
        n := len(y)
        checkPow2(n)
        fwht(y)
        s := 1.0 / math.Sqrt(float64(n))
        for i := range y {
                y[i] *= s * sign(seed, i)
        }
}

//go:build !arm64

package hadamard

// fwht runs the in-place unnormalized Walsh–Hadamard butterfly.
// len(a) must be a power of two.
func fwht(a []float64) {
        n := len(a)
        for h := 1; h < n; h <<= 1 {
                for i := 0; i < n; i += h << 1 {
                        for j := i; j < i+h; j++ {
                                x, y := a[j], a[j+h]
                                a[j], a[j+h] = x+y, x-y
                        }
                }
        }
}

// Package intern provides a small fixed-size string cache used by the
// decoder to deduplicate repeated map keys.
//
// The cache is a direct-mapped table indexed by a hash of the input
// bytes. A hit returns the previously-allocated string with no
// allocation. A miss copies the input once and stores it; collisions
// overwrite the slot rather than chain.
//
// The table is a value type embedded in the Decoder, so it survives
// across Unmarshal calls when the decoder is reused via sync.Pool.
package intern

import (
        "hash/maphash"

        "github.com/alex60217101990/qdf/internal/unsafestr"
)

// Cache is a 256-slot direct-mapped string interner. The zero value is
// ready to use. Lookups are a hash + one read + one comparison. The 256-slot
// table (~4 KiB on 64-bit) lives behind a pointer and is allocated lazily on
// the first Make, so an unused Cache costs ~24 bytes inside its owner — this
// matters for decoders that are constructed per nested value (codegen) and
// never touch a map key. Once allocated it stays in L1 across reuse.
type Cache struct {
        slots *[256]string
        seed  maphash.Seed
        init  bool
}

func (c *Cache) ensure() {
        if !c.init {
                c.seed = maphash.MakeSeed()
                c.slots = new([256]string)
                c.init = true
        }
}

// Make returns a string equal to b. On a cache hit the existing string
// is returned with no allocation. On a miss b is copied into a fresh
// string and stored. The returned string is safe to retain — it does
// not alias b.
//
//go:nosplit
func (c *Cache) Make(b []byte) string {
        if len(b) == 0 {
                return ""
        }
        c.ensure()
        h := maphash.Bytes(c.seed, b)
        slot := h & (uint64(len(c.slots)) - 1)
        if existing := c.slots[slot]; existing != "" && existing == unsafestr.String(b) {
                return existing
        }
        s := string(b)
        c.slots[slot] = s
        return s
}

// Package internarena is a bytes-only bump-pointer allocator for the
// Dense-mode intern table. The encoder used to call strings.Clone on
// every first-occurrence intern, which produces one heap object per
// distinct string and adds GC pressure on streams with thousands of
// unique keys. This arena collapses every interned payload into a
// chain of large slabs and hands out compact integer ids that resolve
// back to byte slices that alias the active slab.
//
// Why hand-roll instead of taking github.com/VoolFI71/go-arena: that
// package targets generic typed allocation (New[T], MakeSlice[T]),
// none of which we use. We need exactly one operation — copy a byte
// payload, return an id — and we want to inline the cursor into our
// encoder state so a Put / Get pair stays a few amortised
// instructions. ~80 LOC beats a v0.0.0-pseudo-version dependency on
// every axis but "did someone else write it for me".
//
// Tricks captured from mcyoung's "Cheating the Reaper in Go"
// (https://mcyoung.xyz/2025/04/21/go-arenas/):
//
//   - cursor as uintptr, not unsafe.Pointer. The store-of-pointer
//     write barrier fires on every Alloc otherwise.
//   - one fixed alignment (8 bytes is overkill for bytes, we drop
//     it — bytes do not need to be aligned). Branchless Put hot path.
//   - chunks []byte slice keeps prior chunks alive across GC even
//     while the cursor (uintptr) cannot itself act as a GC root.
//   - Reset() does not free chunks; the next Put reuses chunks[0]
//     from offset 0. The "6× speedup" pattern from the article's
//     final section. Safe for the encoder pool's reuse cycle — every
//     Reset means the encoder's intern table is logically empty, so
//     handing out the same memory to a new id is correct.
//
// What we do NOT need (and skip):
//
//   - the back-pointer + reflect.StructOf dance. Only required when
//     the arena holds pointer-typed payloads so a pointer into a
//     chunk can keep the whole graph alive. Bytes have no GC scan.
//   - per-size-class sync.Pool, runtime.SetFinalizer. One arena per
//     encState; encoder pool already manages the lifecycle.
//   - one-past-the-end pointer interlock. uintptr cursor sidesteps
//     the issue entirely.
//
// Concurrency: a single Arena is owned by one Encoder. The Encoder
// pool gates concurrent access. The Arena itself is NOT goroutine-
// safe.
//
// Aliasing contract: byte slices returned by Get() alias the chunk
// the id was stored in. A slice stays valid as long as no Reset()
// runs and the encoder does not Put past a chunk-growth boundary
// that retires the chunk the slice points into. In practice the
// only caller is encState, which holds Get() results inside its
// own map keyed by string. Resets clear the map first, then the
// arena, so an aliased string is never read past its arena's life.
package internarena

import (
        "unsafe"
)

// initialChunkBytes is the first slab's capacity. Doubles on every
// growth, matching the runtime's slice-growth heuristic.
const initialChunkBytes = 4 * 1024

// maxChunkBytes caps individual slab growth. The loc-pack format gives each
// chunk a 32-bit offset field (see the locs doc on Arena), so a single chunk
// must never exceed what a uint32 offset can address — otherwise Put would
// truncate the offset at line `uint32(a.off)` and Get would resolve the wrong
// byte range (silent corruption). Doubling growth is unbounded, so without
// this cap a long-lived arena that interns multiple GiB of strings would
// eventually allocate a >4 GiB slab and trip the truncation. Capping well
// below 4 GiB keeps every offset in range; with the 16-bit chunk index
// (65 535 slabs) the arena still addresses ~64 TiB total — far past any real
// payload, and normal workloads never fill even one of these slabs.
const maxChunkBytes = 1 << 30 // 1 GiB

// Arena holds a chain of byte slabs and hands out ids that resolve
// to slices into the active slab.
//
// The two pointer-bearing slices (chunks, locs) lead so the GC pointer-scan
// range stays tight (32 bytes vs 56 for a cursor-first layout); the
// non-pointer bump cursor (off/end/cur) trails.
type Arena struct {
        // chunks owns every slab the arena ever allocated. Each entry
        // is a []byte whose underlying array stays GC-rooted via the
        // slice header. The slice grows by append; doubling.
        chunks [][]byte

        // locs[id] packs (chunk_idx<<48) | (offset<<16) | length so a
        // single uint64 carries every position a caller needs. 16 bits
        // of chunk_idx handles 65 535 slabs (way past practical limits),
        // 32 bits of offset handles 4 GiB chunks, 16 bits of length
        // covers strings up to 65 535 bytes — anything longer falls back
        // to a per-string allocation in the caller.
        locs []uint64

        // off / end are the bump-pointer cursor — but expressed as a
        // byte offset into chunks[cur] (not a raw machine address)
        // so we never round-trip through unsafe.Pointer(uintptr). The
        // chunks slice keeps the slab's backing array GC-rooted; we
        // resolve actual addresses lazily via chunks[cur] on each Put.
        // Same write-barrier savings as the article's uintptr-cursor
        // (no pointer-typed store on the hot path), no go vet warning.
        off uintptr
        end uintptr

        // cur is the index of the chunk the cursor lives in. Reset()
        // rolls this back to 0.
        cur int

        // putBytes accumulates the payload bytes stored via Put since the last
        // Reset — an O(1) running counter so callers can read the current cycle's
        // intern volume without the O(chunks) BytesUsed walk (which also over-counts
        // the full capacity of any chunk grow() skipped because a payload did not
        // fit). The adaptive-retention trigger in encState.reset reads this once per
        // message; ResetWithLimit zeroes it.
        putBytes int
}

// MaxStringLen is the largest single payload the arena can store.
// Callers must handle longer payloads themselves (e.g. by skipping
// the intern table — see encState.lookupOrAssign).
const MaxStringLen = 1<<16 - 1

// Len reports the number of stored ids.
//
//go:nosplit
func (a *Arena) Len() int { return len(a.locs) }

// BytesUsed is the total byte volume the arena has handed out across
// every live chunk. Useful for capacity tuning and debug logs.
func (a *Arena) BytesUsed() int {
        total := 0
        for i, c := range a.chunks {
                if i < a.cur {
                        total += cap(c)
                        continue
                }
                if i == a.cur {
                        // Bytes used in the current chunk = off.
                        total += int(a.off)
                }
        }
        return total
}

// ResidentBytes is the total capacity the arena currently keeps resident
// across every slab it holds — independent of the bump cursor, so it counts
// memory retained past a Reset() (the spike chunks an adaptive caller chooses
// to keep warm). Distinct from BytesUsed, which reports only the volume handed
// out below the live cursor. Used for retention tuning and tests.
//
//go:nosplit
func (a *Arena) ResidentBytes() int {
        total := 0
        for _, c := range a.chunks {
                total += cap(c)
        }
        return total
}

// Put copies s into the arena and returns the assigned id. Caller
// guarantees len(s) <= MaxStringLen; longer payloads must take the
// fallback path.
//
//go:nosplit
func (a *Arena) Put(s string) uint32 {
        n := uintptr(len(s))
        // Lazy first-chunk allocation. Cold arena has chunks==nil and
        // off==end==0; even an empty Put needs chunks[0] so the
        // slab access below does not panic.
        if len(a.chunks) == 0 || a.off+n > a.end {
                a.grow(n)
        }
        cur := a.chunks[a.cur]
        offset := uint32(a.off)
        // Extend the slice header to the slab's true capacity so we can
        // write into the uninitialised tail bytes. The underlying array
        // is owned by chunks[a.cur]; we never escape this aliased slice.
        if n > 0 {
                full := unsafe.Slice(unsafe.SliceData(cur), cap(cur))
                copy(full[a.off:a.off+n], s)
        }
        a.off += n
        a.putBytes += int(n)

        id := uint32(len(a.locs))
        a.locs = append(a.locs, pack(uint16(a.cur), offset, uint16(n)))
        return id
}

// BytesPut is the payload volume stored via Put since the last Reset, in O(1)
// (a running counter, no chunk walk). This is the exact per-cycle intern demand
// — unlike BytesUsed it never counts a skipped chunk's capacity — so it is the
// signal the adaptive-retention policy uses to decide whether to keep the grown
// slabs warm across a Reset.
//
//go:nosplit
func (a *Arena) BytesPut() int { return a.putBytes }

// Get returns the bytes for id. The slice aliases the chunk the
// payload lives in — caller MUST NOT retain it across a Reset() or
// a Put() that triggers chunk growth.
//
//go:nosplit
func (a *Arena) Get(id uint32) []byte {
        chunkIdx, offset, length := unpack(a.locs[id])
        chunk := a.chunks[chunkIdx]
        return chunk[offset : offset+uint32(length)]
}

// DefaultRetainBytes is the soft cap on total chunk capacity the
// arena keeps across Reset() calls. A burst of unusually long
// payloads grows chunks past this; the next Reset drops the extras
// so a long-running encoder pool does not pin spike memory forever.
// Callers may override via ResetWithLimit.
const DefaultRetainBytes = 256 * 1024 // 256 KiB

// Reset is shorthand for ResetWithLimit(DefaultRetainBytes).
func (a *Arena) Reset() { a.ResetWithLimit(DefaultRetainBytes) }

// ResetWithLimit rolls every cursor back to chunks[0] and clears
// the loc table. If the cumulative capacity of all chunks exceeds
// retainBytes the spike chunks (chunks[1..]) are dropped so the
// arena's resident memory stays bounded. chunks[0] — the small
// lazily-allocated initial slab — is always kept warm for the next
// cycle. retainBytes == 0 disables the cap entirely (keep
// everything).
//
// All ids handed out before Reset become invalid; the caller must
// clear its own id-keyed structures before / atomically with the
// call.
func (a *Arena) ResetWithLimit(retainBytes int) {
        a.locs = a.locs[:0]
        a.putBytes = 0
        if len(a.chunks) == 0 {
                a.off = 0
                a.end = 0
                a.cur = 0
                return
        }
        // Drop excess chunks past chunks[0] when total capacity has
        // outgrown the soft cap. Keep chunks[0] — it is the lazy
        // initial slab whose size matches the steady-state workload
        // before any spike happened.
        if retainBytes > 0 && len(a.chunks) > 1 {
                total := 0
                for _, c := range a.chunks {
                        total += cap(c)
                }
                if total > retainBytes {
                        // Clear the dropped slots before truncating so the GC
                        // can reclaim them; the slice header otherwise keeps
                        // their backing arrays alive.
                        for i := 1; i < len(a.chunks); i++ {
                                a.chunks[i] = nil
                        }
                        a.chunks = a.chunks[:1]
                }
        }
        a.cur = 0
        a.off = 0
        a.end = uintptr(cap(a.chunks[0]))
}

// grow advances the cursor to a chunk that can fit need bytes,
// either by walking to the next existing chunk (post-Reset reuse) or
// by allocating a fresh one (doubling growth).
func (a *Arena) grow(need uintptr) {
        // Try to walk to an existing chunk that can hold the request.
        // After Reset, chunks past cur still exist; reuse them before
        // allocating new memory.
        for a.cur+1 < len(a.chunks) {
                a.cur++
                c := a.chunks[a.cur]
                if uintptr(cap(c)) >= need {
                        a.off = 0
                        a.end = uintptr(cap(c))
                        return
                }
        }
        // Allocate a fresh chunk. Doubling growth, but at least `need`.
        size := uintptr(initialChunkBytes)
        if len(a.chunks) > 0 {
                size = uintptr(cap(a.chunks[len(a.chunks)-1])) * 2
        }
        if size < need {
                size = need
        }
        // Round up to the nearest power of two for slab-friendly sizes.
        size = min(
                // Cap slab size so a chunk offset can never exceed the 32-bit field in
                // the loc-pack format. need is always <= MaxStringLen (65 535), far below
                // the cap, so clamping here never starves a request; it only bounds the
                // doubling growth. Growth past this point adds chunks instead of a giant
                // slab (chunkIdx is 16-bit, so ~64 TiB of headroom remains).
                nextPow2(size), maxChunkBytes)
        c := make([]byte, 0, size)
        a.chunks = append(a.chunks, c)
        a.cur = len(a.chunks) - 1
        a.off = 0
        a.end = size
}

// pack folds (chunk, offset, length) into a single uint64.
//
//go:nosplit
func pack(chunkIdx uint16, offset uint32, length uint16) uint64 {
        return uint64(chunkIdx)<<48 | uint64(offset)<<16 | uint64(length)
}

// unpack reverses pack.
//
//go:nosplit
func unpack(loc uint64) (chunkIdx uint16, offset uint32, length uint16) {
        chunkIdx = uint16(loc >> 48)
        offset = uint32(loc>>16) & 0xFFFFFFFF
        length = uint16(loc)
        return
}

// nextPow2 returns the smallest power of two ≥ n. Used to keep chunk
// caps on slab-friendly sizes.
//
//go:nosplit
func nextPow2(n uintptr) uintptr {
        if n == 0 {
                return 1
        }
        n--
        n |= n >> 1
        n |= n >> 2
        n |= n >> 4
        n |= n >> 8
        n |= n >> 16
        n |= n >> 32
        return n + 1
}

// E8 lattice quantizer. E8 is the densest lattice packing in 8 dimensions,
// the union of D8 (integer coords, even sum) and its glue coset D8+½
// (half-integer coords). Its normalized second moment E8G is below the cubic
// lattice's 1/12, so at the same step it quantizes with lower MSE. After the
// Hadamard rotation the data is ~Gaussian, the regime where this packing gain
// is realized.
package lattice

import "math"

// E8G is the normalized second moment of the E8 lattice (vs 1/12 for the cube).
const E8G = 0.071682

// nearestD8 returns the closest point of D8 (integer coordinates with even sum)
// to x. Round each coordinate to the nearest integer; if the sum is odd, move
// the single coordinate with the largest rounding error to its second-nearest
// integer (the exact D8 decoder).
func nearestD8(x *[8]float64) [8]float64 {
        var pt [8]float64
        var sum int
        worstIdx, worstErr := 0, -1.0
        for i := range 8 {
                r := math.RoundToEven(x[i])
                pt[i] = r
                sum += int(r)
                if e := math.Abs(x[i] - r); e > worstErr {
                        worstErr, worstIdx = e, i
                }
        }
        if sum&1 != 0 {
                if x[worstIdx] >= pt[worstIdx] {
                        pt[worstIdx]++
                } else {
                        pt[worstIdx]--
                }
        }
        return pt
}

func dist2(a, b *[8]float64) float64 {
        var s float64
        for i := range 8 {
                d := a[i] - b[i]
                s += d * d
        }
        return s
}

// NearestE8 returns the exact nearest E8 lattice point to x: the closer of the
// nearest D8 point and the nearest D8+½ point.
func NearestE8(x *[8]float64) [8]float64 {
        p0 := nearestD8(x)
        var xs [8]float64
        for i := range 8 {
                xs[i] = x[i] - 0.5
        }
        p1d := nearestD8(&xs)
        var p1 [8]float64
        for i := range 8 {
                p1[i] = p1d[i] + 0.5
        }
        if dist2(x, &p1) < dist2(x, &p0) {
                return p1
        }
        return p0
}

// QuantizeE8 quantizes x (length a multiple of 8) to the E8 lattice at the
// given step. Returns the integer coordinates (floor for the half-integer
// coset) and one coset bit per 8-D block, LSB-first. dst is reused as the
// coords backing when its capacity suffices (pass nil for a fresh slice).
// The overflow return mirrors QuantizeScalar: a coordinate past the int32 range
// is clamped (saturated) and overflow=true is reported so the caller can abort
// to a lossless encoding rather than ship a wrapped/corrupt coordinate.
func QuantizeE8(x []float64, delta float64, dst []int32) (coords []int32, cosets []byte, overflow bool) {
        if len(x)%8 != 0 {
                panic("lattice: QuantizeE8 length must be a multiple of 8")
        }
        nb := len(x) / 8
        if cap(dst) >= len(x) {
                coords = dst[:0]
        } else {
                coords = make([]int32, 0, len(x))
        }
        cosets = make([]byte, (nb+7)/8)
        inv := 1.0 / delta
        var blk [8]float64
        for b := range nb {
                for i := range 8 {
                        blk[i] = x[b*8+i] * inv
                }
                pt := NearestE8(&blk)
                if pt[0]-math.Floor(pt[0]) == 0.5 { // half-integer coset
                        cosets[b/8] |= 1 << (uint(b) & 7)
                        for i := range 8 {
                                c, ov := clampI32(math.Floor(pt[i]))
                                coords = append(coords, c)
                                overflow = overflow || ov
                        }
                } else {
                        for i := range 8 {
                                c, ov := clampI32(pt[i])
                                coords = append(coords, c)
                                overflow = overflow || ov
                        }
                }
        }
        return coords, cosets, overflow
}

// ReconstructE8 inverts QuantizeE8. n must be a multiple of 8 and len(coords)==n.
func ReconstructE8(coords []int32, cosets []byte, delta float64, n int) []float64 {
        out := make([]float64, n)
        nb := n / 8
        for b := range nb {
                off := 0.0
                if cosets[b/8]&(1<<(uint(b)&7)) != 0 {
                        off = 0.5
                }
                for i := range 8 {
                        out[b*8+i] = (float64(coords[b*8+i]) + off) * delta
                }
        }
        return out
}

// Package lattice implements vector quantizers used by the lossy vector codec.
// It currently provides the scalar (integer/cube) lattice. After the Hadamard
// rotation the data is ~Gaussian, so a uniform scalar grid with step Δ is
// already a reasonable quantizer and the entropy stage recovers the rest.
package lattice

import "math"

// ScalarG is the normalized second moment of the 1-D integer lattice (cube):
// the quantization MSE of a uniform scalar quantizer with step Δ is Δ²·ScalarG.
const ScalarG = 1.0 / 12.0

// QuantizeScalar rounds each x[i]/delta to the nearest integer, writing the
// results into dst and returning it. dst is reused when it has enough capacity
// (its existing contents are discarded), otherwise a new slice is allocated.
//
// The second return reports whether any coordinate saturated the int32 range:
// a magnitude past 2^31 (a too-tight budget on a spiky vector shrinking delta)
// cannot be represented, and a raw cast would wrap (sign-flip) and silently
// corrupt the vector. Such values are clamped (saturated) and overflow=true is
// reported so the caller can abort to a lossless encoding instead.
func QuantizeScalar(x []float64, delta float64, dst []int32) (out []int32, overflow bool) {
        if cap(dst) < len(x) {
                dst = make([]int32, 0, len(x))
        }
        dst = dst[:0]
        inv := 1.0 / delta
        for _, v := range x {
                r := math.RoundToEven(v * inv)
                if r > math.MaxInt32 {
                        r, overflow = math.MaxInt32, true
                } else if r < math.MinInt32 {
                        r, overflow = math.MinInt32, true
                }
                dst = append(dst, int32(r))
        }
        return dst, overflow
}

// clampI32 saturates a rounded lattice coordinate to the int32 range, reporting
// whether it was out of range (see QuantizeScalar).
func clampI32(f float64) (int32, bool) {
        if f > math.MaxInt32 {
                return math.MaxInt32, true
        }
        if f < math.MinInt32 {
                return math.MinInt32, true
        }
        return int32(f), false
}

// ReconstructScalar maps each integer index back to float64(q[i])*delta.
func ReconstructScalar(q []int32, delta float64, dst []float64) []float64 {
        if cap(dst) < len(q) {
                dst = make([]float64, 0, len(q))
        }
        dst = dst[:0]
        for _, v := range q {
                dst = append(dst, float64(v)*delta)
        }
        return dst
}

// internal/mapsgen — code generator for typed map encode/decode fast
// paths. Reflect-driven maps are 3–4× slower than typed paths because
// of per-element reflect.Value materialisation. Each generated pair
// inlines WriteX / ReadX calls directly, keeping the encoder inliner
// happy while covering the common map shapes Go programs reach for.
//
// Generated output: ../../maps_fast_generated.go. Re-run via
//
//        go generate ./...
//
// from the repository root.
package main

import (
        "bytes"
        "fmt"
        "go/format"
        "log"
        "os"
        "path/filepath"
        "strings"
)

// kind captures everything the generator needs to emit a write / read
// block for a single Go type.
//
// The three func fields (single pointer words) lead and the two strings
// (each a pointer word + a non-pointer length word) trail, keeping the GC
// pointer-scan range tight (48 bytes vs 56 for the source order).
type kind struct {
        // writeBlock returns a Go statement that emits the value bound
        // to `varName`. Multi-line is fine; the generator wraps each
        // statement inside the map loop body.
        writeBlock func(varName string) string
        // readBlock returns a Go statement that declares a fresh local
        // of type goType, named varName, reading it from the decoder.
        // Block must `return err` on failure. Used for map values.
        readBlock func(varName string) string
        // readKeyBlock is the readBlock variant for the map KEY position.
        // String kinds dedupe via d.keyCache.Make so high-cardinality
        // repeats stay zero-alloc; other kinds delegate to readBlock.
        readKeyBlock func(varName string) string
        // suffix is the camel-case identifier used in the generated
        // function names and dispatch table entries.
        suffix string
        // goType is the in-memory Go type. Always a single token so it
        // fits into `map[K]V` without parentheses.
        goType string
}

func mkReadValue(call, goType string) func(string) string {
        return func(v string) string {
                // ReadInt / ReadUint return int64 / uint64. When the requested
                // goType matches the read's natural width we keep the source
                // shape minimal (`v, err := <call>`); otherwise emit a
                // `<v>Wide` temp and narrow it. The cast site stays uniform
                // (`m[k] = v`) across all pairs so the dispatch table doesn't
                // have to special-case the cast position.
                if goType == "int64" || goType == "uint64" || goType == "string" ||
                        goType == "bool" || goType == "float32" || goType == "float64" ||
                        goType == "[]byte" {
                        return fmt.Sprintf("%s, err := %s\nif err != nil {\nreturn err\n}", v, call)
                }
                return fmt.Sprintf("%sWide, err := %s\nif err != nil {\nreturn err\n}\n%s := %s(%sWide)",
                        v, call, v, goType, v)
        }
}

func mkWriteScalar(call, castTo string) func(string) string {
        return func(v string) string {
                if castTo == "" {
                        return fmt.Sprintf("%s(%s)", call, v)
                }
                return fmt.Sprintf("%s(%s(%s))", call, castTo, v)
        }
}

// kindString — string. Keys dedupe through d.keyCache so repeated
// map keys (the common case for telemetry-style payloads) stay
// allocation-free across many calls. Values use ReadString which
// allocates a Go string per call.
var kindString = kind{
        suffix:     "String",
        goType:     "string",
        writeBlock: func(v string) string { return fmt.Sprintf("e.WriteString(%s)", v) },
        readBlock:  mkReadValue("d.ReadString()", "string"),
        readKeyBlock: func(v string) string {
                return fmt.Sprintf("kb, err := d.readStringBytes()\nif err != nil {\nreturn err\n}\n%s := d.keyCache.Make(kb)", v)
        },
}

var kindBool = kind{
        suffix:     "Bool",
        goType:     "bool",
        writeBlock: func(v string) string { return fmt.Sprintf("e.WriteBool(%s)", v) },
        readBlock:  mkReadValue("d.ReadBool()", "bool"),
}

var kindBytes = kind{
        suffix: "Bytes",
        goType: "[]byte",
        // A nil []byte value travels as tagNil, distinct from an empty []byte —
        // matching the reflect encoder so the nil-vs-empty distinction round-trips.
        writeBlock: func(v string) string {
                return fmt.Sprintf(`if %[1]s == nil {
e.WriteNil()
} else {
e.WriteBytes(%[1]s)
}`, v)
        },
        readBlock: func(v string) string {
                return fmt.Sprintf(`var %[1]s []byte
%[1]sTag, err := d.peekTag()
if err != nil {
return err
}
if %[1]sTag != tagNil {
%[1]s, err = d.ReadBytes()
if err != nil {
return err
}
} else {
d.i++
}`, v)
        },
}

var kindFloat32 = kind{
        suffix:     "Float32",
        goType:     "float32",
        writeBlock: func(v string) string { return fmt.Sprintf("e.WriteFloat32(%s)", v) },
        readBlock:  mkReadValue("d.ReadFloat32()", "float32"),
}

var kindFloat64 = kind{
        suffix:     "Float64",
        goType:     "float64",
        writeBlock: func(v string) string { return fmt.Sprintf("e.WriteFloat64(%s)", v) },
        readBlock:  mkReadValue("d.ReadFloat64()", "float64"),
}

// Integer / unsigned kinds. WriteInt / WriteUint take int64 / uint64;
// in-memory operand width drives the cast on both sides.
func makeIntKind(suffix, goType string) kind {
        return kind{
                suffix:     suffix,
                goType:     goType,
                writeBlock: mkWriteScalar("e.WriteInt", "int64"),
                readBlock:  mkReadValue("d.ReadInt()", goType),
        }
}

func makeUintKind(suffix, goType string) kind {
        return kind{
                suffix:     suffix,
                goType:     goType,
                writeBlock: mkWriteScalar("e.WriteUint", "uint64"),
                readBlock:  mkReadValue("d.ReadUint()", goType),
        }
}

var (
        kindInt8   = makeIntKind("Int8", "int8")
        kindInt16  = makeIntKind("Int16", "int16")
        kindInt32  = makeIntKind("Int32", "int32")
        kindInt    = makeIntKind("Int", "int")
        kindInt64  = makeIntKind("Int64", "int64")
        kindUint8  = makeUintKind("Uint8", "uint8")
        kindUint16 = makeUintKind("Uint16", "uint16")
        kindUint32 = makeUintKind("Uint32", "uint32")
        kindUint   = makeUintKind("Uint", "uint")
        kindUint64 = makeUintKind("Uint64", "uint64")
)

// kindStringSlice — []string value. Encoded as ArrayHeader + each
// WriteString. Used by tag-style payloads (e.g. labels, categories).
var kindStringSlice = kind{
        suffix: "StringSlice",
        goType: "[]string",
        writeBlock: func(v string) string {
                // A nil slice value travels as tagNil (WriteNil), distinct from an empty
                // slice (header 0) — matching the reflect encoder's encodeNilSlice so the
                // fast path and reflect path emit identical wire and the nil-vs-empty
                // distinction survives the round-trip.
                return fmt.Sprintf(`if %[1]s == nil {
e.WriteNil()
} else {
e.WriteArrayHeader(len(%[1]s))
for _, sv := range %[1]s {
e.WriteString(sv)
}
}`, v)
        },
        readBlock: func(v string) string {
                return fmt.Sprintf(`var %[1]s []string
%[1]sTag, err := d.peekTag()
if err != nil {
return err
}
if %[1]sTag != tagNil {
hdrN, err := d.ReadArrayHeader()
if err != nil {
return err
}
if err := d.CheckLength(hdrN, 1); err != nil {
return err
}
%[1]s = make([]string, hdrN)
for i := range hdrN {
sv, err := d.ReadString()
if err != nil {
return err
}
%[1]s[i] = sv
}
} else {
d.i++
}`, v)
        },
}

// kindAny — generic any value. Dispatches through encodeIface (not
// encodeReflect) so the value is encoded in a dynamic-dispatch context:
// encodeIface raises e.ifaceDepth, which suppresses the lossy-vector /
// batched-struct codecs (gated on ifaceDepth==0). A map[K]any value is always
// read back via decodeAny, which cannot decode a tagColVecLossy (0xFD) or
// tagVecBatchStruct (0xFE) block, so those tags must never be emitted here.
// This matches the []any element path (encodeSliceAny → encodeIface).
var kindAny = kind{
        suffix: "Any",
        goType: "any",
        writeBlock: func(v string) string {
                return fmt.Sprintf("if err := encodeIface(e, unsafe.Pointer(&%s)); err != nil {\nreturn err\n}", v)
        },
        readBlock: func(v string) string {
                return fmt.Sprintf("%s, err := decodeAny(d)\nif err != nil {\nreturn err\n}", v)
        },
}

// readKey: returns the readBlock variant when no specialised key
// read exists. Wraps readBlock for non-string kinds.
func (k kind) readKey(varName string) string {
        if k.readKeyBlock != nil {
                return k.readKeyBlock(varName)
        }
        return k.readBlock(varName)
}

type pair struct {
        K, V kind
}

// pairs is the generated set. Keep in sync with the dispatch table
// emitted at the bottom of the file.
//
// Selection: string keys × all common scalar / slice / any values,
// integer keys × the lookup-table variants that actually appear in
// real Go code (enum → label, id → metadata).
var pairs = []pair{
        // string × *
        {kindString, kindString},
        {kindString, kindBool},
        {kindString, kindInt8},
        {kindString, kindInt16},
        {kindString, kindInt32},
        {kindString, kindInt},
        {kindString, kindInt64},
        {kindString, kindUint8},
        {kindString, kindUint16},
        {kindString, kindUint32},
        {kindString, kindUint},
        {kindString, kindUint64},
        {kindString, kindFloat32},
        {kindString, kindFloat64},
        {kindString, kindBytes},
        {kindString, kindStringSlice},
        {kindString, kindAny},

        // int × {string, int, int64, any}
        {kindInt, kindString},
        {kindInt, kindInt},
        {kindInt, kindInt64},
        {kindInt, kindAny},

        // int64 × {string, int64, any}
        {kindInt64, kindString},
        {kindInt64, kindInt64},
        {kindInt64, kindAny},

        // uint64 × {string, uint64, any}
        {kindUint64, kindString},
        {kindUint64, kindUint64},
        {kindUint64, kindAny},
}

func emitPair(buf *bytes.Buffer, p pair) {
        mapTy := fmt.Sprintf("map[%s]%s", p.K.goType, p.V.goType)
        fnName := fmt.Sprintf("Map%s%s", p.K.suffix, p.V.suffix)

        // encode
        fmt.Fprintf(buf, "// ----- %s -----\n\n", mapTy)
        fmt.Fprintf(buf, "func encode%s(e *Encoder, p unsafe.Pointer) error {\n", fnName)
        fmt.Fprintf(buf, "\tm := *(*%s)(p)\n", mapTy)
        fmt.Fprintf(buf, "\tif m == nil {\n\t\te.WriteNil()\n\t\treturn nil\n\t}\n")
        if p.K.suffix == "String" {
                // OptMapShape fast path: recurring key-sets emit a shape header +
                // values in canonical order via the shared generic helper.
                fmt.Fprintf(buf, "\tif len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {\n")
                fmt.Fprintf(buf, "\t\tfor _, k := range mapStringShapeOrder(e, m) {\n")
                fmt.Fprintf(buf, "\t\t\tv := m[k]\n")
                fmt.Fprintf(buf, "\t\t\t%s\n", p.V.writeBlock("v"))
                fmt.Fprintf(buf, "\t\t}\n\t\treturn nil\n\t}\n")
        }
        fmt.Fprintf(buf, "\te.WriteMapHeader(len(m))\n")
        // Canonical fast path: emit keys in sorted order so logically-equal maps
        // serialize byte-identically. The key type is concrete, so slices.Sort is
        // directly monomorphized (no reflect). Reuse the pooled canonKeys* scratch
        // when state is available; fall back to a local slice otherwise. Canonical
        // is independent of the OptMapShape/OptDense shape branch above (which is
        // String-keyed only and already deterministic via mapStringShapeOrder).
        emitCanonicalEncode(buf, p)
        fmt.Fprintf(buf, "\tfor k, v := range m {\n")
        fmt.Fprintf(buf, "\t\t%s\n", indent(p.K.writeBlock("k")))
        fmt.Fprintf(buf, "\t\t%s\n", indent(p.V.writeBlock("v")))
        fmt.Fprintf(buf, "\t}\n\treturn nil\n}\n\n")

        // decode
        fmt.Fprintf(buf, "func decode%s(d *Decoder, p unsafe.Pointer) error {\n", fnName)
        fmt.Fprintf(buf, "\tt, err := d.peekTag()\n\tif err != nil {\n\t\treturn err\n\t}\n")
        fmt.Fprintf(buf, "\tif t == tagNil {\n\t\td.i++\n\t\t*(*%s)(p) = nil\n\t\treturn nil\n\t}\n", mapTy)
        if p.K.suffix == "String" {
                // OptMapShape decode: a tagMapShape header carries the ordered keys;
                // read len(names) values in that order.
                fmt.Fprintf(buf, "\tif t == tagMapShape {\n")
                fmt.Fprintf(buf, "\t\tnames, err := decodeMapStringShapeHeader(d)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n")
                fmt.Fprintf(buf, "\t\tm := reuseOrMakeMap[%s, %s](d, p, len(names))\n", p.K.goType, p.V.goType)
                fmt.Fprintf(buf, "\t\tfor _, k := range names {\n")
                fmt.Fprintf(buf, "\t\t\t%s\n", p.V.readBlock("v"))
                fmt.Fprintf(buf, "\t\t\tm[k] = v\n")
                fmt.Fprintf(buf, "\t\t}\n\t\t*(*%s)(p) = m\n\t\treturn nil\n\t}\n", mapTy)
        }
        fmt.Fprintf(buf, "\tn, err := d.ReadMapHeader()\n\tif err != nil {\n\t\treturn err\n\t}\n")
        fmt.Fprintf(buf, "\tif err := d.CheckLength(n, 2); err != nil {\n\t\treturn err\n\t}\n")
        fmt.Fprintf(buf, "\tm := reuseOrMakeMap[%s, %s](d, p, n)\n", p.K.goType, p.V.goType)
        fmt.Fprintf(buf, "\tfor range n {\n")
        fmt.Fprintf(buf, "\t\t%s\n", indent(p.K.readKey("k")))
        fmt.Fprintf(buf, "\t\t%s\n", indent(p.V.readBlock("v")))
        fmt.Fprintf(buf, "\t\tm[k] = v\n")
        fmt.Fprintf(buf, "\t}\n\t*(*%s)(p) = m\n\treturn nil\n}\n\n", mapTy)
}

// emitCanonicalEncode writes the OptCanonical sorted-key branch for a map
// encoder. The key type is concrete, so the canonical key slice is built with a
// monomorphized slices.Sort. The pooled canonKeys* scratch is reused when the
// encoder carries an encState; otherwise a local slice is allocated.
//
// scratchField / scratchElem map the concrete key type onto the encState scratch
// (string→canonKeysStr, signed int kinds→canonKeysI64, unsigned→canonKeysU64).
// For int (machine width) the scratch element is int64, so the gather casts and
// the map lookup casts back — the sort order is identical (int values fit int64).
func emitCanonicalEncode(buf *bytes.Buffer, p pair) {
        var scratchField, scratchElem string
        switch p.K.goType {
        case "string":
                scratchField, scratchElem = "canonKeysStr", "string"
        case "int", "int64", "int8", "int16", "int32":
                scratchField, scratchElem = "canonKeysI64", "int64"
        case "uint", "uint64", "uint8", "uint16", "uint32", "uintptr":
                scratchField, scratchElem = "canonKeysU64", "uint64"
        default:
                // No canonical fast path for this key kind; the range loop below stays
                // the only emit (such keys are not in the generated pair set today).
                return
        }
        needCast := p.K.goType != scratchElem

        // Re-entrancy: the pooled canonKeys* scratch is shared with the reflect map
        // encoder and the other generated encoders. A map nested inside another map's
        // value (or a reflect map whose value reaches this generated encoder) would
        // clobber the outer map's sorted-key slice while it is still being iterated.
        // canonKeysBusy guards it: when already busy, this encoder allocates a fresh
        // local slice; otherwise it borrows the pool and sets the guard for the
        // duration of the emit loop (the value writes may recurse into another map).
        fmt.Fprintf(buf, "\tif e.opts.Has(OptCanonical) {\n")
        fmt.Fprintf(buf, "\t\tvar keys []%s\n", scratchElem)
        fmt.Fprintf(buf, "\t\tcanonPooled := false\n")
        fmt.Fprintf(buf, "\t\tif e.state != nil && !e.state.canonKeysBusy {\n")
        fmt.Fprintf(buf, "\t\t\tkeys = e.state.%s[:0]\n", scratchField)
        fmt.Fprintf(buf, "\t\t\tcanonPooled = true\n")
        fmt.Fprintf(buf, "\t\t} else {\n")
        fmt.Fprintf(buf, "\t\t\tkeys = make([]%s, 0, len(m))\n", scratchElem)
        fmt.Fprintf(buf, "\t\t}\n")
        if needCast {
                fmt.Fprintf(buf, "\t\tfor k := range m {\n\t\t\tkeys = append(keys, %s(k))\n\t\t}\n", scratchElem)
        } else {
                fmt.Fprintf(buf, "\t\tfor k := range m {\n\t\t\tkeys = append(keys, k)\n\t\t}\n")
        }
        fmt.Fprintf(buf, "\t\tslices.Sort(keys)\n")
        // Release the re-entrancy latch via defer so it clears on EVERY exit —
        // including a mid-loop `return err` from a value write (the *Any pairs).
        // An inline post-loop release leaks busy=true on the error path, pinning a
        // pooled encoder to the fresh-allocation fallback forever. Mirrors the
        // reflect path's `defer e.canonKeysRelease(pooled)`.
        fmt.Fprintf(buf, "\t\tif canonPooled {\n\t\t\te.state.%s = keys\n\t\t\te.state.canonKeysBusy = true\n\t\t\tdefer func() { e.state.canonKeysBusy = false }()\n\t\t}\n", scratchField)
        fmt.Fprintf(buf, "\t\tfor _, sk := range keys {\n")
        // sk is already a per-iteration copy of the range value. Only the narrow-key
        // variants need a real conversion (k := T(sk)); the string/wide-key variants
        // use sk directly — a `k := sk` rename would be dead generated code.
        keyVar := "sk"
        if needCast {
                fmt.Fprintf(buf, "\t\t\tk := %s(sk)\n", p.K.goType)
                keyVar = "k"
        }
        fmt.Fprintf(buf, "\t\t\tv := m[%s]\n", keyVar)
        fmt.Fprintf(buf, "\t\t\t%s\n", indent(indent(p.K.writeBlock(keyVar))))
        fmt.Fprintf(buf, "\t\t\t%s\n", indent(indent(p.V.writeBlock("v"))))
        fmt.Fprintf(buf, "\t\t}\n")
        fmt.Fprintf(buf, "\t\treturn nil\n\t}\n")
}

func emitDispatch(buf *bytes.Buffer) {
        fmt.Fprintf(buf, "// installMapFastPath returns (encode, decode, true) when t matches\n")
        fmt.Fprintf(buf, "// one of the generated typed-map shapes; otherwise (_, _, false). The\n")
        fmt.Fprintf(buf, "// table is generated; edit internal/mapsgen/main.go to add or remove\n")
        fmt.Fprintf(buf, "// pairs.\n")
        fmt.Fprintf(buf, "func installMapFastPath(t reflect.Type) (\n")
        fmt.Fprintf(buf, "\tenc func(*Encoder, unsafe.Pointer) error,\n")
        fmt.Fprintf(buf, "\tdec func(*Decoder, unsafe.Pointer) error,\n")
        fmt.Fprintf(buf, "\tok bool,\n) {\n")
        fmt.Fprintf(buf, "\tswitch t {\n")
        for _, p := range pairs {
                fnName := fmt.Sprintf("Map%s%s", p.K.suffix, p.V.suffix)
                mapTy := fmt.Sprintf("map[%s]%s", p.K.goType, p.V.goType)
                fmt.Fprintf(buf, "\tcase reflect.TypeFor[%s]():\n", mapTy)
                fmt.Fprintf(buf, "\t\treturn encode%s, decode%s, true\n", fnName, fnName)
        }
        fmt.Fprintf(buf, "\t}\n\treturn nil, nil, false\n}\n")
}

// indent reindents a multi-line snippet so the second and later
// lines line up with the two-tab loop-body column. The first-line
// prefix is supplied by the caller via fmt.Fprintf alignment.
func indent(s string) string {
        const prefix = "\t\t"
        lines := strings.Split(s, "\n")
        for i := 1; i < len(lines); i++ {
                lines[i] = prefix + lines[i]
        }
        return strings.Join(lines, "\n")
}

func main() {
        var buf bytes.Buffer
        fmt.Fprintf(&buf, "// Code generated by internal/mapsgen — DO NOT EDIT.\n")
        fmt.Fprintf(&buf, "// Re-run via `go generate ./...` from the repository root.\n\n")
        fmt.Fprintf(&buf, "package qdf\n\n")
        fmt.Fprintf(&buf, "import (\n\t\"reflect\"\n\t\"slices\"\n\t\"unsafe\"\n)\n\n")
        for _, p := range pairs {
                emitPair(&buf, p)
        }
        emitDispatch(&buf)

        src, err := format.Source(buf.Bytes())
        if err != nil {
                // Emit unformatted for easier debugging then fail.
                _, _ = os.Stderr.Write(buf.Bytes())
                log.Fatalf("\nformat: %v", err)
        }

        // Locate the module root by walking up to the go.mod, so the generator
        // works whether invoked from internal/mapsgen or via `go generate ./...`
        // (which runs it with the repo root as the working directory).
        wd, err := os.Getwd()
        if err != nil {
                log.Fatal(err)
        }
        root := wd
        for {
                if _, err := os.Stat(filepath.Join(root, "go.mod")); err == nil {
                        break
                }
                parent := filepath.Dir(root)
                if parent == root {
                        log.Fatalf("could not find go.mod above %s", wd)
                }
                root = parent
        }
        out := filepath.Join(root, "maps_fast_generated.go")
        if err := os.WriteFile(out, src, 0o600); err != nil {
                log.Fatal(err)
        }
        fmt.Fprintf(os.Stderr, "wrote %s (%d bytes, %d pairs)\n", out, len(src), len(pairs))
}

// Package rans is a static order-0 byte range Asymmetric Numeral System
// coder (the canonical rans_byte: 32-bit state, byte renormalization, a
// 12-bit normalized frequency table). It compresses and decompresses a
// single byte slice and has no dependencies on the rest of qdf.
package rans

import (
        "encoding/binary"
        "errors"
        "slices"

        "github.com/alex60217101990/qdf/internal/bufpool"
)

const (
        scaleBits = 12
        scale     = 1 << scaleBits // 4096; frequencies are normalized to sum to this
        ransByteL = 1 << 23        // renormalization lower bound
        // maxRenormBytesPerSym bounds the renorm bytes a single symbol can emit: the
        // state lives in [ransByteL, ransByteL<<8) = [2^23, 2^31) and a freq-1 symbol
        // has xMax = (ransByteL>>scaleBits)<<8 = 2^19, so the `for x >= xMax { x>>=8 }`
        // renorm runs at most ceil((31-19)/8) = 2 times. Output buffers are sized as
        // len*maxRenormBytesPerSym + state + slack so the backward writer never
        // underflows, even when a (sub)stream's local byte distribution is skewed
        // against the frequency table it is encoded under.
        maxRenormBytesPerSym = 2
)

// Format tags for the leading byte of a rANS blob.
// Format tags for the leading byte of a rANS blob. An interleaved tag's value
// equals its stream count N, so the decoder can pass the tag straight to the
// framing parser.
const (
        ransTagSingle = 0 // [tag][table][4-byte state][renorm bytes]
        ransTagInter4 = 4 // interleaved, 4 strided substreams sharing one table
)

// ErrBadTable is returned when a decoded frequency table is malformed
// (frequencies do not sum to scale, or a frequency exceeds scale).
var ErrBadTable = errors.New("rans: invalid frequency table")

// ErrCorrupt is returned when the rANS stream is truncated or inconsistent.
var ErrCorrupt = errors.New("rans: corrupt stream")

// buildFreqs builds the normalized frequency table (sum == scale) and the
// cumulative table from src. Every symbol that occurs in src gets freq >= 1.
func buildFreqs(src []byte) (freq [256]uint32, cum [257]uint32) {
        if len(src) == 0 {
                return freq, cum
        }
        var raw [256]uint32
        for _, b := range src {
                raw[b]++
        }
        n := uint64(len(src))
        var total uint32
        for s := range 256 {
                if raw[s] == 0 {
                        continue
                }
                f := uint32((uint64(raw[s])*scale + n/2) / n)
                if f == 0 {
                        f = 1
                }
                freq[s] = f
                total += f
        }
        // Correct rounding so the frequencies sum to exactly `scale`. Adjust the
        // current largest frequency each step; never drop a used symbol below 1.
        for total > scale {
                bi, bf, found := 0, uint32(0), false
                for s := range 256 {
                        if freq[s] > 1 && freq[s] > bf {
                                bf, bi, found = freq[s], s, true
                        }
                }
                if !found {
                        break // unreachable: total>scale implies some freq>1
                }
                freq[bi]--
                total--
        }
        for total < scale {
                bi, bf, found := 0, uint32(0), false
                for s := range 256 {
                        if freq[s] > bf {
                                bf, bi, found = freq[s], s, true
                        }
                }
                if !found {
                        break // unreachable: non-empty src has at least one freq>=1
                }
                freq[bi]++
                total++
        }
        var c uint32
        for s := range 256 {
                cum[s] = c
                c += freq[s]
        }
        cum[256] = c
        return freq, cum
}

// buildSlot fills the 4096-entry slot->symbol lookup for decode into the
// caller-provided array. Taking the destination by pointer (rather than
// returning a fresh array) lets the caller keep slot on its stack frame: a
// returned &slot would force the 4 KiB array to the heap on every Decode.
func buildSlot(cum *[257]uint32, slot *[scale]byte) {
        for s := range 256 {
                for c := cum[s]; c < cum[s+1]; c++ {
                        slot[c] = byte(s)
                }
        }
}

// encodeStream rANS-encodes src, appends the stream to dst, and returns the
// extended dst. The stream is: 4-byte little-endian final state followed by
// the renormalization bytes in decode order.
// Accepting dst here is not just API style — it closes a data race: the pool
// buffer is returned by the deferred Put only after the append (the copy) has
// completed, so no concurrent encoder can see the data in-flight.
func encodeStream(dst, src []byte, freq *[256]uint32, cum *[257]uint32) []byte {
        // Worst-case output: each symbol emits at most maxRenormBytesPerSym renorm
        // bytes (a freq-1 symbol over the scale=2^12 table renormalizes a state in
        // [2^23, 2^31) down past xMax=2^19, i.e. two >>8 steps), plus the 4-byte final
        // state. len(src)+16 under-sizes this when a stream is dominated by globally-
        // rare bytes — most visible in the interleaved path, where a substream sees a
        // local distribution skewed against the SHARED table (see encodeStreamStrided-
        // Into). Size for the proven bound so the backward writer can never underflow.
        size := len(src)*maxRenormBytesPerSym + 16
        bp := bufpool.Get(size)
        buf := (*bp)[:size]
        defer bufpool.Put(bp)
        pos := size
        x := uint32(ransByteL)
        for _, s := range slices.Backward(src) {

                f := freq[s]
                xMax := ((ransByteL >> scaleBits) << 8) * f
                for x >= xMax {
                        pos--
                        buf[pos] = byte(x)
                        x >>= 8
                }
                x = ((x / f) << scaleBits) + (x % f) + cum[s]
        }
        pos -= 4
        binary.LittleEndian.PutUint32(buf[pos:], x)
        // Copy into dst before the deferred Put returns buf to the pool.
        dst = append(dst, buf[pos:]...)
        return dst
}

// decodeStream reverses encodeStream, emitting exactly n bytes.
func decodeStream(stream []byte, freq *[256]uint32, slot *[scale]byte, cum *[257]uint32, n int) ([]byte, error) {
        if len(stream) < 4 {
                return nil, ErrCorrupt
        }
        x := binary.LittleEndian.Uint32(stream[:4])
        pos := 4
        out := make([]byte, n)
        for i := range n {
                s := slot[x&(scale-1)]
                out[i] = s
                x = freq[s]*(x>>scaleBits) + (x & (scale - 1)) - cum[s]
                for x < ransByteL {
                        if pos >= len(stream) {
                                return nil, ErrCorrupt
                        }
                        x = (x << 8) | uint32(stream[pos])
                        pos++
                }
        }
        return out, nil
}

// appendTable serializes the 256 normalized frequencies as varuints.
func appendTable(dst []byte, freq *[256]uint32) []byte {
        for s := range 256 {
                dst = appendUvarint(dst, uint64(freq[s]))
        }
        return dst
}

// parseTable reads 256 varuint frequencies, validates them, and returns the
// freq/cum tables plus the number of bytes consumed.
func parseTable(src []byte) (freq [256]uint32, cum [257]uint32, used int, err error) {
        pos := 0
        var total uint32
        for s := range 256 {
                v, k := uvarint(src[pos:])
                if k <= 0 {
                        return freq, cum, 0, ErrBadTable
                }
                if v > scale {
                        return freq, cum, 0, ErrBadTable
                }
                freq[s] = uint32(v)
                total += uint32(v)
                pos += k
        }
        if total != scale {
                return freq, cum, 0, ErrBadTable
        }
        var c uint32
        for s := range 256 {
                cum[s] = c
                c += freq[s]
        }
        cum[256] = c
        return freq, cum, pos, nil
}

const (
        // interleaveN is the shipped interleaved stream count (tuned in Task 6).
        // It MUST equal one of the ransTagInterN tag values (the tag == stream count).
        interleaveN = ransTagInter4
        // interleaveMinBytes is the body size at/above which interleaving's extra
        // framing (N states + N-1 lengths) is negligible and the decode-speed win
        // applies. Below it, single-stream (smaller, and small bodies decode fast).
        // Tuned in Task 6.
        interleaveMinBytes = 4096
)

// forceTagForTest overrides the adaptive choice for tests/benches:
//
//        0  ⇒ adaptive (default)
//        -1  ⇒ force single-stream
//        >0  ⇒ force interleaved with that N
var forceTagForTest int

// Encode appends the order-0 rANS encoding of src (frequency table followed by
// the rANS stream) to dst and returns it. The caller stores the original
// length separately and passes it to Decode. For bodies at or above
// interleaveMinBytes, Encode emits an interleaved (multi-stream) blob; smaller
// bodies use the single-stream form.
func Encode(dst, src []byte) []byte {
        freq, cum := buildFreqs(src)
        useInter := len(src) >= interleaveMinBytes
        switch {
        case forceTagForTest < 0:
                useInter = false
        case forceTagForTest > 0:
                useInter = true
        }
        if !useInter {
                dst = append(dst, ransTagSingle)
                dst = appendTable(dst, &freq)
                return encodeStream(dst, src, &freq, &cum)
        }
        n := interleaveN
        if forceTagForTest > 0 {
                n = forceTagForTest
        }
        dst = append(dst, byte(n))
        dst = appendTable(dst, &freq)
        return appendInterleaved(dst, src, &freq, &cum, n)
}

// Decode parses the table from src, rANS-decodes exactly n bytes, and returns
// them. It validates the table and stream and never panics on malformed input.
func Decode(src []byte, n int) ([]byte, error) {
        if n == 0 {
                return []byte{}, nil
        }
        if n < 0 { // honor the never-panics contract: make([]byte, n<0) would panic
                return nil, ErrCorrupt
        }
        if len(src) < 1 {
                return nil, ErrCorrupt
        }
        tag := src[0]
        src = src[1:]
        switch tag {
        case ransTagSingle:
                freq, cum, used, err := parseTable(src)
                if err != nil {
                        return nil, err
                }
                var slot [scale]byte
                buildSlot(&cum, &slot)
                return decodeStream(src[used:], &freq, &slot, &cum, n)
        case ransTagInter4:
                freq, cum, used, err := parseTable(src)
                if err != nil {
                        return nil, err
                }
                var slot [scale]byte
                buildSlot(&cum, &slot)
                var states [maxInterleaveN]uint32
                var regions [maxInterleaveN][]byte
                if err := parseInterleavedRegions(src[used:], int(tag), &states, &regions); err != nil {
                        return nil, err
                }
                return decodeInterleaved4(states[:tag], regions[:tag], &freq, &slot, &cum, n)
        default:
                return nil, ErrCorrupt
        }
}

// appendUvarint / uvarint are local LEB128 helpers (kept self-contained so the
// package has no qdf dependency).
func appendUvarint(b []byte, v uint64) []byte {
        for v >= 0x80 {
                b = append(b, byte(v)|0x80)
                v >>= 7
        }
        return append(b, byte(v))
}

func uvarint(b []byte) (uint64, int) {
        var x uint64
        var s uint
        for i, c := range b {
                if i > 9 {
                        return 0, -1
                }
                if c < 0x80 {
                        // Canonical-form guard (cf. encoding/binary.Uvarint): the 10th byte
                        // (i==9, s==63) may only carry bit 63; c>1 would overflow uint64.
                        if i == 9 && c > 1 {
                                return 0, -1
                        }
                        return x | uint64(c)<<s, i + 1
                }
                x |= uint64(c&0x7f) << s
                s += 7
        }
        return 0, 0
}

package rans

import (
        "encoding/binary"

        "github.com/alex60217101990/qdf/internal/bufpool"
)

// encodeStreamStridedInto rANS-encodes the substream src[k], src[k+n], … into
// the caller-provided buf (which must have len >= m*maxRenormBytesPerSym+16,
// where m is the element count) and returns buf[pos:] = [4-byte LE final state]
// [renorm bytes]. Writing
// into a caller buffer lets appendInterleaved place all N substreams in one
// allocation, avoiding the per-substream make() (and its size-class rounding).
func encodeStreamStridedInto(buf, src []byte, k, n int, freq *[256]uint32, cum *[257]uint32) []byte {
        m := max((len(src)-k+n-1)/n, 0)
        pos := len(buf)
        x := uint32(ransByteL)
        for j := m - 1; j >= 0; j-- {
                s := src[k+j*n]
                f := freq[s]
                xMax := ((ransByteL >> scaleBits) << 8) * f
                for x >= xMax {
                        pos--
                        buf[pos] = byte(x)
                        x >>= 8
                }
                x = ((x / f) << scaleBits) + (x % f) + cum[s]
        }
        pos -= 4
        binary.LittleEndian.PutUint32(buf[pos:], x)
        return buf[pos:]
}

// maxInterleaveN bounds the stack-resident state/region scratch in
// appendInterleaved. The shipped stream count is interleaveN (4); the cap leaves
// headroom without forcing those small arrays to the heap.
const maxInterleaveN = 8

// appendInterleaved appends an interleaved-N body to dst (NOT including the tag
// or the shared freq table, which Encode writes): N 4-byte final states, then
// N-1 uvarint substream byte-lengths (the last substream is the remainder), then
// the concatenated substream byte regions. Each substream is an independent
// single-state rANS over the SHARED freq/cum table. All N substreams encode into
// one scratch allocation, partitioned into per-substream regions.
func appendInterleaved(dst, src []byte, freq *[256]uint32, cum *[257]uint32, n int) []byte {
        var statesArr [maxInterleaveN]uint32
        var subsArr [maxInterleaveN][]byte
        states := statesArr[:n]
        subs := subsArr[:n]
        // One allocation for every substream's output. Each substream is encoded under
        // the SHARED freq table, so its local distribution can be skewed against that
        // table and emit up to maxRenormBytesPerSym bytes per symbol — region k must
        // span m_k*maxRenormBytesPerSym+16 bytes (NOT m_k+16, which assumes <=1 byte/
        // symbol and underflows the backward writer on rare-byte-heavy substreams). The
        // regions sum to len(src)*maxRenormBytesPerSym + n*16.
        scratchSize := len(src)*maxRenormBytesPerSym + n*16
        bp := bufpool.Get(scratchSize)
        scratch := (*bp)[:scratchSize]
        defer bufpool.Put(bp)
        off := 0
        for k := range n {
                m := max((len(src)-k+n-1)/n, 0)
                sz := m*maxRenormBytesPerSym + 16
                region := scratch[off : off+sz]
                blob := encodeStreamStridedInto(region, src, k, n, freq, cum) // [4-byte state][bytes]; always >= 4 bytes
                states[k] = binary.LittleEndian.Uint32(blob[:4])
                subs[k] = blob[4:]
                off += sz
        }
        var s4 [4]byte
        for k := range n {
                binary.LittleEndian.PutUint32(s4[:], states[k])
                dst = append(dst, s4[:]...)
        }
        for k := 0; k < n-1; k++ { // last region length is implied by remainder
                dst = appendUvarint(dst, uint64(len(subs[k])))
        }
        for k := range n {
                dst = append(dst, subs[k]...)
        }
        return dst
}

const ransMask = scale - 1

// parseInterleavedRegions reads n states + (n-1) lengths from src, then splits
// the remaining bytes into n substream regions, writing both into the caller-
// provided arrays. Taking the destinations by pointer lets Decode keep them on
// its stack (the slot table already lives there), so an interleaved decode adds
// no per-call heap allocation beyond the output buffer.
func parseInterleavedRegions(src []byte, n int, states *[maxInterleaveN]uint32, regions *[maxInterleaveN][]byte) error {
        if n < 1 || n > maxInterleaveN || len(src) < n*4 {
                // n < 1 would index lens[n-1] out of range below on a hostile frame.
                return ErrCorrupt
        }
        for k := range n {
                states[k] = binary.LittleEndian.Uint32(src[k*4:])
        }
        src = src[n*4:]
        var lens [maxInterleaveN]int
        // Sum in uint64: on a 32-bit build `int` is 32-bit, and the n-1 per-substream
        // lengths (each individually bounded by len(src), which is not decremented by
        // region bytes here) can sum past 2^31 and wrap negative — making the guard
        // below pass and the remainder length blow up into an out-of-range reslice.
        var total uint64
        for k := 0; k < n-1; k++ {
                v, used := uvarint(src)
                if used <= 0 {
                        return ErrCorrupt
                }
                src = src[used:]
                if v > uint64(len(src)) {
                        return ErrCorrupt
                }
                lens[k] = int(v)
                total += v
        }
        if total > uint64(len(src)) {
                return ErrCorrupt
        }
        lens[n-1] = len(src) - int(total) // remainder (total <= len(src), fits int)
        off := 0
        for k := range n {
                regions[k] = src[off : off+lens[k]]
                off += lens[k]
        }
        return nil
}

// decodeInterleaved4 decodes four interleaved rANS substreams that share one
// freq/cum table: out[i] comes from substream i%4. The main loop is unrolled by
// 4; the remainder loop continues at the same index and dispatches index i to
// substream i&3 so leftover indices land in the correct substream.
func decodeInterleaved4(states []uint32, regions [][]byte, freq *[256]uint32, slot *[scale]byte, cum *[257]uint32, n int) ([]byte, error) {
        out := make([]byte, n)
        var xs [4]uint32
        var rs [4][]byte
        var ps [4]int
        for k := range 4 {
                xs[k] = states[k]
                rs[k] = regions[k]
        }
        i := 0
        for ; i+3 < n; i += 4 {
                s0 := slot[xs[0]&ransMask]
                s1 := slot[xs[1]&ransMask]
                s2 := slot[xs[2]&ransMask]
                s3 := slot[xs[3]&ransMask]
                out[i] = s0
                out[i+1] = s1
                out[i+2] = s2
                out[i+3] = s3
                xs[0] = freq[s0]*(xs[0]>>scaleBits) + (xs[0] & ransMask) - cum[s0]
                xs[1] = freq[s1]*(xs[1]>>scaleBits) + (xs[1] & ransMask) - cum[s1]
                xs[2] = freq[s2]*(xs[2]>>scaleBits) + (xs[2] & ransMask) - cum[s2]
                xs[3] = freq[s3]*(xs[3]>>scaleBits) + (xs[3] & ransMask) - cum[s3]
                for xs[0] < ransByteL {
                        if ps[0] >= len(rs[0]) {
                                return nil, ErrCorrupt
                        }
                        xs[0] = (xs[0] << 8) | uint32(rs[0][ps[0]])
                        ps[0]++
                }
                for xs[1] < ransByteL {
                        if ps[1] >= len(rs[1]) {
                                return nil, ErrCorrupt
                        }
                        xs[1] = (xs[1] << 8) | uint32(rs[1][ps[1]])
                        ps[1]++
                }
                for xs[2] < ransByteL {
                        if ps[2] >= len(rs[2]) {
                                return nil, ErrCorrupt
                        }
                        xs[2] = (xs[2] << 8) | uint32(rs[2][ps[2]])
                        ps[2]++
                }
                for xs[3] < ransByteL {
                        if ps[3] >= len(rs[3]) {
                                return nil, ErrCorrupt
                        }
                        xs[3] = (xs[3] << 8) | uint32(rs[3][ps[3]])
                        ps[3]++
                }
        }
        // remainder: index i maps to substream i&3 (continues from where the main
        // loop stopped, so the modular mapping must use i, not a fresh counter).
        for ; i < n; i++ {
                k := i & 3
                x := xs[k]
                s := slot[x&ransMask]
                out[i] = s
                x = freq[s]*(x>>scaleBits) + (x & ransMask) - cum[s]
                for x < ransByteL {
                        if ps[k] >= len(rs[k]) {
                                return nil, ErrCorrupt
                        }
                        x = (x << 8) | uint32(rs[k][ps[k]])
                        ps[k]++
                }
                xs[k] = x
        }
        return out, nil
}

//go:build !qdf_reflect2

// Package reflectutil holds the reflect-based helpers the qdf
// codec uses to allocate slices and maps without going through
// reflect.Value. The default backend uses the stdlib reflect
// package; the qdf_reflect2 build tag swaps in a
// modern-go/reflect2 implementation that skips the type checks
// on the hot path.
package reflectutil

import (
        "reflect"
        "unsafe"
)

// MakeSlice allocates a slice of element type t.Elem() with length n
// and writes the resulting header into the storage at p.
func MakeSlice(t reflect.Type, n int, p unsafe.Pointer) {
        v := reflect.MakeSlice(t, n, n)
        reflect.NewAt(t, p).Elem().Set(v)
}

// SliceData returns the data pointer of the slice value stored at
// p (whose type is the slice type t).
func SliceData(t reflect.Type, p unsafe.Pointer) unsafe.Pointer {
        v := reflect.NewAt(t, p).Elem()
        return unsafe.Pointer(v.UnsafePointer())
}

// MakeMap allocates a map of type t with the given size hint and
// writes the resulting header into the storage at p.
func MakeMap(t reflect.Type, n int, p unsafe.Pointer) {
        v := reflect.MakeMapWithSize(t, n)
        reflect.NewAt(t, p).Elem().Set(v)
}

package unsafestr

import (
        "unsafe"
        _ "unsafe" // for go:linkname
)

//go:linkname mallocgc runtime.mallocgc
func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer

// DirtBytes allocates a byte slice of length n WITHOUT zeroing the memory
// (runtime.mallocgc with needzero=false). The caller MUST write every byte
// before it is read. Used for bump arenas immediately overwritten by a copy,
// where the zero-fill is pure waste. Mirrors bytedance/gopkg/lang/dirtmake;
// the Go runtime's own string(b) conversion (rawstring) allocates the same way.
func DirtBytes(n int) []byte {
        if n == 0 {
                return nil
        }
        p := mallocgc(uintptr(n), nil, false)
        return unsafe.Slice((*byte)(p), n)
}

// Package unsafestr provides a zero-copy conversion from []byte to
// string. The returned value shares storage with the input and MUST be
// treated as read-only — the input []byte must not be mutated while the
// returned string is in use.
package unsafestr

import "unsafe"

// String returns a string aliasing the input []byte. Read-only.
//
//go:nosplit
func String(b []byte) string {
        if len(b) == 0 {
                return ""
        }
        return unsafe.String(unsafe.SliceData(b), len(b))
}

// Bytes returns a []byte aliasing the input string (zero-copy). The result MUST
// be treated as read-only — mutating it corrupts the source string.
//
//go:nosplit
func Bytes(s string) []byte {
        if len(s) == 0 {
                return nil
        }
        return unsafe.Slice(unsafe.StringData(s), len(s))
}

// Package vecquant orchestrates the lossy vector pipeline: it maps a user
// quality budget to a quantization step, runs Hadamard rotation → scalar
// quantization → rANS entropy coding of the integer coordinates, and verifies
// the achieved error on the real data.
package vecquant

import "math"

// BudgetKind selects how the caller expresses the fidelity target.
type BudgetKind uint8

const (
        KindRelError BudgetKind = iota // max relative L2 error of the vector
        KindCosine                     // min cosine similarity
        KindSNR                        // target SNR in dB
)

// Budget is a single fidelity target. Exactly one is in force per encode.
type Budget struct {
        Kind BudgetKind
        Val  float64
}

// DeltaFor returns the quantization step Δ predicted to meet the budget, given
// the RMS (sigma) of the rotated coordinates and the lattice second moment g.
// Model: per-coordinate MSE ≈ Δ²·g, total rel² ≈ (Δ²·g)/sigma².
func DeltaFor(b Budget, sigma, g float64) float64 {
        switch b.Kind {
        case KindRelError:
                // rel = sqrt(g)*Δ/sigma  =>  Δ = sigma*rel/sqrt(g)
                return sigma * b.Val / math.Sqrt(g)
        case KindCosine:
                // cos ≈ 1 - rel²/2  =>  rel² = 2(1-cos)
                rel2 := 2 * (1 - b.Val)
                if rel2 < 0 {
                        rel2 = 0
                }
                return sigma * math.Sqrt(rel2/g)
        case KindSNR:
                // SNR_dB = 10·log10(sigma²/(Δ²·g))  =>  Δ = sigma/sqrt(g) · 10^(-SNR/20)
                return sigma / math.Sqrt(g) * math.Pow(10, -b.Val/20)
        default:
                return sigma // degenerate: ~1 bit
        }
}

package vecquant

import (
        "math"

        "github.com/alex60217101990/qdf/internal/hadamard"
        "github.com/alex60217101990/qdf/internal/lattice"
)

// hadamardSeed is the fixed rotation seed (stored on the wire so a future
// encoder may randomize it per column without breaking decode).
const hadamardSeed uint64 = 0x51ed270b9f4d8c3a

// Variant identifies the lattice used for a Block's coordinates.
const (
        VariantScalar uint8 = 0
        VariantE8     uint8 = 1
)

// Block is the self-contained encoded form of a set of equal-length vectors.
type Block struct {
        // Slices first so the GC pointer-scan range stops here instead of spanning
        // the whole struct (the trailing scalars are pointer-free).
        Coords  []byte // encodeCoords output over the flattened, row-major indices
        Cosets  []byte // coset bits for VariantE8; nil for VariantScalar
        Dim     int
        Count   int
        Seed    uint64
        Delta   float64
        Variant uint8 // VariantScalar or VariantE8
}

// e8Eligible reports whether the E8 variant is worth attempting for a padded
// dimension. Below 16 (one or two 8-D blocks) the coset/packing overhead
// dominates any packing gain, so we skip the extra encode. This is a perf gate,
// never a correctness gate: scalar is always computed.
func e8Eligible(pdim int) bool { return pdim >= 16 }

// e8TryThreshold is the loosest rel-error at which the E8 variant can still beat
// scalar; above it the coset overhead makes E8 lose, so skip the second encode.
// Set above the measured win boundary (rel ~0.02-0.03) so a winning E8 is never
// skipped.
const e8TryThreshold = 0.04

// e8WorthTrying reports whether the budget is tight enough for E8 to have a
// chance of beating scalar. A perf gate (skips wasted work), never a correctness
// gate: scalar is always computed and the never-worse picker still applies.
func e8WorthTrying(b Budget) bool { return relTarget(b) <= e8TryThreshold }

// rotateInto pads each vector to pdim and rotates into dst (grown as needed),
// returning the flat buffer and the padded dim.
func rotateInto(vectors [][]float64, seed uint64, dst []float64) (flat []float64, pdim int) {
        dim := len(vectors[0])
        pdim = hadamard.NextPow2(dim)
        flat = growF64(dst, len(vectors)*pdim)
        for i, v := range vectors {
                row := flat[i*pdim : i*pdim+pdim]
                copy(row, v)
                for k := len(v); k < pdim; k++ {
                        row[k] = 0 // clear pad (buffer may be reused/dirty)
                }
                hadamard.Forward(row, seed)
        }
        return flat, pdim
}

// rotateAll pads each vector to pdim, rotates in place, returns a flat
// [count*pdim] buffer plus the padded dim.
func rotateAll(vectors [][]float64, seed uint64) (flat []float64, pdim int) {
        return rotateInto(vectors, seed, nil)
}

func rms(flat []float64) float64 {
        var s float64
        for _, v := range flat {
                s += v * v
        }
        if len(flat) == 0 {
                return 1
        }
        return math.Sqrt(s / float64(len(flat)))
}

// encodeScalar runs the scalar verify-loop and returns the chosen delta plus
// the (possibly grown) quantized coord slice so the caller can reuse it.
func encodeScalar(orig [][]float64, flat []float64, pdim, dim int, sigma float64, b Budget, row []float64, q []int32) (float64, []int32, bool) {
        delta := DeltaFor(b, sigma, lattice.ScalarG)
        if delta <= 0 || math.IsNaN(delta) || math.IsInf(delta, 0) {
                delta = sigma
        }
        // Scalar is always emitted as the floor: tighten delta until it meets the
        // budget, then stop. Even at the tightest tried delta it is the fallback.
        // overflow tracks the int32 saturation of the LAST quantization (the q that
        // is returned), so the caller can abort to lossless.
        var overflow bool
        for range 4 {
                q, overflow = lattice.QuantizeScalar(flat, delta, q)
                if budgetMetScalar(orig, q, pdim, dim, delta, b, row) {
                        break
                }
                delta *= 0.6
        }
        return delta, q, overflow
}

// encodeE8 runs the E8 verify-loop; ok=false if it never meets the budget.
// Returns the chosen delta, the integer coords, and the coset bits. qDst is
// reused as the coords backing across the verify-loop iterations (pass nil for
// a fresh slice each call).
func encodeE8(orig [][]float64, flat []float64, pdim, dim int, sigma float64, b Budget, row []float64, qDst []int32) (delta float64, coords []int32, cosets []byte, ok, overflow bool) {
        delta = DeltaFor(b, sigma, lattice.E8G)
        if delta <= 0 || math.IsNaN(delta) || math.IsInf(delta, 0) {
                delta = sigma
        }
        coords = qDst
        for range 4 {
                coords, cosets, overflow = lattice.QuantizeE8(flat, delta, coords)
                if budgetMetE8(orig, coords, cosets, pdim, dim, delta, b, row) {
                        ok = true
                        break
                }
                delta *= 0.6
        }
        return delta, coords, cosets, ok, overflow
}

// budgetMetStream reconstructs one vector at a time into row (len pdim), inverse-
// rotates it, and accumulates the budget metric against orig — without building
// a [][]float64. fill(i) writes the dequantized rotated coords of vector i into
// row[:pdim].
func budgetMetStream(orig [][]float64, dim int, b Budget, row []float64, fill func(i int)) bool {
        cosine := b.Kind == KindCosine
        worst := 0.0
        if cosine {
                worst = math.Inf(1)
        }
        for i := range orig {
                fill(i)
                hadamard.Inverse(row, hadamardSeed)
                o := orig[i]
                if cosine {
                        var dot, na, nb float64
                        for j := range dim {
                                dot += o[j] * row[j]
                                na += o[j] * o[j]
                                nb += row[j] * row[j]
                        }
                        if cos := dot / (math.Sqrt(na)*math.Sqrt(nb) + 1e-30); cos < worst {
                                worst = cos
                        }
                } else {
                        var se, ne float64
                        for j := range dim {
                                d := o[j] - row[j]
                                se += d * d
                                ne += o[j] * o[j]
                        }
                        if rel := math.Sqrt(se / (ne + 1e-30)); rel > worst {
                                worst = rel
                        }
                }
        }
        if cosine {
                return worst >= b.Val
        }
        return worst <= relTarget(b)
}

func budgetMetScalar(orig [][]float64, q []int32, pdim, dim int, delta float64, b Budget, row []float64) bool {
        return budgetMetStream(orig, dim, b, row, func(i int) {
                seg := q[i*pdim : i*pdim+pdim]
                for j := range pdim {
                        row[j] = float64(seg[j]) * delta
                }
        })
}

func budgetMetE8(orig [][]float64, coords []int32, cosets []byte, pdim, dim int, delta float64, b Budget, row []float64) bool {
        nbPerVec := pdim / 8
        return budgetMetStream(orig, dim, b, row, func(i int) {
                for blk := range nbPerVec {
                        bb := i*nbPerVec + blk
                        off := 0.0
                        if cosets[bb/8]&(1<<(uint(bb)&7)) != 0 {
                                off = 0.5
                        }
                        base := i*pdim + blk*8
                        for k := range 8 {
                                row[blk*8+k] = (float64(coords[base+k]) + off) * delta
                        }
                }
        })
}

// Encode is the convenience entry that allocates a one-shot Scratch.
func Encode(vectors [][]float64, b Budget) Block {
        var sc Scratch
        return EncodeWith(vectors, b, &sc)
}

// EncodeWith encodes reusing sc's buffers across the verify-loop and variants.
//
// The returned Block.Coords aliases an sc-owned buffer; it stays valid only
// until the next EncodeWith call on the same sc. Callers must consume it (the
// qdf wire layer copies it into the output before reusing the encoder) before
// re-encoding. Use Encode for a one-shot Block whose Coords is independent.
func EncodeWith(vectors [][]float64, b Budget, sc *Scratch) Block {
        if len(vectors) == 0 {
                sc.Overflowed = false // clear any stale flag from a prior reuse of sc
                return Block{Seed: hadamardSeed, Delta: 1, Variant: VariantScalar}
        }
        dim := len(vectors[0])
        flat, pdim := rotateInto(vectors, hadamardSeed, sc.flat)
        sc.flat = flat
        sc.row = growF64(sc.row, pdim)
        sigma := rms(flat)
        if sigma == 0 {
                sigma = 1
        }

        scDelta, qs, scOver := encodeScalar(vectors, flat, pdim, dim, sigma, b, sc.row, sc.qScalar)
        sc.qScalar = qs
        scCoords, zb, rb := encodeCoordsInto(qs, sc.coordsScalar, sc.zig, sc.ransDst)
        sc.coordsScalar, sc.zig, sc.ransDst = scCoords, zb, rb
        best := Block{
                Dim: dim, Count: len(vectors), Seed: hadamardSeed,
                Delta: scDelta, Coords: scCoords, Variant: VariantScalar,
        }
        bestSize := len(scCoords)
        // Track the chosen variant's int32 saturation so the wire layer can abort the
        // lossy block back to lossless (scalar is the floor, so its overflow seeds it).
        bestOver := scOver

        if e8Eligible(pdim) && e8WorthTrying(b) {
                e8Delta, e8Coords, e8Cosets, ok, e8Over := encodeE8(vectors, flat, pdim, dim, sigma, b, sc.row, sc.qE8)
                sc.qE8 = e8Coords
                if ok {
                        // E8's coord block uses its own scratch buffer so it can coexist with
                        // scalar's for the size comparison; the zig/ransDst staging is reused.
                        e8Bytes, zb2, rb2 := encodeCoordsInto(e8Coords, sc.coordsE8, sc.zig, sc.ransDst)
                        sc.coordsE8, sc.zig, sc.ransDst = e8Bytes, zb2, rb2
                        // Compare true wire sizes: E8 additionally carries the coset stream and
                        // its varuint length prefix, which the wire layer writes.
                        e8Size := len(e8Bytes) + uvarintLen(uint64(len(e8Cosets))) + len(e8Cosets)
                        if e8Size < bestSize {
                                best = Block{
                                        Dim: dim, Count: len(vectors), Seed: hadamardSeed,
                                        Delta: e8Delta, Coords: e8Bytes, Variant: VariantE8, Cosets: e8Cosets,
                                }
                                bestOver = e8Over
                        }
                }
        }
        sc.Overflowed = bestOver
        return best
}

// EncodeForcedE8 encodes with the E8 quantizer only, ignoring the scalar
// alternative. It returns the resulting Block and ok=true when E8 is eligible
// and meets the budget; otherwise ok=false. Intended for measurement/benchmark
// callers that want the real E8 wire size (rANS-coded coords + coset stream),
// not for the production encode path, which uses the never-worse try-both Encode.
func EncodeForcedE8(vectors [][]float64, b Budget) (Block, bool) {
        if len(vectors) == 0 {
                return Block{Seed: hadamardSeed, Delta: 1, Variant: VariantE8}, false
        }
        dim := len(vectors[0])
        flat, pdim := rotateAll(vectors, hadamardSeed)
        if !e8Eligible(pdim) {
                return Block{}, false
        }
        sigma := rms(flat)
        if sigma == 0 {
                sigma = 1
        }
        row := make([]float64, pdim)
        e8Delta, e8Coords, e8Cosets, ok, _ := encodeE8(vectors, flat, pdim, dim, sigma, b, row, nil)
        if !ok {
                return Block{}, false
        }
        return Block{
                Dim: dim, Count: len(vectors), Seed: hadamardSeed,
                Delta: e8Delta, Coords: encodeCoords(e8Coords), Variant: VariantE8, Cosets: e8Cosets,
        }, true
}

// relTarget converts any budget to an equivalent max-rel-error for comparison.
func relTarget(b Budget) float64 {
        switch b.Kind {
        case KindRelError:
                return b.Val
        case KindSNR:
                return math.Pow(10, -b.Val/20)
        case KindCosine:
                return math.Sqrt(2 * (1 - b.Val))
        }
        return b.Val
}

func achievedRelError(orig, recon [][]float64) float64 {
        worst := 0.0
        for i := range orig {
                var se, ne float64
                for j := range orig[i] {
                        d := orig[i][j] - recon[i][j]
                        se += d * d
                        ne += orig[i][j] * orig[i][j]
                }
                rel := math.Sqrt(se / (ne + 1e-30))
                if rel > worst {
                        worst = rel
                }
        }
        return worst
}

// reconstruct dequantizes q and inverse-rotates back to original dim.
func reconstruct(q []int32, pdim, dim, count int, delta float64) [][]float64 {
        outFlat := make([]float64, count*dim)
        out := make([][]float64, count)
        row := make([]float64, pdim)
        for i := range count {
                seg := q[i*pdim : i*pdim+pdim]
                for j := range pdim {
                        row[j] = float64(seg[j]) * delta
                }
                hadamard.Inverse(row, hadamardSeed)
                out[i] = outFlat[i*dim : (i+1)*dim : (i+1)*dim]
                copy(out[i], row[:dim])
        }
        return out
}

// reconstructE8 dequantizes E8 coords+cosets and inverse-rotates to dim.
func reconstructE8(coords []int32, cosets []byte, pdim, dim, count int, delta float64) [][]float64 {
        flat := lattice.ReconstructE8(coords, cosets, delta, count*pdim)
        outFlat := make([]float64, count*dim)
        out := make([][]float64, count)
        row := make([]float64, pdim)
        for i := range count {
                copy(row, flat[i*pdim:i*pdim+pdim])
                hadamard.Inverse(row, hadamardSeed)
                out[i] = outFlat[i*dim : (i+1)*dim : (i+1)*dim]
                copy(out[i], row[:dim])
        }
        return out
}

// Decode inverts the pipeline from the on-wire block.
func (bl Block) Decode() [][]float64 {
        if bl.Count <= 0 || bl.Dim <= 0 {
                return nil
        }
        pdim := hadamard.NextPow2(bl.Dim)
        // Self-defense: the wire layer bounds Count/Dim before building a Block, but
        // guard the exported method so a direct caller's adversarial Block cannot
        // overflow Count*pdim (int multiply) into a small/negative under-read.
        if pdim <= 0 || bl.Count > math.MaxInt/pdim {
                return nil
        }
        q, err := decodeCoords(bl.Coords, bl.Count*pdim)
        if err != nil {
                return nil
        }
        if bl.Variant == VariantE8 {
                // Defense in depth: the coset stream holds one bit per 8-D block, so
                // blocks = count*pdim/8 and the byte length is ceil(blocks/8). The wire
                // layer validates this too, but guard here so a malformed Block can
                // never index past the coset slice.
                blocks := bl.Count * pdim / 8
                if len(bl.Cosets) != (blocks+7)/8 {
                        return nil
                }
                return reconstructE8(q, bl.Cosets, pdim, bl.Dim, bl.Count, bl.Delta)
        }
        return reconstruct(q, pdim, bl.Dim, bl.Count, bl.Delta)
}

package vecquant

// maxRetainedScratch caps reused scratch buffers; a one-off giant vector grows
// them, then Reset drops anything larger so memory is not pinned. Mirrors the
// encoder-side columnar scratch ceiling.
const maxRetainedScratch = 1 << 20

// Scratch holds the reusable buffers for one encode, shared across the verify-
// loop iterations and the scalar/E8 variants. It is owned by a single Encoder;
// never share a Scratch across goroutines.
type Scratch struct {
        flat    []float64 // rotated buffer, len count*pdim
        row     []float64 // streaming reconstruct row, len pdim
        qScalar []int32   // scalar quantizer codes
        qE8     []int32   // E8 quantizer codes

        zig          []byte // transient zigzag-varint staging (shared by both variants)
        ransDst      []byte // transient rANS encode staging (shared)
        coordsScalar []byte // scalar variant's final coord block (coexists with E8's)
        coordsE8     []byte // E8 variant's final coord block

        // Overflowed reports whether the chosen block's quantization saturated the
        // int32 coordinate range (set by EncodeWith). When true the caller must not
        // emit the lossy block — it would violate the fidelity budget — and should
        // keep the lossless encoding instead.
        Overflowed bool
}

// Reset prepares the scratch for the next encode, dropping any buffer that grew
// past the retention ceiling.
func (s *Scratch) Reset() {
        if cap(s.flat) > maxRetainedScratch {
                s.flat = nil
        }
        if cap(s.qScalar) > maxRetainedScratch {
                s.qScalar = nil
        }
        if cap(s.qE8) > maxRetainedScratch {
                s.qE8 = nil
        }
        if cap(s.row) > maxRetainedScratch {
                s.row = nil
        }
        if cap(s.zig) > maxRetainedScratch {
                s.zig = nil
        }
        if cap(s.ransDst) > maxRetainedScratch {
                s.ransDst = nil
        }
        if cap(s.coordsScalar) > maxRetainedScratch {
                s.coordsScalar = nil
        }
        if cap(s.coordsE8) > maxRetainedScratch {
                s.coordsE8 = nil
        }
}

func growF64(b []float64, n int) []float64 {
        if cap(b) < n {
                return make([]float64, n)
        }
        return b[:n]
}

package vecquant

import (
        "encoding/binary"
        "errors"

        "github.com/alex60217101990/qdf/internal/rans"
)

func zigzag32(v int32) uint32   { return uint32((v << 1) ^ (v >> 31)) }
func unzigzag32(u uint32) int32 { return int32(u>>1) ^ -int32(u&1) }

// appendVarintZigzag writes each coordinate as a zigzag uvarint.
func appendVarintZigzag(dst []byte, q []int32) []byte {
        var tmp [binary.MaxVarintLen32]byte
        for _, v := range q {
                n := binary.PutUvarint(tmp[:], uint64(zigzag32(v)))
                dst = append(dst, tmp[:n]...)
        }
        return dst
}

// readVarintZigzag reads exactly n zigzag uvarints.
func readVarintZigzag(src []byte, n int) ([]int32, error) {
        out := make([]int32, 0, n)
        off := 0
        for range n {
                u, k := binary.Uvarint(src[off:])
                if k <= 0 {
                        return nil, errors.New("vecquant: truncated coord stream")
                }
                off += k
                out = append(out, unzigzag32(uint32(u)))
        }
        return out, nil
}

// Stream layout: mode byte (0=raw varint, 1=rANS) | varuint(rawLen) | body.
// encodeCoordsInto is the buffer-reusing form of encodeCoords: it zigzag-varint
// encodes q into the reused zig buffer, rANS-encodes into the reused ransDst
// buffer, and writes the never-larger framing into out (reused). Returns the
// coord block plus the grown staging buffers so the caller can retain them.
func encodeCoordsInto(q []int32, out, zig, ransDst []byte) (res, zigBack, ransBack []byte) {
        zig = appendVarintZigzag(zig[:0], q)
        ransDst = rans.Encode(ransDst[:0], zig)
        out = out[:0]
        var tmp [binary.MaxVarintLen64]byte
        hdr := binary.PutUvarint(tmp[:], uint64(len(zig)))
        if len(ransDst) < len(zig) {
                out = append(out, 1)
                out = append(out, tmp[:hdr]...)
                out = append(out, ransDst...)
        } else {
                out = append(out, 0)
                out = append(out, tmp[:hdr]...)
                out = append(out, zig...)
        }
        return out, zig, ransDst
}

func encodeCoords(q []int32) []byte {
        raw := appendVarintZigzag(nil, q)
        packed := rans.Encode(nil, raw)
        // rans.Encode always returns a rANS blob; pick the smaller of it and the raw
        // zigzag bytes here (never-larger framing).
        var out []byte
        var tmp [binary.MaxVarintLen64]byte
        hdr := binary.PutUvarint(tmp[:], uint64(len(raw)))
        if len(packed) < len(raw) {
                out = append(out, 1)
                out = append(out, tmp[:hdr]...)
                out = append(out, packed...)
        } else {
                out = append(out, 0)
                out = append(out, tmp[:hdr]...)
                out = append(out, raw...)
        }
        return out
}

func decodeCoords(src []byte, count int) ([]int32, error) {
        if len(src) < 1 {
                return nil, errors.New("vecquant: empty coord block")
        }
        mode := src[0]
        rawLen, k := binary.Uvarint(src[1:])
        if k <= 0 {
                return nil, errors.New("vecquant: bad rawLen")
        }
        // Bound rawLen: each coordinate is at most binary.MaxVarintLen32 bytes.
        maxRaw := uint64(count) * uint64(binary.MaxVarintLen32)
        if rawLen > maxRaw {
                return nil, errors.New("vecquant: rawLen exceeds bound")
        }
        body := src[1+k:]
        var raw []byte
        switch mode {
        case 0:
                if uint64(len(body)) < rawLen {
                        return nil, errors.New("vecquant: short raw body")
                }
                raw = body[:rawLen]
        case 1:
                dec, err := rans.Decode(body, int(rawLen))
                if err != nil {
                        return nil, err
                }
                raw = dec
        default:
                return nil, errors.New("vecquant: bad mode")
        }
        return readVarintZigzag(raw, count)
}

// uvarintLen returns the number of bytes binary.PutUvarint would write for v.
func uvarintLen(v uint64) int {
        n := 1
        for v >= 0x80 {
                v >>= 7
                n++
        }
        return n
}

// appendCosets writes a length-prefixed raw coset-bit byte stream.
func appendCosets(dst, cosets []byte) []byte {
        dst = binary.AppendUvarint(dst, uint64(len(cosets)))
        return append(dst, cosets...)
}

// ReadCosets is the exported entry for the qdf wire layer; it reads a
// length-prefixed coset stream and verifies it equals wantBytes.
func ReadCosets(src []byte, wantBytes int) (cosets []byte, used int, err error) {
        return readCosets(src, wantBytes)
}

// readCosets reads a length-prefixed coset stream and verifies the length
// equals wantBytes (the only legal value for the block's shape).
func readCosets(src []byte, wantBytes int) (cosets []byte, used int, err error) {
        n, k := binary.Uvarint(src)
        if k <= 0 {
                return nil, 0, errors.New("vecquant: bad coset length")
        }
        if int(n) != wantBytes {
                return nil, 0, errors.New("vecquant: coset length mismatch")
        }
        if uint64(len(src)-k) < n {
                return nil, 0, errors.New("vecquant: short coset body")
        }
        return src[k : k+int(n)], k + int(n), nil
}

package qdf

import (
        "encoding/binary"
        "errors"
        "math"

        "github.com/alex60217101990/qdf/internal/vecquant"
)

// nextPow2Int returns the smallest power of two >= n (minimum 1). Mirrors
// hadamard.NextPow2 so the wire layer can recompute pdim without importing the
// internal rotation package.
func nextPow2Int(n int) int {
        if n <= 1 {
                return 1
        }
        p := 1
        for p < n {
                p <<= 1
        }
        return p
}

// lossyVecMinElems is the smallest slice length that warrants the codec's
// fixed header overhead. Slices shorter than this go through the lossless path
// even when OptLossyVec is set.
const lossyVecMinElems = 32

// toF64Into widens a []float32 or []float64 into dst (reused when cap suffices)
// and returns the filled slice. For []float64 it returns s unchanged (no copy).
// Any other type returns nil.
func toF64Into(v any, dst []float64) []float64 {
        switch s := v.(type) {
        case []float64:
                return s // already float64; caller must copy if mutation is a concern
        case []float32:
                if cap(dst) < len(s) {
                        dst = make([]float64, len(s))
                }
                dst = dst[:len(s)]
                for i, x := range s {
                        dst[i] = float64(x)
                }
                return dst
        }
        return nil
}

// VectorBudget expresses the fidelity target for the lossy vector codec.
// Construct it with MaxRelError, MinCosine, or TargetSNR.
type VectorBudget struct {
        val  float64
        kind vecquant.BudgetKind
        set  bool
}

// MaxRelError bounds the per-vector relative L2 error (e.g. 1e-3).
func MaxRelError(eps float64) VectorBudget {
        return VectorBudget{kind: vecquant.KindRelError, val: eps, set: true}
}

// MinCosine bounds the minimum cosine similarity (e.g. 0.999) — for embeddings.
func MinCosine(c float64) VectorBudget {
        return VectorBudget{kind: vecquant.KindCosine, val: c, set: true}
}

// TargetSNR targets a signal-to-noise ratio in dB (e.g. 40).
func TargetSNR(db float64) VectorBudget {
        return VectorBudget{kind: vecquant.KindSNR, val: db, set: true}
}

// orDefault returns the budget itself if set, otherwise the package default.
func (b VectorBudget) orDefault() VectorBudget {
        if b.set {
                return b
        }
        return MinCosine(0.999)
}

func toBudget(b VectorBudget) vecquant.Budget {
        b = b.orDefault()
        return vecquant.Budget{Kind: b.kind, Val: b.val}
}

// vecException records a single non-finite value that must be restored after
// lossy quantization. RawBits is interpreted as float32 bits when elemF32 is
// true, or float64 bits otherwise.
type vecException struct {
        vecIdx   uint64
        coordIdx uint64
        rawBits  uint64 // float32bits or float64bits
}

// collectExceptions scans vectors for non-finite (NaN/Inf) values, zeroes them
// out in-place so the lossy pipeline sees finite data, and returns the list of
// exceptions to be restored after decode.
func collectExceptions(vectors [][]float64, elemF32 bool) []vecException {
        var exc []vecException
        for vi, v := range vectors {
                for ci, x := range v {
                        if math.IsNaN(x) || math.IsInf(x, 0) {
                                var bits uint64
                                if elemF32 {
                                        bits = uint64(math.Float32bits(float32(x)))
                                } else {
                                        bits = math.Float64bits(x)
                                }
                                exc = append(exc, vecException{
                                        vecIdx:   uint64(vi),
                                        coordIdx: uint64(ci),
                                        rawBits:  bits,
                                })
                                vectors[vi][ci] = 0
                        }
                }
        }
        return exc
}

// appendLossyVec encodes vectors into dst using the lossy vector codec and
// returns the extended slice. Wire layout:
//
//        0xFD | flags(u8: bit0=elemF32) | varuint(dim) | varuint(count) |
//        u64le seed | f64le delta | varuint(len(coords)) | coords |
//        varuint(nExc) then nExc × (varuint vecIdx, varuint coordIdx, u32/u64 bits)
//
// Non-finite values (NaN/Inf) are stored in the exception list and zeroed for
// quantization; the decoder restores them after bl.Decode().
//
// NOTE: appendLossyVec mutates vectors in place (it zeroes non-finite coords
// before encoding). Callers must pass a slice they own; the current callers
// build it via toF64, which allocates a fresh []float64.
// The ok return is false when quantization saturated the int32 coordinate range
// (an over-tight budget on a spiky vector): the lossy block would silently
// violate the fidelity budget, so the caller must keep the lossless encoding.
func appendLossyVec(vectors [][]float64, elemF32 bool, b vecquant.Budget, sc *vecquant.Scratch) ([]byte, bool) {
        // Collect and zero out non-finite values before encoding.
        exc := collectExceptions(vectors, elemF32)

        bl := vecquant.EncodeWith(vectors, b, sc)
        if sc.Overflowed {
                return nil, false
        }
        var dst []byte
        dst = append(dst, tagColVecLossy)
        var flags byte
        if elemF32 {
                flags |= 1
        }
        flags |= bl.Variant << 1 // bits1-2 carry the lattice variant
        dst = append(dst, flags)
        dst = binary.AppendUvarint(dst, uint64(bl.Dim))
        dst = binary.AppendUvarint(dst, uint64(bl.Count))
        dst = binary.LittleEndian.AppendUint64(dst, bl.Seed)
        dst = binary.LittleEndian.AppendUint64(dst, math.Float64bits(bl.Delta))
        dst = binary.AppendUvarint(dst, uint64(len(bl.Coords)))
        dst = append(dst, bl.Coords...)

        // E8 carries one coset bit per 8-D block, length-prefixed.
        if bl.Variant == vecquant.VariantE8 {
                dst = binary.AppendUvarint(dst, uint64(len(bl.Cosets)))
                dst = append(dst, bl.Cosets...)
        }

        // Append exception list: varuint(nExc) then each entry.
        dst = binary.AppendUvarint(dst, uint64(len(exc)))
        for _, e := range exc {
                dst = binary.AppendUvarint(dst, e.vecIdx)
                dst = binary.AppendUvarint(dst, e.coordIdx)
                if elemF32 {
                        dst = binary.LittleEndian.AppendUint32(dst, uint32(e.rawBits))
                } else {
                        dst = binary.LittleEndian.AppendUint64(dst, e.rawBits)
                }
        }
        return dst, true
}

// readLossyVec decodes a lossy-vec block from src and returns the vectors,
// the elemF32 flag, the number of bytes consumed, and any error.
func readLossyVec(src []byte) (vectors [][]float64, elemF32 bool, used int, err error) {
        if len(src) < 2 || src[0] != tagColVecLossy {
                return nil, false, 0, errors.New("qdf: not a lossy-vec block")
        }
        off := 1
        flags := src[off]
        off++
        elemF32 = flags&1 != 0
        variant := (flags >> 1) & 0x3
        if variant != vecquant.VariantScalar && variant != vecquant.VariantE8 {
                return nil, false, 0, errors.New("qdf: bad lossy-vec variant")
        }
        dim, k := binary.Uvarint(src[off:])
        if k <= 0 {
                return nil, false, 0, errors.New("qdf: bad dim")
        }
        off += k
        count, k := binary.Uvarint(src[off:])
        if k <= 0 {
                return nil, false, 0, errors.New("qdf: bad count")
        }
        off += k
        if off+16 > len(src) {
                return nil, false, 0, errors.New("qdf: short header")
        }
        seed := binary.LittleEndian.Uint64(src[off:])
        off += 8
        delta := math.Float64frombits(binary.LittleEndian.Uint64(src[off:]))
        off += 8
        clen, k := binary.Uvarint(src[off:])
        if k <= 0 {
                return nil, false, 0, errors.New("qdf: bad coords len")
        }
        off += k
        if uint64(len(src)-off) < clen {
                return nil, false, 0, errors.New("qdf: short coords")
        }
        // Bound the output allocation: dim*count float64s must fit within
        // maxColumnarBytes (×8 accounts for float64 element size). dim and count
        // are uint64; guard each factor before the product so the multiply itself
        // cannot wrap for adversarial inputs near 2^32.
        const maxVecFactor = maxColumnarBytes / 8
        if dim > maxVecFactor || count > maxVecFactor || dim*count*8 > maxColumnarBytes {
                return nil, false, 0, errors.New("qdf: lossy-vec output exceeds bound")
        }
        // decodeCoords (Block.Decode) allocates intermediates sized by the PADDED
        // coord count count*pdim (pdim = NextPow2(dim), up to ~2*dim) — the dim*count
        // output bound above does not cover this. The largest such alloc is the rANS
        // rawLen ceiling (count*pdim*MaxVarintLen32), which rans.Decode reserves up
        // front from a possibly tiny compressed coords body. Bound the padded product
        // so that ceiling stays within maxColumnarBytes, else a few-byte blob could
        // claim a ~320MB allocation.
        if count != 0 {
                pdim := uint64(nextPow2Int(int(dim)))
                const maxPadded = maxColumnarBytes / binary.MaxVarintLen32
                if pdim != 0 && count > maxPadded/pdim {
                        return nil, false, 0, errors.New("qdf: lossy-vec coords exceed bound")
                }
        }
        coords := src[off : off+int(clen)]
        off += int(clen)

        // E8 carries a coset stream (one bit per 8-D block) after the coords.
        var cosets []byte
        if variant == vecquant.VariantE8 {
                pdim := nextPow2Int(int(dim))
                // E8 quantizes in 8-D blocks; ReconstructE8 walks nb = n/8 blocks and
                // silently leaves the tail zero when n is not a multiple of 8. The honest
                // encoder only emits E8 at pdim>=16 (e8Eligible), so a pdim<8 block can
                // only come from a corrupt/hostile wire — reject it so E8 decode fails
                // closed instead of returning partially zero-filled vectors.
                if pdim%8 != 0 {
                        return nil, false, 0, errors.New("qdf: E8 requires pdim multiple of 8")
                }
                blocks := int(count) * pdim / 8
                wantCoset := (blocks + 7) / 8
                cs, used2, cerr := vecquant.ReadCosets(src[off:], wantCoset)
                if cerr != nil {
                        return nil, false, 0, cerr
                }
                cosets = cs
                off += used2
        }

        bl := vecquant.Block{
                Dim:     int(dim),
                Count:   int(count),
                Seed:    seed,
                Delta:   delta,
                Coords:  coords,
                Variant: variant,
                Cosets:  cosets,
        }
        vectors = bl.Decode()
        if vectors == nil && count != 0 {
                return nil, false, 0, errors.New("qdf: lossy-vec decode failed")
        }

        // Read exception list: varuint(nExc) then each (vecIdx, coordIdx, bits).
        nExc, k := binary.Uvarint(src[off:])
        if k <= 0 {
                return nil, false, 0, errors.New("qdf: bad exception count")
        }
        off += k
        // Bound to prevent hostile blocks from causing OOM.
        if nExc > dim*count {
                return nil, false, 0, errors.New("qdf: exception count exceeds dim*count")
        }
        for range nExc {
                vi, k := binary.Uvarint(src[off:])
                if k <= 0 {
                        return nil, false, 0, errors.New("qdf: bad exception vecIdx")
                }
                off += k
                ci, k := binary.Uvarint(src[off:])
                if k <= 0 {
                        return nil, false, 0, errors.New("qdf: bad exception coordIdx")
                }
                off += k
                if vi >= count || ci >= dim {
                        return nil, false, 0, errors.New("qdf: exception index out of range")
                }
                if elemF32 {
                        if off+4 > len(src) {
                                return nil, false, 0, errors.New("qdf: short exception bits (f32)")
                        }
                        bits := binary.LittleEndian.Uint32(src[off:])
                        off += 4
                        vectors[vi][ci] = float64(math.Float32frombits(bits))
                } else {
                        if off+8 > len(src) {
                                return nil, false, 0, errors.New("qdf: short exception bits (f64)")
                        }
                        bits := binary.LittleEndian.Uint64(src[off:])
                        off += 8
                        vectors[vi][ci] = math.Float64frombits(bits)
                }
        }
        return vectors, elemF32, off, nil
}

package qdf

import (
        "encoding/binary"
        "errors"
        "math"
        "reflect"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/reflectutil"
        "github.com/alex60217101990/qdf/internal/vecquant"
)

// Batched lossy vector-column codec (tagVecBatchStruct, 0xFE).
//
// A []struct with a []float32/[]float64 vector field, encoded the naive way,
// emits one count=1 lossy block PER ROW — so the fixed block header AND the rANS
// frequency framing are paid per vector and never amortized (~+70% vs batched on
// a typical embedding corpus). This codec gathers each equal-length vector field
// across all N rows into ONE count=N block, encoded once, and decodes by
// scattering the N reconstructed vectors back into the rows. Non-batched fields
// (scalars, strings, varying-length / short vectors) stay row-major.

// encodeVectorBatchStruct batches the equal-length vector fields of a []struct
// (element descriptor td, n rows starting at base, stride bytes apart). It
// returns done=true after writing the full tagVecBatchStruct payload when at
// least one field was batched; done=false (nothing written) when no field is
// batchable, so the caller falls back to the normal []struct encoding.
func (e *Encoder) encodeVectorBatchStruct(td *typeDesc, base unsafe.Pointer, n int, stride uintptr) (done bool, err error) {
        nv := len(td.vecFields)
        if nv == 0 || nv > 8 { // batchedMask is one byte
                return false, nil
        }
        blocks := make([][]byte, nv)
        var mask, polarMask byte
        budget := toBudget(e.vecBudget)
        for vi := range td.vecFields {
                blk, polar, ok := e.buildVecColumnBlock(&td.vecFields[vi], base, n, stride, budget)
                if ok {
                        blocks[vi] = blk
                        mask |= 1 << uint(vi)
                        if polar {
                                polarMask |= 1 << uint(vi)
                        }
                }
        }
        if mask == 0 {
                return false, nil // nothing batchable — let the caller emit row-major
        }

        e.writeHeader()
        e.buf = append(e.buf, tagVecBatchStruct)
        e.buf = appendUvarint(e.buf, uint64(n))
        e.buf = append(e.buf, byte(nv), mask, polarMask)
        // Total struct field count: lets Skip() (schema evolution) walk the per-row
        // non-batched fields without a typeDesc — it knows nf-popcount(mask) fields
        // follow each row and replays their intern/shape state via d.Skip().
        e.buf = appendUvarint(e.buf, uint64(len(td.fields)))
        for vi := range blocks {
                if mask&(1<<uint(vi)) != 0 {
                        e.buf = append(e.buf, blocks[vi]...)
                }
        }

        // Per-row: every NON-batched field in declaration order, via its own codec.
        skip := batchedFieldSet(td, mask)
        for i := range n {
                rowPtr := unsafe.Add(base, uintptr(i)*stride)
                for fi := range td.fields {
                        if skip[fi] {
                                continue
                        }
                        if err := td.fields[fi].desc.encode(e, unsafe.Add(rowPtr, td.fields[fi].offset)); err != nil {
                                return true, err
                        }
                }
        }
        return true, nil
}

// polarMinCV is the smallest per-vector norm coefficient of variation
// (std/mean) at which the polar-split variant is worth attempting. Below it the
// norms are near-constant (unit-norm embeddings), so storing them separately
// only adds overhead — the never-worse compare would reject polar anyway, so we
// skip the second encode. Above it (varying-norm data: weights, raw vectors) a
// shared step over the raw vectors is driven by the largest norm and wastes bits
// on the small ones; normalizing first lets one tight step serve every direction.
const polarMinCV = 0.08

// buildVecColumnBlock gathers field vf across all n rows and returns its batched
// payload plus polar (true when the payload is the polar-split form: a per-vector
// norm stream followed by the lossy block of unit directions). ok=false when the
// field is not batchable (varying length, shorter than lossyVecMinElems, or no
// form beats raw).
func (e *Encoder) buildVecColumnBlock(vf *vecBatchField, base unsafe.Pointer, n int, stride uintptr, b vecquant.Budget) (payload []byte, polar bool, ok bool) {
        dim := -1
        for i := range n {
                fp := unsafe.Add(base, uintptr(i)*stride+vf.offset)
                var l int
                if vf.elemF32 {
                        l = len(*(*[]float32)(fp))
                } else {
                        l = len(*(*[]float64)(fp))
                }
                if i == 0 {
                        dim = l
                } else if l != dim {
                        return nil, false, false // varying length across rows: cannot batch
                }
        }
        if dim < lossyVecMinElems {
                return nil, false, false
        }
        // Bound the gather before n*dim is computed as int (cannot overflow on 32-bit;
        // also keeps the batch within the columnar byte budget). Over the cap, the
        // field stays row-major.
        if int64(n)*int64(dim) > int64(maxColumnarBytes/8) {
                return nil, false, false
        }

        // Gather n vectors into reused scratch (appendLossyVec zeroes NaN/Inf in
        // place, so it must never see the caller's backing arrays).
        if cap(e.vecBatchFlat) < n*dim {
                e.vecBatchFlat = make([]float64, n*dim)
        }
        flat := e.vecBatchFlat[:n*dim]
        if cap(e.vecBatchRows) < n {
                e.vecBatchRows = make([][]float64, n)
        }
        rows := e.vecBatchRows[:n]
        // Gather, and in the same pass compute each vector's L2 norm and whether any
        // coordinate is non-finite — BEFORE the plain encode, which zeroes NaN/Inf in
        // flat (so the norms must be read here, while flat still holds the originals).
        if cap(e.vecBatchNorms) < n {
                e.vecBatchNorms = make([]float64, n)
        }
        norms := e.vecBatchNorms[:n]
        finite := true
        var mean float64
        for i := range n {
                fp := unsafe.Add(base, uintptr(i)*stride+vf.offset)
                dst := flat[i*dim : (i+1)*dim]
                if vf.elemF32 {
                        for j, x := range *(*[]float32)(fp) {
                                dst[j] = float64(x)
                        }
                } else {
                        copy(dst, *(*[]float64)(fp))
                }
                rows[i] = dst
                var s float64
                for _, x := range dst {
                        s += x * x
                }
                nrm := math.Sqrt(s)
                norms[i] = nrm
                if nrm <= 0 || math.IsNaN(nrm) || math.IsInf(nrm, 0) {
                        finite = false
                }
                mean += nrm
        }
        plain, plainOK := appendLossyVec(rows, vf.elemF32, b, &e.vecScratch)
        if !plainOK {
                // Quantization saturated int32 (over-tight budget): keep the field
                // row-major rather than ship a budget-violating lossy block.
                return nil, false, false
        }

        elemSize := 8
        if vf.elemF32 {
                elemSize = 4
        }
        rawBytes := n * dim * elemSize
        best, bestPolar := plain, false

        // Polar candidate: only when every vector is finite and positive-norm (so the
        // plain encode did not zero any coordinate in flat) and the norm spread (CV)
        // is large enough to plausibly beat the per-vector norm overhead.
        if finite {
                mean /= float64(n)
                var varsum float64
                for _, nrm := range norms {
                        d := nrm - mean
                        varsum += d * d
                }
                cv := math.Sqrt(varsum/float64(n)) / (mean + 1e-30)
                if cv > polarMinCV {
                        // Normalize flat in place (no NaN/Inf in this branch, so plain's encode
                        // left it intact) → unit directions; encode those.
                        for i := range n {
                                inv := 1.0 / norms[i]
                                seg := flat[i*dim : (i+1)*dim]
                                for j := range seg {
                                        seg[j] *= inv
                                }
                        }
                        dirsBlk, dirsOK := appendLossyVec(rows, vf.elemF32, b, &e.vecScratch)
                        if dirsOK {
                                polarPayload := appendNormStream(nil, norms)
                                polarPayload = append(polarPayload, dirsBlk...)
                                if len(polarPayload) < len(best) {
                                        best, bestPolar = polarPayload, true
                                }
                        }
                }
        }

        // Never-worse vs raw: if even the smaller form is not below raw, keep the
        // field row-major (its own per-vector never-worse picker applies there).
        if len(best) >= rawBytes {
                return nil, false, false
        }
        return best, bestPolar, true
}

// appendNormStream writes the polar-split norm stream: f64 log-min, f64 log-max,
// then one uint16 per vector quantizing log(norm) over [log-min, log-max]. The
// log domain gives uniform RELATIVE precision, so a wide norm range (1000x)
// still round-trips to ~1e-4. norms must all be finite and positive.
func appendNormStream(dst []byte, norms []float64) []byte {
        if len(norms) == 0 {
                return dst // never reached in practice (n >= columnarMinElems); guards ±Inf header
        }
        logMin, logMax := math.Inf(1), math.Inf(-1)
        for _, nrm := range norms {
                l := math.Log(nrm)
                logMin, logMax = math.Min(logMin, l), math.Max(logMax, l)
        }
        dst = binary.LittleEndian.AppendUint64(dst, math.Float64bits(logMin))
        dst = binary.LittleEndian.AppendUint64(dst, math.Float64bits(logMax))
        span := logMax - logMin
        if span <= 0 {
                span = 1 // all norms equal: every quantum maps to log-min
        }
        for _, nrm := range norms {
                q := math.RoundToEven((math.Log(nrm) - logMin) / span * 65535)
                dst = binary.LittleEndian.AppendUint16(dst, uint16(q))
        }
        return dst
}

// readNormStream reads n quantized norms written by appendNormStream and returns
// them plus the number of bytes consumed.
func readNormStream(src []byte, n int) (norms []float64, used int, err error) {
        if uint64(len(src)) < 16+2*uint64(n) { // uint64 intermediate: 2*n cannot wrap
                return nil, 0, ErrShortBuffer
        }
        logMin := math.Float64frombits(binary.LittleEndian.Uint64(src[0:]))
        logMax := math.Float64frombits(binary.LittleEndian.Uint64(src[8:]))
        if math.IsNaN(logMin) || math.IsNaN(logMax) || math.IsInf(logMin, 0) || math.IsInf(logMax, 0) {
                // Hostile header: NaN/Inf would make span NaN (span<=0 is false for NaN)
                // and yield math.Exp(NaN)=NaN for every decoded norm. The encoder only
                // writes finite logMin/logMax (appendNormStream requires finite norms).
                return nil, 0, ErrInvalidLength
        }
        span := logMax - logMin
        if span <= 0 {
                span = 1
        }
        // Finite header values can still overflow math.Exp: the largest exponent
        // produced below is logMin+span (q=65535), and the honest encoder never
        // writes a log-norm above log(MaxFloat64) (~709.78) because it logs finite
        // norms. Reject anything past that so a hostile header cannot decode to
        // +Inf norms (which would silently corrupt every denormalized vector).
        if logMin+span > math.Log(math.MaxFloat64) {
                return nil, 0, ErrInvalidLength
        }
        norms = make([]float64, n)
        off := 16
        for i := range n {
                q := binary.LittleEndian.Uint16(src[off:])
                off += 2
                norms[i] = math.Exp(logMin + float64(q)/65535*span)
        }
        return norms, off, nil
}

// batchedFieldSet returns a per-field bool: true when that struct field is a
// vector field batched under mask.
func batchedFieldSet(td *typeDesc, mask byte) []bool {
        skip := make([]bool, len(td.fields))
        for vi := range td.vecFields {
                if mask&(1<<uint(vi)) != 0 {
                        skip[td.vecFields[vi].fieldIdx] = true
                }
        }
        return skip
}

// decodeVectorBatchStruct decodes a tagVecBatchStruct payload into a *[]T (t is
// the slice type, td its struct element descriptor). The 0xFE tag has been
// peeked but not consumed.
func (d *Decoder) decodeVectorBatchStruct(t reflect.Type, td *typeDesc, p unsafe.Pointer) error {
        if d.i >= len(d.buf) || d.buf[d.i] != tagVecBatchStruct {
                return ErrBadTag
        }
        d.i++
        n64, k := binary.Uvarint(d.buf[d.i:])
        if k <= 0 {
                return errors.New("qdf: bad vec-batch count")
        }
        d.i += k
        if n64 > uint64(maxColumnarElems) { // uint64->int maxInt clause (32-bit) + ceiling
                return ErrInvalidLength
        }
        n := int(n64)
        // Bound the result allocation by the input: MakeSlice(t, n) below allocates
        // n*stride, so CheckLength(n, 1) is not enough — a hostile count amplifies by
        // the struct stride. Mirror the columnar decoders' count + byte caps.
        if err := checkColumnarN(n); err != nil {
                return err
        }
        if err := checkColumnarBytes(n, t.Elem().Size()); err != nil {
                return err
        }
        if d.i+3 > len(d.buf) {
                return ErrShortBuffer
        }
        nv := int(d.buf[d.i])
        mask := d.buf[d.i+1]
        polarMask := d.buf[d.i+2]
        d.i += 3
        nf64, kf := binary.Uvarint(d.buf[d.i:])
        if kf <= 0 {
                return errors.New("qdf: bad vec-batch field count")
        }
        d.i += kf
        if nv != len(td.vecFields) {
                return errors.New("qdf: vec-batch shape mismatch")
        }
        if int(nf64) != len(td.fields) {
                return errors.New("qdf: vec-batch field count mismatch")
        }
        if polarMask&^mask != 0 { // polar bit on a non-batched field would desync d.i
                return errors.New("qdf: vec-batch polarMask not a subset of mask")
        }

        // Read each batched column's block (count=n vectors). A polar field carries a
        // norm stream before the directions block; rescale to recover the vectors.
        //
        // Each readLossyVec independently caps its own reconstruction at
        // maxColumnarBytes, but up to 8 batched columns would then allow ~8×256MB
        // from a tiny rANS-compressed input. Bound the AGGREGATE across all columns
        // so the batched path amplifies no more than a single columnar decode.
        var totalVecBytes uint64
        batched := make([][][]float64, nv)
        for vi := range nv {
                if mask&(1<<uint(vi)) == 0 {
                        continue
                }
                var norms []float64
                if polarMask&(1<<uint(vi)) != 0 {
                        ns, used, err := readNormStream(d.buf[d.i:], n)
                        if err != nil {
                                return err
                        }
                        d.i += used
                        norms = ns
                }
                vecs, elemF32, used, err := readLossyVec(d.buf[d.i:])
                if err != nil {
                        return err
                }
                d.i += used
                if len(vecs) != n {
                        return errors.New("qdf: vec-batch block row count mismatch")
                }
                if elemF32 != td.vecFields[vi].elemF32 {
                        return ErrTypeMismatch
                }
                if len(vecs) > 0 {
                        totalVecBytes += uint64(len(vecs)) * uint64(len(vecs[0])) * 8
                        if totalVecBytes > maxColumnarBytes {
                                return ErrInvalidLength
                        }
                }
                if norms != nil {
                        for i := range vecs {
                                scale := norms[i]
                                for j := range vecs[i] {
                                        vecs[i][j] *= scale
                                }
                        }
                }
                batched[vi] = vecs
        }

        reflectutil.MakeSlice(t, n, p)
        base := reflectutil.SliceData(t, p)
        stride := t.Elem().Size()

        skip := make([]bool, len(td.fields))
        fieldVi := make([]int, len(td.fields))
        for i := range fieldVi {
                fieldVi[i] = -1
        }
        for vi := range nv {
                fieldVi[td.vecFields[vi].fieldIdx] = vi
                if mask&(1<<uint(vi)) != 0 {
                        skip[td.vecFields[vi].fieldIdx] = true
                }
        }

        // Pre-alloc one flat []float32 slab per elemF32 vector field so the
        // scatter loop below slices into it instead of calling make([]float32,dim)
        // once per row. For n=1000, dim=512 this eliminates 1000 heap allocs of
        // 2 KB each, replacing them with a single 2 MB alloc per field.
        f32Flat := make([][]float32, nv)
        for vi := range nv {
                if batched[vi] == nil || !td.vecFields[vi].elemF32 {
                        continue
                }
                if n > 0 {
                        if dim := len(batched[vi][0]); dim > 0 {
                                f32Flat[vi] = make([]float32, n*dim)
                        }
                }
        }

        for i := range n {
                rowPtr := unsafe.Add(base, uintptr(i)*stride)
                for fi := range td.fields {
                        fp := unsafe.Add(rowPtr, td.fields[fi].offset)
                        if skip[fi] {
                                vi := fieldVi[fi]
                                vec := batched[vi][i]
                                if td.vecFields[vi].elemF32 {
                                        dim := len(vec)
                                        var out []float32
                                        if flat := f32Flat[vi]; flat != nil {
                                                out = flat[i*dim : (i+1)*dim : (i+1)*dim]
                                        } else {
                                                out = make([]float32, dim)
                                        }
                                        for j, x := range vec {
                                                out[j] = float32(x)
                                        }
                                        *(*[]float32)(fp) = out
                                } else {
                                        *(*[]float64)(fp) = vec
                                }
                                continue
                        }
                        if err := td.fields[fi].desc.decode(d, fp); err != nil {
                                return err
                        }
                }
        }
        return nil
}

package qdf

import (
        "math"
        "slices"
)

// Shared helpers for OptMapShape — map key-set interning. The reflect map
// encoder (encodeStringMapShaped in reflect_encode.go) and the generated
// concrete fast paths (maps_fast_generated.go) both reuse the encState /
// decState shape machinery through these. See
// docs/superpowers/specs/2026-06-04-map-shape-interning-design.md.

// mapStringShapeOrder emits the tagMapShape declare-or-reuse header for a
// concrete string-keyed map and returns the canonical (sorted) key order the
// caller must emit values in. Generic over the value type so the generated fast
// paths stay reflection-free.
//
// Hot reuse path: one range for the order-independent set-hash, a membership
// check per bound key (guards a set-hash collision), then the bound order — no
// key slice allocated. Declare path (first sight of a key-set): gather + sort +
// intern the keys; rare. Collision/mismatch degrades to a fresh declaration,
// never to wrong data (cf. hash-key-sampling-collision).
//
// Callers must gate on len(m) > 0, e.state != nil, OptMapShape and OptDense
// before calling.
func mapStringShapeOrder[V any](e *Encoder, m map[string]V) []string {
        // Ensure the stream header precedes the first tag. For a top-level map the
        // plain path emits it via WriteMapHeader, which this shape path bypasses.
        // Idempotent (guarded by e.headerOut).
        e.writeHeader()
        n := len(m)
        st := e.state

        if e.stateSuspended {
                // Inside a never-larger trial (diffKeyedSlice / diffColumnar): emit a
                // self-contained id-0 declaration every time and never bind/reuse a
                // map-shape id — a bound id whose declaration is thrown away with the
                // losing candidate would dangle (ErrUnknownStateID), the same leak the
                // trial suspends interning to avoid. Mirrors Encoder.StructShape under
                // suspension: advance shapeCount (the decoder registers a shape per
                // declaration it reads; the trial re-bases the counter for the discarded
                // candidate) but touch no mapShapes / lastMapShapeID binding.
                keys := make([]string, 0, n)
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                st.shapeDeclareEnc()
                e.buf = append(e.buf, tagMapShape)
                e.buf = appendUvarint(e.buf, 0)
                e.buf = appendUvarint(e.buf, uint64(n))
                for _, k := range keys {
                        e.WriteString(k)
                }
                return keys
        }

        // Fast path: a run of homogeneous rows reuses the previous shape. Verify
        // membership directly (len + every bound key present) — no set-hash, no
        // registry scan. Exact, so collision-proof.
        if st.lastMapShapeID != 0 && len(st.lastMapShapeKeys) == n && mapHasAll(m, st.lastMapShapeKeys) {
                e.buf = append(e.buf, tagMapShape)
                e.buf = appendUvarint(e.buf, uint64(st.lastMapShapeID))
                return st.lastMapShapeKeys
        }

        // Key-set changed: hash to find an earlier shape, else declare.
        var setHash uint64
        for k := range m {
                setHash += internKeyHash(k) // commutative: order-independent
        }
        // Scan all (setHash, n) rows and verify by keys, not just the first: returning
        // the first match and declaring on mismatch re-declares a colliding key-set on
        // every encode (unbounded mapShapes growth under two alternating colliding
        // sets). See the reflect encodeMapStringShape path for the full rationale.
        for i := range st.mapShapes {
                s := &st.mapShapes[i]
                if s.setHash == setHash && s.n == n && mapHasAll(m, s.keys) {
                        st.lastMapShapeID, st.lastMapShapeKeys = s.id, s.keys
                        e.buf = append(e.buf, tagMapShape)
                        e.buf = appendUvarint(e.buf, uint64(s.id))
                        return s.keys
                }
        }
        keys := make([]string, 0, n)
        for k := range m {
                keys = append(keys, k)
        }
        slices.Sort(keys)
        id := st.shapeDeclareEnc()
        st.mapShapeRegister(setHash, n, keys, id)
        st.lastMapShapeID, st.lastMapShapeKeys = id, keys
        e.buf = append(e.buf, tagMapShape)
        e.buf = appendUvarint(e.buf, 0)
        e.buf = appendUvarint(e.buf, uint64(n))
        for _, k := range keys {
                e.WriteString(k)
        }
        return keys
}

// mapHasAll reports whether m contains every key in keys.
func mapHasAll[V any](m map[string]V, keys []string) bool {
        for _, k := range keys {
                if _, ok := m[k]; !ok {
                        return false
                }
        }
        return true
}

// decodeMapStringShapeHeader consumes a tagMapShape header (the caller has
// peeked tag == tagMapShape but not advanced) and returns the ordered key
// names for the shape. On a declaration it reads and registers the keys; on a
// reuse it looks the shape up. The caller then reads len(names) values in order.
// Shared by the reflect decodeMap branch and the generated fast paths.
func decodeMapStringShapeHeader(d *Decoder) ([]string, error) {
        d.i++ // consume tagMapShape
        shapeID, sz := readUvarint(d.buf[d.i:])
        if sz <= 0 {
                return nil, ErrInvalidLength
        }
        // Reject a shape ID that would truncate on the uint32 narrowing below: a
        // crafted id >= 2^32 must not alias a real (id mod 2^32) shape.
        if shapeID > uint64(math.MaxUint32) {
                return nil, ErrUnknownStateID
        }
        d.i += sz
        if d.state == nil {
                d.state = newDecState()
        }
        if shapeID == 0 {
                cnt64, sz := readUvarint(d.buf[d.i:])
                if sz <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += sz
                if cnt64 > uint64(math.MaxInt) { // 32-bit: int(cnt64) would wrap negative
                        return nil, ErrInvalidLength
                }
                cnt := int(cnt64)
                if err := d.CheckLength(cnt, 1); err != nil {
                        return nil, err
                }
                sh := d.state.shapeDeclare()
                keys := make([]string, 0, cnt)
                for range cnt {
                        kb, err := d.readStringBytes()
                        if err != nil {
                                return nil, err
                        }
                        keys = append(keys, d.keyCache.Make(kb))
                }
                sh.names = keys
                return keys, nil
        }
        sh := d.state.shapeLookup(uint32(shapeID))
        if sh == nil {
                return nil, ErrUnknownStateID
        }
        return sh.names, nil
}

// Code generated by internal/mapsgen — DO NOT EDIT.
// Re-run via `go generate ./...` from the repository root.

package qdf

import (
        "reflect"
        "slices"
        "unsafe"
)

// ----- map[string]string -----

func encodeMapStringString(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]string)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteString(v)
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteString(v)
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteString(v)
        }
        return nil
}

func decodeMapStringString(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]string)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, string](d, p, len(names))
                for _, k := range names {
                        v, err := d.ReadString()
                        if err != nil {
                                return err
                        }
                        m[k] = v
                }
                *(*map[string]string)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, string](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                v, err := d.ReadString()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[string]string)(p) = m
        return nil
}

// ----- map[string]bool -----

func encodeMapStringBool(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]bool)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteBool(v)
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteBool(v)
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteBool(v)
        }
        return nil
}

func decodeMapStringBool(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]bool)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, bool](d, p, len(names))
                for _, k := range names {
                        v, err := d.ReadBool()
                        if err != nil {
                                return err
                        }
                        m[k] = v
                }
                *(*map[string]bool)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, bool](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                v, err := d.ReadBool()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[string]bool)(p) = m
        return nil
}

// ----- map[string]int8 -----

func encodeMapStringInt8(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]int8)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteInt(int64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapStringInt8(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]int8)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, int8](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        v := int8(vWide)
                        m[k] = v
                }
                *(*map[string]int8)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, int8](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v := int8(vWide)
                m[k] = v
        }
        *(*map[string]int8)(p) = m
        return nil
}

// ----- map[string]int16 -----

func encodeMapStringInt16(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]int16)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteInt(int64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapStringInt16(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]int16)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, int16](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        v := int16(vWide)
                        m[k] = v
                }
                *(*map[string]int16)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, int16](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v := int16(vWide)
                m[k] = v
        }
        *(*map[string]int16)(p) = m
        return nil
}

// ----- map[string]int32 -----

func encodeMapStringInt32(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]int32)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteInt(int64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapStringInt32(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]int32)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, int32](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        v := int32(vWide)
                        m[k] = v
                }
                *(*map[string]int32)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, int32](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v := int32(vWide)
                m[k] = v
        }
        *(*map[string]int32)(p) = m
        return nil
}

// ----- map[string]int -----

func encodeMapStringInt(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]int)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteInt(int64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapStringInt(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]int)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, int](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        v := int(vWide)
                        m[k] = v
                }
                *(*map[string]int)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, int](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v := int(vWide)
                m[k] = v
        }
        *(*map[string]int)(p) = m
        return nil
}

// ----- map[string]int64 -----

func encodeMapStringInt64(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]int64)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteInt(int64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapStringInt64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]int64)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, int64](d, p, len(names))
                for _, k := range names {
                        v, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        m[k] = v
                }
                *(*map[string]int64)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, int64](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[string]int64)(p) = m
        return nil
}

// ----- map[string]uint8 -----

func encodeMapStringUint8(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]uint8)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteUint(uint64(v))
        }
        return nil
}

func decodeMapStringUint8(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]uint8)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, uint8](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        v := uint8(vWide)
                        m[k] = v
                }
                *(*map[string]uint8)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, uint8](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadUint()
                if err != nil {
                        return err
                }
                v := uint8(vWide)
                m[k] = v
        }
        *(*map[string]uint8)(p) = m
        return nil
}

// ----- map[string]uint16 -----

func encodeMapStringUint16(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]uint16)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteUint(uint64(v))
        }
        return nil
}

func decodeMapStringUint16(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]uint16)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, uint16](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        v := uint16(vWide)
                        m[k] = v
                }
                *(*map[string]uint16)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, uint16](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadUint()
                if err != nil {
                        return err
                }
                v := uint16(vWide)
                m[k] = v
        }
        *(*map[string]uint16)(p) = m
        return nil
}

// ----- map[string]uint32 -----

func encodeMapStringUint32(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]uint32)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteUint(uint64(v))
        }
        return nil
}

func decodeMapStringUint32(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]uint32)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, uint32](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        v := uint32(vWide)
                        m[k] = v
                }
                *(*map[string]uint32)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, uint32](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadUint()
                if err != nil {
                        return err
                }
                v := uint32(vWide)
                m[k] = v
        }
        *(*map[string]uint32)(p) = m
        return nil
}

// ----- map[string]uint -----

func encodeMapStringUint(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]uint)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteUint(uint64(v))
        }
        return nil
}

func decodeMapStringUint(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]uint)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, uint](d, p, len(names))
                for _, k := range names {
                        vWide, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        v := uint(vWide)
                        m[k] = v
                }
                *(*map[string]uint)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, uint](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                vWide, err := d.ReadUint()
                if err != nil {
                        return err
                }
                v := uint(vWide)
                m[k] = v
        }
        *(*map[string]uint)(p) = m
        return nil
}

// ----- map[string]uint64 -----

func encodeMapStringUint64(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]uint64)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteUint(uint64(v))
        }
        return nil
}

func decodeMapStringUint64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]uint64)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, uint64](d, p, len(names))
                for _, k := range names {
                        v, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        m[k] = v
                }
                *(*map[string]uint64)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, uint64](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                v, err := d.ReadUint()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[string]uint64)(p) = m
        return nil
}

// ----- map[string]float32 -----

func encodeMapStringFloat32(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]float32)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteFloat32(v)
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteFloat32(v)
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteFloat32(v)
        }
        return nil
}

func decodeMapStringFloat32(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]float32)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, float32](d, p, len(names))
                for _, k := range names {
                        v, err := d.ReadFloat32()
                        if err != nil {
                                return err
                        }
                        m[k] = v
                }
                *(*map[string]float32)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, float32](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                v, err := d.ReadFloat32()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[string]float32)(p) = m
        return nil
}

// ----- map[string]float64 -----

func encodeMapStringFloat64(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]float64)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        e.WriteFloat64(v)
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        e.WriteFloat64(v)
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                e.WriteFloat64(v)
        }
        return nil
}

func decodeMapStringFloat64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]float64)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, float64](d, p, len(names))
                for _, k := range names {
                        v, err := d.ReadFloat64()
                        if err != nil {
                                return err
                        }
                        m[k] = v
                }
                *(*map[string]float64)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, float64](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                v, err := d.ReadFloat64()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[string]float64)(p) = m
        return nil
}

// ----- map[string][]byte -----

func encodeMapStringBytes(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string][]byte)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        if v == nil {
                                e.WriteNil()
                        } else {
                                e.WriteBytes(v)
                        }
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        if v == nil {
                                e.WriteNil()
                        } else {
                                e.WriteBytes(v)
                        }
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                if v == nil {
                        e.WriteNil()
                } else {
                        e.WriteBytes(v)
                }
        }
        return nil
}

func decodeMapStringBytes(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string][]byte)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, []byte](d, p, len(names))
                for _, k := range names {
                        var v []byte
                        vTag, err := d.peekTag()
                        if err != nil {
                                return err
                        }
                        if vTag != tagNil {
                                v, err = d.ReadBytes()
                                if err != nil {
                                        return err
                                }
                        } else {
                                d.i++
                        }
                        m[k] = v
                }
                *(*map[string][]byte)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, []byte](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                var v []byte
                vTag, err := d.peekTag()
                if err != nil {
                        return err
                }
                if vTag != tagNil {
                        v, err = d.ReadBytes()
                        if err != nil {
                                return err
                        }
                } else {
                        d.i++
                }
                m[k] = v
        }
        *(*map[string][]byte)(p) = m
        return nil
}

// ----- map[string][]string -----

func encodeMapStringStringSlice(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string][]string)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        if v == nil {
                                e.WriteNil()
                        } else {
                                e.WriteArrayHeader(len(v))
                                for _, sv := range v {
                                        e.WriteString(sv)
                                }
                        }
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        if v == nil {
                                e.WriteNil()
                        } else {
                                e.WriteArrayHeader(len(v))
                                for _, sv := range v {
                                        e.WriteString(sv)
                                }
                        }
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                if v == nil {
                        e.WriteNil()
                } else {
                        e.WriteArrayHeader(len(v))
                        for _, sv := range v {
                                e.WriteString(sv)
                        }
                }
        }
        return nil
}

func decodeMapStringStringSlice(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string][]string)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, []string](d, p, len(names))
                for _, k := range names {
                        var v []string
                        vTag, err := d.peekTag()
                        if err != nil {
                                return err
                        }
                        if vTag != tagNil {
                                hdrN, err := d.ReadArrayHeader()
                                if err != nil {
                                        return err
                                }
                                if err := d.CheckLength(hdrN, 1); err != nil {
                                        return err
                                }
                                v = make([]string, hdrN)
                                for i := range hdrN {
                                        sv, err := d.ReadString()
                                        if err != nil {
                                                return err
                                        }
                                        v[i] = sv
                                }
                        } else {
                                d.i++
                        }
                        m[k] = v
                }
                *(*map[string][]string)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, []string](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                var v []string
                vTag, err := d.peekTag()
                if err != nil {
                        return err
                }
                if vTag != tagNil {
                        hdrN, err := d.ReadArrayHeader()
                        if err != nil {
                                return err
                        }
                        if err := d.CheckLength(hdrN, 1); err != nil {
                                return err
                        }
                        v = make([]string, hdrN)
                        for i := range hdrN {
                                sv, err := d.ReadString()
                                if err != nil {
                                        return err
                                }
                                v[i] = sv
                        }
                } else {
                        d.i++
                }
                m[k] = v
        }
        *(*map[string][]string)(p) = m
        return nil
}

// ----- map[string]any -----

func encodeMapStringAny(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[string]any)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        if len(m) > 0 && e.state != nil && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                for _, k := range mapStringShapeOrder(e, m) {
                        v := m[k]
                        if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                                return err
                        }
                }
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []string
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysStr[:0]
                        canonPooled = true
                } else {
                        keys = make([]string, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysStr = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteString(sk)
                        if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                                return err
                        }
                }
                return nil
        }
        for k, v := range m {
                e.WriteString(k)
                if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                        return err
                }
        }
        return nil
}

func decodeMapStringAny(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[string]any)(p) = nil
                return nil
        }
        if t == tagMapShape {
                names, err := decodeMapStringShapeHeader(d)
                if err != nil {
                        return err
                }
                m := reuseOrMakeMap[string, any](d, p, len(names))
                for _, k := range names {
                        v, err := decodeAny(d)
                        if err != nil {
                                return err
                        }
                        m[k] = v
                }
                *(*map[string]any)(p) = m
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[string, any](d, p, n)
        for range n {
                kb, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                k := d.keyCache.Make(kb)
                v, err := decodeAny(d)
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[string]any)(p) = m
        return nil
}

// ----- map[int]string -----

func encodeMapIntString(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[int]string)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []int64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysI64[:0]
                        canonPooled = true
                } else {
                        keys = make([]int64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, int64(k))
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysI64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        k := int(sk)
                        v := m[k]
                        e.WriteInt(int64(k))
                        e.WriteString(v)
                }
                return nil
        }
        for k, v := range m {
                e.WriteInt(int64(k))
                e.WriteString(v)
        }
        return nil
}

func decodeMapIntString(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[int]string)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[int, string](d, p, n)
        for range n {
                kWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                k := int(kWide)
                v, err := d.ReadString()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[int]string)(p) = m
        return nil
}

// ----- map[int]int -----

func encodeMapIntInt(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[int]int)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []int64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysI64[:0]
                        canonPooled = true
                } else {
                        keys = make([]int64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, int64(k))
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysI64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        k := int(sk)
                        v := m[k]
                        e.WriteInt(int64(k))
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteInt(int64(k))
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapIntInt(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[int]int)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[int, int](d, p, n)
        for range n {
                kWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                k := int(kWide)
                vWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v := int(vWide)
                m[k] = v
        }
        *(*map[int]int)(p) = m
        return nil
}

// ----- map[int]int64 -----

func encodeMapIntInt64(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[int]int64)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []int64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysI64[:0]
                        canonPooled = true
                } else {
                        keys = make([]int64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, int64(k))
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysI64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        k := int(sk)
                        v := m[k]
                        e.WriteInt(int64(k))
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteInt(int64(k))
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapIntInt64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[int]int64)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[int, int64](d, p, n)
        for range n {
                kWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                k := int(kWide)
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[int]int64)(p) = m
        return nil
}

// ----- map[int]any -----

func encodeMapIntAny(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[int]any)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []int64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysI64[:0]
                        canonPooled = true
                } else {
                        keys = make([]int64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, int64(k))
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysI64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        k := int(sk)
                        v := m[k]
                        e.WriteInt(int64(k))
                        if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                                return err
                        }
                }
                return nil
        }
        for k, v := range m {
                e.WriteInt(int64(k))
                if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                        return err
                }
        }
        return nil
}

func decodeMapIntAny(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[int]any)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[int, any](d, p, n)
        for range n {
                kWide, err := d.ReadInt()
                if err != nil {
                        return err
                }
                k := int(kWide)
                v, err := decodeAny(d)
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[int]any)(p) = m
        return nil
}

// ----- map[int64]string -----

func encodeMapInt64String(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[int64]string)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []int64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysI64[:0]
                        canonPooled = true
                } else {
                        keys = make([]int64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysI64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteInt(int64(sk))
                        e.WriteString(v)
                }
                return nil
        }
        for k, v := range m {
                e.WriteInt(int64(k))
                e.WriteString(v)
        }
        return nil
}

func decodeMapInt64String(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[int64]string)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[int64, string](d, p, n)
        for range n {
                k, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v, err := d.ReadString()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[int64]string)(p) = m
        return nil
}

// ----- map[int64]int64 -----

func encodeMapInt64Int64(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[int64]int64)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []int64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysI64[:0]
                        canonPooled = true
                } else {
                        keys = make([]int64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysI64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteInt(int64(sk))
                        e.WriteInt(int64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteInt(int64(k))
                e.WriteInt(int64(v))
        }
        return nil
}

func decodeMapInt64Int64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[int64]int64)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[int64, int64](d, p, n)
        for range n {
                k, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[int64]int64)(p) = m
        return nil
}

// ----- map[int64]any -----

func encodeMapInt64Any(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[int64]any)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []int64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysI64[:0]
                        canonPooled = true
                } else {
                        keys = make([]int64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysI64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteInt(int64(sk))
                        if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                                return err
                        }
                }
                return nil
        }
        for k, v := range m {
                e.WriteInt(int64(k))
                if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                        return err
                }
        }
        return nil
}

func decodeMapInt64Any(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[int64]any)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[int64, any](d, p, n)
        for range n {
                k, err := d.ReadInt()
                if err != nil {
                        return err
                }
                v, err := decodeAny(d)
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[int64]any)(p) = m
        return nil
}

// ----- map[uint64]string -----

func encodeMapUint64String(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[uint64]string)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []uint64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysU64[:0]
                        canonPooled = true
                } else {
                        keys = make([]uint64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysU64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteUint(uint64(sk))
                        e.WriteString(v)
                }
                return nil
        }
        for k, v := range m {
                e.WriteUint(uint64(k))
                e.WriteString(v)
        }
        return nil
}

func decodeMapUint64String(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[uint64]string)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[uint64, string](d, p, n)
        for range n {
                k, err := d.ReadUint()
                if err != nil {
                        return err
                }
                v, err := d.ReadString()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[uint64]string)(p) = m
        return nil
}

// ----- map[uint64]uint64 -----

func encodeMapUint64Uint64(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[uint64]uint64)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []uint64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysU64[:0]
                        canonPooled = true
                } else {
                        keys = make([]uint64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysU64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteUint(uint64(sk))
                        e.WriteUint(uint64(v))
                }
                return nil
        }
        for k, v := range m {
                e.WriteUint(uint64(k))
                e.WriteUint(uint64(v))
        }
        return nil
}

func decodeMapUint64Uint64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[uint64]uint64)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[uint64, uint64](d, p, n)
        for range n {
                k, err := d.ReadUint()
                if err != nil {
                        return err
                }
                v, err := d.ReadUint()
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[uint64]uint64)(p) = m
        return nil
}

// ----- map[uint64]any -----

func encodeMapUint64Any(e *Encoder, p unsafe.Pointer) error {
        m := *(*map[uint64]any)(p)
        if m == nil {
                e.WriteNil()
                return nil
        }
        e.WriteMapHeader(len(m))
        if e.opts.Has(OptCanonical) {
                var keys []uint64
                canonPooled := false
                if e.state != nil && !e.state.canonKeysBusy {
                        keys = e.state.canonKeysU64[:0]
                        canonPooled = true
                } else {
                        keys = make([]uint64, 0, len(m))
                }
                for k := range m {
                        keys = append(keys, k)
                }
                slices.Sort(keys)
                if canonPooled {
                        e.state.canonKeysU64 = keys
                        e.state.canonKeysBusy = true
                        defer func() { e.state.canonKeysBusy = false }()
                }
                for _, sk := range keys {
                        v := m[sk]
                        e.WriteUint(uint64(sk))
                        if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                                return err
                        }
                }
                return nil
        }
        for k, v := range m {
                e.WriteUint(uint64(k))
                if err := encodeIface(e, unsafe.Pointer(&v)); err != nil {
                        return err
                }
        }
        return nil
}

func decodeMapUint64Any(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagNil {
                d.i++
                *(*map[uint64]any)(p) = nil
                return nil
        }
        n, err := d.ReadMapHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 2); err != nil {
                return err
        }
        m := reuseOrMakeMap[uint64, any](d, p, n)
        for range n {
                k, err := d.ReadUint()
                if err != nil {
                        return err
                }
                v, err := decodeAny(d)
                if err != nil {
                        return err
                }
                m[k] = v
        }
        *(*map[uint64]any)(p) = m
        return nil
}

// installMapFastPath returns (encode, decode, true) when t matches
// one of the generated typed-map shapes; otherwise (_, _, false). The
// table is generated; edit internal/mapsgen/main.go to add or remove
// pairs.
func installMapFastPath(t reflect.Type) (
        enc func(*Encoder, unsafe.Pointer) error,
        dec func(*Decoder, unsafe.Pointer) error,
        ok bool,
) {
        switch t {
        case reflect.TypeFor[map[string]string]():
                return encodeMapStringString, decodeMapStringString, true
        case reflect.TypeFor[map[string]bool]():
                return encodeMapStringBool, decodeMapStringBool, true
        case reflect.TypeFor[map[string]int8]():
                return encodeMapStringInt8, decodeMapStringInt8, true
        case reflect.TypeFor[map[string]int16]():
                return encodeMapStringInt16, decodeMapStringInt16, true
        case reflect.TypeFor[map[string]int32]():
                return encodeMapStringInt32, decodeMapStringInt32, true
        case reflect.TypeFor[map[string]int]():
                return encodeMapStringInt, decodeMapStringInt, true
        case reflect.TypeFor[map[string]int64]():
                return encodeMapStringInt64, decodeMapStringInt64, true
        case reflect.TypeFor[map[string]uint8]():
                return encodeMapStringUint8, decodeMapStringUint8, true
        case reflect.TypeFor[map[string]uint16]():
                return encodeMapStringUint16, decodeMapStringUint16, true
        case reflect.TypeFor[map[string]uint32]():
                return encodeMapStringUint32, decodeMapStringUint32, true
        case reflect.TypeFor[map[string]uint]():
                return encodeMapStringUint, decodeMapStringUint, true
        case reflect.TypeFor[map[string]uint64]():
                return encodeMapStringUint64, decodeMapStringUint64, true
        case reflect.TypeFor[map[string]float32]():
                return encodeMapStringFloat32, decodeMapStringFloat32, true
        case reflect.TypeFor[map[string]float64]():
                return encodeMapStringFloat64, decodeMapStringFloat64, true
        case reflect.TypeFor[map[string][]byte]():
                return encodeMapStringBytes, decodeMapStringBytes, true
        case reflect.TypeFor[map[string][]string]():
                return encodeMapStringStringSlice, decodeMapStringStringSlice, true
        case reflect.TypeFor[map[string]any]():
                return encodeMapStringAny, decodeMapStringAny, true
        case reflect.TypeFor[map[int]string]():
                return encodeMapIntString, decodeMapIntString, true
        case reflect.TypeFor[map[int]int]():
                return encodeMapIntInt, decodeMapIntInt, true
        case reflect.TypeFor[map[int]int64]():
                return encodeMapIntInt64, decodeMapIntInt64, true
        case reflect.TypeFor[map[int]any]():
                return encodeMapIntAny, decodeMapIntAny, true
        case reflect.TypeFor[map[int64]string]():
                return encodeMapInt64String, decodeMapInt64String, true
        case reflect.TypeFor[map[int64]int64]():
                return encodeMapInt64Int64, decodeMapInt64Int64, true
        case reflect.TypeFor[map[int64]any]():
                return encodeMapInt64Any, decodeMapInt64Any, true
        case reflect.TypeFor[map[uint64]string]():
                return encodeMapUint64String, decodeMapUint64String, true
        case reflect.TypeFor[map[uint64]uint64]():
                return encodeMapUint64Uint64, decodeMapUint64Uint64, true
        case reflect.TypeFor[map[uint64]any]():
                return encodeMapUint64Any, decodeMapUint64Any, true
        }
        return nil, nil, false
}

package qdf

// Marshaler is implemented by types that know how to serialize themselves
// into the QDF wire format. Implementations should append exactly one
// value to dst and return the extended slice.
type Marshaler interface {
        MarshalQDF(dst []byte) ([]byte, error)
}

// EncoderMarshaler is an optional extension of Marshaler that writes directly
// into a shared Encoder. The buffer-based MarshalQDF forces a nested value to
// build its own Encoder on the parent's bytes (one *Encoder heap allocation per
// nested value); EncodeQDF lets a parent thread one Encoder through the whole
// graph. Generated code (cmd/qdfgen) implements it; EncodeNested prefers it.
type EncoderMarshaler interface {
        Marshaler
        EncodeQDF(e *Encoder) error
}

// EncodeNested encodes a nested Marshaler m into the shared encoder e. When m
// implements EncoderMarshaler it writes directly (no new Encoder); otherwise it
// falls back to the buffer-based MarshalQDF + AdoptBuffer. Exported for
// cmd/qdfgen-generated code.
func EncodeNested(e *Encoder, m Marshaler) error {
        if em, ok := m.(EncoderMarshaler); ok {
                return em.EncodeQDF(e)
        }
        b, err := m.MarshalQDF(e.Bytes())
        if err != nil {
                return err
        }
        e.AdoptBuffer(b)
        e.MarkHeaderWritten()
        return nil
}

// DecoderUnmarshaler is the decode counterpart of EncoderMarshaler: it reads its
// value from a SHARED Decoder, advancing it, so a parent can thread one decoder
// through a whole value graph instead of opening one per nested value (one
// *Decoder plus its scratch / intern state per nested value otherwise).
// Generated code (cmd/qdfgen) implements it; DecodeNested prefers it. noCopy and
// arena live on the shared decoder, so a threaded nested decode inherits them
// with no extra arguments.
type DecoderUnmarshaler interface {
        Unmarshaler
        DecodeQDF(d *Decoder) error
}

// DecodeNested decodes one nested value from the shared decoder d. When u
// implements DecoderUnmarshaler it reads directly from d (no new decoder, and it
// inherits d's noCopy / arena). Otherwise it falls back to the buffer-based
// UnmarshalQDF over d's remaining bytes — honoring d's noCopy / arena via the
// Opts / Arena extensions — and advances d by the bytes consumed. Exported for
// cmd/qdfgen-generated code.
func DecodeNested(d *Decoder, u Unmarshaler) error {
        if du, ok := u.(DecoderUnmarshaler); ok {
                return du.DecodeQDF(d)
        }
        src := d.RemainingBytes()
        var n int
        var err error
        switch {
        case d.arena != nil:
                if ua, ok := u.(UnmarshalerArena); ok {
                        n, err = ua.UnmarshalQDFArena(src, d.noCopy, d.arena)
                        break
                }
                if uo, ok := u.(UnmarshalerOpts); ok && d.noCopy {
                        n, err = uo.UnmarshalQDFOpts(src, true)
                } else {
                        n, err = u.UnmarshalQDF(src)
                }
        default:
                if uo, ok := u.(UnmarshalerOpts); ok && d.noCopy {
                        n, err = uo.UnmarshalQDFOpts(src, true)
                } else {
                        n, err = u.UnmarshalQDF(src)
                }
        }
        if err != nil {
                return err
        }
        // Guard the shared cursor against a misbehaving Unmarshaler (mirrors
        // UnmarshalNested): a count past the remaining bytes would push the cursor
        // out of bounds and panic the next read.
        if n < 0 || n > d.Remaining() {
                return ErrShortBuffer
        }
        d.Advance(n)
        return nil
}

// Unmarshaler is implemented by types that know how to deserialize themselves
// from a QDF wire-format slice. Implementations should consume exactly one
// value from src and return the number of bytes consumed.
//
// A custom UnmarshalQDF reads the Fast wire format. Marshal honours this: a
// type implementing Marshaler always emits its Fast-format body and is framed
// as Fast regardless of the Options passed, so Marshaler+Unmarshaler types
// round-trip under any Options. A type that implements Unmarshaler WITHOUT
// also implementing Marshaler is encoded structurally; under a Dense/QPack
// tier that produces a Dense body its Fast-only UnmarshalQDF cannot read.
// Implement both interfaces (or neither) to avoid this — generated code from
// cmd/qdfgen always implements both.
type Unmarshaler interface {
        UnmarshalQDF(src []byte) (n int, err error)
}

// UnmarshalerOpts is an optional extension of Unmarshaler that accepts the
// noCopy flag. When noCopy is true, the implementation should decode string and
// []byte fields as aliases of src (see WithNoCopy) instead of copying. Generated
// code from cmd/qdfgen implements this; the plain UnmarshalQDF delegates to it
// with noCopy=false. Decoders honor it only when the caller opted into noCopy.
type UnmarshalerOpts interface {
        Unmarshaler
        UnmarshalQDFOpts(src []byte, noCopy bool) (n int, err error)
}

// UnmarshalerArena is an optional extension that also accepts a decode Arena, so
// the implementation packs copied string/[]byte fields into it (see WithArena)
// instead of one allocation per field. Generated code from cmd/qdfgen implements
// this; UnmarshalQDFOpts delegates to it with a nil arena. Decoders honor it only
// when the caller passed an arena.
type UnmarshalerArena interface {
        UnmarshalerOpts
        UnmarshalQDFArena(src []byte, noCopy bool, a *Arena) (n int, err error)
}

// UnmarshalNestedArena is UnmarshalNested threading a decode arena: when a is
// non-nil and u implements UnmarshalerArena, the nested decode packs its strings
// into a. Otherwise it falls back to the plain (copying) nested decode.
// Exported for cmd/qdfgen-generated code.
func UnmarshalNestedArena(u Unmarshaler, src []byte, noCopy bool, a *Arena) (int, error) {
        if a != nil {
                if ua, ok := u.(UnmarshalerArena); ok {
                        n, err := ua.UnmarshalQDFArena(src, noCopy, a)
                        if err != nil {
                                return n, err
                        }
                        if n < 0 || n > len(src) {
                                return 0, ErrShortBuffer
                        }
                        return n, nil
                }
        }
        return UnmarshalNested(u, src, noCopy)
}

// UnmarshalNested decodes one nested Unmarshaler value from src, honoring noCopy
// when u also implements UnmarshalerOpts. External Unmarshalers without the Opts
// method fall back to a copying decode. Used by decodeUnmarshaler and by
// cmd/qdfgen-generated code; exported for the latter.
func UnmarshalNested(u Unmarshaler, src []byte, noCopy bool) (int, error) {
        var n int
        var err error
        if uo, ok := u.(UnmarshalerOpts); ok && noCopy {
                n, err = uo.UnmarshalQDFOpts(src, true)
        } else {
                n, err = u.UnmarshalQDF(src)
        }
        if err != nil {
                return n, err
        }
        // Guard the parent cursor against a misbehaving Unmarshaler. Both the
        // reflect path (decodeUnmarshaler) and the generated code advance the
        // parent decoder by the returned count; a count larger than the nested
        // buffer (or negative) would push the cursor out of bounds and panic the
        // next read. Reject it as a short/invalid buffer instead.
        if n < 0 || n > len(src) {
                return 0, ErrShortBuffer
        }
        return n, nil
}

package qdf

import (
        "math"
        "math/bits"
        "reflect"
        "runtime"
        "time"
        "unsafe"
)

// Nullable (optional) columns for the columnar container. A `*T` struct
// field — where T is a scalar (int*/uint*/float*/bool) — would otherwise
// force the whole struct to the row-major fallback, losing the columnar
// codecs on every sibling column. Instead it is stored as a presence
// bitmap (1 bit per row, LSB-first) followed by a dense column of only the
// present values, encoded with T's normal slice codec. The nullable flag
// rides in the columnar shape's kind byte (colKindNullable), so there is no
// separate wire tag; the column slot is `ceil(M/8) mask bytes, <dense
// column>`.

// writeStringColumn emits a gathered string column either as a dictionary
// (when the never-worse gate accepts it) or as M per-value strings. Shared
// by the regular string column path; factored out so nullable columns could
// reuse it (nullable string columns are a follow-up).
func (e *Encoder) writeStringColumn(strs []string) {
        if e.tryWriteStringColumnDict(strs) {
                return
        }
        // FSST runs first (when enabled): on substring-sharing text (URLs, paths, log
        // lines) it beats positional packing, and it declines (never-larger) on random
        // restricted-alphabet IDs — no shared substrings — so those fall through to
        // alphabet-packing next. Alpha-packing sits before raw: on a restricted-
        // alphabet high-cardinality column (hex/base32/base64/decimal IDs) it stores
        // ceil(log2 |A|) bits/char instead of 8. Its own never-larger gate declines
        // (cheaply, after a one-pass alphabet scan that bails the moment |A| exceeds
        // 64) on everything else, so non-ID columns keep their existing form with no
        // wire/CPU regression.
        if e.fsst && e.tryWriteStringColumnFSST(strs) {
                return
        }
        if e.tryWriteStringColumnAlpha(strs) {
                return
        }
        if e.tryWriteStringColumnRaw(strs) {
                return
        }
        // Without OptDense (the codegen/Fast path) the per-value fallback below does
        // NOT intern: WriteString emits every occurrence inline, so the dict/raw
        // never-larger gates — which model the per-value cost as the Dense interned
        // form (distinct once + 1-byte state refs) — under-estimate it and decline,
        // leaving the column wire-bloated and decoding to n string allocations.
        // Mode-aware fallback: a single-distinct column collapses to one value
        // (tagColStrConst), any other column materializes in ONE slab allocation
        // (tagColStrRaw, wire-neutral + a tiny header). The reflect path (OptDense)
        // keeps the interning per-value form, which the gates model correctly — so
        // its wire is unchanged (const/raw-forced are codegen-only; the decoders
        // still read them for cross-path interop).
        //
        // An EMPTY column (len==0, e.g. an all-nil nullable string column's dense
        // part) must emit NOTHING, exactly as the plain loop below does and as the
        // reflect/Dense path does: writeStringColumnRawForced would otherwise lay
        // down a tagColStrRaw block that ReadStringColumn(0) skips (its block-tag
        // dispatch is guarded by n>0), desyncing the decode cursor.
        if !e.opts.Has(OptDense) && len(strs) > 0 {
                if e.tryWriteStringColumnConst(strs) {
                        return
                }
                e.writeStringColumnRawForced(strs)
                return
        }
        for _, v := range strs {
                e.WriteString(v)
        }
}

func loadI64At(p unsafe.Pointer, width uintptr) int64 {
        switch width {
        case 1:
                return int64(*(*int8)(p))
        case 2:
                return int64(*(*int16)(p))
        case 4:
                return int64(*(*int32)(p))
        default:
                return *(*int64)(p)
        }
}

func loadU64At(p unsafe.Pointer, width uintptr) uint64 {
        switch width {
        case 1:
                return uint64(*(*uint8)(p))
        case 2:
                return uint64(*(*uint16)(p))
        case 4:
                return uint64(*(*uint32)(p))
        default:
                return *(*uint64)(p)
        }
}

// loadF64At reads a nullable float64 column value. colKindFloat is float64-only
// (a *float32 field classifies as colKindFloat32, which moves raw bits via
// loadU64At at width 4), so width is always 8 here and needs no switch.
func loadF64At(p unsafe.Pointer, _ uintptr) float64 {
        return *(*float64)(p)
}

func storeI64At(p unsafe.Pointer, width uintptr, v int64) {
        switch width {
        case 1:
                *(*int8)(p) = int8(v)
        case 2:
                *(*int16)(p) = int16(v)
        case 4:
                *(*int32)(p) = int32(v)
        default:
                *(*int64)(p) = v
        }
}

func storeU64At(p unsafe.Pointer, width uintptr, v uint64) {
        switch width {
        case 1:
                *(*uint8)(p) = uint8(v)
        case 2:
                *(*uint16)(p) = uint16(v)
        case 4:
                *(*uint32)(p) = uint32(v)
        default:
                *(*uint64)(p) = v
        }
}

// storeF64At writes a nullable float64 column value. See loadF64At: colKindFloat
// is float64-only, so width is always 8.
func storeF64At(p unsafe.Pointer, _ uintptr, v float64) {
        *(*float64)(p) = v
}

// encodeNullableColumn writes the presence bitmap and the dense present-only
// column for a `*T` field.
func (e *Encoder) encodeNullableColumn(base unsafe.Pointer, plan *columnarPlan, col *colColumn, n int) error {
        st := e.state
        maskBytes := (n + 7) >> 3
        var mask []byte
        if cap(st.colMaskScratch) >= maskBytes {
                mask = st.colMaskScratch[:maskBytes]
                clear(mask)
        } else {
                mask = make([]byte, maskBytes)
        }
        st.colMaskScratch = mask
        stride, off := plan.stride, col.offset

        switch col.kind.base() {
        case colKindInt:
                s := st.colScratchI64[:0]
                for i := range n {
                        pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*stride+off))
                        if pp != nil {
                                mask[i>>3] |= 1 << uint(i&7)
                                s = append(s, loadI64At(pp, col.width))
                        }
                }
                st.colScratchI64 = s
                e.buf = append(e.buf, mask...)
                return encodeSliceInt64(e, unsafe.Pointer(&s))
        case colKindUint:
                s := st.colScratchU64[:0]
                for i := range n {
                        pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*stride+off))
                        if pp != nil {
                                mask[i>>3] |= 1 << uint(i&7)
                                s = append(s, loadU64At(pp, col.width))
                        }
                }
                st.colScratchU64 = s
                e.buf = append(e.buf, mask...)
                return encodeSliceUint64(e, unsafe.Pointer(&s))
        case colKindFloat32:
                // float32: loadU64At(width==4) reads *(*uint32) — the raw f32 bits — so
                // the uint codec carries them losslessly (NaN payloads survive).
                s := st.colScratchU64[:0]
                canon := e.opts.Has(OptCanonical)
                for i := range n {
                        pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*stride+off))
                        if pp != nil {
                                mask[i>>3] |= 1 << uint(i&7)
                                bits := loadU64At(pp, col.width)
                                if canon {
                                        bits = canonicalizeFloat32Bits(bits)
                                }
                                s = append(s, bits)
                        }
                }
                st.colScratchU64 = s
                e.buf = append(e.buf, mask...)
                return encodeSliceUint64(e, unsafe.Pointer(&s))
        case colKindFloat:
                s := st.colScratchF64[:0]
                for i := range n {
                        pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*stride+off))
                        if pp != nil {
                                mask[i>>3] |= 1 << uint(i&7)
                                s = append(s, loadF64At(pp, col.width))
                        }
                }
                st.colScratchF64 = s
                e.buf = append(e.buf, mask...)
                // Lossless: a SCALAR *float64 column must never become lossy under
                // OptLossyVec (which targets genuine []float64/[]float32 VECTOR fields).
                return encodeSliceFloat64Lossless(e, s)
        case colKindBool:
                s := st.colScratchBool[:0]
                for i := range n {
                        pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*stride+off))
                        if pp != nil {
                                mask[i>>3] |= 1 << uint(i&7)
                                s = append(s, *(*bool)(pp))
                        }
                }
                st.colScratchBool = s
                e.buf = append(e.buf, mask...)
                return encodeSliceBool(e, unsafe.Pointer(&s))
        case colKindString:
                s := st.colScratchStr[:0]
                for i := range n {
                        pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*stride+off))
                        if pp != nil {
                                mask[i>>3] |= 1 << uint(i&7)
                                s = append(s, *(*string)(pp))
                        }
                }
                st.colScratchStr = s
                e.buf = append(e.buf, mask...)
                e.writeStringColumn(s)
                return nil
        case colKindTime:
                // Gather present *time.Time values into sec+nsec dense sub-columns,
                // mirroring the non-nullable colKindTime encoder in encodeColumnar.
                sec := st.colScratchI64[:0]
                nsec := st.colScratchU64[:0]
                for i := range n {
                        pp := *(*unsafe.Pointer)(unsafe.Add(base, uintptr(i)*stride+off))
                        if pp != nil {
                                mask[i>>3] |= 1 << uint(i&7)
                                t := (*time.Time)(pp).UTC()
                                sec = append(sec, t.Unix())
                                nsec = append(nsec, uint64(t.Nanosecond()))
                        }
                }
                st.colScratchI64 = sec
                st.colScratchU64 = nsec
                e.buf = append(e.buf, mask...)
                if err := encodeSliceInt64(e, unsafe.Pointer(&sec)); err != nil {
                        return err
                }
                return encodeSliceUint64(e, unsafe.Pointer(&nsec))
        }
        return ErrBadTag
}

// readNullableMask consumes the presence bitmap and returns it plus the
// present count (popcount). The caller decodes the dense column next.
func (d *Decoder) readNullableMask(n int) (mask []byte, present int, err error) {
        maskBytes := (n + 7) >> 3
        if d.i+maskBytes > len(d.buf) {
                return nil, 0, ErrShortBuffer
        }
        mask = d.buf[d.i : d.i+maskBytes]
        d.i += maskBytes
        i := 0
        for ; i+8 <= maskBytes; i += 8 {
                present += bits.OnesCount64(*(*uint64)(unsafe.Pointer(&mask[i])))
        }
        for ; i < maskBytes; i++ {
                present += bits.OnesCount8(mask[i])
        }
        // Reject a mangled bitmap. A valid encoder only sets bits for present rows
        // in [0,n), so the last byte's padding bits (positions [n%8, 8)) are always
        // zero. A hostile mask can set a padding bit while under-filling the in-range
        // bits, keeping the popcount <= n (so the present>n check alone passes) yet
        // making the scatter loop — which consumes only in-range set bits — drop the
        // trailing dense value(s) silently. Reject any set padding bit so the frame
        // errors instead. Mirrors the codegen sibling ReadColNullMask.
        if n&7 != 0 && mask[maskBytes-1]>>uint(n&7) != 0 {
                return nil, 0, ErrInvalidLength
        }
        if present > n {
                return nil, 0, ErrInvalidLength
        }
        return mask, present, nil
}

// decodeNullableColumn reads the mask + dense column and scatters the present
// values back into the `*T` field, allocating all present values in a single
// backing slice that the field pointers reference into.
//
// The int64/uint64/float64/bool scratch fields are reused here too: the dense
// column values are decoded into d.state.colScratch*, used only inside the
// inline set() call, then the scratch is eligible for reuse by the next column.
func (d *Decoder) decodeNullableColumn(base unsafe.Pointer, plan *columnarPlan, col *colColumn, n int) error {
        mask, present, err := d.readNullableMask(n)
        if err != nil {
                return err
        }
        // String columns do not use the typed backing slice — handle them first so
        // the present*sizeof(string) allocation below is not made and discarded, and
        // add the same len(strs)==present guard every other kind has.
        if col.kind.base() == colKindString {
                // This path stores &strs[k] into *string fields, so strs must be a stable
                // owned slice — not the pooled scratch a later column would overwrite.
                d.colStrNoPool = true
                strs, err := d.readStringColumn(present)
                d.colStrNoPool = false
                if err != nil {
                        return err
                }
                if len(strs) != present {
                        return ErrTypeMismatch
                }
                stride, off := plan.stride, col.offset
                k := 0
                for i := range n {
                        fp := unsafe.Add(base, uintptr(i)*stride+off)
                        if mask[i>>3]&(1<<uint(i&7)) != 0 {
                                *(*unsafe.Pointer)(fp) = unsafe.Pointer(&strs[k])
                                k++
                        } else {
                                *(*unsafe.Pointer)(fp) = nil
                        }
                }
                runtime.KeepAlive(strs)
                return nil
        }
        elemSize := col.elemType.Size()
        // Bound the typed backing by the columnar byte ceiling: the outer decode
        // check used the struct stride (8 for a *T field), but elemType.Size() can be
        // larger (time.Time is 24), so present*elemSize can exceed maxColumnarBytes.
        if err := checkColumnarBytes(present, elemSize); err != nil {
                return err
        }
        backing := reflect.MakeSlice(reflect.SliceOf(col.elemType), present, present)
        dataPtr := backing.UnsafePointer()
        stride, off := plan.stride, col.offset
        st := d.state // always non-nil: readColShape initialises it before the loop
        k := 0
        set := func(store func(ea unsafe.Pointer, k int)) {
                for i := range n {
                        fp := unsafe.Add(base, uintptr(i)*stride+off)
                        if mask[i>>3]&(1<<uint(i&7)) != 0 {
                                ea := unsafe.Add(dataPtr, uintptr(k)*elemSize)
                                store(ea, k)
                                *(*unsafe.Pointer)(fp) = ea
                                k++
                        } else {
                                *(*unsafe.Pointer)(fp) = nil
                        }
                }
        }
        switch col.kind.base() {
        case colKindInt:
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return err
                }
                s := st.colScratchI64
                if len(s) != present {
                        return ErrTypeMismatch
                }
                set(func(ea unsafe.Pointer, k int) { storeI64At(ea, col.width, s[k]) })
        case colKindUint:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != present {
                        return ErrTypeMismatch
                }
                set(func(ea unsafe.Pointer, k int) { storeU64At(ea, col.width, s[k]) })
        case colKindFloat32:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                s := st.colScratchU64
                if len(s) != present {
                        return ErrTypeMismatch
                }
                set(func(ea unsafe.Pointer, k int) { storeU64At(ea, col.width, s[k]) })
        case colKindFloat:
                if err := decodeSliceFloat64Into(d, &st.colScratchF64); err != nil {
                        return err
                }
                s := st.colScratchF64
                if len(s) != present {
                        return ErrTypeMismatch
                }
                set(func(ea unsafe.Pointer, k int) { storeF64At(ea, col.width, s[k]) })
        case colKindBool:
                if err := decodeSliceBoolInto(d, &st.colScratchBool); err != nil {
                        return err
                }
                s := st.colScratchBool
                if len(s) != present {
                        return ErrTypeMismatch
                }
                set(func(ea unsafe.Pointer, k int) { *(*bool)(ea) = s[k] })
        case colKindTime:
                // Decode two dense sub-columns (sec []int64, nsec []uint64) for the
                // present count, reconstruct time.Time values, scatter into *time.Time
                // fields using the shared backing slice.
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return err
                }
                sec := st.colScratchI64
                if len(sec) != present {
                        return ErrTypeMismatch
                }
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return err
                }
                nsec := st.colScratchU64
                if len(nsec) != present {
                        return ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return err
                }
                set(func(ea unsafe.Pointer, k int) {
                        *(*time.Time)(ea) = time.Unix(sec[k], int64(nsec[k])).UTC()
                })
        default:
                return ErrBadTag
        }
        runtime.KeepAlive(backing)
        return nil
}

// decodeNullableColumnVals decodes an optional (*T) column like
// decodeNullableColumn (presence mask via readNullableMask, then a dense
// present-only base-kind column), retaining the values expanded to length n
// with cv.present marking the non-nil rows, for later filter + compact.
func (d *Decoder) decodeNullableColumnVals(kind colKind, n int) (colVals, error) {
        var cv colVals
        cv.kind = kind
        mask, present, err := d.readNullableMask(n)
        if err != nil {
                return cv, err
        }
        pres := newBitset(n)
        maskBytes := (n + 7) >> 3
        // mask (LSB-first within byte) and pres (LSB-first within uint64 word) are
        // binary-identical on little-endian: row i lives at bit i&7 of byte i>>3 and
        // at bit i&63 of word i>>6 — the same bit position in the same 8-byte group.
        copy(unsafe.Slice((*byte)(unsafe.Pointer(&pres[0])), maskBytes), mask[:maskBytes])
        cv.present = pres

        // Bound the full-row backing alloc by bytes (max elem = 16, a string header),
        // matching the main decodeNullableColumn path's checkColumnarBytes guard so a
        // compressed row count cannot drive an oversized make([]T, n).
        if err := checkColumnarBytes(n, 16); err != nil {
                return cv, err
        }

        // Decode each dense present-only base column through the pooled colScratch*
        // buffers (mirrors decodeNullableColumn) instead of a fresh per-call slice:
        // s is consumed synchronously into the retained `full` before the next
        // column reuses the scratch, so there is no retained alias.
        st := d.state // non-nil: readColShape initialises it before this decode
        switch kind.base() {
        case colKindInt:
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return cv, err
                }
                s := st.colScratchI64
                if len(s) != present {
                        return cv, ErrTypeMismatch
                }
                full := make([]int64, n)
                k := 0
                for i := range n {
                        if getBit(pres, i) {
                                full[i] = s[k]
                                k++
                        }
                }
                cv.i64 = full
        case colKindUint:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return cv, err
                }
                s := st.colScratchU64
                if len(s) != present {
                        return cv, ErrTypeMismatch
                }
                full := make([]uint64, n)
                k := 0
                for i := range n {
                        if getBit(pres, i) {
                                full[i] = s[k]
                                k++
                        }
                }
                cv.u64 = full
        case colKindFloat32:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return cv, err
                }
                s := st.colScratchU64
                if len(s) != present {
                        return cv, ErrTypeMismatch
                }
                full := make([]uint64, n)
                k := 0
                for i := range n {
                        if getBit(pres, i) {
                                full[i] = s[k]
                                k++
                        }
                }
                cv.u64 = full
        case colKindFloat:
                if err := decodeSliceFloat64Into(d, &st.colScratchF64); err != nil {
                        return cv, err
                }
                s := st.colScratchF64
                if len(s) != present {
                        return cv, ErrTypeMismatch
                }
                full := make([]float64, n)
                k := 0
                for i := range n {
                        if getBit(pres, i) {
                                full[i] = s[k]
                                k++
                        }
                }
                cv.f64 = full
        case colKindBool:
                if err := decodeSliceBoolInto(d, &st.colScratchBool); err != nil {
                        return cv, err
                }
                s := st.colScratchBool
                if len(s) != present {
                        return cv, ErrTypeMismatch
                }
                full := make([]bool, n)
                k := 0
                for i := range n {
                        if getBit(pres, i) {
                                full[i] = s[k]
                                k++
                        }
                }
                cv.b = full
        case colKindString:
                strs, err := d.readStringColumn(present)
                if err != nil {
                        return cv, err
                }
                if len(strs) != present {
                        return cv, ErrTypeMismatch
                }
                full := make([]string, n)
                k := 0
                for i := range n {
                        if getBit(pres, i) {
                                full[i] = strs[k]
                                k++
                        }
                }
                cv.s = full
        case colKindTime:
                // Reuse the pooled colScratch buffers (sec/nsec are consumed synchronously
                // into `full` below before return, like the other kinds above and the
                // non-query decodeNullableColumn sibling) instead of two fresh slices.
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return cv, err
                }
                sec := st.colScratchI64
                if len(sec) != present {
                        return cv, ErrTypeMismatch
                }
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return cv, err
                }
                nsec := st.colScratchU64
                if len(nsec) != present {
                        return cv, ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return cv, err
                }
                // time.Time is 24 bytes; the outer check used the struct stride (8 for a
                // *time.Time field), so n*24 can exceed the columnar byte ceiling.
                if err := checkColumnarBytes(n, unsafe.Sizeof(time.Time{})); err != nil {
                        return cv, err
                }
                full := make([]time.Time, n)
                k := 0
                for i := range n {
                        if getBit(pres, i) {
                                full[i] = time.Unix(sec[k], int64(nsec[k])).UTC()
                                k++
                        }
                }
                cv.ts = full
        default:
                return cv, ErrBadTag
        }
        return cv, nil
}

// scatterNullableRowInto writes cv's optional value at source row src into the
// *T field at compacted row dst. A present value is placed in the caller's
// backing slab at slab+dst*elemSize and the field points into it; an absent
// value leaves the pointer nil. One slab per column replaces the former per-row
// reflect.New (a wide nullable projection's 8k+ allocs collapse to ~one per
// column). The caller MakeSlices the slab and keeps it alive (runtime.KeepAlive)
// until every field pointer has been written.
func (cv *colVals) scatterNullableRowInto(base unsafe.Pointer, plan *columnarPlan, col *colColumn, src, dst int, slab unsafe.Pointer, elemSize uintptr) {
        fp := unsafe.Add(base, uintptr(dst)*plan.stride+col.offset)
        if !getBit(cv.present, src) {
                *(*unsafe.Pointer)(fp) = nil
                return
        }
        ea := unsafe.Add(slab, uintptr(dst)*elemSize) // &slab[dst]
        switch cv.kind.base() {
        case colKindInt:
                storeI64At(ea, col.width, cv.i64[src])
        case colKindUint:
                storeU64At(ea, col.width, cv.u64[src])
        case colKindFloat32:
                storeU64At(ea, col.width, cv.u64[src]) // width==4 ⇒ writes *(*uint32), the f32 bits
        case colKindFloat:
                storeF64At(ea, col.width, cv.f64[src])
        case colKindBool:
                *(*bool)(ea) = cv.b[src]
        case colKindString:
                *(*string)(ea) = cv.s[src]
        case colKindTime:
                *(*time.Time)(ea) = cv.ts[src]
        }
        *(*unsafe.Pointer)(fp) = ea
}

// decodeNullableColumnAny reads the mask + dense column and returns one boxed
// value per row (nil for absent), for the map[string]any decode path.
func (d *Decoder) decodeNullableColumnAny(kind colKind, n int) ([]any, error) {
        mask, present, err := d.readNullableMask(n)
        if err != nil {
                return nil, err
        }
        out := make([]any, n)
        k := 0
        scatter := func(box func(i, k int)) {
                for i := range n {
                        if mask[i>>3]&(1<<uint(i&7)) != 0 {
                                box(i, k)
                                k++
                        }
                }
        }
        // POD dense sub-columns decode into the pooled d.state scratch (like
        // decodeColumnVals): the values are boxed into out by value inside scatter,
        // so reusing the backing across calls carries no aliasing hazard. String
        // and time stay on their own materialization paths.
        st := d.state
        switch kind.base() {
        case colKindInt:
                if err := decodeSliceInt64Into(d, &st.colScratchI64); err != nil {
                        return nil, err
                }
                s := st.colScratchI64
                if len(s) != present {
                        return nil, ErrTypeMismatch
                }
                scatter(func(i, k int) { out[i] = s[k] })
        case colKindUint:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return nil, err
                }
                s := st.colScratchU64
                if len(s) != present {
                        return nil, ErrTypeMismatch
                }
                scatter(func(i, k int) { out[i] = s[k] })
        case colKindFloat32:
                if err := decodeSliceUint64Into(d, &st.colScratchU64); err != nil {
                        return nil, err
                }
                s := st.colScratchU64
                if len(s) != present {
                        return nil, ErrTypeMismatch
                }
                scatter(func(i, k int) { out[i] = math.Float32frombits(uint32(s[k])) })
        case colKindFloat:
                if err := decodeSliceFloat64Into(d, &st.colScratchF64); err != nil {
                        return nil, err
                }
                s := st.colScratchF64
                if len(s) != present {
                        return nil, ErrTypeMismatch
                }
                scatter(func(i, k int) { out[i] = s[k] })
        case colKindBool:
                if err := decodeSliceBoolInto(d, &st.colScratchBool); err != nil {
                        return nil, err
                }
                s := st.colScratchBool
                if len(s) != present {
                        return nil, ErrTypeMismatch
                }
                scatter(func(i, k int) { out[i] = s[k] })
        case colKindString:
                strs, err := d.readStringColumn(present)
                if err != nil {
                        return nil, err
                }
                scatter(func(i, k int) { out[i] = strs[k] })
        case colKindTime:
                var sec []int64
                if err := decodeSliceInt64(d, unsafe.Pointer(&sec)); err != nil {
                        return nil, err
                }
                if len(sec) != present {
                        return nil, ErrTypeMismatch
                }
                var nsec []uint64
                if err := decodeSliceUint64(d, unsafe.Pointer(&nsec)); err != nil {
                        return nil, err
                }
                if len(nsec) != present {
                        return nil, ErrTypeMismatch
                }
                if err := checkNsecColumn(nsec); err != nil {
                        return nil, err
                }
                scatter(func(i, k int) { out[i] = time.Unix(sec[k], int64(nsec[k])).UTC() })
        default:
                return nil, ErrBadTag
        }
        return out, nil
}

// Package qdf is a compact, streaming-friendly binary serialization
// format.
//
// # Quick start
//
// One encode entry point: Marshal(v, opts). The Options bit-mask
// picks which codecs run for that call. Convenience bundles cover
// the common tradeoffs:
//
//        OptSpeed       — Fast mode, no codecs (matches encoding/json shape)
//        OptBalanced    — Dense + QPack + shape interning + Markov-1 + MTF
//        OptCompression — OptBalanced + heavier codecs: Gorilla XOR and ALP for
//                         float series, a final order-0 rANS entropy pass, and
//                         FSST for columnar string columns (large wire reduction
//                         at higher encode CPU)
//
// A single decoder handles every variant; the wire header self-
// describes the dialect.
//
//        b, err := qdf.Marshal(v, qdf.OptSpeed)        // hot path
//        b, err := qdf.Marshal(v, qdf.OptBalanced)     // telemetry / logs
//        b, err := qdf.Marshal(v, qdf.OptCompression)  // backup / archive
//
//        err := qdf.Unmarshal(b, &v)
//
//        enc := qdf.NewStreamEncoder(w, qdf.OptBalanced)
//        dec := qdf.NewStreamDecoder(r)
//
// Marshal returns a freshly-allocated slice owned by the caller.
// AppendMarshal is the zero-extra-copy variant.
//
// # Data shapes and codecs
//
// Under OptBalanced the encoder picks a codec per slice from the data
// itself: integer slices use Frame-of-Reference, delta, run-length or
// dictionary coding; bool slices bit-pack; repeated strings are interned
// and back-referenced. A slice of homogeneous flat structs is transposed
// and compressed column-by-column automatically — numeric columns get the
// integer codecs, an enum-like string column is dictionary-coded (distinct
// table + a bit-packed index per row) when that beats per-value interning,
// an optional (*T, T a scalar or string) field becomes a presence bitmap
// plus a dense present-only column instead of forcing the struct back to
// row-major,
// other repeated string columns collapse — with no flag, and a per-array
// probe falls back to the plain row encoding when columnar would not help. The float codecs that trade encode CPU for size live behind
// OptCompression: Gorilla XOR on smooth series, ALP on quantized/decimal
// float64 grids (quantized telemetry, prices, latencies).
//
// # Concurrency
//
// Marshal, Unmarshal and AppendMarshal are safe for concurrent use: each
// call leases its own encoder/decoder from a pool. A single Encoder,
// Decoder or StreamEncoder value is not safe to share across goroutines —
// use one per goroutine. A Dense document is a sequential stream, so one
// document cannot be encoded or decoded in parallel; to parallelize a
// large dataset, split it into independent shards and Marshal each
// separately.
//
// # Selective decode
//
// Under OptColumnIndex a columnar []struct payload carries a fixed-width
// uint32 column-length index, written after the shape declaration and before
// the column bodies (~4 bytes per column). A reader wanting only some columns
// then advances past the rest using the index instead of decoding them, so
// the cost becomes O(columns you read), not O(all columns). Two entry points:
// decode into a subset struct whose fields are a subset of the wire (matched
// by qdf tag / field name) with plain Unmarshal — wire columns absent from the
// target are skipped, target fields absent from the wire are left zero — or
// name the columns explicitly with UnmarshalColumns (also drives the dynamic
// *[]map[string]any form). The option is a true no-op on non-columnar
// payloads: the flag is backpatched only when the index is actually emitted,
// so the default columnar wire stays byte-identical when it is off. Without
// the index a subset decode still returns correct results by decoding and
// discarding unwanted columns. The index is a single-message feature and is
// not emitted in streaming mode.
//
// # Predicate pushdown
//
// Unmarshal accepts trailing QueryOption arguments that filter and project a
// columnar []struct payload before it is materialized. Where(field, pred)
// keeps only the rows for which a typed predicate holds — func(T) bool over
// the column's element type, AND-ed across multiple Where clauses, with zero
// per-value boxing — and Select(fields...) restricts the output to a named
// subset of columns. The filter field need not appear in the output type, and
// the result can be either *[]Struct or *[]map[string]any. The decoder reads
// each predicate column whole, evaluates it into a row bitmask, ANDs the
// masks, then compacts and materializes only the surviving rows of the
// projected columns; OptColumnIndex lets it seek past skipped columns instead
// of decode-and-discard. A nullable column's nil rows never match. Pushdown
// fires only on columnar payloads — a non-columnar payload (a single struct,
// or a batch the columnar probe declined) returns an error wrapping
// ErrUnsupported; a missing field yields ErrFieldNotFound and a predicate
// whose argument type mismatches the column yields ErrTypeMismatch, both as a
// *QueryError (errors.Is / errors.As). Versus a full decode followed by a
// manual loop, pushdown moves materially fewer bytes (about 4x less on a wide
// batch with 1% selectivity) and runs roughly 2x faster. No other Go
// serializer reads back a filtered, projected subset of a batch without
// decoding the whole thing — see docs/PREDICATE-PUSHDOWN.md for the full
// guide, rationale, and tutorial.
//
// # Structural delta
//
// Diff(old, new, opts) computes a patch carrying only the locations that
// changed; Apply(&base, patch) merges it back in place. The patch is
// self-describing — the receiver never has to know which Options produced it —
// and far smaller than a re-encode because unchanged fields, elements, and keys
// cost no bytes. Slices whose elements have a stable identity field tagged ",key"
// match by key (cheap reorder / middle edit) instead of by position, and an
// equal-length columnar batch is diffed column-by-column when that wins. Two
// fingerprints guard Apply against the wrong type or the wrong base.
// BaselineRegistry[T] applies a chain of patches in a state-sync stream without
// threading the previous value by hand. See docs/DELTA.md.
//
// # Canonical encoding
//
// OptCanonical makes the same logical value serialize to byte-identical output:
// map keys are emitted in sorted order (every key kind) and floats are
// normalized (-0.0 → +0.0, any NaN → one quiet NaN). The bytes are then safe to
// hash, sign, content-address, or deduplicate. It is encode-side only and
// decodes like any other qdf output; it is lossy for the sign of zero and the
// NaN payload, so use the default mode for a bit-exact float round-trip. See
// docs/CANONICAL.md.
//
// See docs/USAGE.md in the repository for a fuller guide.
//
// # Public API surface
//
// The package contract is split across four layers; everything else
// under internal/ is implementation detail and may change between
// releases without notice.
//
// Top-level entry points (file: qdf.go):
//
//        func Marshal(v any, opts Options) ([]byte, error)
//        func AppendMarshal(dst []byte, v any, opts Options) ([]byte, error)
//        func Unmarshal(data []byte, out any) error
//        type Options uint32   // bit-mask of OptDense, OptQPack, OptMTF, …
//
// Selective columnar decode (file: columnar_select.go):
//
//        func UnmarshalColumns(data []byte, out any, fields ...string) error
//
// Predicate pushdown — filter rows and project columns on a columnar
// []struct, AND-ed typed predicates with zero per-value boxing (file:
// query.go):
//
//        func Unmarshal(data []byte, out any, opts ...QueryOption) error
//        func Where[T Queryable](field string, pred func(T) bool) QueryOption
//        func Select(fields ...string) QueryOption
//
// Typed convenience wrappers — generic, zero-extra-reflection (file:
// qdf_generic.go):
//
//        func MarshalT[T any](v T, opts Options) ([]byte, error)
//        func AppendMarshalT[T any](dst []byte, v T, opts Options) ([]byte, error)
//        func UnmarshalT[T any](data []byte, out *T) error
//
// Structural delta — patch / merge two values, key-matched slices, and
// content-addressed baselines for state-sync streams (files: delta.go,
// delta_baseline.go):
//
//        func Diff[T any](old, new T, opts Options) ([]byte, error)
//        func AppendDiff[T any](dst []byte, old, new T, opts Options) ([]byte, error)
//        func Apply[T any](base *T, patch []byte) error
//        func ApplyArena[T any](base *T, patch []byte, arena *Arena) error
//        type BaselineRegistry[T any]   // NewBaselineRegistry / Register / Apply / Len
//
// Low-level encoder / decoder for callers driving the wire directly
// or interoperating with the qdfgen code generator (files:
// encoder.go, decoder.go, stream.go):
//
//        type Encoder struct{ … }
//            NewEncoder(mode Mode) *Encoder
//            NewEncoderWith(opts Options) *Encoder
//            NewEncoderOnBuf(buf []byte, mode Mode) *Encoder
//            (e *Encoder) WriteString / WriteBytes / WriteInt / WriteUint /
//                        WriteFloat32 / WriteFloat64 / WriteBool / WriteNil /
//                        WriteArrayHeader / WriteMapHeader /
//                        WriteTimestamp / WriteStringInline
//            (e *Encoder) AppendBytes / EnsureHeader / Bytes / Take /
//                        Reset / SetBuffer / AdoptBuffer / SetIntern /
//                        SetMaxDepth / SetQPack / QPack / ApplyOpts /
//                        EncodeValue / PreIntern
//        type Decoder struct{ … }
//            NewDecoder() *Decoder
//            NewDecoderOnBuf(buf []byte) *Decoder
//            (d *Decoder) ReadString / ReadStringBytes / ReadBytes /
//                        ReadBool / ReadInt / ReadUint / ReadFloat32 /
//                        ReadFloat64 / ReadNil / ReadArrayHeader /
//                        ReadMapHeader / ReadTimestamp / Skip /
//                        PeekTag / IsNil / Pos / Remaining / RemainingBytes /
//                        Advance / SetInput / SetNoCopy / MarkHeaderRead /
//                        CheckLength / InternKey
//        type StreamEncoder, type StreamDecoder         // io.Writer / io.Reader
//
// User-side hook points (file: marshaler.go):
//
//        type Marshaler interface   { MarshalQDF(dst []byte) ([]byte, error) }
//        type Unmarshaler interface { UnmarshalQDF(src []byte) (int, error) }
//
// The qdfgen code generator (cmd/qdfgen) emits MarshalQDF /
// UnmarshalQDF methods for user struct types; the codegen path
// uses the same Encoder / Decoder primitives listed above and
// requires no reflection at runtime.
package qdf

import (
        "sync"
)

// 4 KiB initial buffer covers most messages without a growth realloc.
// Larger payloads grow once and the grown buffer is recycled through the
// pool. Buffers that pass maxPooledBuf are dropped before being returned
// so idle memory stays bounded.
const (
        initialEncBuf = 4 * 1024
        // maxPooledBuf bounds how big a buffer the encoder pool will
        // retain between Marshal calls. Profiling on large batches
        // (50 k+ records, ~60 MiB output) showed the previous 1 MiB cap
        // caused every iteration to re-grow the buffer from 4 KiB to
        // the final size — runtime.memmove + runtime.madvise dominated
        // the trace. Retaining up to 16 MiB lets large encoders reuse
        // the high-water buffer once it warms up, while still releasing
        // truly outlier payloads instead of pinning them to the pool.
        maxPooledBuf = 16 * 1024 * 1024
        // maxRetainedDeltaScratch bounds the Delta+FOR decode scratch a pooled
        // decoder keeps between calls. Retains it for columns up to ~16 k rows
        // (the maxStateEntries scale) and drops it after a one-off larger decode so
        // the pool never pins an outlier buffer.
        maxRetainedDeltaScratch = 1 << 14
        // maxRetainedWideScratch bounds the int32→int64 / uint32→uint64 widening
        // scratch a pooled encoder keeps between calls (same ~16 k-element scale as
        // the delta scratch); a one-off larger narrow-int slice is dropped so the
        // pool never pins an outlier buffer.
        maxRetainedWideScratch = 1 << 14
)

// Options is a bit-mask of per-call encoder feature toggles. A zero
// value (OptSpeed) gives the fastest path: Fast mode, no codecs, no
// predictors. Or-combine bits with | to opt in to individual codecs.
// Pre-built bundles cover the common "max speed", "balanced", and
// "max compression" use cases without having to remember the layout.
//
// Options carry by value (uint32) so Marshal / AppendMarshal add zero
// allocations over the pool / output-clone overhead. The encoder
// checks each bit with a single AND on the hot path.
type Options uint32

const (
        // OptDense activates the inline intern table. First occurrences of
        // strings and []byte payloads are stored once and back-referenced
        // by ID (1-3 bytes per reuse). Required for any of the state-ref
        // codecs (Repeat / MTF / Pair) and for tagMapShape.
        OptDense Options = 1 << iota

        // OptQPack enables the numeric / boolean slice codecs. Bools
        // bit-pack; 32- and 64-bit integer slices ([]int/int32/int64,
        // []uint/uint32/uint64) run the codec picker — Frame-of-Reference,
        // Delta+FOR, RLE, dictionary, and Patched FOR — widening 32-bit
        // values to 64-bit for selection and narrowing back on decode;
        // float slices stay on raw-LE. Auto-selected per slice; the
        // encoder falls back to raw-LE when nothing wins. Gorilla XOR
        // for floats lives behind OptGorillaFloat (bit 5) — see
        // OptCompression for the bundle that turns it on.
        OptQPack

        // OptShapeIntern routes struct emissions through tagMapShape: each
        // distinct struct type declares its field ordering on first emit;
        // subsequent emits write only the shape ID + values. Requires
        // OptDense (the shape table lives on the intern table side).
        OptShapeIntern

        // OptPairPred enables the Markov-1 successor predictor
        // (tagStatePair, 0xEA). Per previous state-ref ID the encoder stores
        // its most-recent successor (top-1, K=1); a hit emits the transition
        // rank in a single byte (always rank 0). Requires OptDense.
        OptPairPred

        // OptMTF enables Move-to-Front rank coding (tagStateMTF, 0xE9).
        // When the LRU rank of a state-ref ID needs fewer varuint bytes
        // than the raw id, the rank is emitted instead. Requires OptDense.
        OptMTF

        // OptGorillaFloat opts in to the Gorilla XOR codec for []float64
        // (and []float32) slices. Gorilla collapses smooth time-series
        // data dramatically (~75 % wire reduction on quantised metric
        // streams) but trades that for ~10× more CPU on the encode/decode
        // path because the body is bit-level. Off by default in
        // OptBalanced for that reason; included in OptCompression for
        // archive-style workloads where wire size dominates. Requires
        // OptQPack.
        OptGorillaFloat

        // OptRANS adds a final static order-0 rANS entropy pass over the whole
        // encoded body. Opt-in (bundled into OptCompression) because it trades
        // encode/decode CPU for wire size. The encoder applies it only when the
        // rANS form is strictly smaller than the plain body, so it never makes a
        // buffer larger. Whole-buffer, so the streaming encoder ignores it.
        OptRANS

        // OptColumnIndex makes a columnar []struct payload carry a column-length
        // index so readers can decode a subset of columns without decoding the
        // rest. Opt-in, ~4 bytes per column. Only affects payloads the encoder
        // transposes into the columnar container (tagColStruct); has no effect on
        // other shapes. Sets FlagColIndex on the header.
        OptColumnIndex

        // OptFSST opts in to the FSST string codec for columnar string columns
        // (high-cardinality, substring-sharing text: log lines, URLs, paths). A
        // CPU-for-size trade (trains a per-column symbol table), so it is excluded
        // from OptBalanced and bundled into OptCompression. Never-larger: emitted
        // only when strictly smaller than the dictionary / per-value forms.
        OptFSST

        // OptMapShape interns map key-sets the way OptShapeIntern interns struct
        // field-names: the first map with a given set of (string) keys declares the
        // key ordering via tagMapShape; subsequent maps with the same key-set emit
        // only the shape ID + values in canonical (sorted) key order. Cuts encode
        // CPU and wire on maps with recurring keys (telemetry tags, log labels).
        // Requires OptDense (the shape table lives on the intern side). Opt-in in
        // v1; never-worse fallback to a plain map header when a key-set does not
        // recur.
        OptMapShape

        // OptDeltaNoBaseFingerprint disables the patch base fingerprint that Diff/Apply
        // use to detect a mismatched base. With it set, Diff omits the fingerprint and
        // Apply performs no base check — skipping a full reflect walk of the value on
        // BOTH sides (a large speedup for a big base with a tiny patch), at the cost of
        // the wrong-base safety guard. Use only when the caller guarantees Apply's base
        // is exactly the value Diff was computed against. No effect outside Diff/Apply.
        OptDeltaNoBaseFingerprint // bit 10

        // OptCanonical guarantees byte-identical output for logically-equal values:
        // map keys are emitted in sorted order (all key kinds) and floats are
        // normalized (-0.0 → +0.0, any NaN → a canonical quiet NaN). Encode-side
        // only; the bytes are ordinary qdf and decode normally. Intended for hashing
        // / signing / dedup of serialized output. Lossy for the sign of zero and NaN
        // payloads (use the default mode for bit-exact float round-trip).
        OptCanonical // bit 11

        // OptLossyVec enables the opt-in lossy float-vector codec (tag 0xFD) for
        // []float32/[]float64 columns. LOSSY: reconstruction is approximate within
        // the VectorBudget set on the Encoder (default MinCosine(0.999)). Never
        // fires unless this bit is set; never larger than the raw/lossless form.
        OptLossyVec // bit 12

        // OptZoneMap stores eligible columnar integer columns as zone-chunked
        // containers (tag 0xF1): the column is split into 256-row zones, each zone
        // independently codec-picked, prefixed by a per-zone [min,max] zonemap. A
        // bound-carrying predicate (WhereRange / WhereGE / WhereLE / WhereEq) then
        // skips zones that provably cannot match WITHOUT decoding them — a large
        // speedup for range/equality queries over positionally-ordered columns
        // (timestamps, sorted IDs). An explicit size-for-query-speed trade (chunking
        // + zonemap cost wire), so it is opt-in; without the bit the wire is
        // unchanged. v1 covers int/uint columns; string/float are a follow-up.
        OptZoneMap // bit 13

        // Bits 14..31 reserved for future codecs (LZ77, n-gram dictionary, etc.).

        // OptSpeed is the zero-bit preset: Fast mode, no codecs, no
        // predictors. Maximum throughput, smallest CPU footprint.
        OptSpeed Options = 0

        // OptBalanced bundles every codec that does not trade CPU for
        // compression beyond its sweet spot. The right default for
        // telemetry, log batches, and any payload with repetitive
        // strings or numeric slices. Notably excludes OptGorillaFloat —
        // reach for OptCompression when the float slices in the payload
        // are smooth time-series and wire size matters more than encode
        // latency.
        OptBalanced Options = OptDense | OptQPack | OptShapeIntern | OptPairPred | OptMTF

        // OptCompression bundles every codec that the encoder will spend
        // CPU on for wire-size gains. On top of OptBalanced it adds
        // OptGorillaFloat (Gorilla XOR for float slices), OptRANS (a final
        // order-0 rANS entropy pass over the whole body), and OptFSST (the
        // FSST string codec for columnar string columns). Each trades
        // encode/decode CPU for wire size, which is why they stay out of the
        // OptBalanced default; future heavy codecs land in this bundle without
        // breaking the name.
        OptCompression Options = OptBalanced | OptGorillaFloat | OptRANS | OptFSST
)

// Has reports whether the named bit is set. Compiles to a single AND
// + compare. Use on the encoder hot path to gate codec emission.
//
//go:nosplit
func (o Options) Has(bit Options) bool { return o&bit != 0 }

var (
        // encPool serves every Marshal / AppendMarshal call. Encoders are
        // reconfigured via applyOpts on each acquire so a single pool
        // covers any Options combination; the buffer + intern table are
        // reused across calls when the cap fits maxPooledBuf.
        encPool = sync.Pool{
                New: func() any {
                        // state is left nil: Fast/OptSpeed encodes never touch the intern
                        // table, so a pooled encoder serving only OptSpeed pays no state
                        // allocation and no per-call state.reset(). applyOpts lazily
                        // allocates it the first time the encoder runs in Dense mode.
                        return &Encoder{
                                buf:             make([]byte, 0, initialEncBuf),
                                minIntern:       4,
                                maxStateEntries: 1 << 14,
                                maxDepth:        DefaultMaxDepth,
                        }
                },
        }
        decPool = sync.Pool{
                New: func() any { return &Decoder{} },
        }
)

//go:nosplit
func putEnc(enc *Encoder, pool *sync.Pool) {
        if cap(enc.buf) > maxPooledBuf {
                enc.buf = nil
        }
        // Don't pin a spike-sized widening scratch in the pool: a single huge
        // []int32/[]uint32 field would otherwise keep its widened []int64/[]uint64
        // resident on every pooled encoder forever (mirrors the buffer / delta cap).
        if cap(enc.wideI64) > maxRetainedWideScratch {
                enc.wideI64 = nil
        }
        if cap(enc.wideU64) > maxRetainedWideScratch {
                enc.wideU64 = nil
        }
        if cap(enc.blkPlanI64) > maxRetainedWideScratch {
                enc.blkPlanI64 = nil
        }
        if cap(enc.blkPlanU64) > maxRetainedWideScratch {
                enc.blkPlanU64 = nil
        }
        if cap(enc.zoneMM) > maxRetainedWideScratch {
                enc.zoneMM = nil
        }
        pool.Put(enc)
}

// Marshal encodes v with the given option bit-mask. Combine bits with
// | to opt into individual codecs, or use one of the OptSpeed /
// OptBalanced / OptCompression bundles.
//
//        b, _ := qdf.Marshal(event,    qdf.OptSpeed)         // hot path
//        b, _ := qdf.Marshal(batch,    qdf.OptBalanced)      // telemetry
//        b, _ := qdf.Marshal(snapshot, qdf.OptCompression)   // backup
//        b, _ := qdf.Marshal(payload,                        // tuned
//            qdf.OptDense|qdf.OptQPack|qdf.OptShapeIntern)
//
// Big-output detach (in marshalDict): cloning a multi-megabyte buffer to hand
// the caller their own copy used to dominate Large-payload profiles
// (slices.Clone + runtime.memmove). Above marshalDetachThreshold the pool would
// drop the buffer in putEnc anyway (cap exceeds maxPooledBuf), so handing the
// original to the caller and leaving the pool encoder with a nil buf is
// strictly cheaper. Small payloads stay on the clone path so the warm 4 KiB
// pool buffer survives.
func Marshal(v any, opts Options) ([]byte, error) {
        return marshalDict(v, opts, nil)
}

// marshalDetachThreshold is the buffer capacity above which Marshal
// hands the encoder buffer directly to the caller instead of cloning
// it. Chosen at 256 KiB — well above typical message sizes (so the
// hot pool path keeps reusing its buffer) but small enough that any
// payload heading toward the maxPooledBuf cap skips the copy.
const marshalDetachThreshold = 256 * 1024

// AppendMarshal encodes v and appends the result to dst. Reuse the
// returned slice as dst on the next call to avoid per-message
// allocations.
func AppendMarshal(dst []byte, v any, opts Options) ([]byte, error) {
        return appendMarshalDict(dst, v, opts, nil)
}

// Unmarshal decodes data into out, which must be a non-nil pointer.
// The wire dialect is detected from the header — the same Unmarshal
// reads any output produced by Marshal regardless of the encode-side
// option bits.
//
// Slice-backing reuse: when out is a *[]T whose slice already has enough
// capacity for the decoded element count, the decoder reuses that backing
// array instead of allocating a new one — eliminating the result allocation
// (the dominant decode cost) on a decode into a pooled / pre-sized slice. The
// reused backing is overwritten in place (its elements are zeroed first, so no
// stale data leaks), so do not share it with other live slices across the call.
// A nil or too-small destination allocates fresh, so default usage is
// unaffected.
//
// The optional QueryOptions (Where / Select) turn the call into a
// filtering/projecting columnar decode: predicates are AND-ed and the
// named columns projected. They apply only to a columnar []struct,
// []map[string]any or []any payload; on any other shape a query call
// returns a *QueryError wrapping ErrUnsupported. With no options the
// behavior is exactly the plain decode above.
//
// Round-trip fidelity: decoding into the same concrete Go type reproduces the
// value exactly, including the nil-vs-empty distinction for slices, maps and
// pointers (a nil []T decodes as nil, an empty []T{} as empty — like
// encoding/json's null vs []). Two deliberate normalizations are NOT bit-exact:
// decoding into an interface (any / map[string]any) canonicalizes integers to
// int64 / uint64 and structs to map[string]any (the schemaless type contract),
// and a time.Time loses any monotonic-clock reading on encode (as the standard
// library strips it for transport). Values nested deeper than the configured
// max depth, and maps with both int and uint keys of the same small value, are
// the documented exceptions.
func Unmarshal(data []byte, out any, opts ...QueryOption) error {
        if len(opts) == 0 {
                return unmarshal(data, out, nil, false, nil)
        }
        qp, err := buildQueryPlan(opts)
        if err != nil {
                return err
        }
        // A noCopy-only plan (no predicate, no projection) is a plain decode — do
        // NOT route it through the columnar query path, which assumes a columnar
        // container and would reject a row payload.
        if qp.root == nil && qp.selectFields == nil {
                return unmarshal(data, out, nil, qp.noCopy, qp.arena)
        }
        return unmarshalQuery(data, out, qp)
}

// unmarshalQuery is the pooled-decoder dispatch for a filtering/projecting
// decode. The plan's selectFields double as the column projection for the map
// (any) path, reusing the v1 selectFields mechanism.
func unmarshalQuery(data []byte, out any, qp *queryPlan) error {
        dec := decPool.Get().(*Decoder)
        dec.buf = data
        dec.i = 0
        dec.depth = 0 // a prior depth-overflow decode leaks depth>0 (descend errors before its defer ascend); reset so a pooled decoder never carries a stale depth into the next decode
        dec.headerRead = false
        dec.mode = Fast
        // colIndex is set fresh by readHeader; reset defensively so a pooled decoder never carries a stale flag.
        dec.colIndex = false
        // colMaxLen is normally cleared by the reflect columnar defer / generated
        // ClearColMaxLen, but a generated DecodeQDF that errors between
        // ReadColStructHeader and ClearColMaxLen returns the decoder with a stale
        // bound; reset so it can't spuriously clamp the next pooled decode's lengths.
        dec.colMaxLen = 0
        dec.selectFields = qp.selectFields
        dec.query = qp
        dec.noCopy = qp.noCopy
        dec.arena = qp.arena
        clear(dec.mapFreeList) // drop maps recycled by a prior decode into a different target
        if dec.state != nil {
                dec.state.reset()
        }
        err := decodeReflect(dec, out)
        dec.buf = nil
        dec.selectFields = nil
        dec.query = nil
        dec.noCopy = false
        dec.arena = nil // never pin the caller's arena across pooled reuse
        if cap(dec.deltaScratch) > maxRetainedDeltaScratch {
                dec.deltaScratch = nil
        }
        decPool.Put(dec)
        return err
}

// unmarshal is the shared pooled-decoder dispatch behind Unmarshal and
// UnmarshalColumns. When fields is non-nil it restricts the columnar map
// (any) decode to those columns (see Decoder.selectFields).
func unmarshal(data []byte, out any, fields []string, noCopy bool, arena *Arena) error {
        dec := decPool.Get().(*Decoder)
        dec.buf = data
        dec.i = 0
        dec.depth = 0 // a prior depth-overflow decode leaks depth>0 (descend errors before its defer ascend); reset so a pooled decoder never carries a stale depth into the next decode
        dec.headerRead = false
        dec.mode = Fast
        dec.colIndex = false
        dec.colMaxLen = 0 // see unmarshalQuery: a generated DecodeQDF can leak a stale bound on error
        dec.noCopy = noCopy
        dec.arena = arena
        dec.selectFields = fields
        dec.query = nil        // parity with UnmarshalT / SetInput: never inherit a prior query decode's plan from the shared pool
        clear(dec.mapFreeList) // drop maps recycled by a prior decode into a different target
        if dec.state != nil {
                dec.state.reset()
        }
        err := decodeReflect(dec, out)
        dec.buf = nil
        dec.arena = nil // never pin the caller's arena across pooled reuse
        dec.selectFields = nil
        // Reset noCopy so a WithNoCopy() decode never leaves the pooled decoder in
        // aliasing mode for the next acquirer (e.g. UnmarshalT) — that would return
        // buffer-aliased strings the caller never opted into (a silent
        // use-after-free, undetectable by the race detector).
        dec.noCopy = false
        if cap(dec.deltaScratch) > maxRetainedDeltaScratch {
                dec.deltaScratch = nil
        }
        decPool.Put(dec)
        return err
}

package qdf

import "slices"

// Direct entry points for types whose MarshalQDF / UnmarshalQDF are
// emitted by cmd/qdfgen (or hand-written). They bypass the reflect
// descriptor cache entirely: no descOf lookup, no interface conversion,
// no reflect.Value materialisation. The type parameter is constrained
// to the Marshaler / Unmarshaler interface so the compiler can inline
// the method call directly.
//
// Output is byte-identical to Marshal / Unmarshal on the same value
// with OptSpeed. The path emits Fast-mode wire only because that is
// what the generated code produces. Dense / QPack are not available
// through these helpers — use Marshal(v, OptBalanced) or
// Marshal(v, OptQPack) (or the MarshalT generic counterpart) for
// those dialects.

// MarshalDirect serialises a value through its MarshalQDF method.
// Skips the public Marshal entry point's any-boxing, reflect.New, and
// descriptor lookup. Roughly 2-4× faster than Marshal for generated
// types on small payloads, with one fewer allocation per call.
//
// v must be non-nil: MarshalDirect calls v.MarshalQDF directly, so a nil
// pointer-receiver value panics. Use Marshal, which emits an explicit nil, if
// the value may be nil.
func MarshalDirect[T Marshaler](v T) ([]byte, error) {
        enc := encPool.Get().(*Encoder)
        enc.Reset()
        enc.writeHeader()
        out, err := v.MarshalQDF(enc.buf)
        if err != nil {
                putEnc(enc, &encPool)
                return nil, err
        }
        // Big-output detach: above marshalDetachThreshold putEnc would drop the
        // buffer anyway (cap exceeds maxPooledBuf), so hand the original to the
        // caller instead of paying a multi-megabyte slices.Clone. Small payloads
        // stay on the clone path so the warm pool buffer survives. Mirrors Marshal.
        if cap(out) > marshalDetachThreshold {
                enc.buf = nil
                putEnc(enc, &encPool)
                return out, nil
        }
        cloned := slices.Clone(out)
        enc.buf = out[:0]
        putEnc(enc, &encPool) // cap a spike-sized buffer/scratch before pooling
        return cloned, nil
}

// AppendMarshalDirect appends the serialisation of v to dst.
func AppendMarshalDirect[T Marshaler](dst []byte, v T) ([]byte, error) {
        // Header must be present before the receiver's MarshalQDF runs, so
        // emit it through a borrowed encoder (handles the headerOut flag and
        // any future header-byte changes).
        enc := encPool.Get().(*Encoder)
        enc.Reset()
        enc.buf = dst
        enc.writeHeader()
        out, err := v.MarshalQDF(enc.buf)
        if err != nil {
                enc.buf = nil
                putEnc(enc, &encPool)
                return dst, err
        }
        enc.buf = nil
        putEnc(enc, &encPool) // out is the caller's; cap any spike-sized scratch
        return out, nil
}

// UnmarshalDirect dispatches to out.UnmarshalQDF after validating the
// 5-byte header. It is the inverse of MarshalDirect; the same wire
// constraints apply (Fast-mode only).
//
// If the input carries the FlagDense bit, UnmarshalDirect falls back
// to the full Unmarshal path because generated code cannot resolve
// state-ref tags without the decoder's intern table.
//
// Decode-side performance is bounded by the user's UnmarshalQDF
// implementation: the reflect path is heavily pooled (Decoder pool +
// per-decoder key intern cache) and tends to win against naive
// hand-rolled receivers. Generated code from cmd/qdfgen uses
// Decoder.InternKey for map / struct keys and matches or beats the
// reflect path; ad-hoc UnmarshalQDF methods that call NewDecoderOnBuf
// + ReadString in a tight loop will not.
func UnmarshalDirect[T Unmarshaler](data []byte, out T) error {
        if len(data) < 5 {
                return ErrShortBuffer
        }
        if data[0] != Magic0 || data[1] != Magic1 || data[2] != Magic2 {
                return ErrBadMagic
        }
        if data[3] != Version1 {
                return ErrBadVersion
        }
        if data[4]&(FlagDense|FlagQPack|FlagRANS|FlagColIndex) != 0 {
                // Any non-Fast wire (Dense intern table, QPack codec bodies, a rANS
                // entropy pass, or a columnar index) is not what a hand-rolled UnmarshalQDF
                // expects; fall back to the reflect path which handles all of them instead
                // of failing with ErrBadTag on the first codec tag.
                return Unmarshal(data, out)
        }
        _, err := out.UnmarshalQDF(data[5:])
        return err
}

package qdf

import (
        "reflect"
        "slices"
        "unsafe"
)

// Generic entry points. The non-generic Marshal / Unmarshal take any,
// which forces a runtime interface conversion of the caller's value
// (boxing) and, on the encode side, a reflect.New+Set copy for value-
// typed inputs. The generic forms below skip both: T is known at
// instantiation, so reflect.TypeFor[T]() resolves at compile time,
// and unsafe.Pointer(&v) points straight at the function parameter on
// the caller's stack.
//
// Wire output is byte-identical to the non-generic counterparts; the
// generic versions only change the calling convention.

// MarshalT is the generic equivalent of Marshal. T is fixed at the
// call site; opts copies by value, so the call adds zero heap
// allocations over MarshalT itself.
func MarshalT[T any](v T, opts Options) ([]byte, error) {
        enc := encPool.Get().(*Encoder)
        enc.Reset()
        enc.applyOpts(opts)
        if err := encodeT(enc, &v); err != nil {
                putEnc(enc, &encPool)
                return nil, err
        }
        enc.maybeApplyRANS(0)
        // Mirror marshalDict: hand a spike-sized backing straight to the caller
        // instead of cloning it (putEnc would drop it past maxPooledBuf anyway), so
        // MarshalT stays allocation-equivalent to Marshal on large payloads.
        var out []byte
        if cap(enc.buf) > marshalDetachThreshold {
                out = enc.buf
                enc.buf = nil
        } else {
                out = slices.Clone(enc.buf)
        }
        putEnc(enc, &encPool) // cap a spike-sized buffer / widening scratch before pooling
        return out, nil
}

// AppendMarshalT is the generic equivalent of AppendMarshal.
func AppendMarshalT[T any](dst []byte, v T, opts Options) ([]byte, error) {
        enc := encPool.Get().(*Encoder)
        enc.Reset()
        enc.applyOpts(opts)
        start := len(dst)
        enc.buf = dst
        if err := encodeT(enc, &v); err != nil {
                // Detach the caller's dst before pooling: putEnc only nils buf past
                // maxPooledBuf, so a normal-sized dst would stay aliased in the pooled
                // encoder and the next encode would overwrite the caller's backing array.
                enc.buf = nil
                putEnc(enc, &encPool)
                return dst, err
        }
        enc.maybeApplyRANS(start)
        out := enc.buf
        enc.buf = nil
        putEnc(enc, &encPool) // cap a spike-sized widening scratch before pooling
        return out, nil
}

// UnmarshalT is the generic equivalent of Unmarshal. The destination
// pointer must not be nil.
func UnmarshalT[T any](data []byte, out *T) error {
        if out == nil {
                return ErrTypeMismatch
        }
        dec := decPool.Get().(*Decoder)
        dec.buf = data
        dec.i = 0
        dec.depth = 0 // reset stale depth from a prior depth-overflow decode (see unmarshal)
        dec.headerRead = false
        dec.mode = Fast
        // Start from a clean pooled decoder: it is shared with Unmarshal /
        // UnmarshalColumns / query paths, any of which may have left noCopy,
        // selectFields, query, or colIndex set. Inheriting noCopy would silently
        // alias the input buffer; inheriting selectFields/query would mis-project.
        dec.noCopy = false
        dec.colIndex = false
        // A generated columnar DecodeQDF that errors between ReadColStructHeader and
        // ClearColMaxLen returns the pooled decoder with a stale colMaxLen; reset it
        // so this decode's slice codecs are not spuriously clamped (mirrors the reset
        // in unmarshal / unmarshalQuery — UnmarshalT shares the same decoder pool).
        dec.colMaxLen = 0
        dec.selectFields = nil
        dec.query = nil
        clear(dec.mapFreeList) // drop maps recycled by a prior decode into a different target
        if dec.state != nil {
                dec.state.reset()
        }
        t := reflect.TypeFor[T]()
        td, err := descOf(t)
        if err != nil {
                dec.buf = nil
                decPool.Put(dec)
                return err
        }
        err = td.decode(dec, unsafe.Pointer(out))
        dec.buf = nil
        // Don't pin a spike-sized scratch buffer in the pool (mirrors unmarshal).
        if cap(dec.deltaScratch) > maxRetainedDeltaScratch {
                dec.deltaScratch = nil
        }
        decPool.Put(dec)
        return err
}

// encodeT is the shared body of every generic encode entry point. It
// resolves the type-T descriptor at compile time (via reflect.TypeFor)
// and points the encoder straight at vp, which the caller obtained as
// &v on its own stack. No interface boxing, no reflect.New copy.
//
// vp must point at a value of type T. For T = pointer-kind (e.g. *Foo)
// the descriptor's encodePtr will dereference once, matching the
// behavior of the non-generic Marshal(v any) on a *Foo argument.
func encodeT[T any](e *Encoder, vp *T) error {
        if vp == nil {
                e.WriteNil()
                return nil
        }
        t := reflect.TypeFor[T]()
        td, err := descOf(t)
        if err != nil {
                return err
        }
        return td.encode(e, unsafe.Pointer(vp))
}

package qdf

import (
        "math"
        "math/bits"
        "slices"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/bitpack"
        "github.com/alex60217101990/qdf/internal/endian"
)

// QPack raw-LE codec kind byte. Low two bits encode the element width as
// log2(bytes); next two bits encode the type family (uint, int, float).
const (
        qpackRawW1 = 0
        qpackRawW2 = 1
        qpackRawW4 = 2
        qpackRawW8 = 3

        qpackRawFamUint  = 0 << 2
        qpackRawFamInt   = 1 << 2
        qpackRawFamFloat = 2 << 2

        // The kind byte is family|width across all four widths. The encoder never
        // emits the W1/W2 (8/16-bit) forms — int8/int16/uint8/uint16 slices widen to
        // a 32/64-bit column and qpack-FOR then packs them to ≤8/16 bits/value, which
        // is never larger than a narrow raw column. The decoder still honors W1/W2
        // (see qpackRawWidthBytes), so these names complete the wire's kind table and
        // document the formats it can decode; keep them even though no producer emits.
        qpackKindUint8   = qpackRawFamUint | qpackRawW1
        qpackKindUint16  = qpackRawFamUint | qpackRawW2
        qpackKindUint32  = qpackRawFamUint | qpackRawW4
        qpackKindUint64  = qpackRawFamUint | qpackRawW8
        qpackKindInt8    = qpackRawFamInt | qpackRawW1
        qpackKindInt16   = qpackRawFamInt | qpackRawW2
        qpackKindInt32   = qpackRawFamInt | qpackRawW4
        qpackKindInt64   = qpackRawFamInt | qpackRawW8
        qpackKindFloat32 = qpackRawFamFloat | qpackRawW4
        qpackKindFloat64 = qpackRawFamFloat | qpackRawW8
)

func qpackRawWidthBytes(kind byte) int {
        switch kind & 0x03 {
        case qpackRawW1:
                return 1
        case qpackRawW2:
                return 2
        case qpackRawW4:
                return 4
        case qpackRawW8:
                return 8
        }
        return 0
}

// pickU64Codec analyses a []uint64 and decides which QPack codec yields
// the smallest wire form. Cost of the scan is O(n) (min/max + delta
// stats + RLE probe), which is dominated by the encode itself.
func pickU64Codec(s []uint64) (codec qpackCodec, mn uint64, forBits int, first uint64, minDelta int64, deltaBits int, pforBits int, bestCost int) {
        n := len(s)
        codec = qpackRaw
        if n == 0 {
                return
        }
        // raw cost: tag + kind + nVarUint + 8n
        rawCost := 2 + uvarintLen(uint64(n)) + 8*n

        var mx uint64
        mn, mx = minMaxU64(s)
        forBits = bitpack.BitsForDelta(mx - mn)
        bestCost = rawCost
        if forBits <= qpackForMaxBits {
                c := qpackForSizeUnsigned(n, forBits, mn)
                if c < bestCost {
                        bestCost = c
                        codec = qpackFor
                }
        }
        if n >= 4 {
                first, minDelta, deltaBits = computeDeltaStatsU64(s)
                if deltaBits <= qpackForMaxBits {
                        hdr := 3 + uvarintLen(first) + uvarintLen(zigzagEncode64(minDelta)) + uvarintLen(uint64(n))
                        body := 0
                        if n >= 2 {
                                body = ((n - 1) * deltaBits) >> 3
                                if ((n-1)*deltaBits)&7 != 0 {
                                        body++
                                }
                        }
                        c := hdr + body
                        if c < bestCost {
                                bestCost = c
                                codec = qpackDeltaFor
                        }
                }
        }
        // RLE: cheap run-fraction probe over the first 32 elements;
        // skip the full size pass when runs are scarce.
        if n >= 8 {
                probeN := min(32, n)
                probeRuns := 1
                for i := 1; i < probeN; i++ {
                        if s[i] != s[i-1] {
                                probeRuns++
                        }
                }
                // Require avg run length >= 2 (probeRuns <= probeN/2) before
                // we bother with the full pass.
                if probeRuns*2 <= probeN {
                        c := qpackRLESizeU64(s, n)
                        if c < bestCost {
                                bestCost = c
                                codec = qpackRLE
                        }
                }
        }
        // Dict: probe the distinct cardinality and, if it fits the cap,
        // compare against the running best. The probe early-exits at
        // qpackDictMaxDistinct+1 unique values so high-cardinality
        // slices pay an O(n*16) bounded cost in the worst case.
        if n >= 8 {
                var table [qpackDictMaxDistinct + 1]uint64
                if count, ok := probeDistinctU64(s, &table); ok && count > 0 {
                        c := qpackDictSizeU64(table[:count], n)
                        if c < bestCost {
                                bestCost = c
                                codec = qpackDict
                        }
                }
        }
        // PFOR: wins on outlier-heavy slices where plain FOR is forced wide by a
        // few large values. Conservative cost estimate (maxDelta upper bound) keeps
        // it never-worse: chosen only when strictly smaller than every other codec.
        if pb, pc, okp := pforPlanUnsigned(s, mn, forBits); okp && pc < bestCost {
                codec = qpackPFor
                pforBits = pb
                bestCost = pc // see pickI64Codec: keep bestCost in sync with the chosen
                // codec so encodeSliceUint32's never-worse floor picks PFOR over native
                // uint32-raw when PFOR is actually smaller.
        }
        return
}

// qpackRLESizeU64 returns the on-wire byte cost of run-length-encoding
// s as a []uint64. Wire form documented at tagPackRLE.
//
//go:nosplit
func qpackRLESizeU64(s []uint64, n int) int {
        hdr := 2 + uvarintLen(uint64(n))
        body := 0
        runLen := 1
        prev := s[0]
        for i := 1; i < n; i++ {
                if s[i] == prev {
                        runLen++
                        continue
                }
                body += uvarintLen(prev) + uvarintLen(uint64(runLen))
                runLen = 1
                prev = s[i]
        }
        body += uvarintLen(prev) + uvarintLen(uint64(runLen))
        return hdr + body
}

// qpackRLESizeI64 mirrors qpackRLESizeU64 for signed slices. Values
// are zigzag-encoded so small magnitudes still encode in 1-2 bytes.
//
//go:nosplit
func qpackRLESizeI64(s []int64, n int) int {
        hdr := 2 + uvarintLen(uint64(n))
        body := 0
        runLen := 1
        prev := s[0]
        for i := 1; i < n; i++ {
                if s[i] == prev {
                        runLen++
                        continue
                }
                body += uvarintLen(zigzagEncode64(prev)) + uvarintLen(uint64(runLen))
                runLen = 1
                prev = s[i]
        }
        body += uvarintLen(zigzagEncode64(prev)) + uvarintLen(uint64(runLen))
        return hdr + body
}

func pickI64Codec(s []int64) (codec qpackCodec, mn int64, forBits int, first int64, minDelta int64, deltaBits int, pforBits int, bestCost int) {
        n := len(s)
        codec = qpackRaw
        if n == 0 {
                return
        }
        rawCost := 2 + uvarintLen(uint64(n)) + 8*n

        var mx int64
        mn, mx = minMaxI64(s)
        forBits = bitpack.BitsForDelta(uint64(mx) - uint64(mn))
        bestCost = rawCost
        if forBits <= qpackForMaxBits {
                c := qpackForSizeSigned(n, forBits, mn)
                if c < bestCost {
                        bestCost = c
                        codec = qpackFor
                }
        }
        if n >= 4 {
                first, minDelta, deltaBits = computeDeltaStatsI64(s)
                if deltaBits <= qpackForMaxBits {
                        hdr := 3 + uvarintLen(zigzagEncode64(first)) + uvarintLen(zigzagEncode64(minDelta)) + uvarintLen(uint64(n))
                        body := 0
                        if n >= 2 {
                                body = ((n - 1) * deltaBits) >> 3
                                if ((n-1)*deltaBits)&7 != 0 {
                                        body++
                                }
                        }
                        c := hdr + body
                        if c < bestCost {
                                bestCost = c
                                codec = qpackDeltaFor
                        }
                }
        }
        if n >= 8 {
                probeN := min(32, n)
                probeRuns := 1
                for i := 1; i < probeN; i++ {
                        if s[i] != s[i-1] {
                                probeRuns++
                        }
                }
                if probeRuns*2 <= probeN {
                        c := qpackRLESizeI64(s, n)
                        if c < bestCost {
                                bestCost = c
                                codec = qpackRLE
                        }
                }
        }
        if n >= 8 {
                var table [qpackDictMaxDistinct + 1]int64
                if count, ok := probeDistinctI64(s, &table); ok && count > 0 {
                        c := qpackDictSizeI64(table[:count], n)
                        if c < bestCost {
                                bestCost = c
                                codec = qpackDict
                        }
                }
        }
        if pb, pc, okp := pforPlanSigned(s, mn, forBits); okp && pc < bestCost {
                codec = qpackPFor
                pforBits = pb
                bestCost = pc // reflect the chosen codec's cost so the caller's
                // never-worse floor (encodeSliceInt32 vs native int32-raw) sees PFOR's
                // real size and does not fall back to a larger native encoding.
        }
        return
}

// qpackCodec identifies which QPack codec to invoke for a numeric slice.
type qpackCodec uint8

const (
        qpackRaw qpackCodec = iota
        qpackFor
        qpackDeltaFor
        qpackGorilla
        qpackRLE
        qpackDict
        qpackPFor
)

// qpackDictMaxDistinct caps the dictionary codec: a slice with more
// than this many unique values is rarely a dictionary-shape column
// and the probe cost grows linearly with the cap. 64 distinct still
// pack at 6 index bits, covering wide-but-low-cardinality enums (HTTP
// status, state machines) that FOR would force wide; the probe bails
// the moment a (cap+1)th distinct appears, so high-cardinality slices
// stay cheap regardless of the cap.
const qpackDictMaxDistinct = 64

// bitsForDistinct returns the per-index bit width of a dictionary
// codec with the given distinct count. distinct == 1 gives 0 (the
// decoder broadcasts the single dictionary entry).
//
//go:nosplit
func bitsForDistinct(distinct int) int {
        if distinct <= 1 {
                return 0
        }
        return bits.Len(uint(distinct - 1))
}

// qpackProbeSlots is the open-addressed membership set used by the
// distinct probes: the smallest power of two that keeps the load factor
// at or below 1/2 when the table is full (qpackDictMaxDistinct keys). A
// linear membership scan would cost O(n·cap); the set keeps the probe
// O(n) so raising the cap never slows the picker on high- or
// mid-cardinality slices.
const qpackProbeSlots = 128

// probeDistinctU64 walks s up to len(s) elements and returns the
// distinct value table along with the count. The second return is
// false when the cardinality exceeds qpackDictMaxDistinct — in that
// case the table is unspecified and the caller skips the dictionary
// codec entirely. The table buffer is caller-supplied so the picker
// can keep the probe stack-allocated. Not //go:nosplit: the membership
// set's backing arrays exceed the nosplit stack budget, and the probe
// runs once per slice column so the growth-check prologue is noise.
func probeDistinctU64(s []uint64, table *[qpackDictMaxDistinct + 1]uint64) (int, bool) {
        var slots [qpackProbeSlots]uint64
        var used [qpackProbeSlots]bool
        count := 0
        for _, v := range s {
                h := (v * 0x9E3779B97F4A7C15) >> (64 - 7) & (qpackProbeSlots - 1) // top 7 bits → 128 slots
                seen := false
                for used[h] {
                        if slots[h] == v {
                                seen = true
                                break
                        }
                        h = (h + 1) & (qpackProbeSlots - 1)
                }
                if seen {
                        continue
                }
                if count >= qpackDictMaxDistinct {
                        return 0, false
                }
                used[h] = true
                slots[h] = v
                table[count] = v
                count++
        }
        return count, true
}

// probeDistinctIndexU64 is probeDistinctU64 that additionally yields the
// value→index hash map (hslot holds the keys, hidx[slot] the dictionary
// position) the encoder needs to resolve each element. Folding the index build
// into the distinct probe lets dict encode do ONE open-addressed pass instead of
// a probe followed by a separate buildDictIndex pass. Only the encoder needs the
// map; the picker keeps the lighter probeDistinctU64.
func probeDistinctIndexU64(s []uint64, table *[qpackDictMaxDistinct + 1]uint64) (count int, ok bool, hslot [qpackProbeSlots]uint64, hidx [qpackProbeSlots]int16) {
        var used [qpackProbeSlots]bool
        for _, v := range s {
                h := (v * 0x9E3779B97F4A7C15) >> (64 - 7) & (qpackProbeSlots - 1)
                seen := false
                for used[h] {
                        if hslot[h] == v {
                                seen = true
                                break
                        }
                        h = (h + 1) & (qpackProbeSlots - 1)
                }
                if seen {
                        continue
                }
                if count >= qpackDictMaxDistinct {
                        return 0, false, hslot, hidx
                }
                used[h] = true
                hslot[h] = v
                hidx[h] = int16(count)
                table[count] = v
                count++
        }
        return count, true, hslot, hidx
}

// probeDistinctIndexI64 mirrors probeDistinctIndexU64 for signed slices.
func probeDistinctIndexI64(s []int64, table *[qpackDictMaxDistinct + 1]int64) (count int, ok bool, hslot [qpackProbeSlots]int64, hidx [qpackProbeSlots]int16) {
        var used [qpackProbeSlots]bool
        for _, v := range s {
                h := (uint64(v) * 0x9E3779B97F4A7C15) >> (64 - 7) & (qpackProbeSlots - 1)
                seen := false
                for used[h] {
                        if hslot[h] == v {
                                seen = true
                                break
                        }
                        h = (h + 1) & (qpackProbeSlots - 1)
                }
                if seen {
                        continue
                }
                if count >= qpackDictMaxDistinct {
                        return 0, false, hslot, hidx
                }
                used[h] = true
                hslot[h] = v
                hidx[h] = int16(count)
                table[count] = v
                count++
        }
        return count, true, hslot, hidx
}

// probeDistinctI64 mirrors probeDistinctU64 for signed slices, hashing
// the two's-complement bit pattern.
func probeDistinctI64(s []int64, table *[qpackDictMaxDistinct + 1]int64) (int, bool) {
        var slots [qpackProbeSlots]int64
        var used [qpackProbeSlots]bool
        count := 0
        for _, v := range s {
                h := (uint64(v) * 0x9E3779B97F4A7C15) >> (64 - 7) & (qpackProbeSlots - 1)
                seen := false
                for used[h] {
                        if slots[h] == v {
                                seen = true
                                break
                        }
                        h = (h + 1) & (qpackProbeSlots - 1)
                }
                if seen {
                        continue
                }
                if count >= qpackDictMaxDistinct {
                        return 0, false
                }
                used[h] = true
                slots[h] = v
                table[count] = v
                count++
        }
        return count, true
}

// qpackDictSizeU64 returns the wire byte cost of dictionary-coding s
// with the supplied distinct table.
//
//go:nosplit
func qpackDictSizeU64(distinct []uint64, n int) int {
        hdr := 2 + uvarintLen(uint64(len(distinct)))
        for _, v := range distinct {
                hdr += uvarintLen(v)
        }
        hdr += uvarintLen(uint64(n))
        bp := bitsForDistinct(len(distinct))
        body := (n*bp + 7) >> 3
        return hdr + body
}

// qpackDictSizeI64 mirrors qpackDictSizeU64 with zigzag-encoded
// dictionary values.
//
//go:nosplit
func qpackDictSizeI64(distinct []int64, n int) int {
        hdr := 2 + uvarintLen(uint64(len(distinct)))
        for _, v := range distinct {
                hdr += uvarintLen(zigzagEncode64(v))
        }
        hdr += uvarintLen(uint64(n))
        bp := bitsForDistinct(len(distinct))
        body := (n*bp + 7) >> 3
        return hdr + body
}

// pickF64Codec selects between the raw-LE bulk codec and the
// Gorilla XOR codec for a []float64 by probing the XOR
// distribution of the first sliceProbe consecutive pairs and
// projecting the per-element bit cost. It also returns the
// projected Gorilla wire size in bytes, which the float64 picker
// compares against the ALP estimate. The byte projection is only
// meaningful when the returned codec is qpackGorilla; when
// qpackRaw it is set to a raw-equivalent value.
//
// Raw cost: 64 bits / element. Gorilla worst case approaches raw;
// best case (smooth time series with repeated XOR windows) drops
// well below — single equal-value step is 1 control bit. The
// probe samples enough pairs to estimate the average meaningful-
// bit count cheaply (~30 ns total), then compares against a
// threshold that accounts for Gorilla's per-sample control bits
// and header overhead.
//
// Returns qpackGorilla on hit, qpackRaw otherwise. The decoder
// already handles tagPackGorilla via readPackedGorillaFloat64Slice
// so the choice is wire-compatible — older decoders that don't
// know the tag would fail with ErrBadTag, but they can't decode
// any Dense / QPack stream anyway.
//
//go:nosplit
func pickF64Codec(s []float64) (qpackCodec, int) {
        n := len(s)
        rawBytes := 12 + n*8
        if n < 8 {
                // Gorilla overhead (kind + first u64 + numBits varuint)
                // dominates on tiny slices.
                return qpackRaw, rawBytes
        }
        probe := min(32, n-1)
        var total uint64
        prev := math.Float64bits(s[0])
        for i := 1; i <= probe; i++ {
                cur := math.Float64bits(s[i])
                x := cur ^ prev
                if x == 0 {
                        // Repeat: Gorilla writes a single control bit.
                        total++
                } else {
                        // Meaningful bits + ~14 control + window-header bits
                        // in the average case (4-bit no-window-update + 11
                        // bit new-window). Slightly pessimistic.
                        meaningful := uint64(64 - bits.LeadingZeros64(x) - bits.TrailingZeros64(x))
                        total += meaningful + 14
                }
                prev = cur
        }
        // Projected average bits per element for Gorilla. Raw is 64.
        // Pick Gorilla only when the projection is comfortably below
        // raw to absorb fixed-header overhead (kind + first value +
        // numBits varuint = ~10 bytes ≈ 80 bits amortised).
        avgBits := total / uint64(probe)
        gorBytes := 12 + (int(avgBits)*n+7)/8
        if avgBits+1 < 48 {
                return qpackGorilla, gorBytes
        }
        return qpackRaw, rawBytes
}

// pickF32Codec is the float32 twin of pickF64Codec: it projects Gorilla's
// average bits-per-element from a sample prefix and recommends Gorilla only when
// that projection is comfortably below the raw 32 bits/elem. Raw bytes are exact
// (tag + kind + uvarint(n) + 4n); the Gorilla figure is an estimate — the caller
// emits Gorilla for real and re-checks against raw, so this only decides whether
// the attempt is worth making. The XOR window uses 4-bit leading-zero / 5-bit
// meaningful-bit fields here (vs 5/6 for float64), so the per-new-window control
// overhead is ~11 bits.
func pickF32Codec(s []float32) (qpackCodec, int) {
        n := len(s)
        rawBytes := 2 + uvarintLen(uint64(n)) + n*4
        if n < 8 {
                // Gorilla's fixed overhead (kind + first u32 + numBits varuint) dominates
                // on tiny slices.
                return qpackRaw, rawBytes
        }
        probe := min(32, n-1)
        var total uint64
        prev := math.Float32bits(s[0])
        for i := 1; i <= probe; i++ {
                cur := math.Float32bits(s[i])
                x := cur ^ prev
                if x == 0 {
                        total++ // repeat: one control bit
                } else {
                        meaningful := uint64(32 - bits.LeadingZeros32(x) - bits.TrailingZeros32(x))
                        total += meaningful + 11
                }
                prev = cur
        }
        avgBits := total / uint64(probe)
        gorBytes := 2 + uvarintLen(uint64(n)) + 4 + (int(avgBits)*n+7)/8
        // Keep the same 75%-of-raw threshold as float64 (48/64): 24/32.
        if avgBits+1 < 24 {
                return qpackGorilla, gorBytes
        }
        return qpackRaw, rawBytes
}

// QPack codec helpers. Each codec emits a single self-described tagged
// payload that replaces the per-element tag stream for one slice. The
// codecs are opt-in (Encoder.SetQPack); decoders accept the new tags
// unconditionally so a payload written with QPack can always be read.

// writePackedBool encodes a []bool as one bit per element, LSB-first within
// each byte. Wire form: tagPackBool, varuint(n), ceil(n/8) bytes. Two
// fewer bytes than the per-element tag form once n >= 2, and 8× smaller
// past n >= 16.
func (e *Encoder) writePackedBool(s []bool) {
        e.writeHeader()
        n := len(s)
        nBytes := (n + 7) >> 3
        // Worst-case header is 1 (tag) + 10 (varuint) bytes. Reserve once so
        // the body append is a single memmove past the grow.
        out := slices.Grow(e.buf, 11+nBytes)
        out = append(out, tagPackBool)
        out = appendUvarint(out, uint64(n))
        start := len(out)
        out = out[:start+nBytes]
        body := out[start : start+nBytes]
        clear(body)
        // bitpack.PackBoolsLSB dispatches to AVX2 (VPSLLW + VPMOVMSKB, 32
        // bools per iteration) under qdf_simd on amd64; everything else
        // falls back to a scalar bit-by-bit pack.
        bitpack.PackBoolsLSB(body, s, n)
        e.buf = out
}

// writePackedRawBytes emits a tagPackRaw header followed by the supplied
// little-endian payload. n is the element count; payload length must
// equal n * widthFromKind(kind).
func (e *Encoder) writePackedRawBytes(kind byte, n int, payload []byte) {
        e.writeHeader()
        out := slices.Grow(e.buf, 2+10+len(payload))
        out = append(out, tagPackRaw, kind)
        out = appendUvarint(out, uint64(n))
        out = append(out, payload...)
        e.buf = out
}

// writePackedUint64Slice writes []uint64 as a raw-LE bulk payload. On a
// little-endian target this is a single memmove; on big-endian, an
// element-wise LE emit loop.
func (e *Encoder) writePackedUint64Slice(s []uint64) {
        n := len(s)
        if endian.NativeIsLittle && n > 0 {
                body := unsafe.Slice((*byte)(unsafe.Pointer(&s[0])), n*8)
                e.writePackedRawBytes(qpackKindUint64, n, body)
                return
        }
        e.writeHeader()
        out := slices.Grow(e.buf, 2+10+n*8)
        out = append(out, tagPackRaw, qpackKindUint64)
        out = appendUvarint(out, uint64(n))
        for _, v := range s {
                out = appendU64(out, v)
        }
        e.buf = out
}

func (e *Encoder) writePackedInt64Slice(s []int64) {
        n := len(s)
        if endian.NativeIsLittle && n > 0 {
                body := unsafe.Slice((*byte)(unsafe.Pointer(&s[0])), n*8)
                e.writePackedRawBytes(qpackKindInt64, n, body)
                return
        }
        e.writeHeader()
        out := slices.Grow(e.buf, 2+10+n*8)
        out = append(out, tagPackRaw, qpackKindInt64)
        out = appendUvarint(out, uint64(n))
        for _, v := range s {
                out = appendU64(out, uint64(v))
        }
        e.buf = out
}

func (e *Encoder) writePackedUint32Slice(s []uint32) {
        n := len(s)
        if endian.NativeIsLittle && n > 0 {
                body := unsafe.Slice((*byte)(unsafe.Pointer(&s[0])), n*4)
                e.writePackedRawBytes(qpackKindUint32, n, body)
                return
        }
        e.writeHeader()
        out := slices.Grow(e.buf, 2+10+n*4)
        out = append(out, tagPackRaw, qpackKindUint32)
        out = appendUvarint(out, uint64(n))
        for _, v := range s {
                out = appendU32(out, v)
        }
        e.buf = out
}

func (e *Encoder) writePackedInt32Slice(s []int32) {
        n := len(s)
        if endian.NativeIsLittle && n > 0 {
                body := unsafe.Slice((*byte)(unsafe.Pointer(&s[0])), n*4)
                e.writePackedRawBytes(qpackKindInt32, n, body)
                return
        }
        e.writeHeader()
        out := slices.Grow(e.buf, 2+10+n*4)
        out = append(out, tagPackRaw, qpackKindInt32)
        out = appendUvarint(out, uint64(n))
        for _, v := range s {
                out = appendU32(out, uint32(v))
        }
        e.buf = out
}

func (e *Encoder) writePackedFloat32Slice(s []float32) {
        n := len(s)
        if endian.NativeIsLittle && n > 0 {
                body := unsafe.Slice((*byte)(unsafe.Pointer(&s[0])), n*4)
                e.writePackedRawBytes(qpackKindFloat32, n, body)
                return
        }
        e.writeHeader()
        out := slices.Grow(e.buf, 2+10+n*4)
        out = append(out, tagPackRaw, qpackKindFloat32)
        out = appendUvarint(out, uint64(n))
        for _, v := range s {
                out = appendU32(out, math.Float32bits(v))
        }
        e.buf = out
}

func (e *Encoder) writePackedFloat64Slice(s []float64) {
        n := len(s)
        if endian.NativeIsLittle && n > 0 {
                body := unsafe.Slice((*byte)(unsafe.Pointer(&s[0])), n*8)
                e.writePackedRawBytes(qpackKindFloat64, n, body)
                return
        }
        e.writeHeader()
        out := slices.Grow(e.buf, 2+10+n*8)
        out = append(out, tagPackRaw, qpackKindFloat64)
        out = appendUvarint(out, uint64(n))
        for _, v := range s {
                out = appendU64(out, math.Float64bits(v))
        }
        e.buf = out
}

// readPackedRawHeader consumes the kind byte and varuint length that
// follow a tagPackRaw tag (the tag itself must already be consumed). It
// returns the element count and a slice aliasing the LE payload. The
// caller's expected kind must match the on-wire kind.
func (d *Decoder) readPackedRawHeader(expectKind byte) (int, []byte, error) {
        if d.i >= len(d.buf) {
                return 0, nil, ErrShortBuffer
        }
        k := d.buf[d.i]
        d.i++
        if k != expectKind {
                return 0, nil, ErrTypeMismatch
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, nil, ErrInvalidLength
        }
        d.i += nr
        w := qpackRawWidthBytes(k)
        if w == 0 {
                return 0, nil, ErrBadTag
        }
        if n64 > uint64(len(d.buf)-d.i)/uint64(w) {
                return 0, nil, ErrShortBuffer
        }
        n := int(n64)
        nBytes := n * w
        body := d.buf[d.i : d.i+nBytes]
        d.i += nBytes
        return n, body, nil
}

func (d *Decoder) readPackedUint64Slice() ([]uint64, error) {
        n, body, err := d.readPackedRawHeader(qpackKindUint64)
        if err != nil {
                return nil, err
        }
        out := make([]uint64, n)
        if n == 0 {
                return out, nil
        }
        if endian.NativeIsLittle {
                dst := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*8)
                copy(dst, body)
                return out, nil
        }
        for i := range n {
                out[i] = readU64(body[i*8:])
        }
        return out, nil
}

func (d *Decoder) readPackedInt64Slice() ([]int64, error) {
        n, body, err := d.readPackedRawHeader(qpackKindInt64)
        if err != nil {
                return nil, err
        }
        out := make([]int64, n)
        if n == 0 {
                return out, nil
        }
        if endian.NativeIsLittle {
                dst := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*8)
                copy(dst, body)
                return out, nil
        }
        for i := range n {
                out[i] = int64(readU64(body[i*8:]))
        }
        return out, nil
}

func (d *Decoder) readPackedUint32Slice() ([]uint32, error) {
        n, body, err := d.readPackedRawHeader(qpackKindUint32)
        if err != nil {
                return nil, err
        }
        out := make([]uint32, n)
        if n == 0 {
                return out, nil
        }
        if endian.NativeIsLittle {
                dst := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*4)
                copy(dst, body)
                return out, nil
        }
        for i := range n {
                out[i] = readU32(body[i*4:])
        }
        return out, nil
}

func (d *Decoder) readPackedInt32Slice() ([]int32, error) {
        n, body, err := d.readPackedRawHeader(qpackKindInt32)
        if err != nil {
                return nil, err
        }
        out := make([]int32, n)
        if n == 0 {
                return out, nil
        }
        if endian.NativeIsLittle {
                dst := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*4)
                copy(dst, body)
                return out, nil
        }
        for i := range n {
                out[i] = int32(readU32(body[i*4:]))
        }
        return out, nil
}

func (d *Decoder) readPackedFloat32Slice() ([]float32, error) {
        n, body, err := d.readPackedRawHeader(qpackKindFloat32)
        if err != nil {
                return nil, err
        }
        out := make([]float32, n)
        if n == 0 {
                return out, nil
        }
        if endian.NativeIsLittle {
                dst := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*4)
                copy(dst, body)
                return out, nil
        }
        for i := range n {
                out[i] = math.Float32frombits(readU32(body[i*4:]))
        }
        return out, nil
}

func (d *Decoder) readPackedFloat64Slice() ([]float64, error) {
        n, body, err := d.readPackedRawHeader(qpackKindFloat64)
        if err != nil {
                return nil, err
        }
        out := make([]float64, n)
        if n == 0 {
                return out, nil
        }
        if endian.NativeIsLittle {
                dst := unsafe.Slice((*byte)(unsafe.Pointer(&out[0])), n*8)
                copy(dst, body)
                return out, nil
        }
        for i := range n {
                out[i] = math.Float64frombits(readU64(body[i*8:]))
        }
        return out, nil
}

// readPackedBool decodes a bool slice written by writePackedBool. The tag
// byte must already have been consumed.
func (d *Decoder) readPackedBool() ([]bool, error) {
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if n64 > uint64(len(d.buf)-d.i)*8 {
                // Length is in elements, payload is in bytes. Reject claims that
                // cannot possibly fit even if every byte carried 8 valid bits.
                return nil, ErrShortBuffer
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8 lets n64 exceed MaxInt -> int(n64) wraps negative
                return nil, ErrInvalidLength
        }
        n := int(n64)
        nBytes := (n + 7) >> 3
        if d.i+nBytes > len(d.buf) {
                return nil, ErrShortBuffer
        }
        out := make([]bool, n)
        base := d.i
        i := 0
        for ; i+8 <= n; i += 8 {
                b := d.buf[base+(i>>3)]
                out[i] = b&(1<<0) != 0
                out[i+1] = b&(1<<1) != 0
                out[i+2] = b&(1<<2) != 0
                out[i+3] = b&(1<<3) != 0
                out[i+4] = b&(1<<4) != 0
                out[i+5] = b&(1<<5) != 0
                out[i+6] = b&(1<<6) != 0
                out[i+7] = b&(1<<7) != 0
        }
        for ; i < n; i++ {
                out[i] = d.buf[base+(i>>3)]&(1<<uint(i&7)) != 0
        }
        d.i += nBytes
        return out, nil
}

package qdf

import (
        "math"
        "math/bits"
        "slices"

        "github.com/alex60217101990/qdf/internal/bitpack"
)

// ALP (Adaptive Lossless floating-Point, CWI 2023), decimal path. Encodes a
// []float64 whose values are decimals in disguise (quantized telemetry, prices,
// percentages, latencies) as integer mantissas I = round(v * 10^d) bit-packed
// with Frame-of-Reference, plus an exception list for any value that does not
// reconstruct bit-for-bit (NaN/±Inf/-0 and genuinely-irrational floats). Wins
// big over Gorilla on quantized data and decodes far faster; the float64 picker
// keeps Gorilla/raw for pure-smooth data where ALP would lose.

const alpMaxExp = 18

var (
        alpPow10 [alpMaxExp + 1]float64
        alpInv10 [alpMaxExp + 1]float64
)

func init() {
        p := 1.0
        for i := 0; i <= alpMaxExp; i++ {
                alpPow10[i] = p
                alpInv10[i] = 1.0 / p
                p *= 10
        }
}

// alpMaxExpSearch caps the effective-exponent search. 0..15 covers every
// practical decimal resolution; 16..18 only ever add exceptions.
const alpMaxExpSearch = 15

// alpMaxElems caps the element count ALP will encode or accept. A constant
// (width==0) slice carries no per-element bytes, so n is otherwise unbounded by
// the buffer; capping on both sides keeps a hostile header from forcing an
// oversized allocation while never rejecting anything the encoder emits.
const alpMaxElems = 1 << 24

// alpFloatPlan is the chosen encoding parameters for one []float64 block.
type alpFloatPlan struct {
        d      int   // effective decimal exponent (e-f)
        forMin int64 // FOR reference of the non-exception integer mantissas
        width  uint8 // bits per packed element, 0..56
        exc    int   // exception count
}

// alpScoreExp evaluates one effective exponent d over s: returns the FOR min,
// bit width, exception count. Round-trip uses exact float64 equality.
func alpScoreExp(s []float64, d int) (forMin int64, width uint8, exc int) {
        pe := alpPow10[d]
        ie := alpInv10[d]
        mn := int64(math.MaxInt64)
        mx := int64(math.MinInt64)
        for _, v := range s {
                I := int64(math.RoundToEven(v * pe))
                // Bit-exact round-trip check. Arithmetic == would treat -0.0 as equal
                // to +0.0 and silently drop the sign bit, and NaN != NaN; comparing
                // bits makes -0.0/NaN/±Inf land in the exception list as the spec
                // requires (exact float64 bit equality).
                if math.Float64bits(float64(I)*ie) != math.Float64bits(v) {
                        exc++
                        continue
                }
                if I < mn {
                        mn = I
                }
                if I > mx {
                        mx = I
                }
        }
        if mn > mx { // all exceptions
                return 0, 0, exc
        }
        return mn, uint8(bits.Len64(uint64(mx - mn))), exc
}

// alpChooseExp samples ~32 evenly-spaced values to pick the cheapest effective
// exponent in 0..alpMaxExpSearch by FOR-width + exception cost on the sample.
func alpChooseExp(s []float64) int {
        const sampleN = 32
        sample := s
        if len(s) > sampleN {
                var sampleBuf [sampleN]float64
                sample = sampleBuf[:0]
                step := len(s) / sampleN
                for i := 0; i < len(s); i += step {
                        sample = append(sample, s[i])
                }
        }
        bestD, bestCost := 0, math.MaxInt // platform max sentinel (int, 32-bit safe)
        for d := 0; d <= alpMaxExpSearch; d++ {
                _, w, exc := alpScoreExp(sample, d)
                cost := (int(w)*len(sample)+7)/8 + exc*10
                if cost < bestCost {
                        bestCost, bestD = cost, d
                }
        }
        return bestD
}

// alpPlanFloat64 chooses the exponent, scores the full block, and returns a
// conservative upper bound on the encoded byte size. ok is false when ALP is
// not applicable: width exceeds the 56-bit FOR cap, or so many values are
// exceptions that ALP can never beat raw. The byte estimate over-counts
// exception positions (5-byte varuint) so the picker never chooses ALP when it
// would actually grow the wire.
func alpPlanFloat64(s []float64) (plan alpFloatPlan, estBytes int, ok bool) {
        n := len(s)
        if n == 0 {
                return alpFloatPlan{}, 2 + 1, true // tag+kind+varuint(0)
        }
        if n > alpMaxElems {
                return alpFloatPlan{}, 0, false // too large; raw/Gorilla (buffer-bounded) handle it
        }
        d := alpChooseExp(s)
        forMin, width, exc := alpScoreExp(s, d)
        if width > qpackForMaxBits {
                return alpFloatPlan{}, 0, false
        }
        if exc >= n { // nothing packs; raw/Gorilla strictly better
                return alpFloatPlan{}, 0, false
        }
        plan = alpFloatPlan{d: d, forMin: forMin, width: width, exc: exc}
        body := (int(width)*n + 7) / 8
        estBytes = 2 + uvarintLen(uint64(n)) + 1 + uvarintLen(zigzagEncode64(forMin)) +
                1 + body + uvarintLen(uint64(exc)) + exc*(5+8)
        return plan, estBytes, true
}

// writePackedALPFloat64Slice emits s under the ALP decimal codec using a
// pre-computed plan (from alpPlanFloat64). The caller guarantees plan.ok.
func (e *Encoder) writePackedALPFloat64Slice(s []float64, plan alpFloatPlan) {
        e.writeHeader()
        n := len(s)
        out := slices.Grow(e.buf, 2+10+1+10+1+(int(plan.width)*n+7)/8+10+plan.exc*(5+8))
        out = append(out, tagPackALP, qpackKindFloat64)
        out = appendUvarint(out, uint64(n))
        if n == 0 {
                e.buf = out
                return
        }
        out = append(out, byte(plan.d))
        out = appendUvarint(out, zigzagEncode64(plan.forMin))
        out = append(out, plan.width)

        pe := alpPow10[plan.d]
        ie := alpInv10[plan.d]

        // Acquire exception-index scratch. Populated during the pack pass so the
        // exception-emit phase below never needs a second O(n) re-scan of s.
        excIdx := e.alpExcScratch[:0]

        if plan.width > 0 {
                // Pack I_i - forMin; exceptions occupy a 0 slot. Reuse pooled scratch
                // instead of a per-call make; clear is REQUIRED because exception slots
                // are never written in the loop below and must read back as 0 (a reused
                // buffer would otherwise leak a prior column's mantissa into them).
                if cap(e.alpScratch) < n {
                        e.alpScratch = make([]uint64, n)
                }
                packed := e.alpScratch[:n]
                clear(packed)
                for i, v := range s {
                        I := int64(math.RoundToEven(v * pe))
                        if math.Float64bits(float64(I)*ie) == math.Float64bits(v) {
                                packed[i] = uint64(I - plan.forMin)
                        } else {
                                excIdx = append(excIdx, i) // collect exception index for emit below
                        }
                }
                bodyBytes := (int(plan.width)*n + 7) / 8
                base := len(out)
                // Extend into the capacity slices.Grow already reserved; bitpack.Pack
                // overwrites every one of bodyBytes bytes, so no zero-fill is needed (a
                // make([]byte, bodyBytes) here would alloc + zero a throwaway slice).
                out = out[:base+bodyBytes]
                bitpack.Pack(out[base:base+bodyBytes], packed, int(plan.width))
        } else if plan.exc > 0 {
                // plan.width == 0: all non-exception values share the same mantissa
                // (constant column). No pack body, but still collect exception indices
                // for the emit pass below.
                for i, v := range s {
                        I := int64(math.RoundToEven(v * pe))
                        if math.Float64bits(float64(I)*ie) != math.Float64bits(v) {
                                excIdx = append(excIdx, i)
                        }
                }
        }

        // Save scratch back for the next call (avoids re-alloc when the exception
        // count is stable across batches).
        e.alpExcScratch = excIdx

        // Exception list. When there are no exceptions (the common case for clean
        // quantized telemetry) the guard skips the emit entirely. Otherwise emit
        // from the collected index scratch — no second O(n) re-scan of s.
        out = appendUvarint(out, uint64(plan.exc))
        if len(excIdx) > 0 {
                for _, i := range excIdx {
                        out = appendUvarint(out, uint64(i))
                        out = appendU64(out, math.Float64bits(s[i]))
                }
        }
        e.buf = out
}

// --- float32 ALP (qpackKindFloat32) ---
//
// Identical scheme to the float64 path: integer mantissas I = round(v * 10^d)
// FOR-bit-packed, plus an exception list for any value that does not reconstruct
// bit-for-bit. The mantissa is computed in float64 (exact for the integers ALP
// targets) and the round-trip is checked against the float32 bits; exception
// values are stored as 4-byte float32 patterns, not 8. The 23-bit float32
// mantissa makes the typical FOR width narrower than float64.

// alpMantissaF32 returns the integer mantissa round(v*10^d) (pe=10^d) and whether
// it reconstructs v bit-for-bit as a float32 (ie=10^-d) — i.e. whether the value
// packs as a mantissa or falls to the exception list. The mantissa is computed
// through float64 so it stays exact for values needing more than float32's 24-bit
// significand (a float32-only round would mis-round those into extra exceptions).
// Bit comparison (not ==) keeps -0.0/NaN/±Inf as exceptions. Centralises the one
// float32↔float64↔int64 chain the score / pack / exception passes all share.
func alpMantissaF32(v float32, pe, ie float64) (mant int64, exact bool) {
        I := int64(math.RoundToEven(float64(v) * pe))
        return I, math.Float32bits(float32(float64(I)*ie)) == math.Float32bits(v)
}

// alpScoreExpF32 evaluates one effective exponent d over s (float32 bit-exact
// round-trip). Mirrors alpScoreExp.
func alpScoreExpF32(s []float32, d int) (forMin int64, width uint8, exc int) {
        pe := alpPow10[d]
        ie := alpInv10[d]
        mn := int64(math.MaxInt64)
        mx := int64(math.MinInt64)
        for _, v := range s {
                I, ok := alpMantissaF32(v, pe, ie)
                if !ok {
                        exc++
                        continue
                }
                if I < mn {
                        mn = I
                }
                if I > mx {
                        mx = I
                }
        }
        if mn > mx {
                return 0, 0, exc
        }
        return mn, uint8(bits.Len64(uint64(mx - mn))), exc
}

// alpChooseExpF32 samples ~32 values to pick the cheapest effective exponent.
func alpChooseExpF32(s []float32) int {
        const sampleN = 32
        sample := s
        if len(s) > sampleN {
                var sampleBuf [sampleN]float32
                sample = sampleBuf[:0]
                step := len(s) / sampleN
                for i := 0; i < len(s); i += step {
                        sample = append(sample, s[i])
                }
        }
        bestD, bestCost := 0, math.MaxInt // platform max sentinel (int, 32-bit safe)
        for d := 0; d <= alpMaxExpSearch; d++ {
                _, w, exc := alpScoreExpF32(sample, d)
                cost := (int(w)*len(sample)+7)/8 + exc*6 // exc value is 4 bytes here
                if cost < bestCost {
                        bestCost, bestD = cost, d
                }
        }
        return bestD
}

// alpPlanFloat32 chooses the exponent, scores the block, and returns a
// conservative upper bound on the encoded size (exception value = 4 bytes).
func alpPlanFloat32(s []float32) (plan alpFloatPlan, estBytes int, ok bool) {
        n := len(s)
        if n == 0 {
                return alpFloatPlan{}, 2 + 1, true
        }
        if n > alpMaxElems {
                return alpFloatPlan{}, 0, false
        }
        d := alpChooseExpF32(s)
        forMin, width, exc := alpScoreExpF32(s, d)
        if width > qpackForMaxBits {
                return alpFloatPlan{}, 0, false
        }
        if exc >= n {
                return alpFloatPlan{}, 0, false
        }
        plan = alpFloatPlan{d: d, forMin: forMin, width: width, exc: exc}
        body := (int(width)*n + 7) / 8
        estBytes = 2 + uvarintLen(uint64(n)) + 1 + uvarintLen(zigzagEncode64(forMin)) +
                1 + body + uvarintLen(uint64(exc)) + exc*(5+4)
        return plan, estBytes, true
}

// writePackedALPFloat32Slice emits s under ALP using a pre-computed plan.
func (e *Encoder) writePackedALPFloat32Slice(s []float32, plan alpFloatPlan) {
        e.writeHeader()
        n := len(s)
        out := slices.Grow(e.buf, 2+10+1+10+1+(int(plan.width)*n+7)/8+10+plan.exc*(5+4))
        out = append(out, tagPackALP, qpackKindFloat32)
        out = appendUvarint(out, uint64(n))
        if n == 0 {
                e.buf = out
                return
        }
        out = append(out, byte(plan.d))
        out = appendUvarint(out, zigzagEncode64(plan.forMin))
        out = append(out, plan.width)

        pe := alpPow10[plan.d]
        ie := alpInv10[plan.d]

        // Acquire exception-index scratch (shared with float64 path; reset to [:0]).
        excIdx := e.alpExcScratch[:0]

        if plan.width > 0 {
                if cap(e.alpScratch) < n {
                        e.alpScratch = make([]uint64, n)
                }
                packed := e.alpScratch[:n]
                clear(packed) // exception slots stay 0 (never written in the loop)
                for i, v := range s {
                        if I, ok := alpMantissaF32(v, pe, ie); ok {
                                packed[i] = uint64(I - plan.forMin)
                        } else {
                                excIdx = append(excIdx, i) // collect exception index for emit below
                        }
                }
                bodyBytes := (int(plan.width)*n + 7) / 8
                base := len(out)
                // Extend into the capacity slices.Grow already reserved; bitpack.Pack
                // overwrites every one of bodyBytes bytes, so no zero-fill is needed (a
                // make([]byte, bodyBytes) here would alloc + zero a throwaway slice).
                out = out[:base+bodyBytes]
                bitpack.Pack(out[base:base+bodyBytes], packed, int(plan.width))
        } else if plan.exc > 0 {
                // plan.width == 0: constant column with exceptions — collect indices.
                for i, v := range s {
                        if _, ok := alpMantissaF32(v, pe, ie); !ok {
                                excIdx = append(excIdx, i)
                        }
                }
        }

        // Save scratch back for the next call.
        e.alpExcScratch = excIdx

        // Exception list: emit from collected index scratch — no second O(n) re-scan.
        out = appendUvarint(out, uint64(plan.exc))
        if len(excIdx) > 0 {
                for _, i := range excIdx {
                        out = appendUvarint(out, uint64(i))
                        out = appendU32(out, math.Float32bits(s[i]))
                }
        }
        e.buf = out
}

// readPackedALPFloat32Slice decodes a float32 ALP payload. The cursor (d.i) must
// point at the kind byte. Bounds mirror readPackedALPFloat64Slice; exception
// values are 4 bytes.
func (d *Decoder) readPackedALPFloat32Slice() ([]float32, error) {
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        if d.buf[d.i] != qpackKindFloat32 {
                return nil, ErrTypeMismatch
        }
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if n64 == 0 {
                return []float32{}, nil
        }
        if n64 > alpMaxElems {
                return nil, ErrInvalidLength
        }
        // Bound by the columnar row count before make([]float32, n) (see the float64
        // reader); no-op for standalone decode (colMaxLen==0).
        if !d.colLenOK(n64) {
                return nil, ErrInvalidLength
        }
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        expD := int(d.buf[d.i])
        d.i++
        if expD > alpMaxExp {
                return nil, ErrBadTag
        }
        fm64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        forMin := zigzagDecode64(fm64)
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        width := int(d.buf[d.i])
        d.i++
        if width > qpackForMaxBits {
                return nil, ErrBadTag
        }
        rem := uint64(len(d.buf) - d.i)
        if width > 0 && n64 > rem*8/uint64(width) {
                return nil, ErrShortBuffer
        }
        n := int(n64)
        ie := alpInv10[expD]
        out := make([]float32, n)
        if width > 0 {
                bodyBytes := int((n64*uint64(width) + 7) / 8)
                if cap(d.deltaScratch) < n {
                        d.deltaScratch = make([]uint64, n)
                }
                packed := d.deltaScratch[:n]
                bitpack.Unpack(packed, d.buf[d.i:d.i+bodyBytes], width)
                d.i += bodyBytes
                for i := range out {
                        out[i] = float32(float64(int64(packed[i])+forMin) * ie)
                }
        } else {
                fv := float32(float64(forMin) * ie)
                for i := range out {
                        out[i] = fv
                }
        }
        excN, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if excN > n64 {
                return nil, ErrInvalidLength
        }
        for range excN {
                pos64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if pos64 >= n64 {
                        return nil, ErrInvalidLength
                }
                if d.i+4 > len(d.buf) {
                        return nil, ErrShortBuffer
                }
                out[pos64] = math.Float32frombits(readU32(d.buf[d.i:]))
                d.i += 4
        }
        return out, nil
}

// readPackedALPFloat64Slice decodes an ALP payload. The cursor (d.i) must point
// at the kind byte (the caller consumed the tag). Bounds-checked against hostile
// input: width capped at 56, body size validated overflow-safe, exception
// positions validated < n.
func (d *Decoder) readPackedALPFloat64Slice() ([]float64, error) {
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        if d.buf[d.i] != qpackKindFloat64 {
                return nil, ErrTypeMismatch
        }
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if n64 == 0 {
                return []float64{}, nil
        }
        if n64 > alpMaxElems {
                return nil, ErrInvalidLength
        }
        // Inside a columnar float column the element count must equal the row count;
        // gate before the make([]float64, n) below (mirrors readPackedGorillaHeader).
        // The constant (width==0) path carries no per-element body, so without this a
        // tiny header could claim alpMaxElems rows and force a ~128 MB allocation that
        // only the post-decode len(s)!=n check would reject. Standalone decode has
        // colMaxLen==0, so colLenOK is a no-op there (the alpMaxElems cap still holds).
        if !d.colLenOK(n64) {
                return nil, ErrInvalidLength
        }
        // Header: d(1), forMin zigzag-varuint, width(1).
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        expD := int(d.buf[d.i])
        d.i++
        if expD > alpMaxExp {
                return nil, ErrBadTag
        }
        fm64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        forMin := zigzagDecode64(fm64)
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        width := int(d.buf[d.i])
        d.i++
        if width > qpackForMaxBits {
                return nil, ErrBadTag
        }
        // Body, overflow-safe size check (mirrors readPackedForHeader).
        rem := uint64(len(d.buf) - d.i)
        if width > 0 && n64 > rem*8/uint64(width) {
                return nil, ErrShortBuffer
        }
        n := int(n64)
        ie := alpInv10[expD]
        out := make([]float64, n)
        if width > 0 {
                bodyBytes := int((n64*uint64(width) + 7) / 8)
                // Reuse the shared transient unpack scratch (as readPackedDictUint64Slice
                // does): packed is fully written by Unpack before any read, only consumed
                // into out, never aliased into the returned slice. This branch runs only
                // when width > 0, so Unpack always writes all n slots — the width == 0
                // constant path below never touches deltaScratch, so no stale-tail leak.
                if cap(d.deltaScratch) < n {
                        d.deltaScratch = make([]uint64, n)
                }
                packed := d.deltaScratch[:n]
                bitpack.Unpack(packed, d.buf[d.i:d.i+bodyBytes], width)
                d.i += bodyBytes
                for i := range out {
                        out[i] = float64(int64(packed[i])+forMin) * ie
                }
        } else {
                fv := float64(forMin) * ie
                for i := range out {
                        out[i] = fv
                }
        }
        // Exception list.
        excN, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if excN > n64 {
                return nil, ErrInvalidLength
        }
        for range excN {
                pos64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if pos64 >= n64 {
                        return nil, ErrInvalidLength
                }
                if d.i+8 > len(d.buf) {
                        return nil, ErrShortBuffer
                }
                out[pos64] = math.Float64frombits(readU64(d.buf[d.i:]))
                d.i += 8
        }
        return out, nil
}

package qdf

import (
        "encoding/binary"
        "math"
        "sync/atomic"

        "github.com/alex60217101990/qdf/internal/bitpack"
)

// Per-block adaptive integer codec (wire tag tagPackBlock, 0xF0).
//
// A long integer column is split into fixed-size blocks; each block is encoded
// independently with the existing whole-column picker (pickI64Codec /
// pickU64Codec → FOR / Delta+FOR / RLE / dict / PFOR / raw). A column whose
// statistics shift along its length (sorted-then-random, bursty timestamps,
// mixed-cardinality IDs, counters with resets) packs each region optimally
// instead of paying one codec for the whole. An offset table prefixes the
// blocks so a predicate query can seek to and decode only the blocks covering
// its matched rows.
//
// Strictly never-larger: the block form is emitted only when its exact byte
// cost is smaller than the whole-column codec. The block-cost estimate is exact
// (each block's pickXXXCodec.bestCost is exact for the codec it picks, and the
// header + offset table are sized exactly), and the whole-column cost is exact
// for non-constant columns — constant columns are filtered out by the flat
// pre-gate before any per-block work — so the comparison cannot be wrong in a
// way that inflates the wire.

const (
        // blockCodecMinLen is the smallest column the block codec considers: two
        // blocks of the smaller size, below which there is nothing to adapt.
        blockCodecMinLen = 2 * blockSizeSmall
        blockSizeSmall   = 256
        blockSizeLarge   = 1024
        blkLogSmall      = 8  // log2(256)
        blkLogLarge      = 10 // log2(1024)

        blockKindInt  = 0x00
        blockKindUint = 0x01

        // The cheap pre-gate samples blockGateSamples blocks, measures each one's FOR
        // width over just a blockGateWindow-element window (one tiny min/max pass —
        // orders of magnitude cheaper than a full pickXXXCodec), and fires when the
        // spread between the widest and narrowest sampled block is >= blockGateBitsSpread.
        // It keys on block-to-block HETEROGENEITY, not block-vs-whole tightness, so a
        // globally monotonic column (small per-block FOR width but delta-optimal as a
        // whole) is correctly skipped instead of triggering a wasted full probe. The
        // gate only filters probe CPU — the full probe + never-larger still decide, so
        // a gate miss costs size, never correctness.
        blockGateSamples    = 8
        blockGateWindow     = 64
        blockGateBitsSpread = 4

        // blockGateWideBits is the cheapest gate: a column whose whole-column FOR and
        // delta widths are BOTH below this is already compact (delta/RLE/dict or a
        // narrow FOR — the common flat column), so blocks have no headroom and the
        // sampling pass is skipped entirely. Only genuinely wide columns (the kind a
        // regime shift can shrink) ever reach the sampling gate.
        blockGateWideBits = 12
)

// blockSizeFor maps a wire blkLog byte to its block size, rejecting any value
// other than the two the encoder emits.
func blockSizeFor(blkLog byte) (int, bool) {
        switch blkLog {
        case blkLogSmall:
                return blockSizeSmall, true
        case blkLogLarge:
                return blockSizeLarge, true
        }
        return 0, false
}

// blockRegimeLikelyI64 cheaply tests whether s is heterogeneous enough to be
// worth a full per-block probe. It samples up to blockGateSamples blocks and
// compares the average per-block FOR width (one min/max pass each — far cheaper
// than a full pickI64Codec) to the whole-column width: blocks that pack markedly
// tighter mean local structure a single codec can't exploit. Catches trend
// (sorted-then-random), periodic (bursty, mixed-card), and constant-run
// (rle-then-noise) regimes; misses only the rare case where width is uniform but
// codec choice shifts — a size miss, never a correctness one.
func blockRegimeLikelyI64(s []int64, wholeForBits int) bool {
        if wholeForBits <= 2 {
                return false // already near-incompressible by FOR; blocks can't help
        }
        n := len(s)
        nb := (n + blockSizeSmall - 1) / blockSizeSmall
        step := max(nb/blockGateSamples, 1)
        minB, maxB := 64, -1
        for b := 0; b < nb; b += step {
                i := b * blockSizeSmall
                j := min(i+blockGateWindow, n)
                mn, mx := minMaxI64(s[i:j])
                w := bitpack.BitsForDelta(uint64(mx) - uint64(mn))
                minB, maxB = min(minB, w), max(maxB, w)
        }
        return maxB-minB >= blockGateBitsSpread
}

func blockRegimeLikelyU64(s []uint64, wholeForBits int) bool {
        if wholeForBits <= 2 {
                return false
        }
        n := len(s)
        nb := (n + blockSizeSmall - 1) / blockSizeSmall
        step := max(nb/blockGateSamples, 1)
        minB, maxB := 64, -1
        for b := 0; b < nb; b += step {
                i := b * blockSizeSmall
                j := min(i+blockGateWindow, n)
                mn, mx := minMaxU64(s[i:j])
                w := bitpack.BitsForDelta(mx - mn)
                minB, maxB = min(minB, w), max(maxB, w)
        }
        return maxB-minB >= blockGateBitsSpread
}

// ---- encode (int64) ----

// tryWriteBlockInt64 emits the per-block adaptive form of s when it is strictly
// smaller than the whole-column codec, returning true. It returns false (having
// written nothing) when the column is too short, statistically flat, or the
// block form would not win — the caller then falls back to writeQPackInt64.
// blockCodecEnabled gates the per-block codec. Always true in production; tests
// flip it to obtain a whole-column baseline wire for never-larger assertions.
var blockCodecEnabled = true

// blockSelectiveBlocksDecoded counts blocks materialized by the selective decode
// path. Test-only instrumentation (to assert untouched blocks are skipped);
// atomic so concurrent Decoders running selective decode do not race on it.
var blockSelectiveBlocksDecoded atomic.Int64

// tryWriteBlockInt64 emits the smaller of the per-block form and the whole-column
// form when the cheap gates say the block form is plausible, returning true (the
// column is fully written). It returns false when the gates reject the column —
// the caller then emits the whole-column form itself.
//
// The whole-column codec is passed in (picked once by the caller). The size
// decision is made by EMITTING BOTH forms and keeping the smaller, not by
// comparing predicted costs: pickI64Codec's PFOR cost is a conservative UPPER
// bound, so a predicted-cost compare could keep a block form that is actually
// larger than the real whole-column emit (a never-larger violation). Emitting and
// measuring is exact. The gates ensure this double-emit only happens on the rare
// regime column, never on flat columns.
func (e *Encoder) tryWriteBlockInt64(s []int64, codec qpackCodec, mn int64, forBits int, first, minDelta int64, deltaBits, pforBits int) bool {
        if !blockCodecEnabled || e.rans {
                // Skip under the outer rANS pass: the per-block offset table (high-entropy
                // uint32s) and repeated sub-headers rANS-compress worse than the homogeneous
                // whole-column form, so even a raw-smaller block can inflate the final wire.
                // Keeping the whole-column form leaves OptCompression byte-identical to before.
                return false
        }
        if len(s) < blockCodecMinLen {
                return false
        }
        // Cheapest gate first: a compact column (narrow FOR and narrow delta) has no
        // headroom — skip before sampling.
        if forBits < blockGateWideBits && deltaBits < blockGateWideBits {
                return false
        }
        if !blockRegimeLikelyI64(s, forBits) {
                return false
        }
        // Emit whole first (the safe fallback), then the block form, and keep whichever
        // is actually smaller — block usually wins when the gates fire, so it is emitted
        // last and re-emission only happens on a gate false-positive. planBlocksI64
        // caches the per-block picks so writeBlockInt64 does not re-pick.
        start := len(e.buf)
        hdr, flag := e.headerOut, e.headerFlagAt
        e.emitQPackInt64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
        wholeSz := len(e.buf) - start
        e.buf = e.buf[:start]
        e.headerOut, e.headerFlagAt = hdr, flag
        e.planBlocksI64(s, blockSizeSmall)
        e.writeBlockInt64(s, blockSizeSmall)
        if len(e.buf)-start >= wholeSz {
                // Block not smaller — roll back and re-emit the whole-column form. Restore
                // the header latch so a rolled-away top-level stream header is re-emitted
                // (mirrors the Gorilla never-worse rollback).
                e.buf = e.buf[:start]
                e.headerOut, e.headerFlagAt = hdr, flag
                e.emitQPackInt64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
        }
        return true
}

// blockPlanI64 caches one block's picked codec so the emit pass does not re-pick.
type blockPlanI64 struct {
        mn, first, minDelta          int64
        codec                        qpackCodec
        forBits, deltaBits, pforBits int
}

// planBlocksI64 picks the codec for every block of s (size blk) and caches the
// choices in e.blkPlanI64 so writeBlockInt64 emits without re-picking (pickI64Codec
// is the dominant encode cost).
func (e *Encoder) planBlocksI64(s []int64, blk int) {
        n := len(s)
        nBlocks := (n + blk - 1) / blk
        if cap(e.blkPlanI64) < nBlocks {
                e.blkPlanI64 = make([]blockPlanI64, nBlocks)
        }
        plans := e.blkPlanI64[:nBlocks]
        bi := 0
        for i := 0; i < n; i += blk {
                j := min(i+blk, n)
                codec, mn, forBits, first, minDelta, deltaBits, pforBits, _ := pickI64Codec(s[i:j])
                plans[bi] = blockPlanI64{mn: mn, first: first, minDelta: minDelta, codec: codec, forBits: forBits, deltaBits: deltaBits, pforBits: pforBits}
                bi++
        }
        e.blkPlanI64 = plans
}

func (e *Encoder) writeBlockInt64(s []int64, blk int) {
        // Emit the stream header before appending directly to e.buf, exactly as the
        // other QPack writers do — a bare top-level slice has no header yet, and the
        // per-block emitQPackInt64 calls would otherwise insert it after the tag.
        // Idempotent (no-op for the struct-field case where it already exists).
        e.writeHeader()
        n := len(s)
        nBlocks := (n + blk - 1) / blk
        plans := e.blkPlanI64[:nBlocks] // filled by planBlocksI64 (same s, same blk)
        e.buf = append(e.buf, tagPackBlock, blockKindInt, blkLogSmall)
        e.buf = appendUvarint(e.buf, uint64(n))
        offAt := len(e.buf)
        e.buf = append(e.buf, make([]byte, 4*nBlocks)...)
        bodyStart := len(e.buf)
        bi := 0
        for i := 0; i < n; i += blk {
                j := min(i+blk, n)
                binary.LittleEndian.PutUint32(e.buf[offAt+4*bi:], uint32(len(e.buf)-bodyStart))
                p := &plans[bi]
                e.emitQPackInt64(s[i:j], p.codec, p.mn, p.forBits, p.first, p.minDelta, p.deltaBits, p.pforBits)
                bi++
        }
}

// ---- encode (uint64) ----

func (e *Encoder) tryWriteBlockUint64(s []uint64, codec qpackCodec, mn uint64, forBits int, first uint64, minDelta int64, deltaBits, pforBits int) bool {
        if !blockCodecEnabled || e.rans {
                return false // see tryWriteBlockInt64: offset table is rANS-hostile
        }
        if len(s) < blockCodecMinLen {
                return false
        }
        if forBits < blockGateWideBits && deltaBits < blockGateWideBits {
                return false
        }
        if !blockRegimeLikelyU64(s, forBits) {
                return false
        }
        // Emit-measure-keep-smaller (see tryWriteBlockInt64 for why predicted costs
        // are not trustworthy for the never-larger decision).
        start := len(e.buf)
        hdr, flag := e.headerOut, e.headerFlagAt
        e.emitQPackUint64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
        wholeSz := len(e.buf) - start
        e.buf = e.buf[:start]
        e.headerOut, e.headerFlagAt = hdr, flag
        e.planBlocksU64(s, blockSizeSmall)
        e.writeBlockUint64(s, blockSizeSmall)
        if len(e.buf)-start >= wholeSz {
                e.buf = e.buf[:start]
                e.headerOut, e.headerFlagAt = hdr, flag
                e.emitQPackUint64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
        }
        return true
}

// blockPlanU64 caches one block's picked codec so the emit pass does not re-pick.
type blockPlanU64 struct {
        mn, first                    uint64
        minDelta                     int64
        codec                        qpackCodec
        forBits, deltaBits, pforBits int
}

func (e *Encoder) planBlocksU64(s []uint64, blk int) {
        n := len(s)
        nBlocks := (n + blk - 1) / blk
        if cap(e.blkPlanU64) < nBlocks {
                e.blkPlanU64 = make([]blockPlanU64, nBlocks)
        }
        plans := e.blkPlanU64[:nBlocks]
        bi := 0
        for i := 0; i < n; i += blk {
                j := min(i+blk, n)
                codec, mn, forBits, first, minDelta, deltaBits, pforBits, _ := pickU64Codec(s[i:j])
                plans[bi] = blockPlanU64{mn: mn, first: first, minDelta: minDelta, codec: codec, forBits: forBits, deltaBits: deltaBits, pforBits: pforBits}
                bi++
        }
        e.blkPlanU64 = plans
}

func (e *Encoder) writeBlockUint64(s []uint64, blk int) {
        e.writeHeader()
        n := len(s)
        nBlocks := (n + blk - 1) / blk
        plans := e.blkPlanU64[:nBlocks]
        e.buf = append(e.buf, tagPackBlock, blockKindUint, blkLogSmall)
        e.buf = appendUvarint(e.buf, uint64(n))
        offAt := len(e.buf)
        e.buf = append(e.buf, make([]byte, 4*nBlocks)...)
        bodyStart := len(e.buf)
        bi := 0
        for i := 0; i < n; i += blk {
                j := min(i+blk, n)
                binary.LittleEndian.PutUint32(e.buf[offAt+4*bi:], uint32(len(e.buf)-bodyStart))
                p := &plans[bi]
                e.emitQPackUint64(s[i:j], p.codec, p.mn, p.forBits, p.first, p.minDelta, p.deltaBits, p.pforBits)
                bi++
        }
}

// ---- decode header (shared) ----

// readBlockHeader consumes the block-container header (the tag is already
// consumed by the caller), validates it, and returns the block size, total
// element count, the byte offset where the offset table begins, and the byte
// offset where the block bodies begin. It does NOT consume the offset table.
func (d *Decoder) readBlockHeader(expectKind byte) (blk, n, offBase, bodyStart, nBlocks int, err error) {
        if d.i+2 > len(d.buf) {
                return 0, 0, 0, 0, 0, ErrShortBuffer
        }
        if d.buf[d.i] != expectKind {
                return 0, 0, 0, 0, 0, ErrTypeMismatch
        }
        d.i++
        blk, ok := blockSizeFor(d.buf[d.i])
        if !ok {
                return 0, 0, 0, 0, 0, ErrBadTag
        }
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, 0, 0, 0, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) || n64 == 0 || n64 > uint64(math.MaxInt) {
                return 0, 0, 0, 0, 0, ErrInvalidLength
        }
        n = int(n64)
        // Bound the make([]T, n) below by the same 256 MiB ceiling the columnar path
        // uses. A standalone block wire (top-level []int64) has no colMaxLen, so n is
        // otherwise capped only by the offset table (4 B/block) — constant sub-blocks
        // pack 256 elems into ~5 B, so a hostile tiny buffer could request a huge
        // slice. This caps the pre-block allocation regardless. (In the columnar path
        // colMaxLen already bounds n to the row count; this is redundant but cheap.)
        if checkColumnarBytes(n, 8) != nil {
                return 0, 0, 0, 0, 0, ErrInvalidLength
        }
        nBlocks = (n + blk - 1) / blk
        offBase = d.i
        if d.i+4*nBlocks > len(d.buf) {
                return 0, 0, 0, 0, 0, ErrShortBuffer
        }
        d.i += 4 * nBlocks
        bodyStart = d.i
        // Validate the offset table once: offsets[0]==0, strictly increasing, each
        // within the buffer. Selective and decode-all both rely on this.
        prev := -1
        for b := 0; b < nBlocks; b++ {
                off := int(binary.LittleEndian.Uint32(d.buf[offBase+4*b:]))
                if b == 0 && off != 0 {
                        return 0, 0, 0, 0, 0, ErrInvalidLength
                }
                if off <= prev {
                        return 0, 0, 0, 0, 0, ErrInvalidLength
                }
                if bodyStart+off > len(d.buf) {
                        return 0, 0, 0, 0, 0, ErrShortBuffer
                }
                prev = off
        }
        return blk, n, offBase, bodyStart, nBlocks, nil
}

// blockLen returns the element count of block b (the last block holds the
// remainder).
func blockLen(b, nBlocks, n, blk int) int {
        if b == nBlocks-1 {
                return n - b*blk
        }
        return blk
}

// ---- decode-all (int64) ----

func (d *Decoder) readBlockInt64() ([]int64, error) {
        var s []int64
        if err := d.readBlockInt64Into(&s); err != nil {
                return nil, err
        }
        return s, nil
}

// readBlockInt64Into is the scratch-reusing form: it grows *dst to the column
// length and decodes every block into it. The tag is already consumed.
func (d *Decoder) readBlockInt64Into(dst *[]int64) error {
        // A block body could itself carry tagPackBlock or tagZoneChunk (both dispatched
        // by readQPackInt64); bound the nesting so a hostile payload of nested blocks
        // cannot overflow the goroutine stack.
        if err := d.descend(); err != nil {
                return err
        }
        defer d.ascend()
        blk, n, offBase, bodyStart, nBlocks, err := d.readBlockHeader(blockKindInt)
        if err != nil {
                return err
        }
        // Bound each inner sub-block's constant-codec count to the block length so a
        // malformed sub-block header cannot drive an oversized make() before the
        // per-block length check rejects it (the standalone path has colMaxLen==0).
        oldMax := d.colMaxLen
        d.colMaxLen = blk
        defer func() { d.colMaxLen = oldMax }()
        growI64(dst, n)
        out := *dst
        for b := range nBlocks {
                off := int(binary.LittleEndian.Uint32(d.buf[offBase+4*b:]))
                d.i = bodyStart + off
                t, err := d.peekTag()
                if err != nil {
                        return err
                }
                v, err := d.readQPackInt64(t)
                if err != nil {
                        return err
                }
                if len(v) != blockLen(b, nBlocks, n, blk) {
                        return ErrInvalidLength
                }
                copy(out[b*blk:], v)
        }
        return nil
}

// ---- selective decode (predicate-matched rows only) ----

// decodeBlockColumnSelective decodes only the blocks of a block-tagged column
// that cover the matched row indices, scattering their values into a colVals of
// length n (matched positions filled, the rest left zero). matched must be
// ascending (matchedIndices guarantees this), so each needed block is decoded
// at most once and in wire order. The column starts at byte offset start;
// d.i is restored to its entry value on return. kind must be colKindInt or
// colKindUint and the column must be non-nullable (the caller gates this).
func (d *Decoder) decodeBlockColumnSelective(start int, kind colKind, n int, matched []int) (colVals, error) {
        save := d.i
        defer func() { d.i = save }()
        d.i = start + 1 // skip the tagPackBlock byte (recorded start points at it)

        expect := byte(blockKindInt)
        if kind == colKindUint {
                expect = blockKindUint
        }
        blk, hn, offBase, bodyStart, nBlocks, err := d.readBlockHeader(expect)
        if err != nil {
                return colVals{}, err
        }
        if hn != n {
                return colVals{}, ErrTypeMismatch
        }
        log := blkLogSmall
        if blk == blockSizeLarge {
                log = blkLogLarge
        }

        // Tighten the sub-block allocation bound to blk for the nested codec reads
        // (a constant sub-block otherwise inherits colMaxLen==n, the whole column).
        // The decode-all paths (readBlockInt64Into/readBlockUint64Into) do the same.
        oldMax := d.colMaxLen
        d.colMaxLen = blk
        defer func() { d.colMaxLen = oldMax }()

        cv := colVals{kind: kind}
        if kind == colKindInt {
                cv.i64 = make([]int64, n)
        } else {
                cv.u64 = make([]uint64, n)
        }

        mi := 0
        for mi < len(matched) {
                b := matched[mi] >> log
                if b >= nBlocks { // defensive; matched indices are < n so this cannot trip
                        return colVals{}, ErrInvalidLength
                }
                off := int(binary.LittleEndian.Uint32(d.buf[offBase+4*b:]))
                d.i = bodyStart + off
                blockSelectiveBlocksDecoded.Add(1)
                t, err := d.peekTag()
                if err != nil {
                        return colVals{}, err
                }
                want := blockLen(b, nBlocks, n, blk)
                bstart := b * blk
                if kind == colKindInt {
                        v, err := d.readQPackInt64(t)
                        if err != nil {
                                return colVals{}, err
                        }
                        if len(v) != want {
                                return colVals{}, ErrInvalidLength
                        }
                        for mi < len(matched) && matched[mi]>>log == b {
                                cv.i64[matched[mi]] = v[matched[mi]-bstart]
                                mi++
                        }
                } else {
                        v, err := d.readQPackUint64(t)
                        if err != nil {
                                return colVals{}, err
                        }
                        if len(v) != want {
                                return colVals{}, ErrInvalidLength
                        }
                        for mi < len(matched) && matched[mi]>>log == b {
                                cv.u64[matched[mi]] = v[matched[mi]-bstart]
                                mi++
                        }
                }
        }
        return cv, nil
}

// ---- decode-all (uint64) ----

func (d *Decoder) readBlockUint64() ([]uint64, error) {
        var s []uint64
        if err := d.readBlockUint64Into(&s); err != nil {
                return nil, err
        }
        return s, nil
}

func (d *Decoder) readBlockUint64Into(dst *[]uint64) error {
        if err := d.descend(); err != nil {
                return err
        }
        defer d.ascend()
        blk, n, offBase, bodyStart, nBlocks, err := d.readBlockHeader(blockKindUint)
        if err != nil {
                return err
        }
        // Bound each inner sub-block's constant-codec count to the block length (see
        // readBlockInt64Into).
        oldMax := d.colMaxLen
        d.colMaxLen = blk
        defer func() { d.colMaxLen = oldMax }()
        growU64(dst, n)
        out := *dst
        for b := range nBlocks {
                off := int(binary.LittleEndian.Uint32(d.buf[offBase+4*b:]))
                d.i = bodyStart + off
                t, err := d.peekTag()
                if err != nil {
                        return err
                }
                v, err := d.readQPackUint64(t)
                if err != nil {
                        return err
                }
                if len(v) != blockLen(b, nBlocks, n, blk) {
                        return ErrInvalidLength
                }
                copy(out[b*blk:], v)
        }
        return nil
}

package qdf

import (
        "math"
        "slices"

        "github.com/alex60217101990/qdf/internal/bitpack"
)

// Delta + FOR codec for integer slices.
//
// First value is stored verbatim. For i >= 1 we compute Δᵢ = aᵢ - aᵢ₋₁,
// find (minΔ, maxΔ), and bit-pack (Δᵢ - minΔ) into ceil(bits/8) per slot.
// For monotonic data (timestamps, IDs, counters) Δᵢ are tiny and clustered
// so bitsPer often collapses to 1..16 bits. For mixed-direction data the
// zigzag bias rolled into minΔ keeps the bit width compact.
//
// Wire payload (after the tag and kind byte):
//   bits           (1 byte, 0..56)
//   firstVal       (varuint for unsigned kinds; zigzag varuint for signed)
//   minDelta       (zigzag varuint, signed)
//   n              (varuint)
//   body           ceil((n-1)*bits/8) bytes when n>=2, else absent
//
// Arithmetic is performed in uint64 for the unsigned variant (wraps
// modulo 2^64, which is the inverse of the modular subtraction the
// encoder used). The signed variant promotes everything to int64.

// computeDeltaStatsU64 inspects an unsigned monotonic-or-not slice and
// returns the first value, the minimum delta (as int64), and the bits
// required to represent (Δᵢ - minΔ) for every i >= 1.
func computeDeltaStatsU64(s []uint64) (first uint64, minDelta int64, bitsPer int) {
        if len(s) == 0 {
                return 0, 0, 0
        }
        first = s[0]
        if len(s) == 1 {
                return first, 0, 0
        }
        minD := int64(s[1] - s[0])
        maxD := minD
        prev := s[1]
        for i := 2; i < len(s); i++ {
                d := int64(s[i] - prev)
                if d < minD {
                        minD = d
                }
                if d > maxD {
                        maxD = d
                }
                prev = s[i]
        }
        bitsPer = bitpack.BitsForDelta(uint64(maxD) - uint64(minD))
        return first, minD, bitsPer
}

func computeDeltaStatsI64(s []int64) (first int64, minDelta int64, bitsPer int) {
        if len(s) == 0 {
                return 0, 0, 0
        }
        first = s[0]
        if len(s) == 1 {
                return first, 0, 0
        }
        minD := s[1] - s[0]
        maxD := minD
        prev := s[1]
        for i := 2; i < len(s); i++ {
                d := s[i] - prev
                if d < minD {
                        minD = d
                }
                if d > maxD {
                        maxD = d
                }
                prev = s[i]
        }
        bitsPer = bitpack.BitsForDelta(uint64(maxD) - uint64(minD))
        return first, minD, bitsPer
}

// writePackedDeltaForUint64Slice emits the delta+FOR codec for s. Caller
// must have called computeDeltaStatsU64 already to pick first/minDelta/
// bitsPer; bitsPer must be <= qpackForMaxBits.
func (e *Encoder) writePackedDeltaForUint64Slice(s []uint64, first uint64, minDelta int64, bitsPer int) {
        e.writeHeader()
        n := len(s)
        bodyN := 0
        if n >= 2 {
                bodyN = ((n - 1) * bitsPer) >> 3
                if ((n-1)*bitsPer)&7 != 0 {
                        bodyN++
                }
        }
        out := slices.Grow(e.buf, 3+10+10+10+bodyN)
        out = append(out, tagPackDeltaFor, qpackKindUint64, byte(bitsPer))
        out = appendUvarint(out, first)
        out = appendUvarint(out, zigzagEncode64(minDelta))
        out = appendUvarint(out, uint64(n))
        if n < 2 || bitsPer == 0 {
                e.buf = out
                return
        }
        start := len(out)
        out = out[:start+bodyN]
        body := out[start : start+bodyN]
        clear(body)
        minU := uint64(minDelta)
        var chunk [64]uint64
        prev := s[0]
        wrote := 0
        for i := 1; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                k := 0
                for j := i; j < end; j++ {
                        d := s[j] - prev
                        chunk[k] = d - minU
                        prev = s[j]
                        k++
                }
                bitpack.PackChunk(body, chunk[:k], bitsPer, wrote)
                wrote += k
        }
        e.buf = out
}

func (e *Encoder) writePackedDeltaForInt64Slice(s []int64, first int64, minDelta int64, bitsPer int) {
        e.writeHeader()
        n := len(s)
        bodyN := 0
        if n >= 2 {
                bodyN = ((n - 1) * bitsPer) >> 3
                if ((n-1)*bitsPer)&7 != 0 {
                        bodyN++
                }
        }
        out := slices.Grow(e.buf, 3+10+10+10+bodyN)
        out = append(out, tagPackDeltaFor, qpackKindInt64, byte(bitsPer))
        out = appendUvarint(out, zigzagEncode64(first))
        out = appendUvarint(out, zigzagEncode64(minDelta))
        out = appendUvarint(out, uint64(n))
        if n < 2 || bitsPer == 0 {
                e.buf = out
                return
        }
        start := len(out)
        out = out[:start+bodyN]
        body := out[start : start+bodyN]
        clear(body)
        minU := uint64(minDelta)
        var chunk [64]uint64
        prev := s[0]
        wrote := 0
        for i := 1; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                k := 0
                for j := i; j < end; j++ {
                        d := s[j] - prev
                        chunk[k] = uint64(d) - minU
                        prev = s[j]
                        k++
                }
                bitpack.PackChunk(body, chunk[:k], bitsPer, wrote)
                wrote += k
        }
        e.buf = out
}

// readPackedDeltaForHeader consumes the kind, bits, firstVal, minDelta,
// count fields after a tagPackDeltaFor tag (the tag itself already
// consumed). signedFirst is set for signed kinds; unsignedFirst for
// unsigned kinds.
func (d *Decoder) readPackedDeltaForHeader(expectKind byte) (bitsPer int, unsignedFirst uint64, signedFirst int64, minDelta int64, n int, body []byte, err error) {
        if d.i+2 > len(d.buf) {
                return 0, 0, 0, 0, 0, nil, ErrShortBuffer
        }
        k := d.buf[d.i]
        d.i++
        if k != expectKind {
                return 0, 0, 0, 0, 0, nil, ErrTypeMismatch
        }
        bp := d.buf[d.i]
        d.i++
        if int(bp) > qpackForMaxBits {
                return 0, 0, 0, 0, 0, nil, ErrBadTag
        }
        bitsPer = int(bp)
        first64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, 0, 0, 0, nil, ErrInvalidLength
        }
        d.i += nr
        if k&(1<<2) != 0 {
                signedFirst = zigzagDecode64(first64)
        } else {
                unsignedFirst = first64
        }
        mn64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, 0, 0, 0, nil, ErrInvalidLength
        }
        d.i += nr
        minDelta = zigzagDecode64(mn64)
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, 0, 0, 0, nil, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return 0, 0, 0, 0, 0, nil, ErrInvalidLength
        }
        if n64 < 2 {
                return bitsPer, unsignedFirst, signedFirst, minDelta, int(n64), nil, nil
        }
        // Validate in uint64 before the signed cast; the same body-shape
        // overflow that produced a reverse-range panic in readPackedForHeader
        // applies here.
        rem := uint64(len(d.buf) - d.i)
        if bitsPer > 0 {
                if (n64 - 1) > rem*8/uint64(bitsPer) {
                        return 0, 0, 0, 0, 0, nil, ErrShortBuffer
                }
        } else if n64 > qpackMaxStandaloneCount {
                // bitsPer == 0 (constant deltas): empty body, no per-element bound.
                return 0, 0, 0, 0, 0, nil, ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8 lets n64 exceed MaxInt -> int(n64) wraps negative
                return 0, 0, 0, 0, 0, nil, ErrInvalidLength
        }
        n = int(n64)
        bodyBytes := int(((n64-1)*uint64(bitsPer) + 7) / 8)
        body = d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        return bitsPer, unsignedFirst, signedFirst, minDelta, n, body, nil
}

func (d *Decoder) readPackedDeltaForUint64Slice() ([]uint64, error) {
        bitsPer, first, _, minDelta, n, body, err := d.readPackedDeltaForHeader(qpackKindUint64)
        if err != nil {
                return nil, err
        }
        out := make([]uint64, n)
        if n == 0 {
                return out, nil
        }
        out[0] = first
        if n == 1 {
                return out, nil
        }
        if bitsPer == 0 {
                step := uint64(minDelta)
                v := first
                for i := 1; i < n; i++ {
                        v += step
                        out[i] = v
                }
                return out, nil
        }
        if cap(d.deltaScratch) < n-1 {
                d.deltaScratch = make([]uint64, n-1)
        }
        tmp := d.deltaScratch[:n-1]
        bitpack.Unpack(tmp, body, bitsPer)
        minU := uint64(minDelta)
        v := first
        for i, dv := range tmp {
                v += dv + minU
                out[i+1] = v
        }
        return out, nil
}

func (d *Decoder) readPackedDeltaForInt64Slice() ([]int64, error) {
        bitsPer, _, first, minDelta, n, body, err := d.readPackedDeltaForHeader(qpackKindInt64)
        if err != nil {
                return nil, err
        }
        out := make([]int64, n)
        if n == 0 {
                return out, nil
        }
        out[0] = first
        if n == 1 {
                return out, nil
        }
        if bitsPer == 0 {
                v := first
                for i := 1; i < n; i++ {
                        v += minDelta
                        out[i] = v
                }
                return out, nil
        }
        if cap(d.deltaScratch) < n-1 {
                d.deltaScratch = make([]uint64, n-1)
        }
        tmp := d.deltaScratch[:n-1]
        bitpack.Unpack(tmp, body, bitsPer)
        minU := uint64(minDelta)
        v := uint64(first)
        for i, dv := range tmp {
                v += dv + minU
                out[i+1] = int64(v)
        }
        return out, nil
}

package qdf

import (
        "math"
        "slices"

        "github.com/alex60217101990/qdf/internal/bitpack"
)

// Dictionary-coded integer slice codec. Wire format documented at
// tagPackDict in wire.go. Wins on enum-like columns where the
// distinct count is small (≤ qpackDictMaxDistinct) but the values
// are spread far enough that the FOR / Delta+FOR bitpack can't
// reach competitive density.
//
// The encoder re-runs the distinct probe because the picker discards
// the table after the size estimate; both the probe and the per-element
// index resolution use an open-addressed hash set, so each is O(n)
// regardless of the distinct count.

// writePackedDictUint64Slice emits s as a tagPackDict payload. The
// caller has already decided via pickU64Codec that dict wins.
func (e *Encoder) writePackedDictUint64Slice(s []uint64) {
        e.writeHeader()
        n := len(s)
        var table [qpackDictMaxDistinct + 1]uint64
        // One open-addressed pass yields both the distinct table and the value→index
        // map, so the encoder skips the separate buildDictIndex pass.
        count, ok, hslot, hidx := probeDistinctIndexU64(s, &table)
        if !ok || count == 0 {
                // Fall back to raw — picker mis-selected or the slice grew
                // distinct values between probe and encode. The fallback
                // keeps the wire well-formed; the only cost is a missed
                // dict win, not a panic.
                e.writePackedUint64Slice(s)
                return
        }
        bp := bitsForDistinct(count)
        bodyBytes := (n*bp + 7) >> 3
        out := slices.Grow(e.buf, 2+10+10*count+10+bodyBytes)
        out = append(out, tagPackDict, qpackKindUint64)
        out = appendUvarint(out, uint64(count))
        for i := range count {
                out = appendUvarint(out, table[i])
        }
        out = appendUvarint(out, uint64(n))
        if bp == 0 {
                // Single distinct value — body is empty, decoder broadcasts.
                e.buf = out
                return
        }
        start := len(out)
        out = out[:start+bodyBytes]
        body := out[start : start+bodyBytes]
        clear(body)
        // hslot/hidx (the value→index map) came from probeDistinctIndexU64 above.
        // Stage indices in a chunk buffer so bitpack.PackChunk can
        // reuse its existing LSB-first packing routine.
        var chunk [64]uint64
        for i := 0; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                for j, v := range s[i:end] {
                        h := (v * 0x9E3779B97F4A7C15) >> (64 - 7) & (qpackProbeSlots - 1)
                        for hslot[h] != v { // present by construction
                                h = (h + 1) & (qpackProbeSlots - 1)
                        }
                        chunk[j] = uint64(hidx[h])
                }
                bitpack.PackChunk(body, chunk[:end-i], bp, i)
        }
        e.buf = out
}

// writePackedDictInt64Slice mirrors the unsigned variant. Dictionary
// entries are zigzag-encoded.
func (e *Encoder) writePackedDictInt64Slice(s []int64) {
        e.writeHeader()
        n := len(s)
        var table [qpackDictMaxDistinct + 1]int64
        count, ok, hslot, hidx := probeDistinctIndexI64(s, &table)
        if !ok || count == 0 {
                e.writePackedInt64Slice(s)
                return
        }
        bp := bitsForDistinct(count)
        bodyBytes := (n*bp + 7) >> 3
        out := slices.Grow(e.buf, 2+10+10*count+10+bodyBytes)
        out = append(out, tagPackDict, qpackKindInt64)
        out = appendUvarint(out, uint64(count))
        for i := range count {
                out = appendUvarint(out, zigzagEncode64(table[i]))
        }
        out = appendUvarint(out, uint64(n))
        if bp == 0 {
                e.buf = out
                return
        }
        start := len(out)
        out = out[:start+bodyBytes]
        body := out[start : start+bodyBytes]
        clear(body)
        // hslot/hidx (the value→index map) came from probeDistinctIndexI64 above.
        var chunk [64]uint64
        for i := 0; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                for j, v := range s[i:end] {
                        h := (uint64(v) * 0x9E3779B97F4A7C15) >> (64 - 7) & (qpackProbeSlots - 1)
                        for hslot[h] != v { // present by construction
                                h = (h + 1) & (qpackProbeSlots - 1)
                        }
                        chunk[j] = uint64(hidx[h])
                }
                bitpack.PackChunk(body, chunk[:end-i], bp, i)
        }
        e.buf = out
}

// readPackedDictHeader parses the kind byte and distinct count after
// a tagPackDict tag. The tag itself must already be consumed. The
// caller walks the distinct values and the element count next, then
// the bitsPer-packed body — distinct value parsing is caller-specific
// because of signed zigzag, and bitsPer is derivable from count.
func (d *Decoder) readPackedDictHeader(expectKind byte) (count int, bitsPer int, err error) {
        if d.i >= len(d.buf) {
                return 0, 0, ErrShortBuffer
        }
        k := d.buf[d.i]
        d.i++
        if k != expectKind {
                return 0, 0, ErrTypeMismatch
        }
        c64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, ErrInvalidLength
        }
        d.i += nr
        if c64 == 0 || c64 > qpackDictMaxDistinct {
                return 0, 0, ErrBadTag
        }
        count = int(c64)
        bitsPer = bitsForDistinct(count)
        return count, bitsPer, nil
}

// readPackedDictUint64Slice consumes a tagPackDict body emitted with
// qpackKindUint64.
func (d *Decoder) readPackedDictUint64Slice() ([]uint64, error) {
        count, bitsPer, err := d.readPackedDictHeader(qpackKindUint64)
        if err != nil {
                return nil, err
        }
        var table [qpackDictMaxDistinct]uint64
        for i := range count {
                v, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                table[i] = v
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return nil, ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8/bitsPer lets n64 exceed MaxInt -> int(n64) wraps negative -> make panics
                return nil, ErrInvalidLength
        }
        if bitsPer == 0 && n64 > qpackMaxStandaloneCount {
                // Single distinct value (count == 1 ⇒ bitsPer == 0): empty index body,
                // no per-element bound. Cap before make() (columnar bounded by colLenOK).
                return nil, ErrInvalidLength
        }
        n := int(n64)
        if bitsPer == 0 {
                out := make([]uint64, n)
                v := table[0]
                for i := range out {
                        out[i] = v
                }
                return out, nil
        }
        // bitsPer > 0: the index body is n*bitsPer bits. Bound n against the
        // remaining buffer BEFORE allocating — the original order ran make() first,
        // so a tiny header claiming a huge n drove a multi-GB allocation before this
        // check could reject it.
        rem := uint64(len(d.buf) - d.i)
        if n64 > rem*8/uint64(bitsPer) {
                return nil, ErrShortBuffer
        }
        out := make([]uint64, n)
        bodyBytes := (n*bitsPer + 7) >> 3
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        // Reuse the shared transient unpack scratch (as the Delta+FOR readers do):
        // idx holds the bit-unpacked dictionary indices, fully written by Unpack and
        // only read to map table->out, never aliased into the returned slice.
        if cap(d.deltaScratch) < n {
                d.deltaScratch = make([]uint64, n)
        }
        idx := d.deltaScratch[:n]
        bitpack.Unpack(idx, body, bitsPer)
        for i, k := range idx {
                if k >= uint64(count) {
                        return nil, ErrBadTag
                }
                out[i] = table[k]
        }
        return out, nil
}

// readPackedDictInt64Slice consumes a tagPackDict body emitted with
// qpackKindInt64.
func (d *Decoder) readPackedDictInt64Slice() ([]int64, error) {
        count, bitsPer, err := d.readPackedDictHeader(qpackKindInt64)
        if err != nil {
                return nil, err
        }
        var table [qpackDictMaxDistinct]int64
        for i := range count {
                v, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                table[i] = zigzagDecode64(v)
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return nil, ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8/bitsPer lets n64 exceed MaxInt -> int(n64) wraps negative -> make panics
                return nil, ErrInvalidLength
        }
        if bitsPer == 0 && n64 > qpackMaxStandaloneCount {
                // Single distinct value (count == 1 ⇒ bitsPer == 0): empty index body,
                // no per-element bound. Cap before make() (columnar bounded by colLenOK).
                return nil, ErrInvalidLength
        }
        n := int(n64)
        if bitsPer == 0 {
                out := make([]int64, n)
                v := table[0]
                for i := range out {
                        out[i] = v
                }
                return out, nil
        }
        // bitsPer > 0: bound n against the remaining buffer BEFORE allocating — the
        // original order ran make() first, so a tiny header claiming a huge n drove
        // a multi-GB allocation before this check could reject it.
        rem := uint64(len(d.buf) - d.i)
        if n64 > rem*8/uint64(bitsPer) {
                return nil, ErrShortBuffer
        }
        out := make([]int64, n)
        bodyBytes := (n*bitsPer + 7) >> 3
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        // Reuse the shared transient unpack scratch (mirrors readPackedDictUint64Slice
        // and the Delta+FOR readers); idx is fully written then mapped into out.
        if cap(d.deltaScratch) < n {
                d.deltaScratch = make([]uint64, n)
        }
        idx := d.deltaScratch[:n]
        bitpack.Unpack(idx, body, bitsPer)
        for i, k := range idx {
                if k >= uint64(count) {
                        return nil, ErrBadTag
                }
                out[i] = table[k]
        }
        return out, nil
}

package qdf

import (
        "math"
        "slices"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/bitpack"
)

// Frame-of-Reference (FOR) bit-packing for integer slices.
//
// For a slice with min and max values, every element fits in
// ceil(log2(max-min+1)) bits once min has been subtracted. The encoded
// payload is one min reference plus a tightly packed bit-stream of the
// deltas, LSB-first within each byte. With clustered integer data this
// crushes the per-element cost from 64 bits to anywhere from 0 (all
// equal) to ~16-24 bits without measurably slowing the decoder.
//
// The packer caps bits at 56 so the running 64-bit accumulator never
// overflows during the partial-byte drain (worst case: 7 carry-over bits
// + 56 new bits = 63 < 64). Slices that genuinely need 57-64 bits per
// delta should pick the raw-LE codec, which strictly beats FOR there. The
// pure bit-packing kernels live in internal/bitpack.

const qpackForMaxBits = 56

// qpackMaxStandaloneCount caps the element count a constant-value codec
// (bitsPer == 0) may claim outside a columnar column. Such codecs carry an
// empty body, so the per-element byte bound that guards the bitsPer > 0 path
// does not exist; without this ceiling a ~14-byte header could drive a
// multi-GB make(). Columnar columns are already bounded by colLenOK; this
// mirrors the standalone cap RLE applies (qpack_rle.go).
//
// Pinned to maxColumnarElems (1<<24, 128 MiB for an int64 slice) — the same
// ceiling columnar already enforces for a constant column. The previous 1<<30
// allowed an 8 GiB allocation from a tiny hostile header (OOM-DoS); a constant
// slice above 16M elements is not a real workload and callers that large must
// stream/shard regardless.
const qpackMaxStandaloneCount = maxColumnarElems

// qpackForSizeUnsigned estimates the wire size, in bytes, of a FOR-packed
// encoding for n unsigned values whose delta range needs bits per slot
// and whose minimum value is m. Used to choose between raw and FOR.
func qpackForSizeUnsigned(n int, bitsPer int, m uint64) int {
        // tag(1) + kind(1) + bits(1) + min varuint (1..10) + count varuint (1..5 for n<=2^28)
        // + ceil(n*bits/8) body.
        hdr := 3 + uvarintLen(m) + uvarintLen(uint64(n))
        body := (n*bitsPer + 7) >> 3
        return hdr + body
}

func qpackForSizeSigned(n int, bitsPer int, m int64) int {
        hdr := 3 + uvarintLen(zigzagEncode64(m)) + uvarintLen(uint64(n))
        body := (n*bitsPer + 7) >> 3
        return hdr + body
}

// minMaxU64 returns the (min, max) of s. Caller must ensure len(s) > 0.
func minMaxU64(s []uint64) (uint64, uint64) {
        mn, mx := s[0], s[0]
        for _, v := range s[1:] {
                if v < mn {
                        mn = v
                }
                if v > mx {
                        mx = v
                }
        }
        return mn, mx
}

func minMaxI64(s []int64) (int64, int64) {
        mn, mx := s[0], s[0]
        for _, v := range s[1:] {
                if v < mn {
                        mn = v
                }
                if v > mx {
                        mx = v
                }
        }
        return mn, mx
}

// writePackedForUint64Slice emits a FOR-packed []uint64. Caller should
// already have chosen FOR over raw (e.g. via qpackForSizeUnsigned <
// raw-size). bitsPer must be in [0, 56]. min must equal the slice's min.
func (e *Encoder) writePackedForUint64Slice(s []uint64, mn uint64, bitsPer int) {
        e.writeHeader()
        n := len(s)
        bodyBytes := (n*bitsPer + 7) >> 3
        out := slices.Grow(e.buf, 3+10+10+bodyBytes)
        out = append(out, tagPackFor, qpackKindUint64, byte(bitsPer))
        out = appendUvarint(out, mn)
        out = appendUvarint(out, uint64(n))
        if bitsPer == 0 {
                e.buf = out
                return
        }
        start := len(out)
        out = out[:start+bodyBytes]
        body := out[start : start+bodyBytes]
        clear(body)
        // Subtract min on the fly to avoid an extra allocation.
        // Reuse a small stack-allocated chunk buffer for the delta values.
        var chunk [64]uint64
        for i := 0; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                for j, v := range s[i:end] {
                        chunk[j] = v - mn
                }
                bitpack.PackChunk(body, chunk[:end-i], bitsPer, i)
        }
        e.buf = out
}

// writePackedForInt64Slice mirrors writePackedForUint64Slice with a
// signed min stored as zigzag varuint.
func (e *Encoder) writePackedForInt64Slice(s []int64, mn int64, bitsPer int) {
        e.writeHeader()
        n := len(s)
        bodyBytes := (n*bitsPer + 7) >> 3
        out := slices.Grow(e.buf, 3+10+10+bodyBytes)
        out = append(out, tagPackFor, qpackKindInt64, byte(bitsPer))
        out = appendUvarint(out, zigzagEncode64(mn))
        out = appendUvarint(out, uint64(n))
        if bitsPer == 0 {
                e.buf = out
                return
        }
        start := len(out)
        out = out[:start+bodyBytes]
        body := out[start : start+bodyBytes]
        clear(body)
        mnU := uint64(mn)
        var chunk [64]uint64
        for i := 0; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                for j, v := range s[i:end] {
                        chunk[j] = uint64(v) - mnU
                }
                bitpack.PackChunk(body, chunk[:end-i], bitsPer, i)
        }
        e.buf = out
}

// readPackedForHeader consumes the kind, bits, min, count fields after a
// tagPackFor tag. The tag itself must already have been consumed. The
// returned signedMin is meaningful only when the kind family is signed.
func (d *Decoder) readPackedForHeader(expectKind byte) (bitsPer int, unsignedMin uint64, signedMin int64, n int, body []byte, err error) {
        if d.i+2 > len(d.buf) {
                return 0, 0, 0, 0, nil, ErrShortBuffer
        }
        k := d.buf[d.i]
        d.i++
        if k != expectKind {
                return 0, 0, 0, 0, nil, ErrTypeMismatch
        }
        b := d.buf[d.i]
        d.i++
        if int(b) > qpackForMaxBits {
                return 0, 0, 0, 0, nil, ErrBadTag
        }
        bitsPer = int(b)
        mn64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, 0, 0, nil, ErrInvalidLength
        }
        d.i += nr
        if k&(1<<2) != 0 {
                signedMin = zigzagDecode64(mn64)
        } else {
                unsignedMin = mn64
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, 0, 0, nil, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return 0, 0, 0, 0, nil, ErrInvalidLength
        }
        // Validate body size in uint64 BEFORE the signed cast. A hostile
        // varuint with n64 > MaxInt would otherwise wrap int(n64) to
        // negative and the bounds check in the original form
        // (`d.i + bodyBytes > len(d.buf)`) accepts a negative bodyBytes,
        // then `d.buf[d.i : d.i+bodyBytes]` panics with a reverse-range
        // slice. Same shape as the Skip-path overflow fixed earlier;
        // applies to the body-reading path here too.
        rem := uint64(len(d.buf) - d.i)
        if bitsPer > 0 {
                if n64 > rem*8/uint64(bitsPer) {
                        return 0, 0, 0, 0, nil, ErrShortBuffer
                }
        } else if n64 > qpackMaxStandaloneCount {
                // bitsPer == 0 (constant slice): empty body, so the per-element
                // bound above does not apply. Cap an implausible standalone count.
                return 0, 0, 0, 0, nil, ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8 lets n64 exceed MaxInt -> int(n64) wraps negative -> make panics
                return 0, 0, 0, 0, nil, ErrInvalidLength
        }
        n = int(n64)
        bodyBytes := int((n64*uint64(bitsPer) + 7) / 8)
        body = d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        return bitsPer, unsignedMin, signedMin, n, body, nil
}

func (d *Decoder) readPackedForUint64Slice() ([]uint64, error) {
        bitsPer, mn, _, n, body, err := d.readPackedForHeader(qpackKindUint64)
        if err != nil {
                return nil, err
        }
        out := make([]uint64, n)
        if bitsPer == 0 {
                for i := range out {
                        out[i] = mn
                }
                return out, nil
        }
        bitpack.Unpack(out, body, bitsPer)
        if mn != 0 {
                for i := range out {
                        out[i] += mn
                }
        }
        return out, nil
}

func (d *Decoder) readPackedForInt64Slice() ([]int64, error) {
        bitsPer, _, mn, n, body, err := d.readPackedForHeader(qpackKindInt64)
        if err != nil {
                return nil, err
        }
        out := make([]int64, n)
        if bitsPer == 0 {
                for i := range out {
                        out[i] = mn
                }
                return out, nil
        }
        // int64 and uint64 share a layout, so decode straight into out viewed
        // as []uint64 and add the reference in place — no separate scratch
        // buffer or copy pass (mirrors the unsigned reader above).
        u := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(out))), n)
        bitpack.Unpack(u, body, bitsPer)
        if mnU := uint64(mn); mnU != 0 {
                for i := range u {
                        u[i] += mnU
                }
        }
        return out, nil
}

package qdf

import (
        "sync"

        "github.com/alex60217101990/qdf/internal/fsst"
        "github.com/alex60217101990/qdf/internal/unsafestr"
)

const (
        // fsstMinElems: below this a per-column table cannot pay off.
        fsstMinElems = 16
        // fsstMaxDecompPerByte bounds decode expansion: each compressed byte
        // yields at most one symbol of ≤8 bytes.
        fsstMaxDecompPerByte = 8
        // fsstProbeRounds trains a coarser table for the columnar probe's size
        // estimate (a rough table is enough to decide columnar-vs-row-major); the
        // encoder still uses the full buildRounds for the table it emits. Keeps the
        // per-column probe training off the OptCompression hot path.
        fsstProbeRounds = 1
        // fsstReuseInterval is the number of consecutive batches that reuse the
        // cached FSST symbol table (fsstCachedTbl) before retraining. Matches
        // retainReleaseStreak (8) so a steady workload holds its table across 8
        // batches before a full retrain checks whether the string distribution has
        // shifted. Only applies to streaming encoders (same Encoder reused across
        // calls); pool-backed Marshal resets the cache on every acquire.
        fsstReuseInterval = 8
)

// fsstBuilderPool reuses FSST training scratch across the per-column probe
// estimates and encode attempts, so a column where FSST is evaluated but not
// chosen does not re-allocate the trainer's counter + first-byte index on every
// call (the source of the OptCompression encode-allocation regression).
var fsstBuilderPool = sync.Pool{New: func() any { return fsst.NewBuilder() }}

// fsstTablePool reuses SymbolTable objects across per-column decodes. Each
// pooled table retains its symbols/cands backing arrays; UnmarshalInto resets
// them and refills in-place, avoiding the three allocations that a fresh
// newSymbolTable call would otherwise make.
var fsstTablePool = sync.Pool{New: func() any { return &fsst.SymbolTable{} }}

// tryWriteStringColumnFSST trains an FSST table on strs, compresses them, and
// emits a tagColStrFSST block — but only when the block is strictly smaller
// than the raw per-value size (sum(len)+n framing). It is invoked only after
// the dictionary codec has bailed (i.e. high-cardinality columns), where the
// per-value path is dominated by the raw bytes, so raw size is a sound
// never-larger baseline. Returns true iff the block was written.
func (e *Encoder) tryWriteStringColumnFSST(strs []string) bool {
        n := len(strs)
        if n < fsstMinElems {
                return false
        }

        // A pre-trained dictionary (FSSTDict.Marshal) skips the per-batch training,
        // which is the dominant FSST encode cost; otherwise train on this column.
        tbl := e.fsstDict
        if tbl == nil {
                // Streaming cache: reuse the table trained on a recent batch to skip
                // the Build() cost (~140–311 µs) when the same Encoder is reused across
                // consecutive batches. We retrain every fsstReuseInterval batches so the
                // table adapts to drifting distributions. The wire format is unchanged:
                // the table is always fully serialised (MarshalTo below) so the decoder
                // needs no protocol changes — caching is encoder-side only.
                if e.fsstCachedTbl != nil && e.fsstBatchCount < fsstReuseInterval {
                        tbl = e.fsstCachedTbl
                        e.fsstBatchCount++
                } else {
                        // (Re)train: build via the pooled builder (reuses counter/cand scratch),
                        // then clone to an independent copy before the builder is returned to
                        // the pool. bld.Build returns a table aliasing bld's internal storage;
                        // that alias is invalidated the next time any goroutine calls Build on
                        // the same Builder. Clone() copies the three fields (symbols, cands,
                        // first) into a fresh allocation that survives beyond this call.
                        samples := e.state.fsstSamples[:0]
                        for _, s := range strs {
                                samples = append(samples, unsafestr.Bytes(s))
                        }
                        e.state.fsstSamples = samples
                        bld := fsstBuilderPool.Get().(*fsst.Builder)
                        defer fsstBuilderPool.Put(bld)
                        tbl = bld.Build(samples)
                        e.fsstCachedTbl = tbl.Clone() // independent copy; bld safe to pool
                        e.fsstBatchCount = 0
                }
        }

        // Compress all rows into one scratch buffer, recording per-row lengths.
        comp := e.state.fsstScratch[:0]
        compLens := e.state.fsstLens[:0]
        decompTotal := 0
        for _, s := range strs {
                before := len(comp)
                comp = tbl.Compress(unsafestr.Bytes(s), comp)
                compLens = append(compLens, len(comp)-before)
                decompTotal += len(s)
        }
        e.state.fsstScratch = comp
        e.state.fsstLens = compLens

        // Block size = tag + table + uvarint(n) + uvarint(decompTotal)
        //            + sum( uvarint(compLen) + compLen ). SerializedSize avoids
        //            materializing the table just to measure it.
        size := 1 + tbl.SerializedSize() + uvarintLen(uint64(n)) + uvarintLen(uint64(decompTotal))
        for _, cl := range compLens {
                size += uvarintLen(uint64(cl)) + cl
        }
        // Never-larger baseline: raw per-value bytes + one framing byte each.
        // decompTotal is the sum of len(s) over strs (accumulated in the encode
        // loop above), identical to a separate raw-total pass.
        if size >= decompTotal+n {
                return false
        }

        e.writeHeader()
        out := e.buf
        out = append(out, tagColStrFSST)
        out = tbl.MarshalTo(out) // serialize the table straight into the output
        out = appendUvarint(out, uint64(n))
        out = appendUvarint(out, uint64(decompTotal))
        pos := 0
        for _, cl := range compLens {
                out = appendUvarint(out, uint64(cl))
                out = append(out, comp[pos:pos+cl]...)
                pos += cl
        }
        e.buf = out
        return true
}

// readStringColumnFSST decodes a tagColStrFSST block (tag at d.i) into n
// strings, all backed by a single per-column slab. Every length is validated
// before allocation.
func (d *Decoder) readStringColumnFSST(n int) ([]string, error) {
        d.i++ // consume tagColStrFSST
        tbl := fsstTablePool.Get().(*fsst.SymbolTable)
        defer fsstTablePool.Put(tbl)
        used, err := fsst.UnmarshalInto(d.buf[d.i:], tbl)
        if err != nil {
                return nil, err
        }
        d.i += used

        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if int(n64) != n {
                return nil, ErrTypeMismatch
        }

        dt64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        rem := uint64(len(d.buf) - d.i)
        if dt64 > rem*fsstMaxDecompPerByte {
                return nil, ErrShortBuffer
        }
        // Hard cap: decompTotal must not exceed maxColumnarElems * maxSymLen (8).
        // This prevents a hostile varint from driving a huge slab alloc.
        if dt64 > uint64(maxColumnarElems)*8 {
                return nil, ErrShortBuffer
        }

        slab := make([]byte, 0, int(dt64))
        out := d.colStrScratch(n)
        for i := range n {
                cl64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if cl64 > uint64(len(d.buf)-d.i) {
                        return nil, ErrShortBuffer
                }
                cl := int(cl64)
                start := len(slab)
                var ok bool
                // Bounded decode: never let the slab grow past its pre-sized capacity
                // (dt64), so a malformed row cannot trigger a transient over-allocation.
                slab, ok = tbl.DecompressN(d.buf[d.i:d.i+cl], slab, int(dt64))
                if !ok {
                        return nil, ErrShortBuffer
                }
                d.i += cl
                out[i] = unsafestr.String(slab[start:])
        }
        return out, nil
}

// readStringColumnFSSTInto decodes a tagColStrFSST block (tag at d.i) DIRECTLY
// into the batch slab, writing a per-row Str handle into out[0:n]. It mirrors
// readStringColumnFSST's header parse and EVERY bounds guard verbatim (the
// dt64 caps, per-row compressed-length checks, and DecompressN's absolute-length
// limit), but instead of a temp scratch buffer it decompresses each row straight
// onto slab.buf — DecompressN appends in place — eliminating the ~dt64-byte temp
// alloc and the redundant second copy the general readStringColumnHandles path
// pays for FSST columns.
//
// Bounds parity is load-bearing here (this is decompress + unsafe slab writes):
// the slab reservation is bounded by dt64, which is itself bounded by the input;
// the slab's uint32 offset overflow guard (maxBatchSlabBytes, matching
// slab.append) is honored BEFORE any reservation. A malformed/hostile column
// errors — never OOM, never OOB write into the slab.
func readStringColumnFSSTInto(d *Decoder, n int, slab *batchSlab, out []Str) error {
        d.i++ // consume tagColStrFSST
        tbl := fsstTablePool.Get().(*fsst.SymbolTable)
        defer fsstTablePool.Put(tbl)
        used, err := fsst.UnmarshalInto(d.buf[d.i:], tbl)
        if err != nil {
                return err
        }
        d.i += used

        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if int(n64) != n {
                return ErrTypeMismatch
        }

        dt64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        rem := uint64(len(d.buf) - d.i)
        if dt64 > rem*fsstMaxDecompPerByte {
                return ErrShortBuffer
        }
        // Hard cap: decompTotal must not exceed maxColumnarElems * maxSymLen (8).
        // This prevents a hostile varint from driving a huge slab reservation.
        if dt64 > uint64(maxColumnarElems)*8 {
                return ErrShortBuffer
        }

        // Honor the slab's uint32 offset cap BEFORE reserving: the reserved region
        // grows slab.buf by dt64, and a Str handle's offset must fit uint32. This is
        // the same overflow guard slab.append enforces (maxBatchSlabBytes) — a direct
        // write that would exceed it errors instead of silently wrapping the offset.
        base := len(slab.buf)
        if uint64(base)+dt64 > maxBatchSlabBytes {
                return ErrInvalidLength
        }
        // Reserve dt64 bytes ONCE so per-row DecompressN appends land in place with
        // no mid-column reallocation (offsets stay stable). limit is the ABSOLUTE
        // len(dst) cap DecompressN enforces; base+dt64 mirrors readStringColumnFSST's
        // int(dt64) cap on a slab that started empty.
        slab.grow(int(dt64))
        limit := base + int(dt64)
        for i := range n {
                cl64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return ErrInvalidLength
                }
                d.i += nr
                if cl64 > uint64(len(d.buf)-d.i) {
                        return ErrShortBuffer
                }
                cl := int(cl64)
                start := len(slab.buf)
                var ok bool
                // Bounded decode: never let slab.buf grow past base+dt64, so a malformed
                // row cannot trigger a transient over-allocation or an OOB slab write.
                slab.buf, ok = tbl.DecompressN(d.buf[d.i:d.i+cl], slab.buf, limit)
                if !ok {
                        return ErrShortBuffer
                }
                d.i += cl
                ln := len(slab.buf) - start
                if ln == 0 {
                        // Match slab.append's empty-string convention: the zero handle.
                        out[i] = Str{}
                        continue
                }
                out[i] = Str{off: uint32(start), len: uint32(ln)}
        }
        return nil
}

package qdf

import (
        "math"
        "math/bits"
        "slices"
)

// Gorilla XOR codec for float slices, from
// "Gorilla: A Fast, Scalable, In-Memory Time Series Database" (FB, VLDB
// 2015). For slow-varying floats (sensor reads, metrics, monotonic
// timestamps cast to float) it crushes the per-sample cost from 64 bits
// down to ~1-12 bits without lossy compression: NaN and ±Inf survive
// unchanged because the codec operates on the IEEE-754 bit pattern.
//
// Per sample:
//   - xor = bits(curr) ^ bits(prev)
//   - if xor == 0: write a single 0 bit
//   - else: write a 1 bit, then either
//       0  + meaningful bits inside the previous lz/tz window, or
//       1  + 5-bit lz + 6-bit (mbLen-1) + meaningful bits.
//
// Bit ordering is MSB-first within each byte to match the original paper
// and make the bit-stream comparable to other Gorilla implementations.

// bitWriter / bitReader are MSB-first scratchpads. They are local to the
// gorilla codec and do not allocate on the hot path.
type bitWriter struct {
        buf   []byte
        nbits int // total bits written, independent of buf's base offset
        cur   byte
        used  uint8 // bits used in cur (0..7)
}

func (bw *bitWriter) writeBit(v bool) {
        if v {
                bw.cur |= 1 << (7 - bw.used)
        }
        bw.used++
        bw.nbits++
        if bw.used == 8 {
                bw.buf = append(bw.buf, bw.cur)
                bw.cur = 0
                bw.used = 0
        }
}

func (bw *bitWriter) writeBits(v uint64, count uint8) {
        if count == 0 {
                return
        }
        bw.nbits += int(count)

        avail := 8 - bw.used // free bits remaining in bw.cur

        // Fast path: entire payload fits in the current byte.
        if count <= avail {
                shift := avail - count
                bw.cur |= byte(v&((1<<count)-1)) << shift
                bw.used += count
                if bw.used == 8 {
                        bw.buf = append(bw.buf, bw.cur)
                        bw.cur = 0
                        bw.used = 0
                }
                return
        }

        // Fill the remaining bits of the current byte with the top `avail` bits of v.
        // Top `avail` bits of v (MSB-first) = v >> (count - avail), truncated to `avail` bits.
        bw.cur |= byte(v >> (count - avail))
        bw.buf = append(bw.buf, bw.cur)
        bw.cur = 0
        bw.used = 0
        remaining := count - avail

        // Append full bytes directly (no branch per bit, no append overhead per byte).
        for remaining >= 8 {
                remaining -= 8
                bw.buf = append(bw.buf, byte(v>>remaining))
        }

        // Store trailing partial bits in bw.cur.
        if remaining > 0 {
                bw.cur = byte(v&((1<<remaining)-1)) << (8 - remaining)
                bw.used = remaining
        }
}

// flush finalises any partial byte. Returns the total bit count written. The
// count is tracked incrementally (nbits) rather than derived from len(buf) so
// the writer can append directly into a caller buffer that already holds a
// header prefix.
func (bw *bitWriter) flush() int {
        total := bw.nbits
        if bw.used > 0 {
                bw.buf = append(bw.buf, bw.cur)
                bw.cur = 0
                bw.used = 0
        }
        return total
}

// finishGorillaBody finalises a Gorilla body that was written in-place into out
// starting at bodyStart (no separate scratch buffer), prepending its
// uvarint(totalBits) length via a single memmove of the body. This avoids the
// per-call scratch []byte that grew from nil and the final copy into out.
func finishGorillaBody(out []byte, bodyStart, totalBits int) []byte {
        bodyEnd := len(out)
        ul := uvarintLen(uint64(totalBits))
        out = slices.Grow(out, ul)
        out = out[:bodyEnd+ul]
        copy(out[bodyStart+ul:], out[bodyStart:bodyEnd])
        // Write the uvarint into the gap we just opened; cap is guaranteed, so
        // appendUvarint fills out[bodyStart:bodyStart+ul] without reallocating.
        _ = appendUvarint(out[bodyStart:bodyStart], uint64(totalBits))
        return out
}

type bitReader struct {
        buf  []byte
        pos  int   // byte index
        used uint8 // bits consumed in buf[pos] (0..7)
}

func (br *bitReader) readBit() (bool, bool) {
        if br.pos >= len(br.buf) {
                return false, false
        }
        v := (br.buf[br.pos] >> (7 - br.used)) & 1
        br.used++
        if br.used == 8 {
                br.used = 0
                br.pos++
        }
        return v == 1, true
}

func (br *bitReader) readBits(count uint8) (uint64, bool) {
        if count == 0 {
                return 0, true
        }
        // Upfront bounds check: compute bytes needed from br.pos.
        need := (int(br.used) + int(count) + 7) / 8
        if br.pos+need > len(br.buf) {
                return 0, false
        }

        avail := uint8(8) - br.used // bits remaining in current byte (1..8)
        var out uint64

        if avail >= count {
                // Fast path: all bits fit within the current byte.
                shift := avail - count
                out = uint64(br.buf[br.pos]>>shift) & ((1 << count) - 1)
                br.used += count
                if br.used == 8 {
                        br.used = 0
                        br.pos++
                }
                return out, true
        }

        // Consume remaining bits of the current byte (avail bits, MSB-first).
        out = uint64(br.buf[br.pos]) & ((1 << avail) - 1)
        br.pos++
        br.used = 0
        remaining := count - avail

        // Consume full bytes.
        for remaining >= 8 {
                out = (out << 8) | uint64(br.buf[br.pos])
                br.pos++
                remaining -= 8
        }

        // Consume the partial trailing byte.
        if remaining > 0 {
                shift := 8 - remaining
                out = (out << remaining) | uint64(br.buf[br.pos]>>shift)
                br.used = remaining
        }

        return out, true
}

// writePackedGorillaFloat64Slice emits a Gorilla XOR-coded []float64.
// n=0 writes only the tag/kind/n; n=1 also writes the first value but no
// XOR body.
func (e *Encoder) writePackedGorillaFloat64Slice(s []float64) {
        e.writeHeader()
        n := len(s)
        // Reserve generously: 4 byte body per sample is a comfortable upper
        // bound for any non-pathological input; the slow path appends if it
        // exceeds.
        out := slices.Grow(e.buf, 2+10+8+10+4*n)
        out = append(out, tagPackGorilla, qpackKindFloat64)
        out = appendUvarint(out, uint64(n))
        if n == 0 {
                e.buf = out
                return
        }
        first := math.Float64bits(s[0])
        out = appendU64(out, first)
        if n == 1 {
                out = appendUvarint(out, 0)
                e.buf = out
                return
        }
        bodyStart := len(out)
        bw := bitWriter{buf: out}
        prev := first
        // prevLZ == 64 marks "no previous window".
        prevLZ := uint8(64)
        prevTZ := uint8(0)
        for i := 1; i < n; i++ {
                cur := math.Float64bits(s[i])
                x := cur ^ prev
                if x == 0 {
                        bw.writeBit(false)
                        prev = cur
                        continue
                }
                bw.writeBit(true)
                lz := uint8(bits.LeadingZeros64(x))
                tz := uint8(bits.TrailingZeros64(x))
                if lz > 31 {
                        lz = 31
                }
                if prevLZ != 64 && lz >= prevLZ && tz >= prevTZ {
                        bw.writeBit(false)
                        mb := 64 - int(prevLZ) - int(prevTZ)
                        bw.writeBits(x>>prevTZ, uint8(mb))
                } else {
                        bw.writeBit(true)
                        bw.writeBits(uint64(lz), 5)
                        mb := 64 - int(lz) - int(tz)
                        bw.writeBits(uint64(mb-1), 6)
                        bw.writeBits(x>>tz, uint8(mb))
                        prevLZ = lz
                        prevTZ = tz
                }
                prev = cur
        }
        totalBits := bw.flush()
        out = finishGorillaBody(bw.buf, bodyStart, totalBits)
        e.buf = out
}

func (e *Encoder) writePackedGorillaFloat32Slice(s []float32) {
        e.writeHeader()
        n := len(s)
        out := slices.Grow(e.buf, 2+10+4+10+2*n)
        out = append(out, tagPackGorilla, qpackKindFloat32)
        out = appendUvarint(out, uint64(n))
        if n == 0 {
                e.buf = out
                return
        }
        first := math.Float32bits(s[0])
        out = appendU32(out, first)
        if n == 1 {
                out = appendUvarint(out, 0)
                e.buf = out
                return
        }
        bodyStart := len(out)
        bw := bitWriter{buf: out}
        prev := first
        prevLZ := uint8(32)
        prevTZ := uint8(0)
        for i := 1; i < n; i++ {
                cur := math.Float32bits(s[i])
                x := cur ^ prev
                if x == 0 {
                        bw.writeBit(false)
                        prev = cur
                        continue
                }
                bw.writeBit(true)
                lz := uint8(bits.LeadingZeros32(x))
                tz := uint8(bits.TrailingZeros32(x))
                if lz > 15 {
                        lz = 15
                }
                if prevLZ != 32 && lz >= prevLZ && tz >= prevTZ {
                        bw.writeBit(false)
                        mb := 32 - int(prevLZ) - int(prevTZ)
                        bw.writeBits(uint64(x>>prevTZ), uint8(mb))
                } else {
                        bw.writeBit(true)
                        bw.writeBits(uint64(lz), 4)
                        mb := 32 - int(lz) - int(tz)
                        bw.writeBits(uint64(mb-1), 5)
                        bw.writeBits(uint64(x>>tz), uint8(mb))
                        prevLZ = lz
                        prevTZ = tz
                }
                prev = cur
        }
        totalBits := bw.flush()
        out = finishGorillaBody(bw.buf, bodyStart, totalBits)
        e.buf = out
}

// readPackedGorillaHeader consumes kind, n, firstVal, numBits, body.
// The tag itself must already be consumed. The numBits varuint is
// validated and consumed but not returned — it is used only to compute
// bodyBytes = ceil(numBits/8) and slice the body from the wire buffer;
// the bitReader is then bounded by that byte-sliced body length, not by
// a tracked bit count. Surfacing numBits made it unused (caught by unparam).
func (d *Decoder) readPackedGorillaHeader(expectKind byte) (n int, firstU64 uint64, body []byte, err error) {
        if d.i >= len(d.buf) {
                return 0, 0, nil, ErrShortBuffer
        }
        k := d.buf[d.i]
        d.i++
        if k != expectKind {
                return 0, 0, nil, ErrTypeMismatch
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, nil, ErrInvalidLength
        }
        d.i += nr
        // Bound the element count before make([]float, n). In a columnar column
        // it must equal the row count (colLenOK); standalone, every element past
        // the first costs >=1 control bit in the body, so a valid n cannot exceed
        // the remaining bits — this stops a tiny payload from claiming billions of
        // elements and triggering an OOM (the body-bytes check below does not
        // bound n, which is read independently).
        if !d.colLenOK(n64) {
                return 0, 0, nil, ErrInvalidLength
        }
        if rem := uint64(len(d.buf) - d.i); n64 > rem*8+1 {
                return 0, 0, nil, ErrShortBuffer
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8 lets n64 exceed MaxInt -> int(n64) wraps negative -> make panics
                return 0, 0, nil, ErrInvalidLength
        }
        n = int(n64)
        if n == 0 {
                return n, 0, nil, nil
        }
        w := qpackRawWidthBytes(k)
        if d.i+w > len(d.buf) {
                return 0, 0, nil, ErrShortBuffer
        }
        switch w {
        case 4:
                firstU64 = uint64(readU32(d.buf[d.i:]))
        case 8:
                firstU64 = readU64(d.buf[d.i:])
        default:
                return 0, 0, nil, ErrBadTag
        }
        d.i += w
        if n == 1 {
                // numBits must be 0 but still encoded.
                nb64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return 0, 0, nil, ErrInvalidLength
                }
                d.i += nr
                if nb64 != 0 {
                        return 0, 0, nil, ErrBadTag
                }
                return n, firstU64, nil, nil
        }
        nb64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, 0, nil, ErrInvalidLength
        }
        d.i += nr
        // uint64 bounds check before signed cast.
        rem := uint64(len(d.buf) - d.i)
        if nb64 > rem*8 {
                return 0, 0, nil, ErrShortBuffer
        }
        bodyBytes := int((nb64 + 7) / 8)
        body = d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        return n, firstU64, body, nil
}

func (d *Decoder) readPackedGorillaFloat64Slice() ([]float64, error) {
        n, firstU, body, err := d.readPackedGorillaHeader(qpackKindFloat64)
        if err != nil {
                return nil, err
        }
        out := make([]float64, n)
        if n == 0 {
                return out, nil
        }
        out[0] = math.Float64frombits(firstU)
        if n == 1 {
                return out, nil
        }
        br := bitReader{buf: body}
        prev := firstU
        prevLZ := uint8(64)
        prevTZ := uint8(0)
        for i := 1; i < n; i++ {
                ctl, ok := br.readBit()
                if !ok {
                        return nil, ErrShortBuffer
                }
                if !ctl {
                        out[i] = math.Float64frombits(prev)
                        continue
                }
                ctl2, ok := br.readBit()
                if !ok {
                        return nil, ErrShortBuffer
                }
                var x uint64
                if !ctl2 {
                        mb := 64 - int(prevLZ) - int(prevTZ)
                        // A reuse-window with zero meaningful bits is malformed: a value equal
                        // to prev is encoded as ctl=0, never ctl=1/ctl2=0. In particular the
                        // initial sentinel window (prevLZ=64) gives mb=0, so a hostile stream
                        // opening with ctl=1/ctl2=0 would readBits(0)=0 and silently emit a
                        // duplicate of the previous value. The encoder guards this (a reuse
                        // window is only written once a real window exists); enforce it here.
                        if mb <= 0 {
                                return nil, ErrInvalidLength
                        }
                        mbBits, ok := br.readBits(uint8(mb))
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        x = mbBits << prevTZ
                } else {
                        lz64, ok := br.readBits(5)
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        mbLen64, ok := br.readBits(6)
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        mb := mbLen64 + 1
                        // A well-formed window always has lz + mb <= 64 (lz + mb + tz = 64).
                        // Reject a hostile stream where it exceeds the word width before the
                        // unsigned `64 - lz64 - mb` underflows: that would make the shift
                        // below collapse to 0 (silent wrong value) and poison prevTZ for the
                        // rest of the slice.
                        if lz64+mb > 64 {
                                return nil, ErrInvalidLength
                        }
                        tz := 64 - lz64 - mb
                        mbBits, ok := br.readBits(uint8(mb))
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        x = mbBits << tz
                        prevLZ = uint8(lz64)
                        prevTZ = uint8(tz)
                }
                cur := prev ^ x
                out[i] = math.Float64frombits(cur)
                prev = cur
        }
        return out, nil
}

func (d *Decoder) readPackedGorillaFloat32Slice() ([]float32, error) {
        n, firstU, body, err := d.readPackedGorillaHeader(qpackKindFloat32)
        if err != nil {
                return nil, err
        }
        out := make([]float32, n)
        if n == 0 {
                return out, nil
        }
        out[0] = math.Float32frombits(uint32(firstU))
        if n == 1 {
                return out, nil
        }
        br := bitReader{buf: body}
        prev := uint32(firstU)
        prevLZ := uint8(32)
        prevTZ := uint8(0)
        for i := 1; i < n; i++ {
                ctl, ok := br.readBit()
                if !ok {
                        return nil, ErrShortBuffer
                }
                if !ctl {
                        out[i] = math.Float32frombits(prev)
                        continue
                }
                ctl2, ok := br.readBit()
                if !ok {
                        return nil, ErrShortBuffer
                }
                var x uint32
                if !ctl2 {
                        mb := 32 - int(prevLZ) - int(prevTZ)
                        // See the float64 path: mb==0 (the initial prevLZ=32 sentinel) means a
                        // hostile ctl=1/ctl2=0 opener would silently duplicate the prior value.
                        if mb <= 0 {
                                return nil, ErrInvalidLength
                        }
                        mbBits, ok := br.readBits(uint8(mb))
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        x = uint32(mbBits) << prevTZ
                } else {
                        lz64, ok := br.readBits(4)
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        mbLen64, ok := br.readBits(5)
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        mb := mbLen64 + 1
                        // As in the float64 path: a valid window has lz + mb <= 32; reject a
                        // hostile stream before the unsigned subtraction underflows.
                        if lz64+mb > 32 {
                                return nil, ErrInvalidLength
                        }
                        tz := 32 - lz64 - mb
                        mbBits, ok := br.readBits(uint8(mb))
                        if !ok {
                                return nil, ErrShortBuffer
                        }
                        x = uint32(mbBits) << tz
                        prevLZ = uint8(lz64)
                        prevTZ = uint8(tz)
                }
                cur := prev ^ x
                out[i] = math.Float32frombits(cur)
                prev = cur
        }
        return out, nil
}

package qdf

import (
        "math"
        "math/bits"
        "slices"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/bitpack"
)

// Patched Frame-of-Reference. Packs (v-min) at a reduced width b chosen so
// that the few values that don't fit (outliers) are cheaper to store in an
// exception list than to widen every slot. Selected by the picker only when
// strictly smaller than every other codec, so it never regresses.

// pforPlanUnsigned scans s (min already known, forBits = plain-FOR width) and
// returns the bit width b that minimizes the PFOR wire cost, that cost, and
// whether PFOR is a sensible candidate (ok=false for n<8 or forBits<2, where
// there is no room to patch). The exception value byte cost uses maxDelta as a
// conservative UPPER bound, so PFOR is never chosen when it would be larger.
func pforPlanUnsigned(s []uint64, mn uint64, forBits int) (b int, cost int, ok bool) {
        n := len(s)
        if n < 8 || forBits < 2 || forBits > qpackForMaxBits {
                return 0, 0, false
        }
        var hist [65]int
        var maxDelta uint64
        for _, v := range s {
                d := v - mn
                hist[bits.Len64(d)]++
                if d > maxDelta {
                        maxDelta = d
                }
        }
        valLen := uvarintLen(maxDelta) // conservative upper bound per exception value
        // Each exception is written as uvarint(i-prev) + value; the index gap is at
        // most n, so uvarintLen(n) is a safe per-exception upper bound. Charging 1
        // byte (as before) under-counted far-apart outliers, letting PFOR be picked
        // while its real output exceeded the runner-up — a never-larger violation.
        gapLen := uvarintLen(uint64(n))
        hdr := 3 + uvarintLen(uint64(n)) + uvarintLen(mn)
        bestB, bestCost := -1, math.MaxInt
        suffix := 0 // number of values with width > cand, accumulated as cand descends
        for cand := forBits - 1; cand >= 0; cand-- {
                suffix += hist[cand+1]
                body := (n*cand + 7) >> 3
                c := hdr + body + uvarintLen(uint64(suffix)) + suffix*(gapLen+valLen)
                if c < bestCost {
                        bestCost, bestB = c, cand
                }
        }
        if bestB < 0 {
                return 0, 0, false
        }
        return bestB, bestCost, true
}

// writePackedPForUint64Slice emits s as a tagPackPFor payload at width b.
// mn must equal the slice min; b must come from pforPlanUnsigned.
func (e *Encoder) writePackedPForUint64Slice(s []uint64, mn uint64, b int) {
        e.writeHeader()
        n := len(s)
        mask := uint64(1)<<uint(b) - 1
        bodyBytes := (n*b + 7) >> 3
        out := slices.Grow(e.buf, 3+10+10+bodyBytes)
        out = append(out, tagPackPFor, qpackKindUint64)
        out = appendUvarint(out, uint64(n))
        out = append(out, byte(b))
        out = appendUvarint(out, mn)
        start := len(out)
        out = out[:start+bodyBytes] // capacity guaranteed by slices.Grow at line 66
        body := out[start : start+bodyBytes]
        // Zero-initialize body bytes since PackChunk may read-modify-write on byte boundaries
        clear(body)
        // Collect exceptions during the pack pass so no second O(n) scan of s is
        // needed. excPos stores pre-computed relative positions (i+j - prev); excDelta
        // stores the corresponding delta values. Both slices reuse e.pforExcPos /
        // e.pforExcDelta across calls — same lifecycle as alpScratch.
        excPos := e.pforExcPos[:0]
        excDelta := e.pforExcDelta[:0]
        prev := 0
        var chunk [64]uint64
        for i := 0; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                for j, v := range s[i:end] {
                        d := v - mn
                        chunk[j] = d & mask
                        if pforIsException(d, b) {
                                excPos = append(excPos, uint64(i+j-prev))
                                excDelta = append(excDelta, d)
                                prev = i + j
                        }
                }
                bitpack.PackChunk(body, chunk[:end-i], b, i)
        }
        e.pforExcPos = excPos
        e.pforExcDelta = excDelta
        out = appendUvarint(out, uint64(len(excPos)))
        for i, rp := range excPos {
                out = appendUvarint(out, rp)
                out = appendUvarint(out, excDelta[i])
        }
        e.buf = out
}

// pforIsException reports whether delta d does not fit in b bits (and so must
// be patched). d==0 never needs patching (it fits any width, including b==0).
func pforIsException(d uint64, b int) bool {
        if d == 0 {
                return false
        }
        return b == 0 || d>>uint(b) != 0
}

// pforPlanSigned mirrors pforPlanUnsigned for int64 (deltas computed in
// wrapping uint64 space relative to min, min stored zigzag).
func pforPlanSigned(s []int64, mn int64, forBits int) (b int, cost int, ok bool) {
        n := len(s)
        if n < 8 || forBits < 2 || forBits > qpackForMaxBits {
                return 0, 0, false
        }
        mnU := uint64(mn)
        var hist [65]int
        var maxDelta uint64
        for _, v := range s {
                d := uint64(v) - mnU
                hist[bits.Len64(d)]++
                if d > maxDelta {
                        maxDelta = d
                }
        }
        valLen := uvarintLen(maxDelta)
        gapLen := uvarintLen(uint64(n)) // uvarint(i-prev) <= uvarint(n); see pforPlanUnsigned
        hdr := 3 + uvarintLen(uint64(n)) + uvarintLen(zigzagEncode64(mn))
        bestB, bestCost := -1, math.MaxInt
        suffix := 0
        for cand := forBits - 1; cand >= 0; cand-- {
                suffix += hist[cand+1]
                body := (n*cand + 7) >> 3
                c := hdr + body + uvarintLen(uint64(suffix)) + suffix*(gapLen+valLen)
                if c < bestCost {
                        bestCost, bestB = c, cand
                }
        }
        if bestB < 0 {
                return 0, 0, false
        }
        return bestB, bestCost, true
}

// writePackedPForInt64Slice emits s as a tagPackPFor payload at width b.
// mn must equal the slice min; b must come from pforPlanSigned.
func (e *Encoder) writePackedPForInt64Slice(s []int64, mn int64, b int) {
        e.writeHeader()
        n := len(s)
        mnU := uint64(mn)
        mask := uint64(1)<<uint(b) - 1
        bodyBytes := (n*b + 7) >> 3
        out := slices.Grow(e.buf, 3+10+10+bodyBytes)
        out = append(out, tagPackPFor, qpackKindInt64)
        out = appendUvarint(out, uint64(n))
        out = append(out, byte(b))
        out = appendUvarint(out, zigzagEncode64(mn))
        start := len(out)
        out = out[:start+bodyBytes] // capacity guaranteed by slices.Grow at line 153
        body := out[start : start+bodyBytes]
        // Zero-initialize body bytes since PackChunk may read-modify-write on byte boundaries
        clear(body)
        // Collect exceptions during the pack pass (same pattern as the uint64 writer).
        excPos := e.pforExcPos[:0]
        excDelta := e.pforExcDelta[:0]
        prev := 0
        var chunk [64]uint64
        for i := 0; i < n; i += len(chunk) {
                end := min(i+len(chunk), n)
                for j, v := range s[i:end] {
                        d := uint64(v) - mnU
                        chunk[j] = d & mask
                        if pforIsException(d, b) {
                                excPos = append(excPos, uint64(i+j-prev))
                                excDelta = append(excDelta, d)
                                prev = i + j
                        }
                }
                bitpack.PackChunk(body, chunk[:end-i], b, i)
        }
        e.pforExcPos = excPos
        e.pforExcDelta = excDelta
        out = appendUvarint(out, uint64(len(excPos)))
        for i, rp := range excPos {
                out = appendUvarint(out, rp)
                out = appendUvarint(out, excDelta[i])
        }
        e.buf = out
}

// readPackedPForInt64Slice decodes a tagPackPFor int64 payload (tag already
// consumed) into a []int64.
func (d *Decoder) readPackedPForInt64Slice() ([]int64, error) {
        if d.i+1 > len(d.buf) || d.buf[d.i] != qpackKindInt64 {
                return nil, ErrTypeMismatch
        }
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return nil, ErrInvalidLength
        }
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        b := int(d.buf[d.i])
        d.i++
        if b > qpackForMaxBits {
                return nil, ErrBadTag
        }
        mz, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        mnU := uint64(zigzagDecode64(mz))
        rem := uint64(len(d.buf) - d.i)
        if b > 0 {
                if n64 > rem*8/uint64(b) {
                        return nil, ErrShortBuffer
                }
        } else if n64 > qpackMaxStandaloneCount {
                // b == 0 (constant base): empty packed body, no per-element bound.
                return nil, ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8 lets n64 exceed MaxInt -> int(n64) wraps negative
                return nil, ErrInvalidLength
        }
        n := int(n64)
        bodyBytes := (n*b + 7) >> 3
        if d.i+bodyBytes > len(d.buf) {
                return nil, ErrShortBuffer
        }
        out := make([]int64, n)
        // Decode straight into out viewed as []uint64 and add the reference in
        // place — no separate scratch buffer or copy pass (mirrors the unsigned
        // reader). The exception loop below overwrites individual slots.
        u := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(out))), n)
        if b > 0 {
                bitpack.Unpack(u, d.buf[d.i:d.i+bodyBytes], b)
        }
        d.i += bodyBytes
        if mnU != 0 {
                for k := range u {
                        u[k] += mnU
                }
        }
        excN64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if excN64 > n64 {
                return nil, ErrInvalidLength
        }
        pos := 0
        for range excN64 {
                dp, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                delta, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if dp > uint64(n-pos) {
                        return nil, ErrInvalidLength
                }
                pos += int(dp)
                if pos < 0 || pos >= n {
                        return nil, ErrInvalidLength
                }
                out[pos] = int64(mnU + delta)
        }
        return out, nil
}

// readPackedPForUint64Slice decodes a tagPackPFor payload (tag already
// consumed) into a []uint64.
func (d *Decoder) readPackedPForUint64Slice() ([]uint64, error) {
        if d.i+1 > len(d.buf) {
                return nil, ErrShortBuffer
        }
        if d.buf[d.i] != qpackKindUint64 {
                return nil, ErrTypeMismatch
        }
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) {
                return nil, ErrInvalidLength
        }
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        b := int(d.buf[d.i])
        d.i++
        if b > qpackForMaxBits {
                return nil, ErrBadTag
        }
        mn, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        rem := uint64(len(d.buf) - d.i)
        if b > 0 {
                if n64 > rem*8/uint64(b) {
                        return nil, ErrShortBuffer
                }
        } else if n64 > qpackMaxStandaloneCount {
                // b == 0 (constant base): empty packed body, no per-element bound.
                return nil, ErrInvalidLength
        }
        if n64 > uint64(math.MaxInt) { // 32-bit: rem*8 lets n64 exceed MaxInt -> int(n64) wraps negative
                return nil, ErrInvalidLength
        }
        n := int(n64)
        bodyBytes := (n*b + 7) >> 3
        if d.i+bodyBytes > len(d.buf) {
                return nil, ErrShortBuffer
        }
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        out := make([]uint64, n)
        if b > 0 {
                bitpack.Unpack(out, body, b)
        }
        if mn != 0 {
                for k := range out {
                        out[k] += mn
                }
        }
        excN64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if excN64 > n64 {
                return nil, ErrInvalidLength
        }
        pos := 0
        for range excN64 {
                dp, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                delta, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if dp > uint64(n-pos) {
                        return nil, ErrInvalidLength
                }
                pos += int(dp)
                if pos < 0 || pos >= n {
                        return nil, ErrInvalidLength
                }
                out[pos] = mn + delta
        }
        return out, nil
}

package qdf

import "slices"

// Run-length encoded integer slice codec. Compact when many consecutive
// values repeat — common shape in telemetry: HTTP Status columns, log
// Level enum-likes, sparse counter snapshots where most readings sit at
// zero. Wire format documented at tagPackRLE in wire.go.
//
// The encoder writes one (value, runLen) pair per run; the decoder
// unrolls until the declared element count is consumed.

// writePackedRLEUint64Slice emits s as a tagPackRLE payload. n == 0
// still writes the tag/kind/n triplet so a round-trip distinguishes an
// empty slice from no slice. Caller already decided via pickU64Codec
// that RLE wins; no probe is repeated here.
func (e *Encoder) writePackedRLEUint64Slice(s []uint64) {
        e.writeHeader()
        n := len(s)
        // Pre-grow by the actual run count, not n: RLE is chosen precisely for
        // run-dominated input, so 20*n over-reserves by orders of magnitude (a single
        // 1M-element constant run would reserve ~20MB for ~22 bytes). One extra scan
        // (cheap next to the encode) sizes the reservation to 20 bytes per run.
        runs := 0
        for i := range n {
                if i == 0 || s[i] != s[i-1] {
                        runs++
                }
        }
        out := slices.Grow(e.buf, 2+10+20*runs)
        out = append(out, tagPackRLE, qpackKindUint64)
        out = appendUvarint(out, uint64(n))
        if n == 0 {
                e.buf = out
                return
        }
        runLen := uint64(1)
        prev := s[0]
        for i := 1; i < n; i++ {
                if s[i] == prev {
                        runLen++
                        continue
                }
                out = appendUvarint(out, prev)
                out = appendUvarint(out, runLen)
                runLen = 1
                prev = s[i]
        }
        out = appendUvarint(out, prev)
        out = appendUvarint(out, runLen)
        e.buf = out
}

// writePackedRLEInt64Slice mirrors the unsigned variant but zigzag-
// encodes each run value so small negatives stay one byte.
func (e *Encoder) writePackedRLEInt64Slice(s []int64) {
        e.writeHeader()
        n := len(s)
        // Pre-grow by run count, not n (see writePackedRLEUint64Slice).
        runs := 0
        for i := range n {
                if i == 0 || s[i] != s[i-1] {
                        runs++
                }
        }
        out := slices.Grow(e.buf, 2+10+20*runs)
        out = append(out, tagPackRLE, qpackKindInt64)
        out = appendUvarint(out, uint64(n))
        if n == 0 {
                e.buf = out
                return
        }
        runLen := uint64(1)
        prev := s[0]
        for i := 1; i < n; i++ {
                if s[i] == prev {
                        runLen++
                        continue
                }
                out = appendUvarint(out, zigzagEncode64(prev))
                out = appendUvarint(out, runLen)
                runLen = 1
                prev = s[i]
        }
        out = appendUvarint(out, zigzagEncode64(prev))
        out = appendUvarint(out, runLen)
        e.buf = out
}

// readPackedRLEHeader parses the kind byte and element count. The tag
// itself must already be consumed; the caller dispatches on kind to
// pick the signed / unsigned body decoder.
func (d *Decoder) readPackedRLEHeader(expectKind byte) (n int, err error) {
        if d.i >= len(d.buf) {
                return 0, ErrShortBuffer
        }
        k := d.buf[d.i]
        d.i++
        if k != expectKind {
                return 0, ErrTypeMismatch
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return 0, ErrInvalidLength
        }
        d.i += nr
        // In a columnar column every codec must yield exactly the struct count
        // (colMaxLen); RLE can legitimately claim far more elements than remaining
        // bytes (a long run is 2 bytes), so unlike the body-bounded codecs it MUST
        // be gated by colLenOK or a tiny body could claim a multi-GB element count.
        if !d.colLenOK(n64) {
                return 0, ErrInvalidLength
        }
        // Standalone decode (colMaxLen == 0): RLE has no per-element body bound (a
        // long run is 2 bytes), so the element count must be capped UNCONDITIONALLY.
        // Nesting the cap under `n64 > remaining` defeated it — a large input buffer
        // makes that guard vacuous, letting a tiny body claim up to `remaining`
        // elements (an 8x-amplification OOM). Mirrors the FOR/Dict constant-codec cap.
        if d.colMaxLen == 0 && n64 > qpackMaxStandaloneCount {
                return 0, ErrInvalidLength
        }
        return int(n64), nil
}

// readPackedRLEUint64Slice consumes a tagPackRLE body emitted with
// qpackKindUint64 and returns the unrolled []uint64.
func (d *Decoder) readPackedRLEUint64Slice() ([]uint64, error) {
        n, err := d.readPackedRLEHeader(qpackKindUint64)
        if err != nil {
                return nil, err
        }
        out := make([]uint64, n)
        idx := 0
        for idx < n {
                v, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                runLen, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if runLen == 0 || runLen > uint64(n-idx) {
                        return nil, ErrInvalidLength
                }
                end := idx + int(runLen)
                out[idx] = v
                for size := 1; idx+size < end; size <<= 1 {
                        copy(out[idx+size:end], out[idx:idx+size])
                }
                idx = end
        }
        return out, nil
}

// readPackedRLEInt64Slice consumes a tagPackRLE body emitted with
// qpackKindInt64 and returns the unrolled []int64. Each value comes
// back through zigzagDecode64.
func (d *Decoder) readPackedRLEInt64Slice() ([]int64, error) {
        n, err := d.readPackedRLEHeader(qpackKindInt64)
        if err != nil {
                return nil, err
        }
        out := make([]int64, n)
        idx := 0
        for idx < n {
                v64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                runLen, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if runLen == 0 || runLen > uint64(n-idx) {
                        return nil, ErrInvalidLength
                }
                v := zigzagDecode64(v64)
                end := idx + int(runLen)
                out[idx] = v
                for size := 1; idx+size < end; size <<= 1 {
                        copy(out[idx+size:end], out[idx:idx+size])
                }
                idx = end
        }
        return out, nil
}

package qdf

import (
        "math"
        "slices"

        "github.com/alex60217101990/qdf/internal/bitpack"
        "github.com/alex60217101990/qdf/internal/unsafestr"
)

// Alphabet-aware bit-packed string columns (tagColStrAlpha) for the columnar
// container. See wire.go for the format and rationale. This is the high-card,
// restricted-alphabet counterpart to tagColStrRaw: it targets ID columns whose
// bytes are all drawn from a small alphabet (hex/base32/base64/decimal — trace,
// span, request IDs, hashes, GUIDs), a class dict, front-coding and FSST all
// miss. Each character costs ceil(log2 |A|) bits instead of 8.

// qpackStrAlphaMaxAlphabet caps the alphabet size at 64: |A| <= 64 keeps the
// per-char code at <= 6 bits (a strict win over 8), and the one-pass scan bails
// the instant a 65th distinct byte appears, so a full-alphabet (text) column is
// rejected after reading only its first ~dozens of bytes — no full scan, no
// regression on the common non-ID column.
const qpackStrAlphaMaxAlphabet = 64

// alphaMinDistinctPct is the high-cardinality floor: alpha-packing only fires
// when at least this percentage of a leading sample is distinct. Low-card
// columns (constants, enums) are far cheaper as const/dict/interned references,
// and alpha's raw-floor gate alone would not catch that (a constant column
// packs smaller than raw yet far larger than a single interned value). Mirrors
// the dict probe's high-cardinality threshold.
const alphaMinDistinctPct = 70

// alphaProbeMinAvgLen is the minimum average value length (bytes) for the
// intern-aware columnar probe to credit alpha-packing. Short tokens pack no
// better than a dictionary once the per-row length prefix is paid, so crediting
// them could flip a borderline struct into columnar for no real gain. The ID
// columns alpha targets (hex/uuid/base32 — 16+ chars) clear this comfortably.
const alphaProbeMinAvgLen = 8

// tryWriteStringColumnAlpha attempts to emit strs as a tagColStrAlpha block.
// It returns true when the alphabet-packed form was written (and is strictly
// smaller than the raw per-value floor), false when the caller should fall
// through to the next codec. The alphabet scan is a single pass that bails the
// moment the alphabet exceeds 64 distinct bytes, so non-restricted columns pay
// only a short prefix scan before falling through unchanged.
func (e *Encoder) tryWriteStringColumnAlpha(strs []string) bool {
        n := len(strs)
        if n < qpackStrDictMinElems {
                return false
        }

        // Cheap high-cardinality gate first (shared with the dict pre-bail): a low-card
        // column (constant, enum, run-heavy) is cheaper as const/dict/interned
        // references than as packed chars, and the raw-floor gate below would not catch
        // that — so bail before the full alphabet scan. alpha wants HIGH cardinality,
        // the inverse of the dict bail (same 64-row window / 70% threshold).
        if !dictSampleHighCard(strs) {
                return false
        }

        // One pass: build the alphabet (byte -> dense code), and accumulate the raw
        // per-value floor, the total character count, fixed-length detection, and the
        // per-value length-prefix bytes (used only if the column is not fixed-length).
        // seen/code are stack arrays (zero-initialised, non-escaping); code[c] is only
        // read for bytes already marked in seen, so it needs no separate reset.
        var seen [256]bool
        var code [256]uint8
        var alphabet [qpackStrAlphaMaxAlphabet]byte
        a := 0
        totalChars := 0
        rawFloor := 0
        lenPrefixBytes := 0
        fixedLen := true
        firstLen := -1
        for _, s := range strs {
                l := len(s)
                if firstLen < 0 {
                        firstLen = l
                } else if l != firstLen {
                        fixedLen = false
                }
                totalChars += l
                rawFloor += uvarintLen(uint64(l)) + l
                lenPrefixBytes += uvarintLen(uint64(l))
                for i := range l {
                        c := s[i]
                        if !seen[c] {
                                if a >= qpackStrAlphaMaxAlphabet {
                                        return false // alphabet too large to pack below 8 bits/char
                                }
                                seen[c] = true
                                code[c] = uint8(a)
                                alphabet[a] = c
                                a++
                        }
                }
        }
        if a < 2 {
                // a == 0 (all-empty) or a == 1 (single distinct byte): no positional
                // packing is possible/useful — the const/raw paths handle these.
                return false
        }
        cbits := bitsForDistinct(a) // ceil(log2 a), >= 1 since a >= 2

        bodyBytes := (totalChars*cbits + 7) >> 3
        overhead := 1 + uvarintLen(uint64(a)) + a + uvarintLen(uint64(n)) + 1 // tag,a,alphabet,n,flags
        if fixedLen {
                overhead += uvarintLen(uint64(firstLen))
        } else {
                overhead += lenPrefixBytes
        }
        // Never-larger gate: the packed block (header + body) must beat the raw
        // per-value floor (sum of length prefixes + bytes). rawFloor is the byte cost
        // of the per-value/raw path this codec replaces, so this guarantees the wire
        // never grows when the codec fires.
        if overhead+bodyBytes >= rawFloor {
                return false
        }

        e.writeHeader()
        out := e.buf
        out = append(out, tagColStrAlpha)
        out = appendUvarint(out, uint64(a))
        out = append(out, alphabet[:a]...)
        out = appendUvarint(out, uint64(n))
        var flags byte
        if fixedLen {
                flags |= 1
        }
        out = append(out, flags)
        if fixedLen {
                out = appendUvarint(out, uint64(firstLen))
        } else {
                for _, s := range strs {
                        out = appendUvarint(out, uint64(len(s)))
                }
        }

        // Grow without the make([]byte) zero-fill: every body byte is written below,
        // so the (possibly stale) grown region needs no clearing.
        start := len(out)
        out = slices.Grow(out, bodyBytes)[:start+bodyBytes]
        body := out[start : start+bodyBytes]
        if cbits == 4 {
                // Hex / 16-symbol alphabet — the dominant ID case. Pack two nibbles per
                // byte (LSB-first: first char low, second char high).
                if fixedLen && firstLen&1 == 0 {
                        // Fixed even length (32-char trace IDs, 16-char span IDs): every value
                        // is a whole number of bytes, so nibbles never straddle a value boundary
                        // — pack two-at-a-time with no carry branch.
                        pos := 0
                        for _, s := range strs {
                                for i := 0; i < len(s); i += 2 {
                                        body[pos] = code[s[i]] | code[s[i+1]]<<4
                                        pos++
                                }
                        }
                        e.buf = out
                        return true
                }
                // Variable or odd length: carry a pending low nibble across boundaries.
                pos, pend := 0, -1
                for _, s := range strs {
                        for i := 0; i < len(s); i++ {
                                c := int(code[s[i]])
                                if pend < 0 {
                                        pend = c
                                } else {
                                        body[pos] = byte(pend | c<<4)
                                        pos++
                                        pend = -1
                                }
                        }
                }
                if pend >= 0 {
                        body[pos] = byte(pend)
                }
                e.buf = out
                return true
        }
        // General LSB-first bit-writer over the char codes — identical layout to
        // bitpack.Pack, so bitpack.Unpack reverses it on decode (codes are < a <=
        // 2^cbits, so no masking is needed).
        var acc uint64
        var have uint
        pos := 0
        for _, s := range strs {
                for i := 0; i < len(s); i++ {
                        acc |= uint64(code[s[i]]) << have
                        have += uint(cbits)
                        for have >= 8 {
                                body[pos] = byte(acc)
                                acc >>= 8
                                have -= 8
                                pos++
                        }
                }
        }
        if have > 0 {
                body[pos] = byte(acc)
        }
        e.buf = out
        return true
}

// readStringColumnAlpha decodes a tagColStrAlpha block (tag at d.i) into the n
// per-row strings. All characters are materialised into ONE slab (a single
// allocation for the whole column) and returned as views into it; the bit-
// unpacked codes reuse the shared transient scratch. Bounds mirror the other
// columnar string readers so a hostile header cannot drive an oversized alloc.
func (d *Decoder) readStringColumnAlpha(n int) ([]string, error) {
        d.i++ // consume tagColStrAlpha
        a64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if a64 < 2 || a64 > qpackStrAlphaMaxAlphabet {
                // a >= 2 keeps cbits >= 1 (non-empty body), so totalChars is bounded by
                // the remaining buffer below before any allocation.
                return nil, ErrBadTag
        }
        a := int(a64)
        if a > len(d.buf)-d.i {
                return nil, ErrShortBuffer
        }
        alphabet := d.buf[d.i : d.i+a] // alias; only read, never returned
        d.i += a

        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if int(n64) != n {
                return nil, ErrTypeMismatch
        }
        if d.i >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        flags := d.buf[d.i]
        d.i++
        fixedLen := flags&1 != 0
        cbits := bitsForDistinct(a)

        // Lengths. fixedLen stores one shared length; otherwise n per-row varints.
        // For the varint case the region is scanned twice (once to sum totalChars and
        // bound it before allocating, once to slice the slab) — re-reading varints is
        // cheap and avoids a per-row lengths allocation, keeping decode allocs on par
        // with the raw path it replaces.
        rem := uint64(len(d.buf) - d.i)
        // Each char costs cbits bits in the packed body, so the total character count
        // is bounded by rem*8/cbits (NOT rem: cbits < 8 means totalChars can
        // legitimately exceed the remaining byte count). Cap that bound by maxInt as
        // well so the int(tc) narrowings below cannot truncate a multi-GiB count to a
        // negative length on a 32-bit build (which would bypass the body check and
        // panic), mirroring the origLen guard in decodeRANS.
        maxChars := rem * 8 / uint64(cbits)
        if maxInt := uint64(math.MaxInt); maxChars > maxInt {
                maxChars = maxInt
        }
        var (
                totalChars int
                fixedL     int
                lenStart   int
        )
        if fixedLen {
                l64, k := readUvarint(d.buf[d.i:])
                if k <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += k
                // totalChars = n*L; guard against multiplication overflow and the bound.
                tc := uint64(n) * l64
                if l64 != 0 && tc/l64 != uint64(n) {
                        return nil, ErrInvalidLength // overflow
                }
                if tc > maxChars {
                        return nil, ErrShortBuffer
                }
                totalChars = int(tc)
                fixedL = int(l64)
        } else {
                lenStart = d.i
                // Guard inside the loop so a hostile run of huge length varints bails
                // immediately rather than accumulating past the bound.
                var tc uint64
                for range n {
                        l64, k := readUvarint(d.buf[d.i:])
                        if k <= 0 {
                                return nil, ErrInvalidLength
                        }
                        d.i += k
                        tc += l64
                        if tc > maxChars {
                                return nil, ErrShortBuffer
                        }
                }
                totalChars = int(tc)
        }

        // Widen the multiply: totalChars is capped at maxInt and cbits at 6, so on a
        // 32-bit build totalChars*cbits as int could overflow to a negative bodyBytes.
        // The true value is <= rem (totalChars <= rem*8/cbits), so it fits int once the
        // intermediate is computed in 64 bits.
        bodyBytes := int((uint64(totalChars)*uint64(cbits) + 7) >> 3)
        if bodyBytes > len(d.buf)-d.i {
                return nil, ErrShortBuffer
        }
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes

        slab := make([]byte, totalChars)
        if cbits == 4 {
                // Hex / 16-symbol alphabet — the dominant ID case. Fuse the unpack and the
                // alphabet scatter into one pass over the body (two nibbles per byte),
                // skipping the uint64 scratch and the general bit-window decoder.
                if a64 == 16 {
                        // Every nibble is a valid alphabet index (no range check) → hand the
                        // whole column to the SIMD-accelerated nibble→LUT kernel (VPSHUFB on
                        // amd64, TBL on arm64, scalar otherwise; all bit-identical).
                        bitpack.DecodeHex4(slab, body, (*[16]byte)(alphabet))
                } else {
                        // a < 16: a nibble may exceed the alphabet and must be rejected.
                        full := totalChars &^ 1
                        k := 0
                        for ; k < full; k += 2 {
                                b := body[k>>1]
                                lo, hi := b&0xf, b>>4
                                if uint64(lo) >= a64 || uint64(hi) >= a64 {
                                        return nil, ErrBadTag
                                }
                                slab[k] = alphabet[lo]
                                slab[k+1] = alphabet[hi]
                        }
                        if k < totalChars { // trailing odd nibble (low half of the last byte)
                                lo := body[k>>1] & 0xf
                                if uint64(lo) >= a64 {
                                        return nil, ErrBadTag
                                }
                                slab[k] = alphabet[lo]
                        }
                }
        } else {
                // General path: unpack the char codes into the shared transient scratch,
                // then map each code to its alphabet byte.
                if cap(d.deltaScratch) < totalChars {
                        d.deltaScratch = make([]uint64, totalChars)
                }
                codes := d.deltaScratch[:totalChars]
                bitpack.Unpack(codes, body, cbits)
                for k, c := range codes {
                        if c >= a64 {
                                return nil, ErrBadTag
                        }
                        slab[k] = alphabet[c]
                }
        }

        out := d.colStrScratch(n)
        off := 0
        if fixedLen {
                for i := range n {
                        out[i] = unsafestr.String(slab[off : off+fixedL])
                        off += fixedL
                }
        } else {
                j := lenStart
                for i := range n {
                        l64, k := readUvarint(d.buf[j:])
                        j += k
                        l := int(l64)
                        out[i] = unsafestr.String(slab[off : off+l])
                        off += l
                }
        }
        return out, nil
}

// readStringColumnAlphaInto decodes a tagColStrAlpha column (tag at d.i) DIRECTLY
// into the batch slab, writing a per-row Str handle into out[0:n]. It mirrors
// readStringColumnAlpha's header parse and body-unpack verbatim, but unpacks the
// alphabet characters straight onto the slab's backing instead of a fresh
// make([]byte, totalChars) scratch, and emits (off,len) handles instead of
// []string views — eliminating the temp allocation and the second copy the
// general readStringColumnHandles path pays for an alpha column. Every bound
// readStringColumnAlpha enforces is preserved (alphabet range, totalChars/body
// caps, per-row length re-scan), plus the slab's uint32 offset-overflow guard
// before the reservation. readStringColumnAlpha itself is unchanged — the reflect
// Unmarshal path still uses it.
func (d *Decoder) readStringColumnAlphaInto(n int, bs *batchSlab, out []Str) error {
        d.i++ // consume tagColStrAlpha
        a64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if a64 < 2 || a64 > qpackStrAlphaMaxAlphabet {
                return ErrBadTag
        }
        a := int(a64)
        if a > len(d.buf)-d.i {
                return ErrShortBuffer
        }
        alphabet := d.buf[d.i : d.i+a]
        d.i += a

        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return ErrInvalidLength
        }
        d.i += nr
        if int(n64) != n {
                return ErrTypeMismatch
        }
        if d.i >= len(d.buf) {
                return ErrShortBuffer
        }
        flags := d.buf[d.i]
        d.i++
        fixedLen := flags&1 != 0
        cbits := bitsForDistinct(a)

        rem := uint64(len(d.buf) - d.i)
        maxChars := rem * 8 / uint64(cbits)
        if maxInt := uint64(math.MaxInt); maxChars > maxInt {
                maxChars = maxInt
        }
        var (
                totalChars int
                fixedL     int
                lenStart   int
        )
        if fixedLen {
                l64, k := readUvarint(d.buf[d.i:])
                if k <= 0 {
                        return ErrInvalidLength
                }
                d.i += k
                tc := uint64(n) * l64
                if l64 != 0 && tc/l64 != uint64(n) {
                        return ErrInvalidLength
                }
                if tc > maxChars {
                        return ErrShortBuffer
                }
                totalChars = int(tc)
                fixedL = int(l64)
        } else {
                lenStart = d.i
                var tc uint64
                for range n {
                        l64, k := readUvarint(d.buf[d.i:])
                        if k <= 0 {
                                return ErrInvalidLength
                        }
                        d.i += k
                        tc += l64
                        if tc > maxChars {
                                return ErrShortBuffer
                        }
                }
                totalChars = int(tc)
        }

        bodyBytes := int((uint64(totalChars)*uint64(cbits) + 7) >> 3)
        if bodyBytes > len(d.buf)-d.i {
                return ErrShortBuffer
        }
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes

        // Reserve totalChars on the slab's backing and unpack straight into it. The
        // uint32 offset-overflow guard (matching slab.append/maxBatchSlabBytes) runs
        // BEFORE the reservation so a direct write can never wrap a handle offset.
        base := len(bs.buf)
        if uint64(base)+uint64(totalChars) > maxBatchSlabBytes {
                return ErrInvalidLength
        }
        bs.grow(totalChars)
        dst := bs.buf[base : base+totalChars]
        if cbits == 4 {
                if a64 == 16 {
                        bitpack.DecodeHex4(dst, body, (*[16]byte)(alphabet))
                } else {
                        full := totalChars &^ 1
                        k := 0
                        for ; k < full; k += 2 {
                                b := body[k>>1]
                                lo, hi := b&0xf, b>>4
                                if uint64(lo) >= a64 || uint64(hi) >= a64 {
                                        return ErrBadTag
                                }
                                dst[k] = alphabet[lo]
                                dst[k+1] = alphabet[hi]
                        }
                        if k < totalChars {
                                lo := body[k>>1] & 0xf
                                if uint64(lo) >= a64 {
                                        return ErrBadTag
                                }
                                dst[k] = alphabet[lo]
                        }
                }
        } else {
                if cap(d.deltaScratch) < totalChars {
                        d.deltaScratch = make([]uint64, totalChars)
                }
                codes := d.deltaScratch[:totalChars]
                bitpack.Unpack(codes, body, cbits)
                for k, c := range codes {
                        if c >= a64 {
                                return ErrBadTag
                        }
                        dst[k] = alphabet[c]
                }
        }

        // Commit the unpacked bytes onto the slab and emit per-row handles.
        bs.buf = bs.buf[:base+totalChars]
        off := 0
        if fixedLen {
                for i := range n {
                        if fixedL == 0 {
                                out[i] = Str{}
                                continue
                        }
                        out[i] = Str{off: uint32(base + off), len: uint32(fixedL)}
                        off += fixedL
                }
        } else {
                j := lenStart
                for i := range n {
                        l64, k := readUvarint(d.buf[j:])
                        j += k
                        l := int(l64)
                        if l == 0 {
                                out[i] = Str{}
                                continue
                        }
                        out[i] = Str{off: uint32(base + off), len: uint32(l)}
                        off += l
                }
        }
        return nil
}

package qdf

import (
        "cmp"
        "slices"

        "github.com/alex60217101990/qdf/internal/bitpack"
        "github.com/alex60217101990/qdf/internal/unsafestr"
)

// Dictionary-coded string columns for the columnar container. A string
// column whose values are drawn from a small set (enum-like dimensions —
// log level, service, region, status) is cheaper to store as a distinct
// table plus a bitpacked index per row than as M interned values, because
// each interned reference still costs at least a byte while an index costs
// ceil(log2 distinct) bits. Wire format documented at tagColStrDict.
//
// The codec is gated so it never grows the wire: it is chosen only when the
// bitpacked index body is smaller than the per-value floor of n bytes (the
// gathered-column fallback emits at least a 1-byte string/intern-ref tag per
// row and does NOT run-length-collapse consecutive identical rows). Because the
// distinct count is capped at 256, the index spends at most 8 bits/row, so a
// bitpacked index beats per-value refs for every column the high-cardinality
// probe does not already reject — including skewed (clustered) low-card columns,
// which an earlier run-count floor wrongly sent to the per-value path.

const (
        // qpackStrDictMaxDistinct caps the distinct count. 256 keeps the index
        // at <= 8 bits and the distinct probe bounded.
        qpackStrDictMaxDistinct = 256
        // qpackStrDictMinElems skips the probe for tiny columns where the table
        // overhead cannot pay off.
        qpackStrDictMinElems = 16
        // Cheap high-cardinality bail: if the first sample rows are mostly
        // distinct, the column is ID/text-like — abandon before the full scan.
        qpackStrDictSampleN      = 64
        qpackStrDictSampleMaxPct = 70 // distinct% over the sample above which we bail
)

// tryWriteStringColumnDict attempts to emit strs as a tagColStrDict block.
// It returns true when the dictionary form was written (and is strictly
// smaller than the per-value n-byte floor), false when the caller should fall
// back to per-value WriteString. Encode-side scratch is reused across
// columns to avoid per-column allocation.
func (e *Encoder) tryWriteStringColumnDict(strs []string) bool {
        n := len(strs)
        if n < qpackStrDictMinElems {
                return false
        }
        m := e.state.strDictMap
        if m == nil {
                m = make(map[string]uint32, qpackStrDictMaxDistinct)
                e.state.strDictMap = m
        } else {
                clear(m)
        }
        table := e.state.colDictTable[:0]
        idx := e.state.colScratchU64[:0]
        // gp tracks the longest byte prefix shared by EVERY distinct value, computed
        // for free as the table is built (no sort, no extra pass). It is a sort-free
        // signal for the front-coded table form: gp >= 2 guarantees the front-coded
        // table is strictly smaller than the plain one (it saves >= (d-1)*(gp-1)
        // bytes), so the encoder pays the sort ONLY when front-coding is sure to win
        // — a non-prefix-shared column skips it entirely (no CPU regression).
        gp := 0
        for i, s := range strs {
                id, ok := m[s]
                if !ok {
                        if len(m) >= qpackStrDictMaxDistinct {
                                e.state.colDictTable = table
                                e.state.colScratchU64 = idx
                                return false
                        }
                        id = uint32(len(m))
                        m[s] = id
                        table = append(table, s)
                        if len(table) == 1 {
                                gp = len(s)
                        } else {
                                gp = commonPrefixLen(table[0][:gp], s)
                        }
                }
                idx = append(idx, uint64(id))
                // High-cardinality bail: after the sample, if the column is mostly
                // distinct it will not dict-compress — stop before scanning the rest.
                if i+1 == qpackStrDictSampleN && len(m)*100 > qpackStrDictSampleN*qpackStrDictSampleMaxPct {
                        e.state.colDictTable = table
                        e.state.colScratchU64 = idx
                        return false
                }
        }
        e.state.colDictTable = table
        e.state.colScratchU64 = idx

        d := len(table)
        if d < 2 {
                // A single distinct value never beats the per-value path (one interned
                // string + a state-repeat run). The gate below would reject it anyway,
                // but make the invariant explicit: the decoder relies on count >= 2 (so
                // bits >= 1, so a non-empty index body) to keep the row count bounded by
                // the buffer. A count==1 dict would carry no body and let a tiny input
                // claim a huge n.
                return false
        }
        bits := bitsForDistinct(d)
        bodyBytes := (n*bits + 7) >> 3
        // Never-worse gate. The distinct table bytes are paid by both forms (the
        // per-value fallback emits each distinct string once too), so they cancel;
        // compare only the dict's fixed overhead + index body against the per-value
        // floor. That floor is n, not the run count: the gathered-column fallback is
        // a per-value loop (writeStringColumn) that does NOT run-length-collapse
        // consecutive identical rows — each of the n rows emits at least a 1-byte
        // string/intern-ref tag, so the fallback costs >= n bytes after the table
        // cancels. (An earlier `runs` floor assumed RLE collapse that never happens;
        // it wrongly rejected dict for skewed low-card columns whose dominant value
        // clusters into few runs, bloating them ~2-3x.) Since distinct <= 256, the
        // index spends <= 8 bits/row, so a bitpacked index strictly beats 1-byte refs
        // for every d < 256 and ties (rejects) at d == 256 — exactly the n floor.
        overhead := 1 + uvarintLen(uint64(d)) + uvarintLen(uint64(n))
        if bodyBytes+overhead >= n {
                return false
        }

        // Table representation: plain (first-seen) vs front-coded (sorted, each entry
        // stored as shared-prefix-len + suffix). The per-row index body is
        // byte-identical between the two forms, so only the table bytes differ. The
        // front-coded form is attempted ONLY when the sort-free global-prefix signal
        // (gp) guarantees it wins: gp >= 2 ⇒ (d-1)*(gp-1) >= d-1 bytes saved, so the
        // encoder never sorts a column front-coding cannot help — no CPU regression
        // on non-prefix-shared data (which keeps the plain dictionary unchanged).
        e.writeHeader()
        out := e.buf
        useFC := false
        if gp >= 2 {
                plainTbl := 0
                for _, s := range table {
                        plainTbl += uvarintLen(uint64(len(s))) + len(s)
                }
                var order, inv [qpackStrDictMaxDistinct]uint16
                for k := range d {
                        order[k] = uint16(k)
                }
                slices.SortFunc(order[:d], func(a, b uint16) int { return cmp.Compare(table[a], table[b]) })
                fcTbl, prevS := 0, ""
                for k := range d {
                        s := table[order[k]]
                        pfx := commonPrefixLen(prevS, s)
                        fcTbl += uvarintLen(uint64(pfx)) + uvarintLen(uint64(len(s)-pfx)) + (len(s) - pfx)
                        inv[order[k]] = uint16(k)
                        prevS = s
                }
                if fcTbl < plainTbl { // guaranteed by gp >= 2; the explicit never-larger gate
                        useFC = true
                        out = append(out, tagColStrDictFC)
                        out = appendUvarint(out, uint64(d))
                        prevS = ""
                        for k := range d {
                                s := table[order[k]]
                                pfx := commonPrefixLen(prevS, s)
                                out = appendUvarint(out, uint64(pfx))
                                out = appendUvarint(out, uint64(len(s)-pfx))
                                out = append(out, s[pfx:]...)
                                prevS = s
                        }
                        for i := range idx { // remap row indices into the sorted table
                                idx[i] = uint64(inv[idx[i]])
                        }
                }
        }
        if !useFC {
                // Per-row index codec: flat ceil(log2 d)-bit pack (tagColStrDict) vs the
                // QPack integer picker (tagColStrDictQ). The picker's chosen-codec byte
                // cost (qCost, which already includes the QPack block framing) is compared
                // against the flat body; the QPack form is emitted only when strictly
                // smaller, so the column never grows. The picker captures run-length /
                // dictionary structure a skewed index has but the flat pack cannot exploit.
                // QPack is paired only with the plain table: a front-coded table fires on
                // high-card sorted-prefix data whose index is near-uniform, where the
                // picker cannot win, so the FC branch keeps the flat index.
                codec, mn, forBits, first, minDelta, deltaBits, pforBits, qCost := pickU64Codec(idx)
                // The !qpackConstantOverCap guard keeps qCost honest: when n exceeds the
                // standalone cap, emitQPackUint64 downgrades a constant-body codec (RLE /
                // Dict / const-FOR) to raw, so the emitted block would be ~8n bytes, not
                // qCost — choosing DictQ then could grow the wire past the flat index. It is
                // currently unreachable here (n <= maxColumnarElems == qpackMaxStandaloneCount
                // for every columnar/gathered caller), but gating on it decouples the
                // never-larger guarantee from that constant coincidence.
                if qCost < bodyBytes && !qpackConstantOverCap(n, codec, forBits, deltaBits, pforBits) {
                        out = append(out, tagColStrDictQ)
                        out = appendUvarint(out, uint64(d))
                        for _, s := range table {
                                out = appendUvarint(out, uint64(len(s)))
                                out = append(out, s...)
                        }
                        // emitQPackUint64 appends the index block (with its own row count) to
                        // e.buf; writeHeader is idempotent so the mid-block call is a no-op for
                        // the header already emitted above.
                        e.buf = out
                        e.emitQPackUint64(idx, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
                        return true
                }
                out = append(out, tagColStrDict)
                out = appendUvarint(out, uint64(d))
                for _, s := range table {
                        out = appendUvarint(out, uint64(len(s)))
                        out = append(out, s...)
                }
        }
        out = appendUvarint(out, uint64(n))
        if bits > 0 {
                start := len(out)
                out = append(out, make([]byte, bodyBytes)...)
                body := out[start : start+bodyBytes]
                var chunk [64]uint64
                for i := 0; i < n; i += len(chunk) {
                        end := min(i+len(chunk), n)
                        bitpack.PackChunk(body, idx[i:end], bits, i)
                }
        }
        e.buf = out
        return true
}

// dictSampleHighCard reports whether a leading sample of strs is mostly distinct
// (> qpackStrDictSampleMaxPct), using a fixed stack array and a linear scan — no
// map, no allocation. It mirrors the in-loop high-cardinality bail of
// tryWriteStringColumnDict so a mostly-distinct column can exit before the hash
// map is built. The linear O(sample²) compares are cheap because distinct values
// (random IDs) diverge in their first bytes, so most comparisons fail fast.
func dictSampleHighCard(strs []string) bool {
        sampleN := min(len(strs), qpackStrDictSampleN)
        var seen [qpackStrDictSampleN]string
        distinct := 0
        for i := range sampleN {
                s := strs[i]
                fresh := true
                for j := 0; j < distinct; j++ {
                        if seen[j] == s {
                                fresh = false
                                break
                        }
                }
                if fresh {
                        seen[distinct] = s
                        distinct++
                }
        }
        return distinct*100 > sampleN*qpackStrDictSampleMaxPct
}

// commonPrefixLen returns the length of the longest byte prefix shared by a and
// b. Used by the front-coded string-dictionary codec.
func commonPrefixLen(a, b string) int {
        n := min(len(a), len(b))
        i := 0
        for i < n && a[i] == b[i] {
                i++
        }
        return i
}

// readStringColumn decodes a string column of n values written by
// writeStringColumn: a tagColStrDict dictionary block, a tagColStrFSST block,
// or n per-value strings.
func (d *Decoder) readStringColumn(n int) ([]string, error) {
        if n > 0 && d.i < len(d.buf) && d.buf[d.i] == tagColStrConst {
                return d.readStringColumnConst(n)
        }
        // FSST allocates its own output slab+slice, so dispatch before the per-value
        // make to avoid a dead []string allocation on the FSST path.
        if n > 0 && d.i < len(d.buf) && d.buf[d.i] == tagColStrFSST {
                return d.readStringColumnFSST(n)
        }
        if n > 0 && d.i < len(d.buf) && d.buf[d.i] == tagColStrRaw {
                return d.readStringColumnRaw(n)
        }
        if n > 0 && d.i < len(d.buf) && d.buf[d.i] == tagColStrAlpha {
                return d.readStringColumnAlpha(n)
        }
        out := d.colStrScratch(n)
        if n > 0 && d.i < len(d.buf) && (d.buf[d.i] == tagColStrDict || d.buf[d.i] == tagColStrDictFC || d.buf[d.i] == tagColStrDictQ) {
                var (
                        table []string
                        idx   []uint32
                        err   error
                )
                switch d.buf[d.i] {
                case tagColStrDictFC:
                        table, idx, err = d.readStringColumnDictFC(n)
                case tagColStrDictQ:
                        table, idx, err = d.readStringColumnDictQ(n)
                default:
                        table, idx, err = d.readStringColumnDict(n)
                }
                if err != nil {
                        return nil, err
                }
                for i := range n {
                        out[i] = table[idx[i]]
                }
                return out, nil
        }
        for i := range n {
                sb, err := d.readStringBytes()
                if err != nil {
                        return nil, err
                }
                out[i] = d.materializeStr(sb) // aliases input under noCopy / arena
        }
        return out, nil
}

// colDictTableScratch / colDictIdxScratch return reused per-column dict scratch.
// Both are transient — the only caller (readStringColumn) copies table[idx[i]]
// into the column result before the next column read — so reusing them across
// columns is safe, and they are distinct buffers from the result scratch
// (colScratchStr), so the table->result copy never self-overwrites.
func (d *Decoder) colDictTableScratch(count int) []string {
        st := d.colStateDec()
        if cap(st.colDictTableScr) < count {
                st.colDictTableScr = make([]string, count)
        }
        st.colDictTableScr = st.colDictTableScr[:count]
        return st.colDictTableScr
}

func (d *Decoder) colDictIdxScratch(n int) []uint32 {
        st := d.colStateDec()
        if cap(st.colDictIdxScr) < n {
                st.colDictIdxScr = make([]uint32, n)
        }
        st.colDictIdxScr = st.colDictIdxScr[:n]
        return st.colDictIdxScr
}

// readStringColumnDict decodes a tagColStrDict block (tag at d.i) into a
// distinct table and a per-row index slice of length n. Each row's string
// is table[idx[i]]; the distinct strings are allocated once and shared by
// every row that references them. n is the row count the caller expects
// from the columnar header; the block's own count must match.
func (d *Decoder) readStringColumnDict(n int) (table []string, idx []uint32, err error) {
        d.i++ // consume tagColStrDict
        c64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, nil, ErrInvalidLength
        }
        d.i += nr
        if c64 < 2 || c64 > qpackStrDictMaxDistinct {
                // The encoder never emits a single-distinct (count==1) dictionary —
                // per-value wins, so the never-worse gate rejects it. Requiring
                // count >= 2 here is therefore lossless for valid streams and, crucially,
                // guarantees bits >= 1 so the index body is non-empty and the row count n
                // is bounded by the remaining buffer below. A count==1 dict would carry
                // no body and let a tiny hostile input drive a huge n / allocation.
                return nil, nil, ErrBadTag
        }
        count := int(c64)
        table = d.colDictTableScratch(count)
        for i := range count {
                l64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, nil, ErrInvalidLength
                }
                d.i += nr
                if l64 > uint64(len(d.buf)-d.i) {
                        return nil, nil, ErrShortBuffer
                }
                l := int(l64)
                table[i] = d.materializeStr(d.buf[d.i : d.i+l]) // aliases input under noCopy / arena
                d.i += l
        }
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, nil, ErrInvalidLength
        }
        d.i += nr
        if int(n64) != n {
                return nil, nil, ErrTypeMismatch
        }
        // count >= 2 ⇒ bits >= 1 ⇒ the index body is non-empty, so this check
        // bounds n by the remaining buffer BEFORE any n-sized allocation; a tiny
        // input can no longer drive a large idx/tmp/output slice.
        bits := bitsForDistinct(count)
        rem := uint64(len(d.buf) - d.i)
        if n64 > rem*8/uint64(bits) {
                return nil, nil, ErrShortBuffer
        }
        bodyBytes := (n*bits + 7) >> 3
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        idx = d.colDictIdxScratch(n)
        // Reuse the shared transient unpack scratch (as readPackedDictUint64Slice
        // does): tmp holds the bit-unpacked dictionary indices, fully written by
        // Unpack below before any read, and only mapped into idx — never aliased
        // into the returned slices. count >= 2 ⇒ bits >= 1 (checked above), so
        // Unpack always writes all n slots; no stale-tail under-fill is possible.
        if cap(d.deltaScratch) < n {
                d.deltaScratch = make([]uint64, n)
        }
        tmp := d.deltaScratch[:n]
        bitpack.Unpack(tmp, body, bits)
        for i, v := range tmp {
                if v >= c64 {
                        return nil, nil, ErrBadTag
                }
                idx[i] = uint32(v)
        }
        return table, idx, nil
}

// readStringColumnDictQ decodes a tagColStrDictQ block (tag at d.i): a plain
// distinct table (identical layout to tagColStrDict) followed by a QPack-coded
// per-row index instead of a flat bitpack. The index block carries its own row
// count; the decoded length must equal n (the columnar header count) and every
// index must address a table slot. The QPack reader bounds its own allocation by
// the remaining buffer, so a hostile inner count cannot force an oversized slice.
func (d *Decoder) readStringColumnDictQ(n int) (table []string, idx []uint32, err error) {
        d.i++ // consume tagColStrDictQ
        c64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, nil, ErrInvalidLength
        }
        d.i += nr
        if c64 < 2 || c64 > qpackStrDictMaxDistinct {
                // count >= 2 mirrors tagColStrDict: the encoder never emits a single-
                // distinct dictionary (per-value wins), so this is lossless for valid
                // streams while rejecting a hostile count.
                return nil, nil, ErrBadTag
        }
        count := int(c64)
        table = d.colDictTableScratch(count)
        for i := range count {
                l64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, nil, ErrInvalidLength
                }
                d.i += nr
                if l64 > uint64(len(d.buf)-d.i) {
                        return nil, nil, ErrShortBuffer
                }
                l := int(l64)
                table[i] = d.materializeStr(d.buf[d.i : d.i+l]) // aliases input under noCopy / arena
                d.i += l
        }
        // QPack index block: peek the codec tag, decode, validate length and range.
        if d.i >= len(d.buf) {
                return nil, nil, ErrShortBuffer
        }
        idx64, err := d.readQPackUint64(d.buf[d.i])
        if err != nil {
                return nil, nil, err
        }
        if len(idx64) != n {
                return nil, nil, ErrTypeMismatch
        }
        idx = d.colDictIdxScratch(n)
        for i, v := range idx64 {
                if v >= c64 {
                        return nil, nil, ErrBadTag
                }
                idx[i] = uint32(v)
        }
        return table, idx, nil
}

// readStringColumnDictFC decodes a tagColStrDictFC block (tag at d.i): a sorted,
// front-coded distinct table plus a per-row index slice of length n. Each table
// entry is reconstructed as prev[:sharedPrefixLen] + suffix; all distinct strings
// are materialised into ONE slab (a single allocation for the whole table) and
// returned as views into it. Bounds mirror readStringColumnDict.
func (d *Decoder) readStringColumnDictFC(n int) (table []string, idx []uint32, err error) {
        d.i++ // consume tagColStrDictFC
        c64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, nil, ErrInvalidLength
        }
        d.i += nr
        if c64 < 2 || c64 > qpackStrDictMaxDistinct {
                // count >= 2 keeps bits >= 1 (non-empty index body) so n is buffer-bounded.
                return nil, nil, ErrBadTag
        }
        count := int(c64)

        // Reconstruct the sorted front-coded table into one growing slab: entry k is
        // prev[:prefixLen] + suffix. starts/lens record each entry's region in the
        // FINAL slab (offsets are stable across the appends' reallocations).
        var starts, lens [qpackStrDictMaxDistinct]int
        slab := make([]byte, 0, 64)
        prevStart, prevLen := 0, 0
        for i := range count {
                p64, k := readUvarint(d.buf[d.i:])
                if k <= 0 {
                        return nil, nil, ErrInvalidLength
                }
                d.i += k
                s64, k := readUvarint(d.buf[d.i:])
                if k <= 0 {
                        return nil, nil, ErrInvalidLength
                }
                d.i += k
                // The shared prefix cannot exceed the previous entry's length (entry 0
                // has prevLen 0 → prefix 0); the suffix is bounded by the remaining buffer.
                if p64 > uint64(prevLen) || s64 > uint64(len(d.buf)-d.i) {
                        return nil, nil, ErrBadTag
                }
                p, s := int(p64), int(s64)
                // Cap cumulative slab growth: the prefix is copied from the previous entry
                // (no wire cost), so a hostile table (entry 0 suffix S, then N entries with
                // prefix=S, suffix=0) amplifies to N*S with no per-entry guard catching it.
                // The uncompressed distinct table cannot legitimately exceed the columnar
                // byte ceiling.
                if int64(len(slab))+int64(p)+int64(s) > int64(maxColumnarBytes) {
                        return nil, nil, ErrInvalidLength
                }
                start := len(slab)
                slab = append(slab, slab[prevStart:prevStart+p]...) // shared prefix from prev
                slab = append(slab, d.buf[d.i:d.i+s]...)            // suffix from wire
                d.i += s
                starts[i], lens[i] = start, p+s
                prevStart, prevLen = start, p+s
        }
        table = d.colDictTableScratch(count)
        for i := range count {
                table[i] = unsafestr.String(slab[starts[i] : starts[i]+lens[i]]) // view into the owned slab
        }

        // Index body: identical layout to tagColStrDict. Bound n by the remaining
        // buffer (bits >= 1 since count >= 2) BEFORE the n-sized allocations.
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, nil, ErrInvalidLength
        }
        d.i += nr
        if int(n64) != n {
                return nil, nil, ErrTypeMismatch
        }
        bits := bitsForDistinct(count)
        rem := uint64(len(d.buf) - d.i)
        if n64 > rem*8/uint64(bits) {
                return nil, nil, ErrShortBuffer
        }
        bodyBytes := (n*bits + 7) >> 3
        body := d.buf[d.i : d.i+bodyBytes]
        d.i += bodyBytes
        idx = d.colDictIdxScratch(n)
        if cap(d.deltaScratch) < n {
                d.deltaScratch = make([]uint64, n)
        }
        tmp := d.deltaScratch[:n]
        bitpack.Unpack(tmp, body, bits)
        for i, v := range tmp {
                if v >= c64 {
                        return nil, nil, ErrBadTag
                }
                idx[i] = uint32(v)
        }
        return table, idx, nil
}

package qdf

import "github.com/alex60217101990/qdf/internal/unsafestr"

// Bulk-materialized ("raw") string columns for the columnar container — the
// high-cardinality counterpart to the dictionary codec in qpack_strdict.go.
//
// A mostly-distinct string column (IDs, GUIDs, emails, free text) cannot
// dict-compress: dictionary coding wins only when a small distinct set repeats.
// Written per value it also costs one heap allocation per row on decode, because
// each distinct string is materialized into its own backing array. tagColStrRaw
// instead lays every value down once, length-prefixed, so the decoder copies the
// whole column into ONE slab and every row is a sub-slice of it: the distinct
// bytes are still stored once (wire-neutral against the per-value form, which
// also stores each distinct value once and cannot intern-dedup a distinct
// column) but decode drops from n allocations to one slab plus the result slice.
//
// The codec is chosen automatically (no option) for high-cardinality columns;
// low/medium-cardinality columns keep the per-value path, where intern dedup
// shrinks the wire more than a flat layout. See tagColStrRaw for the wire format.

// qpackStrRawMinElems skips the bulk form for tiny columns, where the per-value
// path's handful of allocations is already cheap.
const qpackStrRawMinElems = 16

// tryWriteStringColumnRaw emits strs as a tagColStrRaw block when the bulk form
// is no larger on the wire than the per-value path, returning true on emit. The
// bulk form drops decode from n allocations to one slab, so it is preferred
// wherever it is wire-safe — that is the high-cardinality case, where per-value
// interning has few repeats to dedup. A single pass estimates both sizes (a
// distinct value costs its bytes once plus an intern tag; a repeat costs a state
// ref) and the bulk form is declined when interning would pack the column
// smaller, so the codec never grows the wire.
func (e *Encoder) tryWriteStringColumnRaw(strs []string) bool {
        n := len(strs)
        if n < qpackStrRawMinElems {
                return false
        }
        m := e.state.strDictMap
        if m == nil {
                m = make(map[string]uint32, min(n, 1024))
                e.state.strDictMap = m
        } else {
                clear(m)
        }
        total := 0
        bulkBytes := 0 // Σ (uvarint(len) + len)
        perVal := 0    // distinct: intern tag + uvarint(len) + len; repeat: state ref (>= 1)
        for _, s := range strs {
                l := len(s)
                total += l
                lp := uvarintLen(uint64(l))
                bulkBytes += lp + l
                if l < e.minIntern {
                        // WriteString does NOT intern sub-minIntern strings: it emits each
                        // occurrence inline (fixstr/str8/...) with no dedup. Charge that real
                        // cost per occurrence — modelling them as interned (distinct tag /
                        // 1-byte repeat) over-counts perVal and could fire the bulk form even
                        // though it is larger by the bulk header, breaking never-larger.
                        perVal += stringInlineHeaderLen(l) + l
                        continue
                }
                if _, seen := m[s]; seen {
                        perVal++ // a Dense state ref is at least one byte
                } else {
                        m[s] = 0
                        perVal += 1 + lp + l // first occurrence: tag + length + bytes
                }
        }
        bulkHdr := 1 + uvarintLen(uint64(n)) + uvarintLen(uint64(total))
        if bulkBytes+bulkHdr > perVal {
                return false // interning packs this column smaller — keep per-value
        }

        e.writeHeader()
        out := e.buf
        out = append(out, tagColStrRaw)
        out = appendUvarint(out, uint64(n))
        out = appendUvarint(out, uint64(total))
        for _, s := range strs {
                out = appendUvarint(out, uint64(len(s)))
                out = append(out, s...)
        }
        e.buf = out
        return true
}

// writeStringColumnRawForced ALWAYS emits strs as a tagColStrRaw block, with no
// size/cardinality decline. It is the unconditional emit half of
// tryWriteStringColumnRaw (same wire shape, decoded by readStringColumnRaw for
// any n >= 0). Used by the column-diff path, which needs a wire-stateless string
// codec (raw is fully self-contained) it can build on a shared encoder state
// without polluting the intern state table.
func (e *Encoder) writeStringColumnRawForced(strs []string) {
        n := len(strs)
        total := 0
        for _, s := range strs {
                total += len(s)
        }
        e.writeHeader()
        out := e.buf
        out = append(out, tagColStrRaw)
        out = appendUvarint(out, uint64(n))
        out = appendUvarint(out, uint64(total))
        for _, s := range strs {
                out = appendUvarint(out, uint64(len(s)))
                out = append(out, s...)
        }
        e.buf = out
}

// readStringColumnRaw decodes a tagColStrRaw block (tag at d.i) into n strings.
// Under noCopy each row aliases the input buffer directly (zero allocation past
// the result slice); otherwise every row is a sub-slice of one owned slab pre-
// sized from the block's total, so the column materializes in a single backing
// allocation. Every length is validated against the remaining buffer and the
// running slab bound before use, so a hostile block can neither read out of
// range nor force the slab to reallocate (which would dangle an earlier alias).
func (d *Decoder) readStringColumnRaw(n int) ([]string, error) {
        d.i++ // consume tagColStrRaw
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        if int(n64) != n {
                return nil, ErrTypeMismatch
        }
        total64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return nil, ErrInvalidLength
        }
        d.i += nr
        // The value bytes (total) plus their length prefixes must fit in what is
        // left, so total alone cannot exceed the remaining buffer — bound it before
        // the slab allocation so a hostile varint cannot drive a huge make.
        if total64 > uint64(len(d.buf)-d.i) {
                return nil, ErrShortBuffer
        }
        total := int(total64)
        out := d.colStrScratch(n)

        if d.noCopy {
                for i := range n {
                        l64, nr := readUvarint(d.buf[d.i:])
                        if nr <= 0 {
                                return nil, ErrInvalidLength
                        }
                        d.i += nr
                        if l64 > uint64(len(d.buf)-d.i) {
                                return nil, ErrShortBuffer
                        }
                        l := int(l64)
                        out[i] = unsafestr.String(d.buf[d.i : d.i+l]) // alias the input
                        d.i += l
                }
                return out, nil
        }

        // One owned slab; appends are bounded to cap == total so the backing never
        // reallocates and every prior sub-slice alias stays valid.
        slab := make([]byte, 0, total)
        for i := range n {
                l64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return nil, ErrInvalidLength
                }
                d.i += nr
                if l64 > uint64(len(d.buf)-d.i) {
                        return nil, ErrShortBuffer
                }
                l := int(l64)
                start := len(slab)
                if start+l > total {
                        return nil, ErrShortBuffer // lengths sum past the declared total
                }
                slab = append(slab, d.buf[d.i:d.i+l]...)
                out[i] = unsafestr.String(slab[start : start+l])
                d.i += l
        }
        return out, nil
}

package qdf

import (
        "encoding/binary"
        "math"
        "sync/atomic"
)

// Zone-chunked integer column with a min/max zonemap (wire tag tagZoneChunk,
// 0xF1, OptZoneMap).
//
// The column is split into fixed 256-row zones (same chunk size as the block
// codec, so the per-zone codec picks are shared via e.blkPlanI64). Each zone is
// an independent QPack int sub-slice; a uint32 offset table and a per-zone
// [min,max] zonemap precede the bodies. A bound-carrying predicate
// (WhereRange/GE/LE/Eq) skips zones whose [min,max] cannot intersect its bounds
// WITHOUT decoding them — the whole point of the feature. See decodeZoneSkip* in
// columnar.go for the query-side use.
//
// Unlike the block codec this is an explicit size-for-query-speed trade: it is
// emitted only under OptZoneMap, never auto, and carries no never-larger gate
// (chunking an ordered column costs more than a single whole-column codec — that
// is the price of zone-skippability). Without OptZoneMap the wire is unchanged.

const (
        zoneKindInt   = 0x00
        zoneKindUint  = 0x01
        zoneKindFloat = 0x02 // float64
        // zoneChunkMinLen: a column shorter than two zones has nothing to chunk.
        zoneChunkMinLen = 2 * blockSizeSmall // reuse the block codec's 256-row zone
)

// Zonemap encoding (the byte after kind). minmax is the per-zone [min,max] index;
// linear is a single learned model pos≈c·value+d (±epsP) over a SORTED int/uint
// column, which collapses the whole per-zone index to ~18 bytes. The encoder
// picks linear only when the column is monotonic, the fit holds within ~one zone
// (so zone-skip is preserved), and the model is strictly smaller than the minmax
// index (never-worse). See fitLinearZmap.
const (
        zmapMinMax = 0x00
        zmapLinear = 0x01
)

// linearFit is a learned zonemap: the row position of a value v is predicted by
// c*v+d within ±epsP positions. A query range [lo,hi] therefore maps to the row
// range [c*lo+d-epsP, c*hi+d+epsP], hence to a contiguous zone range — every
// matching row provably lands inside it, so no zone with a match is ever skipped.
type linearFit struct {
        c, d float64
        epsP int
}

// fitLinearZmap fits a single line pos≈c*value+d over a monotonic non-decreasing
// integer column (values widened to float64) and returns it iff (a) the column is
// sorted, (b) the values are not all equal (c>0), and (c) the worst-case position
// residual is at most blk — i.e. the model locates any value to within one zone,
// so zone-skip stays tight. epsP carries a +1 margin to absorb float rounding in
// the query-side re-evaluation. The caller still compares the encoded size
// against the minmax index and keeps whichever is smaller.
func fitLinearZmap[T int64 | uint64](s []T, blk int) (linearFit, bool) {
        n := len(s)
        if n < 2 {
                return linearFit{}, false
        }
        // Monotonic non-decreasing? (position == rank requires a sorted column.)
        for i := 1; i < n; i++ {
                if s[i] < s[i-1] {
                        return linearFit{}, false
                }
        }
        // Least-squares regression of position (y=i) on value (x=s[i]), in float64.
        var sx, sy, sxx, sxy float64
        fn := float64(n)
        for i := range n {
                x := float64(s[i])
                y := float64(i)
                sx += x
                sy += y
                sxx += x * x
                sxy += x * y
        }
        denom := fn*sxx - sx*sx
        if denom <= 0 { // all values equal (or degenerate) → no usable slope
                return linearFit{}, false
        }
        c := (fn*sxy - sx*sy) / denom
        d := (sy - c*sx) / fn
        // Reject any non-finite model. The decoder (readZoneChunkHeader) rejects
        // IsInf(d) too, so emitting d=±Inf here (reachable when c*sx overflows) would
        // write a column its own decoder refuses to read.
        if !(c > 0) || math.IsInf(c, 0) || math.IsNaN(d) || math.IsInf(d, 0) {
                return linearFit{}, false
        }
        // Worst-case position residual.
        var maxRes float64
        for i := range n {
                r := math.Abs(float64(i) - (c*float64(s[i]) + d))
                if r > maxRes {
                        maxRes = r
                }
        }
        epsP := int(math.Ceil(maxRes)) + 1 // +1: float-rounding margin for query re-eval
        if epsP > blk {
                return linearFit{}, false // fit too loose → zone-skip would degrade
        }
        return linearFit{c: c, d: d, epsP: epsP}, true
}

// zoneRangeFor maps a predicate value range [lo,hi] to the inclusive zone range
// [zlo,zhi] that can contain a matching row, per the linear model. Returns
// ok=false when the range cannot intersect the column (no zone needed).
func (lf linearFit) zoneRangeFor(lo, hi float64, n, blk int) (zlo, zhi int, ok bool) {
        pLo := lf.c*lo + lf.d - float64(lf.epsP)
        pHi := lf.c*hi + lf.d + float64(lf.epsP)
        if math.IsNaN(pLo) || math.IsNaN(pHi) || pHi < 0 || pLo > float64(n-1) {
                return 0, 0, false
        }
        // Clamp to [0, n-1] in the FLOAT domain before the int conversion: an
        // open-ended GE/LE bound makes pHi/pLo overflow int64 range, and an
        // out-of-range float→int yields MinInt64 on amd64, which would wrap to a
        // bogus negative zone and silently drop every matching row.
        if pLo < 0 {
                pLo = 0
        }
        if pHi > float64(n-1) {
                pHi = float64(n - 1)
        }
        loI := int(math.Floor(pLo))
        hiI := int(math.Floor(pHi))
        return loI / blk, hiI / blk, true
}

// finiteMinMaxF64 returns the min and max of s over non-NaN values. ±Inf are
// ordered and included. An all-NaN (or empty) slice yields the empty interval
// (+Inf, -Inf), which intersects no finite predicate range — so such a zone is
// correctly skipped (NaN never matches a comparison). Caller passes len(s) > 0.
func finiteMinMaxF64(s []float64) (mn, mx float64) {
        mn, mx = math.Inf(1), math.Inf(-1)
        for _, v := range s {
                if v != v { // NaN: builtin min/max would propagate it, so skip explicitly
                        continue
                }
                mn, mx = min(mn, v), max(mx, v)
        }
        return mn, mx
}

// ---- encode (int64) ----

// linearZmapBytes is the encoded size of a linear zonemap (2 float64 + epsP).
func linearZmapBytes(f linearFit) int { return 16 + uvarintLen(uint64(f.epsP)) }

func zmapByte(linear bool) byte {
        if linear {
                return zmapLinear
        }
        return zmapMinMax
}

func (e *Encoder) writeZoneChunkInt64(s []int64) {
        e.writeHeader()
        n := len(s)
        zoneCount := (n + blockSizeSmall - 1) / blockSizeSmall
        e.planBlocksI64(s, blockSizeSmall) // cache per-zone codec picks (no re-pick on emit)
        plans := e.blkPlanI64[:zoneCount]

        // Pick the learned linear zonemap over per-zone min/max only when it is both
        // valid (monotonic, tight fit) and strictly smaller (never-worse).
        fit, linOK := fitLinearZmap(s, blockSizeSmall)
        // zmm holds the per-zone zigzag(min),zigzag(max) payload, computed here only
        // when the linear map is a candidate; reused by the else branch below so the
        // min/max pass runs once even when the linear map loses the never-worse test.
        zmm := e.zoneMM[:0]
        if linOK {
                var mmBytes int
                for i := 0; i < n; i += blockSizeSmall {
                        j := min(i+blockSizeSmall, n)
                        mn, mx := minMaxI64(s[i:j])
                        zmn, zmx := zigzagEncode64(mn), zigzagEncode64(mx)
                        zmm = append(zmm, zmn, zmx)
                        mmBytes += uvarintLen(zmn) + uvarintLen(zmx)
                }
                e.zoneMM = zmm
                linOK = linearZmapBytes(fit) < mmBytes
        }

        e.buf = append(e.buf, tagZoneChunk, zoneKindInt, zmapByte(linOK), blkLogSmall)
        e.buf = appendUvarint(e.buf, uint64(n))
        offAt := len(e.buf)
        e.buf = append(e.buf, make([]byte, 4*zoneCount)...)
        switch {
        case linOK:
                e.buf = binary.LittleEndian.AppendUint64(e.buf, math.Float64bits(fit.c))
                e.buf = binary.LittleEndian.AppendUint64(e.buf, math.Float64bits(fit.d))
                e.buf = appendUvarint(e.buf, uint64(fit.epsP))
        case len(zmm) == 2*zoneCount:
                // zonemap: per-zone min,max as zigzag-varint, reusing the pass above.
                for _, z := range zmm {
                        e.buf = appendUvarint(e.buf, z)
                }
        default:
                // linear was never a candidate: single min/max pass here.
                for i := 0; i < n; i += blockSizeSmall {
                        j := min(i+blockSizeSmall, n)
                        mn, mx := minMaxI64(s[i:j])
                        e.buf = appendUvarint(e.buf, zigzagEncode64(mn))
                        e.buf = appendUvarint(e.buf, zigzagEncode64(mx))
                }
        }
        bodyStart := len(e.buf)
        zi := 0
        for i := 0; i < n; i += blockSizeSmall {
                j := min(i+blockSizeSmall, n)
                binary.LittleEndian.PutUint32(e.buf[offAt+4*zi:], uint32(len(e.buf)-bodyStart))
                p := &plans[zi]
                e.emitQPackInt64(s[i:j], p.codec, p.mn, p.forBits, p.first, p.minDelta, p.deltaBits, p.pforBits)
                zi++
        }
}

// ---- encode (uint64) ----

func (e *Encoder) writeZoneChunkUint64(s []uint64) {
        e.writeHeader()
        n := len(s)
        zoneCount := (n + blockSizeSmall - 1) / blockSizeSmall
        e.planBlocksU64(s, blockSizeSmall)
        plans := e.blkPlanU64[:zoneCount]

        fit, linOK := fitLinearZmap(s, blockSizeSmall)
        // zmm holds the per-zone min,max payload (raw uvarint), reused by the else
        // branch so the min/max pass runs once when the linear map loses.
        zmm := e.zoneMM[:0]
        if linOK {
                var mmBytes int
                for i := 0; i < n; i += blockSizeSmall {
                        j := min(i+blockSizeSmall, n)
                        mn, mx := minMaxU64(s[i:j])
                        zmm = append(zmm, mn, mx)
                        mmBytes += uvarintLen(mn) + uvarintLen(mx)
                }
                e.zoneMM = zmm
                linOK = linearZmapBytes(fit) < mmBytes
        }

        e.buf = append(e.buf, tagZoneChunk, zoneKindUint, zmapByte(linOK), blkLogSmall)
        e.buf = appendUvarint(e.buf, uint64(n))
        offAt := len(e.buf)
        e.buf = append(e.buf, make([]byte, 4*zoneCount)...)
        switch {
        case linOK:
                e.buf = binary.LittleEndian.AppendUint64(e.buf, math.Float64bits(fit.c))
                e.buf = binary.LittleEndian.AppendUint64(e.buf, math.Float64bits(fit.d))
                e.buf = appendUvarint(e.buf, uint64(fit.epsP))
        case len(zmm) == 2*zoneCount:
                for _, z := range zmm {
                        e.buf = appendUvarint(e.buf, z)
                }
        default:
                for i := 0; i < n; i += blockSizeSmall {
                        j := min(i+blockSizeSmall, n)
                        mn, mx := minMaxU64(s[i:j])
                        e.buf = appendUvarint(e.buf, mn)
                        e.buf = appendUvarint(e.buf, mx)
                }
        }
        bodyStart := len(e.buf)
        zi := 0
        for i := 0; i < n; i += blockSizeSmall {
                j := min(i+blockSizeSmall, n)
                binary.LittleEndian.PutUint32(e.buf[offAt+4*zi:], uint32(len(e.buf)-bodyStart))
                p := &plans[zi]
                e.emitQPackUint64(s[i:j], p.codec, p.mn, p.forBits, p.first, p.minDelta, p.deltaBits, p.pforBits)
                zi++
        }
}

// ---- encode (float64) ----

func (e *Encoder) writeZoneChunkFloat64(s []float64) {
        e.writeHeader()
        n := len(s)
        zoneCount := (n + blockSizeSmall - 1) / blockSizeSmall
        // float64 uses the per-zone min/max index only (no linear model in v1).
        e.buf = append(e.buf, tagZoneChunk, zoneKindFloat, zmapMinMax, blkLogSmall)
        e.buf = appendUvarint(e.buf, uint64(n))
        offAt := len(e.buf)
        e.buf = append(e.buf, make([]byte, 4*zoneCount)...)
        // zonemap: per-zone finite min,max as 8-byte IEEE-754 bits (LE).
        for i := 0; i < n; i += blockSizeSmall {
                j := min(i+blockSizeSmall, n)
                mn, mx := finiteMinMaxF64(s[i:j])
                e.buf = binary.LittleEndian.AppendUint64(e.buf, math.Float64bits(mn))
                e.buf = binary.LittleEndian.AppendUint64(e.buf, math.Float64bits(mx))
        }
        bodyStart := len(e.buf)
        zi := 0
        for i := 0; i < n; i += blockSizeSmall {
                j := min(i+blockSizeSmall, n)
                binary.LittleEndian.PutUint32(e.buf[offAt+4*zi:], uint32(len(e.buf)-bodyStart))
                ss := s[i:j]
                _ = encodeSliceFloat64Lossless(e, ss) // never errors for a plain []float64
                zi++
        }
}

// ---- decode header (shared) ----

// zoneChunkHeader is the decoded geometry + zonemap of a zone-chunked column.
// minI64/maxI64 (or minU64/maxU64) hold the per-zone bounds for predicate
// zone-skip; the other-kind slices stay nil.
type zoneChunkHeader struct {
        minI64, maxI64 []int64
        minU64, maxU64 []uint64
        minF64, maxF64 []float64
        lin            linearFit // valid when zmap == zmapLinear (int/uint only)
        offBase        int       // byte offset of the uint32 offset table
        bodyStart      int       // byte offset of the first zone body
        n              int
        zoneCount      int
        blk            int
        kind           byte
        zmap           byte // zmapMinMax | zmapLinear
}

// readZoneChunkHeader consumes the tag-less header (caller consumed the tag),
// validates it, decodes the offset table position and (when loadZonemap) the
// per-zone min/max zonemap, and returns the geometry. It leaves d.i at bodyStart.
//
// Only the predicate zone-skip path needs the zonemap; the full-decode and
// selective paths read all/matched zones regardless, so they pass loadZonemap
// false — the zonemap bytes are still walked to locate bodyStart (the int/uint
// entries are variable-length varints), but no slices are allocated and no
// values are decoded.
func (d *Decoder) readZoneChunkHeader(loadZonemap bool) (zoneChunkHeader, error) {
        var h zoneChunkHeader
        if d.i+3 > len(d.buf) {
                return h, ErrShortBuffer
        }
        h.kind = d.buf[d.i]
        if h.kind != zoneKindInt && h.kind != zoneKindUint && h.kind != zoneKindFloat {
                return h, ErrTypeMismatch
        }
        d.i++
        h.zmap = d.buf[d.i]
        // A linear zonemap is only defined for int/uint columns.
        if h.zmap != zmapMinMax && h.zmap != zmapLinear {
                return h, ErrBadTag
        }
        if h.zmap == zmapLinear && h.kind == zoneKindFloat {
                return h, ErrBadTag
        }
        d.i++
        blk, ok := blockSizeFor(d.buf[d.i])
        if !ok {
                return h, ErrBadTag
        }
        h.blk = blk
        d.i++
        n64, nr := readUvarint(d.buf[d.i:])
        if nr <= 0 {
                return h, ErrInvalidLength
        }
        d.i += nr
        if !d.colLenOK(n64) || n64 == 0 || n64 > uint64(math.MaxInt) {
                return h, ErrInvalidLength
        }
        h.n = int(n64)
        if checkColumnarBytes(h.n, 8) != nil {
                return h, ErrInvalidLength
        }
        h.zoneCount = (h.n + blk - 1) / blk

        // offset table
        h.offBase = d.i
        if d.i+4*h.zoneCount > len(d.buf) {
                return h, ErrShortBuffer
        }
        d.i += 4 * h.zoneCount

        // Linear zonemap: a single model (2 float64 + epsP varint) replaces the whole
        // per-zone min/max index. Read it (or skip its fixed bytes) and return.
        if h.zmap == zmapLinear {
                if d.i+16 > len(d.buf) {
                        return h, ErrShortBuffer
                }
                if loadZonemap {
                        h.lin.c = math.Float64frombits(binary.LittleEndian.Uint64(d.buf[d.i:]))
                        h.lin.d = math.Float64frombits(binary.LittleEndian.Uint64(d.buf[d.i+8:]))
                }
                d.i += 16
                eps64, nr := readUvarint(d.buf[d.i:])
                if nr <= 0 {
                        return h, ErrInvalidLength
                }
                d.i += nr
                if eps64 > uint64(math.MaxInt) {
                        return h, ErrInvalidLength
                }
                if loadZonemap {
                        h.lin.epsP = int(eps64)
                        // A non-finite model or non-positive slope cannot bound positions.
                        if !(h.lin.c > 0) || math.IsInf(h.lin.c, 0) || math.IsNaN(h.lin.d) || math.IsInf(h.lin.d, 0) {
                                return h, ErrInvalidLength
                        }
                }
                h.bodyStart = d.i
                if err := h.validateOffsets(d); err != nil {
                        return h, err
                }
                return h, nil
        }

        // zonemap (per-kind min,max)
        switch h.kind {
        case zoneKindInt:
                if loadZonemap {
                        h.minI64 = make([]int64, h.zoneCount)
                        h.maxI64 = make([]int64, h.zoneCount)
                }
                for z := 0; z < h.zoneCount; z++ {
                        mnZ, nr := readUvarint(d.buf[d.i:])
                        if nr <= 0 {
                                return h, ErrInvalidLength
                        }
                        d.i += nr
                        mxZ, nr2 := readUvarint(d.buf[d.i:])
                        if nr2 <= 0 {
                                return h, ErrInvalidLength
                        }
                        d.i += nr2
                        if loadZonemap {
                                h.minI64[z] = zigzagDecode64(mnZ)
                                h.maxI64[z] = zigzagDecode64(mxZ)
                                if h.minI64[z] > h.maxI64[z] {
                                        return h, ErrInvalidLength
                                }
                        }
                }
        case zoneKindUint:
                if loadZonemap {
                        h.minU64 = make([]uint64, h.zoneCount)
                        h.maxU64 = make([]uint64, h.zoneCount)
                }
                for z := 0; z < h.zoneCount; z++ {
                        mnZ, nr := readUvarint(d.buf[d.i:])
                        if nr <= 0 {
                                return h, ErrInvalidLength
                        }
                        d.i += nr
                        mxZ, nr2 := readUvarint(d.buf[d.i:])
                        if nr2 <= 0 {
                                return h, ErrInvalidLength
                        }
                        d.i += nr2
                        if loadZonemap {
                                h.minU64[z] = mnZ
                                h.maxU64[z] = mxZ
                                if h.minU64[z] > h.maxU64[z] {
                                        return h, ErrInvalidLength
                                }
                        }
                }
        default: // zoneKindFloat: 8-byte min + 8-byte max per zone (IEEE-754 bits LE)
                if d.i+16*h.zoneCount > len(d.buf) {
                        return h, ErrShortBuffer
                }
                if loadZonemap {
                        h.minF64 = make([]float64, h.zoneCount)
                        h.maxF64 = make([]float64, h.zoneCount)
                        for z := 0; z < h.zoneCount; z++ {
                                h.minF64[z] = math.Float64frombits(binary.LittleEndian.Uint64(d.buf[d.i:]))
                                h.maxF64[z] = math.Float64frombits(binary.LittleEndian.Uint64(d.buf[d.i+8:]))
                                d.i += 16
                                // No min<=max check: an all-NaN zone stores the empty interval
                                // (+Inf, -Inf) on purpose, and NaN bounds are not ordered.
                        }
                } else {
                        d.i += 16 * h.zoneCount
                }
        }

        h.bodyStart = d.i
        if err := h.validateOffsets(d); err != nil {
                return h, err
        }
        return h, nil
}

// validateOffsets checks the offset table: offsets[0]==0, strictly increasing,
// each in-buffer. Call after h.bodyStart is set.
func (h *zoneChunkHeader) validateOffsets(d *Decoder) error {
        prev := -1
        for z := 0; z < h.zoneCount; z++ {
                off := int(binary.LittleEndian.Uint32(d.buf[h.offBase+4*z:]))
                if z == 0 && off != 0 {
                        return ErrInvalidLength
                }
                if off <= prev {
                        return ErrInvalidLength
                }
                if h.bodyStart+off > len(d.buf) {
                        return ErrShortBuffer
                }
                prev = off
        }
        return nil
}

// zoneOff returns the absolute byte offset of zone z's body.
func (h *zoneChunkHeader) zoneOff(d *Decoder, z int) int {
        return h.bodyStart + int(binary.LittleEndian.Uint32(d.buf[h.offBase+4*z:]))
}

// zoneLen returns the element count of zone z.
func (h *zoneChunkHeader) zoneLen(z int) int {
        if z == h.zoneCount-1 {
                return h.n - z*h.blk
        }
        return h.blk
}

// ---- decode-all (float64) ----

func (d *Decoder) readZoneChunkFloat64() ([]float64, error) {
        var s []float64
        if err := d.readZoneChunkFloat64Into(&s); err != nil {
                return nil, err
        }
        return s, nil
}

func (d *Decoder) readZoneChunkFloat64Into(dst *[]float64) error {
        if err := d.descend(); err != nil {
                return err
        }
        defer d.ascend()
        h, err := d.readZoneChunkHeader(false)
        if err != nil {
                return err
        }
        if h.kind != zoneKindFloat {
                return ErrTypeMismatch
        }
        growF64(dst, h.n)
        out := *dst
        var tmp []float64
        for z := range h.zoneCount {
                d.i = h.zoneOff(d, z)
                if err := decodeSliceFloat64Into(d, &tmp); err != nil {
                        return err
                }
                if len(tmp) != h.zoneLen(z) {
                        return ErrInvalidLength
                }
                copy(out[z*h.blk:], tmp)
        }
        return nil
}

// zoneSkippedZones counts zones the query path proved cannot match and did not
// decode. Test-only instrumentation; atomic so concurrent Decoders zone-skipping
// on different goroutines do not race on it.
var zoneSkippedZones atomic.Int64

// decodeZoneChunkQuery decodes only the zones of a zone-chunked column that
// intersect at least one of the referencing leaves' bounds and evaluates each
// leaf's predicate over those zones into the leaf's precompT mask (zones that
// cannot match are skipped — their rows stay FALSE). It is FILTER-only: it never
// materialises projected values, because a row matched via an OR/NOT sibling can
// live in a zone this column's bounds skipped, so projection must follow the
// FINAL matched set (decodeZoneChunkSelective), not the leaf bounds. The column
// starts at byte offset start (pointing at the tag); the caller repositions d.i
// afterwards via the column-length index.
func (d *Decoder) decodeZoneChunkQuery(start int, leaves []*cnode, n int) error {
        d.i = start + 1 // skip tag
        h, err := d.readZoneChunkHeader(true)
        if err != nil {
                return err
        }
        if h.n != n {
                return ErrTypeMismatch
        }
        for _, lf := range leaves {
                lf.precompT = newBitset(n)
        }
        // For the linear zonemap, each leaf's value range maps once to a contiguous
        // zone range; a zone is then needed iff it falls inside some leaf's range.
        type zr struct {
                zlo, zhi int
                ok       bool
        }
        var lzr []zr
        if h.zmap == zmapLinear {
                lzr = make([]zr, len(leaves))
                for li, lf := range leaves {
                        var lo, hi float64
                        if h.kind == zoneKindInt {
                                lo, hi = float64(lf.term.loI64), float64(lf.term.hiI64)
                        } else {
                                lo, hi = float64(lf.term.loU64), float64(lf.term.hiU64)
                        }
                        a, b, ok := h.lin.zoneRangeFor(lo, hi, h.n, h.blk)
                        lzr[li] = zr{a, b, ok}
                }
        }
        for z := range h.zoneCount {
                needed := false
                for li, lf := range leaves {
                        if h.zmap == zmapLinear {
                                if lzr[li].ok && z >= lzr[li].zlo && z <= lzr[li].zhi {
                                        needed = true
                                }
                                if needed {
                                        break
                                }
                                continue
                        }
                        // Zone z intersects the leaf's bounds iff zoneMax >= lo && zoneMin <= hi.
                        switch h.kind {
                        case zoneKindInt:
                                if h.maxI64[z] >= lf.term.loI64 && h.minI64[z] <= lf.term.hiI64 {
                                        needed = true
                                }
                        case zoneKindUint:
                                if h.maxU64[z] >= lf.term.loU64 && h.minU64[z] <= lf.term.hiU64 {
                                        needed = true
                                }
                        default: // float: empty-interval (all-NaN) zones never intersect → skipped
                                if h.maxF64[z] >= lf.term.loF64 && h.minF64[z] <= lf.term.hiF64 {
                                        needed = true
                                }
                        }
                        if needed {
                                break
                        }
                }
                if !needed {
                        zoneSkippedZones.Add(1)
                        continue
                }
                d.i = h.zoneOff(d, z)
                tg, err := d.peekTag()
                if err != nil {
                        return err
                }
                base := z * h.blk
                want := h.zoneLen(z)
                switch h.kind {
                case zoneKindInt:
                        v, err := d.readQPackInt64(tg)
                        if err != nil {
                                return err
                        }
                        if len(v) != want {
                                return ErrInvalidLength
                        }
                        for _, lf := range leaves {
                                p := lf.term.pI64
                                m := lf.precompT
                                for r, x := range v {
                                        if p(x) {
                                                setBit(m, base+r)
                                        }
                                }
                        }
                case zoneKindUint:
                        v, err := d.readQPackUint64(tg)
                        if err != nil {
                                return err
                        }
                        if len(v) != want {
                                return ErrInvalidLength
                        }
                        for _, lf := range leaves {
                                p := lf.term.pU64
                                m := lf.precompT
                                for r, x := range v {
                                        if p(x) {
                                                setBit(m, base+r)
                                        }
                                }
                        }
                default: // float64
                        var v []float64
                        if err := decodeSliceFloat64Into(d, &v); err != nil {
                                return err
                        }
                        if len(v) != want {
                                return ErrInvalidLength
                        }
                        for _, lf := range leaves {
                                p := lf.term.pF64
                                m := lf.precompT
                                for r, x := range v {
                                        if p(x) {
                                                setBit(m, base+r)
                                        }
                                }
                        }
                }
        }
        return nil
}

// decodeZoneChunkSelective decodes only the zones spanning at least one matched
// row and fills a length-n colVals with their values (rows in undecoded zones
// stay zero — they are not matched, so are never scattered). It is the projection
// counterpart to decodeZoneChunkQuery's filter pass: a row matched via an OR/NOT
// sibling can fall in a zone this column's bounds skipped, so projection must
// follow the FINAL matched set rather than the leaf bounds. start points at the
// tag; matched holds the surviving row indices.
func (d *Decoder) decodeZoneChunkSelective(start, n int, matched []int) (*colVals, error) {
        d.i = start + 1 // skip tag
        h, err := d.readZoneChunkHeader(false)
        if err != nil {
                return nil, err
        }
        if h.n != n {
                return nil, ErrTypeMismatch
        }
        var cv *colVals
        switch h.kind {
        case zoneKindInt:
                cv = &colVals{kind: colKindInt, i64: make([]int64, n)}
        case zoneKindUint:
                cv = &colVals{kind: colKindUint, u64: make([]uint64, n)}
        default:
                cv = &colVals{kind: colKindFloat, f64: make([]float64, n)}
        }
        need := make([]bool, h.zoneCount)
        for _, r := range matched {
                need[r/h.blk] = true
        }
        var ftmp []float64
        for z := range h.zoneCount {
                if !need[z] {
                        continue
                }
                d.i = h.zoneOff(d, z)
                base := z * h.blk
                want := h.zoneLen(z)
                switch h.kind {
                case zoneKindInt:
                        tg, err := d.peekTag()
                        if err != nil {
                                return nil, err
                        }
                        v, err := d.readQPackInt64(tg)
                        if err != nil {
                                return nil, err
                        }
                        if len(v) != want {
                                return nil, ErrInvalidLength
                        }
                        copy(cv.i64[base:], v)
                case zoneKindUint:
                        tg, err := d.peekTag()
                        if err != nil {
                                return nil, err
                        }
                        v, err := d.readQPackUint64(tg)
                        if err != nil {
                                return nil, err
                        }
                        if len(v) != want {
                                return nil, ErrInvalidLength
                        }
                        copy(cv.u64[base:], v)
                default: // float64
                        if err := decodeSliceFloat64Into(d, &ftmp); err != nil {
                                return nil, err
                        }
                        if len(ftmp) != want {
                                return nil, ErrInvalidLength
                        }
                        copy(cv.f64[base:], ftmp)
                }
        }
        return cv, nil
}

// ---- decode-all (int64) ----

func (d *Decoder) readZoneChunkInt64() ([]int64, error) {
        var s []int64
        if err := d.readZoneChunkInt64Into(&s); err != nil {
                return nil, err
        }
        return s, nil
}

func (d *Decoder) readZoneChunkInt64Into(dst *[]int64) error {
        // A zone body could itself carry tagZoneChunk; bound the nesting so a hostile
        // payload of nested zone chunks cannot overflow the goroutine stack.
        if err := d.descend(); err != nil {
                return err
        }
        defer d.ascend()
        h, err := d.readZoneChunkHeader(false)
        if err != nil {
                return err
        }
        if h.kind != zoneKindInt {
                return ErrTypeMismatch
        }
        growI64(dst, h.n)
        out := *dst
        for z := range h.zoneCount {
                d.i = h.zoneOff(d, z)
                t, err := d.peekTag()
                if err != nil {
                        return err
                }
                v, err := d.readQPackInt64(t)
                if err != nil {
                        return err
                }
                if len(v) != h.zoneLen(z) {
                        return ErrInvalidLength
                }
                copy(out[z*h.blk:], v)
        }
        return nil
}

// ---- decode-all (uint64) ----

func (d *Decoder) readZoneChunkUint64() ([]uint64, error) {
        var s []uint64
        if err := d.readZoneChunkUint64Into(&s); err != nil {
                return nil, err
        }
        return s, nil
}

func (d *Decoder) readZoneChunkUint64Into(dst *[]uint64) error {
        if err := d.descend(); err != nil {
                return err
        }
        defer d.ascend()
        h, err := d.readZoneChunkHeader(false)
        if err != nil {
                return err
        }
        if h.kind != zoneKindUint {
                return ErrTypeMismatch
        }
        growU64(dst, h.n)
        out := *dst
        for z := range h.zoneCount {
                d.i = h.zoneOff(d, z)
                t, err := d.peekTag()
                if err != nil {
                        return err
                }
                v, err := d.readQPackUint64(t)
                if err != nil {
                        return err
                }
                if len(v) != h.zoneLen(z) {
                        return ErrInvalidLength
                }
                copy(out[z*h.blk:], v)
        }
        return nil
}

package qdf

import (
        "math/bits"
        "slices"
)

// Queryable is the set of column element types a predicate can match. Exact
// base types only: Where dispatches on the concrete func(T) bool type once at
// construction, which matches base types but not user-defined named types.
type Queryable interface {
        int | int8 | int16 | int32 | int64 |
                uint | uint8 | uint16 | uint32 | uint64 | uintptr |
                float32 | float64 | string | bool
}

// predTerm is one resolved predicate: a column name, the column kind its
// predicate expects, and exactly one native predicate (the field matching
// want is non-nil). Native predicates take the decoder's wide scratch type so
// evaluation is a direct typed call with zero per-value interface boxing.
type predTerm struct {
        pI64  func(int64) bool
        pU64  func(uint64) bool
        pF64  func(float64) bool
        pStr  func(string) bool
        pBool func(bool) bool
        field string
        // Optional comparison bounds, set by the WhereRange/GE/GT/LE/LT/Eq
        // constructors so a zone-chunked column can skip zones whose [min,max] cannot
        // intersect [loI64,hiI64] (int) / [loU64,hiU64] (uint). hasBounds is false for
        // the opaque Where(func) path (no zone-skip) and for float/string in v1.
        loI64, hiI64 int64
        loU64, hiU64 uint64
        loF64, hiF64 float64
        hasBounds    bool
        want         colKind // 1-byte tail: kept last to avoid padding before the pointers
}

// QueryOption configures a filtering/projecting Unmarshal. Construct with Where,
// And, Or, Not (predicate tree) or Select (projection). Exactly one of node /
// selectFields is set.
type QueryOption struct {
        node         *condNode
        arena        *Arena
        selectFields []string
        noCopy       bool
}

// WithArena makes the decode pack copied inline string values into a, instead
// of allocating one string per field — near-zero allocations across an epoch of
// decodes (see Arena). The decoded strings alias a's memory and follow Arena's
// lifetime contract. Ignored together with WithNoCopy (aliasing already skips
// the copy). Composes with predicates / Select / WithNoCopy.
func WithArena(a *Arena) QueryOption { return QueryOption{arena: a} }

// condOp is the kind of a predicate-tree node.
type condOp uint8

const (
        condLeaf condOp = iota // single typed predicate (term set)
        condAnd                // all kids true
        condOr                 // any kid true
        condNot                // single kid negated
)

// condNode is one node of the boolean predicate tree. A leaf carries exactly
// one typed predTerm; And/Or carry n kids; Not carries one. err records a
// construction misuse (e.g. a Select passed into a combinator) surfaced at
// buildQueryPlan time.
type condNode struct {
        term *predTerm
        err  error
        kids []*condNode
        op   condOp // 1-byte tail: kept last to avoid padding before the pointer fields
}

// Where keeps only rows whose field column satisfies pred. T must match the
// column's native kind (any integer width, float32/64, string, or bool); a
// mismatch is reported as ErrTypeMismatch at decode time. If field is absent
// from the wire the call returns ErrFieldNotFound. Multiple Where options are
// AND-ed. The predicate is called once per row with the native value — no
// interface boxing per value.
func Where[T Queryable](field string, pred func(T) bool) QueryOption {
        t := &predTerm{field: field}
        switch p := any(pred).(type) {
        case func(int) bool:
                t.want, t.pI64 = colKindInt, func(v int64) bool { return p(int(v)) }
        case func(int8) bool:
                t.want, t.pI64 = colKindInt, func(v int64) bool { return p(int8(v)) }
        case func(int16) bool:
                t.want, t.pI64 = colKindInt, func(v int64) bool { return p(int16(v)) }
        case func(int32) bool:
                t.want, t.pI64 = colKindInt, func(v int64) bool { return p(int32(v)) }
        case func(int64) bool:
                t.want, t.pI64 = colKindInt, p
        case func(uint) bool:
                t.want, t.pU64 = colKindUint, func(v uint64) bool { return p(uint(v)) }
        case func(uint8) bool:
                t.want, t.pU64 = colKindUint, func(v uint64) bool { return p(uint8(v)) }
        case func(uint16) bool:
                t.want, t.pU64 = colKindUint, func(v uint64) bool { return p(uint16(v)) }
        case func(uint32) bool:
                t.want, t.pU64 = colKindUint, func(v uint64) bool { return p(uint32(v)) }
        case func(uint64) bool:
                t.want, t.pU64 = colKindUint, p
        case func(uintptr) bool:
                t.want, t.pU64 = colKindUint, func(v uint64) bool { return p(uintptr(v)) }
        case func(float32) bool:
                // float32 columns carry colKindFloat32 (raw bits); the eval path widens
                // each f32 back to float64 before this predicate narrows it again.
                t.want, t.pF64 = colKindFloat32, func(v float64) bool { return p(float32(v)) }
        case func(float64) bool:
                t.want, t.pF64 = colKindFloat, p
        case func(string) bool:
                t.want, t.pStr = colKindString, p
        case func(bool) bool:
                t.want, t.pBool = colKindBool, p
        }
        return QueryOption{node: &condNode{op: condLeaf, term: t}}
}

// Select restricts decoding to the named columns. Top-level only; nesting a
// Select inside And/Or/Not is reported as ErrUnsupported.
func Select(fields ...string) QueryOption {
        return QueryOption{selectFields: append([]string(nil), fields...)}
}

// WithNoCopy makes the decode return string and []byte values that ALIAS the
// input buffer instead of copying them. On string-heavy payloads this is ~2x
// faster with near-zero allocations.
//
// DANGER — lifetime contract: the returned values are valid ONLY while the
// input buffer passed to Unmarshal stays alive and is never modified or reused.
// Do NOT use WithNoCopy when the input may be recycled, mutated, or freed before
// you finish reading the decoded values — e.g. a pooled HTTP request body. The
// decoded strings would silently become garbage. This is a manual-memory
// use-after-free, not a data race: the race detector will NOT catch it.
//
// Safe for caller-owned, immutable input such as an mmap or a file read fully
// into memory.
//
// Also safe — and the common zero-copy high-throughput pattern — when the input
// buffer is allocated FRESH per message and never pooled or overwritten: e.g.
// io.ReadAll(body), or a make([]byte, n) you read one message into and do not
// reuse. You do NOT have to track the buffer's lifetime manually: the aliasing
// string/[]byte headers in the decoded value keep the backing buffer alive
// through the garbage collector (Go marks the whole allocation from an interior
// pointer), so the buffer lives exactly as long as the values that point into
// it. The hazard is exclusively buffer REUSE (a sync.Pool buffer returned after
// the handler, a scratch slice you overwrite on the next read) or mutation —
// not lifetime. If every decode reads into its own fresh slice, WithNoCopy is
// safe and gives the full ~2x / near-zero-alloc win for free.
//
// Scope: WithNoCopy affects the reflect decode path. A type that implements
// Unmarshaler (including codegen-generated UnmarshalQDF) decodes through its own
// decoder via the byte-only UnmarshalQDF(data) interface, which cannot inherit
// this flag, so such types still copy. Use SetNoCopy on a Decoder/StreamDecoder
// you drive directly if you need zero-copy there.
func WithNoCopy() QueryOption { return QueryOption{noCopy: true} }

// combine builds an And/Or node from option kids, flagging any non-predicate
// (Select) kid as an ErrUnsupported misuse.
func combine(op condOp, opts []QueryOption) QueryOption {
        n := &condNode{op: op}
        for _, o := range opts {
                if o.node == nil {
                        n.err = &QueryError{Op: "predicate combinator", Err: ErrUnsupported}
                        continue
                }
                n.kids = append(n.kids, o.node)
        }
        return QueryOption{node: n}
}

// And keeps rows where every sub-predicate is true. Multiple top-level Where
// options are an implicit And; And is the explicit, nestable form.
func And(opts ...QueryOption) QueryOption { return combine(condAnd, opts) }

// Or keeps rows where at least one sub-predicate is true.
func Or(opts ...QueryOption) QueryOption { return combine(condOr, opts) }

// Not keeps rows where the sub-predicate is not true. Under three-valued NULL
// semantics a nil (absent) nullable row is UNKNOWN, so Not(pred) excludes nil
// rows just as the bare predicate does.
func Not(opt QueryOption) QueryOption {
        n := &condNode{op: condNot}
        if opt.node == nil {
                n.err = &QueryError{Op: "predicate combinator", Err: ErrUnsupported}
        } else {
                n.kids = []*condNode{opt.node}
        }
        return QueryOption{node: n}
}

// queryPlan is the resolved set of options for one decode: a boolean predicate
// tree (root, nil = no filter) and an optional projection.
type queryPlan struct {
        root         *condNode
        arena        *Arena
        selectFields []string
        noCopy       bool
}

// firstCondErr returns the first construction error in the tree, or nil.
func firstCondErr(n *condNode) error {
        if n == nil {
                return nil
        }
        if n.err != nil {
                return n.err
        }
        for _, k := range n.kids {
                if e := firstCondErr(k); e != nil {
                        return e
                }
        }
        return nil
}

func buildQueryPlan(opts []QueryOption) (*queryPlan, error) {
        qp := &queryPlan{}
        var nodes []*condNode
        for _, o := range opts {
                if o.noCopy {
                        qp.noCopy = true
                }
                if o.arena != nil {
                        qp.arena = o.arena
                }
                switch {
                case o.node != nil:
                        nodes = append(nodes, o.node)
                case o.selectFields != nil:
                        qp.selectFields = append(qp.selectFields, o.selectFields...)
                }
        }
        switch len(nodes) {
        case 0:
                // no filter
        case 1:
                qp.root = nodes[0]
        default:
                qp.root = &condNode{op: condAnd, kids: nodes}
        }
        qp.root = simplifyCond(qp.root)
        if err := firstCondErr(qp.root); err != nil {
                return nil, err
        }
        return qp, nil
}

// simplifyCond rewrites the tree into an equivalent smaller form: it folds
// double negation, flattens nested same-op And/Or, unwraps single-child
// And/Or, and dedups identical sibling leaves (same field + same predicate
// closure, keyed by predTerm pointer). Purely an optimisation — semantics are
// unchanged.
func simplifyCond(n *condNode) *condNode {
        if n == nil || n.op == condLeaf {
                return n
        }
        if n.op == condNot {
                // An err-bearing Not built from a non-predicate option (e.g. Not(Select))
                // carries no kid; keep it verbatim so firstCondErr surfaces its error.
                // Indexing kids[0] here would panic.
                if len(n.kids) == 0 {
                        return n
                }
                c := simplifyCond(n.kids[0])
                if c.op == condNot && len(c.kids) > 0 { // Not(Not(x)) -> x
                        return c.kids[0]
                }
                return &condNode{op: condNot, kids: []*condNode{c}, err: n.err}
        }
        // And / Or: simplify kids, flatten same-op, dedup sibling leaves.
        var kids []*condNode
        err := n.err
        for _, k := range n.kids {
                k = simplifyCond(k)
                if k.op == n.op { // flatten associative same-op
                        if k.err != nil && err == nil {
                                err = k.err // flattening drops k itself, so carry its error up
                        }
                        kids = append(kids, k.kids...)
                        continue
                }
                kids = append(kids, k)
        }
        seen := make(map[*predTerm]bool)
        out := kids[:0]
        for _, k := range kids {
                if k.op == condLeaf {
                        if seen[k.term] {
                                continue
                        }
                        seen[k.term] = true
                }
                out = append(out, k)
        }
        if len(out) == 1 && err == nil { // single-child And/Or -> the child (only when no error on this node)
                return out[0]
        }
        return &condNode{op: n.op, kids: out, err: err}
}

// --- bitset: row-match masks. LSB-first within each uint64 word. ---

func newBitset(n int) []uint64 { return make([]uint64, (n+63)>>6) }

// fullBitset returns an n-row bitset with every valid bit set (the "match all"
// / AND identity), bits >= n cleared. Word-fill, not a per-row setBit loop.
func fullBitset(n int) []uint64 {
        b := newBitset(n)
        for i := range b {
                b[i] = ^uint64(0)
        }
        if r := n & 63; r != 0 && len(b) > 0 {
                b[len(b)-1] = (uint64(1) << uint(r)) - 1
        }
        return b
}

func setBit(b []uint64, i int) { b[i>>6] |= 1 << (uint(i) & 63) }

func getBit(b []uint64, i int) bool { return b[i>>6]&(1<<(uint(i)&63)) != 0 }

// bitsetAnd sets a &= b (a and b same length).
func bitsetAnd(a, b []uint64) {
        for i := range a {
                a[i] &= b[i]
        }
}

// bitsetOr sets a |= b (a and b same length).
func bitsetOr(a, b []uint64) {
        for i := range a {
                a[i] |= b[i]
        }
}

// bitsetAndNot sets a &^= b (a and b same length).
func bitsetAndNot(a, b []uint64) {
        for i := range a {
                a[i] &^= b[i]
        }
}

// notMask returns a fresh complement of m over n rows, with bits >= n cleared
// so popcount and whole-word ops stay meaningful.
func notMask(m []uint64, n int) []uint64 {
        out := make([]uint64, len(m))
        for i := range m {
                out[i] = ^m[i]
        }
        if r := n & 63; r != 0 && len(out) > 0 {
                out[len(out)-1] &= (uint64(1) << uint(r)) - 1
        }
        return out
}

func popcount(b []uint64) int {
        n := 0
        for _, w := range b {
                n += bits.OnesCount64(w)
        }
        return n
}

// matchedIndices appends the set-bit indices (< n) of b to dst in order. It
// enumerates per word via bits.TrailingZeros64, skipping whole zero words, so a
// sparse (selective) result mask costs ~one append per match instead of n
// per-bit tests — the common case for a selective columnar-query predicate.
// Bit layout is LSB-first (see getBit), so the lowest set bit is the lowest
// index and the emitted order is ascending, identical to the per-bit scan.
func matchedIndices(b []uint64, n int, dst []int) []int {
        full := n >> 6 // count of words wholly in range [0, n)
        for w := range full {
                x := b[w]
                base := w << 6
                for x != 0 {
                        dst = append(dst, base+bits.TrailingZeros64(x))
                        x &= x - 1 // clear lowest set bit
                }
        }
        // Tail word: keep only the bits below n.
        if r := n & 63; r != 0 && full < len(b) {
                x := b[full] & ((uint64(1) << uint(r)) - 1)
                base := full << 6
                for x != 0 {
                        dst = append(dst, base+bits.TrailingZeros64(x))
                        x &= x - 1
                }
        }
        return dst
}

// cnode is a flattened predicate-tree node with a dense id. The flat form is
// built per decode (local — it never mutates the shared QueryOption tree, so
// the same tree stays safe to reuse across concurrent Unmarshal calls) and is
// indexed by int, avoiding the per-node allocations and pointer hashing a
// map[*condNode]… would cost on a small tree.
type cnode struct {
        term     *predTerm // leaf only
        cv       *colVals  // leaf only: decoded column values (nil until bound)
        precompT []uint64  // leaf only: precomputed T mask from zone-skip (nil = eval normally)
        kids     []int     // child ids (And/Or: n; Not: 1)
        col      int       // leaf only: resolved wire column index (-1 until resolved)
        op       condOp    // 1-byte tails kept last to avoid padding before the pointers above
        unk      bool      // subtree can produce SQL UNKNOWN (contains a nullable leaf)
}

// flattenCond linearises the tree in pre-order (a parent always precedes its
// descendants), returning the flat slice; the root is index 0 when non-empty.
func flattenCond(root *condNode) []cnode {
        if root == nil {
                return nil
        }
        var flat []cnode
        var walk func(*condNode) int
        walk = func(n *condNode) int {
                id := len(flat)
                flat = append(flat, cnode{op: n.op, term: n.term, col: -1})
                if len(n.kids) > 0 {
                        kids := make([]int, len(n.kids))
                        for i, k := range n.kids {
                                kids[i] = walk(k)
                        }
                        flat[id].kids = kids
                }
                return id
        }
        walk(root)
        return flat
}

// singleBoundedLeafForCol returns the lone leaf resolved to wire column c when
// EXACTLY ONE leaf references it and that leaf carries comparison bounds — the
// case zone-skip handles. Multiple leaves on one column are declined (nil): each
// leaf's bounds are decoded independently (union), so two complementary
// half-ranges — e.g. WhereGE+WhereLE — would union to the whole domain and skip
// nothing; expressing a range as a single WhereRange keeps the skip. Returns nil
// for zero / multiple / unbounded leaves.
func singleBoundedLeafForCol(flat []cnode, c int) *cnode {
        var found *cnode
        for i := range flat {
                if flat[i].op == condLeaf && flat[i].col == c {
                        if found != nil || !flat[i].term.hasBounds {
                                return nil
                        }
                        found = &flat[i]
                }
        }
        return found
}

// markUnknown sets unk on every node whose subtree contains a nullable leaf (a
// source of SQL UNKNOWN); only such subtrees carry an explicit F mask. The
// pre-order ids let a single reverse pass finish every child before its parent.
func markUnknown(flat []cnode) {
        // NOTE: do not "modernize" this to slices.Backward — the body mutates
        // flat[i].unk through the index, and `for _, f := range slices.Backward`
        // binds f to a COPY of the value element ([]cnode), so the writes would be
        // lost (every node's unk stays false, silently breaking 3VL NULL handling).
        // Covered by TestMarkUnknownNullableProp.
        for i := len(flat) - 1; i >= 0; i-- {
                if flat[i].op == condLeaf {
                        flat[i].unk = flat[i].cv != nil && flat[i].cv.present != nil
                        continue
                }
                for _, k := range flat[i].kids {
                        if flat[k].unk {
                                flat[i].unk = true
                                break
                        }
                }
        }
}

// evalCond returns the T mask (rows where node id is TRUE) and, when the subtree
// can produce UNKNOWN (unk), the F mask (rows where it is FALSE); a nil F means
// "no unknowns, F == complement of T".
func evalCond(flat []cnode, id, n int) (t, f []uint64) {
        nd := &flat[id]
        switch nd.op {
        case condLeaf:
                if nd.precompT != nil {
                        // Zone-skip produced this leaf's T mask directly (skipped zones are
                        // FALSE). The column is non-nullable (zone-chunk), so F is implicit.
                        return nd.precompT, nil
                }
                return nd.cv.evalMasks(nd.term, n)
        case condAnd:
                return evalAnd(flat, id, n)
        case condOr:
                return evalOr(flat, id, n)
        case condNot:
                ct, cf := evalCond(flat, nd.kids[0], n)
                if cf == nil {
                        cf = notMask(ct, n) // child had no unknowns: its F is the exact complement
                }
                if !nd.unk {
                        return cf, nil // no unknowns: keep F implicit (nil) per the protocol
                }
                // NOT swaps TRUE/FALSE; UNKNOWN rows stay in neither mask.
                return cf, ct
        default:
                panic("qdf: evalCond: unknown op")
        }
}

// evalOr combines kids: T = OR of kid T; F = AND of kid F (or ~kidT when a kid
// has no unknowns). F stays nil when the whole subtree has no unknowns. With no
// kids the OR identity (no rows) is returned.
func evalOr(flat []cnode, id, n int) (t, f []uint64) {
        nd := &flat[id]
        t = newBitset(n)
        first := true
        for _, k := range nd.kids {
                kt, kf := evalCond(flat, k, n)
                bitsetOr(t, kt)
                if nd.unk {
                        if kf == nil {
                                kf = notMask(kt, n)
                        }
                        if first {
                                f = slices.Clone(kf)
                        } else {
                                bitsetAnd(f, kf)
                        }
                }
                first = false
        }
        return t, f
}

// evalAnd combines kids: T = AND of kid T; F = OR of kid F (or ~kidT when a kid
// has no unknowns). F stays nil when the whole subtree has no unknowns. With no
// kids the AND identity (all rows true) is returned.
func evalAnd(flat []cnode, id, n int) (t, f []uint64) {
        nd := &flat[id]
        t = fullBitset(n)
        for _, k := range nd.kids {
                kt, kf := evalCond(flat, k, n)
                bitsetAnd(t, kt)
                if nd.unk {
                        if kf == nil {
                                kf = notMask(kt, n)
                        }
                        if f == nil {
                                f = slices.Clone(kf)
                        } else {
                                bitsetOr(f, kf)
                        }
                }
        }
        return t, f
}

package qdf

import (
        "math"
        "reflect"

        "github.com/alex60217101990/qdf/internal/bitflag"
)

// Bound-carrying predicates. Unlike Where(func) — whose closure is opaque and so
// cannot drive zone-skip — WhereCmp and WhereRange carry the predicate's
// comparison bounds, so a zone-chunked column (OptZoneMap) can skip zones whose
// [min,max] cannot intersect the bounds without decoding them. They evaluate
// identically to the equivalent Where(func) per row; the bounds are an extra used
// only by the zone-skip fast path. v1 carries bounds for int/uint columns;
// float/string get the per-row predicate only (no skip yet) — still correct.
//
// Strict comparators (GT/LT) record a CONSERVATIVE inclusive bound (e.g. GT(b)
// uses [b, +inf]) so a zone is skipped only when provably empty; the exact strict
// test still runs per row, so the result is exact, just occasionally decoding a
// zone that contributes no rows.

// CmpOp is a comparison operator for WhereCmp, built from three primitive bits —
// "accept less-than", "accept equal", "accept greater-than" — so the named
// operators compose and the per-row test and zone bounds derive directly from the
// bits (no per-operator switch). Combine bits for custom relations, e.g. NE.
type CmpOp uint8

const (
        cmpLT CmpOp = 1 << iota // value <  bound accepted
        cmpEQ                   // value == bound accepted
        cmpGT                   // value >  bound accepted
)

const (
        LT CmpOp = cmpLT         // field <  val
        LE CmpOp = cmpLT | cmpEQ // field <= val
        EQ CmpOp = cmpEQ         // field == val
        GE CmpOp = cmpGT | cmpEQ // field >= val
        GT CmpOp = cmpGT         // field >  val
        NE CmpOp = cmpLT | cmpGT // field != val
)

// Ordered is Queryable minus bool — the kinds that support <,<=,>,>=.
type Ordered interface {
        ~int | ~int8 | ~int16 | ~int32 | ~int64 |
                ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr |
                ~float32 | ~float64 | ~string
}

// boundKind classifies a Queryable value into its column kind and the value in
// the widened form the eval path uses. Boxing happens once, at construction. ok
// is false only for a value whose kind is none of int/uint/float/string — which
// the Ordered constraint forbids, so it is a defensive guard, not a normal path.
//
// The concrete type switch is the fast path; a NAMED type (e.g. `type Status
// int32`) does not match it and is resolved by the reflect fallback. Without that
// fallback a named signed-int silently classified as colKindInt(0) with value 0,
// turning WhereCmp/WhereRange into a `field == 0` predicate with no error.
func boundKind(x any) (kind colKind, i int64, u uint64, f float64, s string, ok bool) {
        switch v := x.(type) {
        case int:
                return colKindInt, int64(v), 0, 0, "", true
        case int8:
                return colKindInt, int64(v), 0, 0, "", true
        case int16:
                return colKindInt, int64(v), 0, 0, "", true
        case int32:
                return colKindInt, int64(v), 0, 0, "", true
        case int64:
                return colKindInt, v, 0, 0, "", true
        case uint:
                return colKindUint, 0, uint64(v), 0, "", true
        case uint8:
                return colKindUint, 0, uint64(v), 0, "", true
        case uint16:
                return colKindUint, 0, uint64(v), 0, "", true
        case uint32:
                return colKindUint, 0, uint64(v), 0, "", true
        case uint64:
                return colKindUint, 0, v, 0, "", true
        case uintptr:
                return colKindUint, 0, uint64(v), 0, "", true
        case float32:
                return colKindFloat32, 0, 0, float64(v), "", true
        case float64:
                return colKindFloat, 0, 0, v, "", true
        case string:
                return colKindString, 0, 0, 0, v, true
        }
        // Named type: resolve the underlying kind via reflect.
        switch rv := reflect.ValueOf(x); rv.Kind() {
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                return colKindInt, rv.Int(), 0, 0, "", true
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                return colKindUint, 0, rv.Uint(), 0, "", true
        case reflect.Float32:
                return colKindFloat32, 0, 0, rv.Float(), "", true
        case reflect.Float64:
                return colKindFloat, 0, 0, rv.Float(), "", true
        case reflect.String:
                return colKindString, 0, 0, 0, rv.String(), true
        }
        return 0, 0, 0, 0, "", false
}

func leaf(t *predTerm) QueryOption { return QueryOption{node: &condNode{op: condLeaf, term: t}} }

// WhereCmp keeps rows where (field op val): one of GE / GT / LE / LT / EQ. It is
// the bound-carrying form of a single comparison, enabling zone-skip on
// OptZoneMap int/uint columns. For a two-sided range use WhereRange.
func WhereCmp[T Ordered](field string, op CmpOp, val T) QueryOption {
        k, vI, vU, vF, vS, ok := boundKind(any(val))
        if !ok {
                return QueryOption{node: &condNode{op: condLeaf, err: &QueryError{Op: "WhereCmp", Field: field, Err: ErrTypeMismatch}}}
        }
        t := &predTerm{field: field, want: k}
        switch k {
        case colKindInt:
                t.pI64 = func(v int64) bool {
                        if v < vI {
                                return bitflag.Has(op, cmpLT)
                        }
                        if v > vI {
                                return bitflag.Has(op, cmpGT)
                        }
                        return bitflag.Has(op, cmpEQ)
                }
                // Conservative inclusive bounds: open the low side only if "<" is accepted,
                // the high side only if ">" is accepted; otherwise clamp to val.
                t.loI64, t.hiI64, t.hasBounds = math.MinInt64, math.MaxInt64, true
                if !bitflag.Has(op, cmpLT) {
                        t.loI64 = vI
                }
                if !bitflag.Has(op, cmpGT) {
                        t.hiI64 = vI
                }
        case colKindUint:
                t.pU64 = func(v uint64) bool {
                        if v < vU {
                                return bitflag.Has(op, cmpLT)
                        }
                        if v > vU {
                                return bitflag.Has(op, cmpGT)
                        }
                        return bitflag.Has(op, cmpEQ)
                }
                t.loU64, t.hiU64, t.hasBounds = 0, math.MaxUint64, true
                if !bitflag.Has(op, cmpLT) {
                        t.loU64 = vU
                }
                if !bitflag.Has(op, cmpGT) {
                        t.hiU64 = vU
                }
        case colKindFloat, colKindFloat32:
                t.pF64 = func(v float64) bool {
                        if v != v { // NaN matches nothing
                                return false
                        }
                        if v < vF {
                                return bitflag.Has(op, cmpLT)
                        }
                        if v > vF {
                                return bitflag.Has(op, cmpGT)
                        }
                        return bitflag.Has(op, cmpEQ)
                }
                // Float bounds enable zone-skip on float64 zone-chunked columns. An all-NaN
                // zone stores an empty [+Inf,-Inf] interval, so it is skipped (NaN never
                // matches), and a partly-NaN zone is governed by its finite min/max.
                t.loF64, t.hiF64, t.hasBounds = math.Inf(-1), math.Inf(1), true
                if !bitflag.Has(op, cmpLT) {
                        t.loF64 = vF
                }
                if !bitflag.Has(op, cmpGT) {
                        t.hiF64 = vF
                }
        case colKindString:
                t.pStr = func(v string) bool {
                        if v < vS {
                                return bitflag.Has(op, cmpLT)
                        }
                        if v > vS {
                                return bitflag.Has(op, cmpGT)
                        }
                        return bitflag.Has(op, cmpEQ)
                }
        }
        return leaf(t)
}

// WhereRange keeps rows where lo <= field <= hi (inclusive). Bound-carrying:
// enables zone-skip on OptZoneMap int/uint columns.
func WhereRange[T Ordered](field string, lo, hi T) QueryOption {
        k, loI, loU, loF, loS, ok := boundKind(any(lo))
        _, hiI, hiU, hiF, hiS, _ := boundKind(any(hi))
        if !ok {
                return QueryOption{node: &condNode{op: condLeaf, err: &QueryError{Op: "WhereRange", Field: field, Err: ErrTypeMismatch}}}
        }
        t := &predTerm{field: field, want: k}
        switch k {
        case colKindInt:
                t.pI64 = func(v int64) bool { return v >= loI && v <= hiI }
                t.loI64, t.hiI64, t.hasBounds = loI, hiI, true
        case colKindUint:
                t.pU64 = func(v uint64) bool { return v >= loU && v <= hiU }
                t.loU64, t.hiU64, t.hasBounds = loU, hiU, true
        case colKindFloat, colKindFloat32:
                t.pF64 = func(v float64) bool { return v >= loF && v <= hiF } // NaN → false
                t.loF64, t.hiF64, t.hasBounds = loF, hiF, true
        case colKindString:
                t.pStr = func(v string) bool { return v >= loS && v <= hiS }
        }
        return leaf(t)
}

package qdf

import (
        "reflect"
        "strings"
        "sync"
        "time"
        "unsafe"
)

// typeDesc is the compiled descriptor for a reflect.Type. Looked up via the
// typeCache map keyed by the Type's runtime pointer (cheap and stable).
// Field order keeps the GC pointer-scan range tight (64 bytes vs 96 for the
// source order): the single-word pointer fields lead, the multi-word `fields`
// slice (whose len/cap words are non-pointer) follows, and the non-pointer
// scalars (schemaFP/keyOff) plus the 1-byte tails trail.
type typeDesc struct {
        rType reflect.Type
        elem  *typeDesc // slice/array/map-value/ptr
        // fpElem is the element descriptor for a primitive-slice fast-path type
        // (installSliceFastPath leaves elem nil so hashDesc/schemaFP stay unchanged
        // and delta patches keep their on-wire schema fingerprint). fpHash reads it
        // instead of re-resolving via descOf (a sync.Map load) on every call.
        fpElem  *typeDesc
        colPlan *columnarPlan // non-nil on a []struct whose element is columnar-eligible

        // encode is the specialized encoder for this type. It receives a pointer
        // to a value of the type (NOT an interface) so it can dereference via
        // unsafe.Pointer without paying for reflect.Value.
        encode func(e *Encoder, p unsafe.Pointer) error
        decode func(d *Decoder, p unsafe.Pointer) error

        // keyDesc describes the key field's type for a struct element's identity key
        // (the field tagged `qdf:"...,key"`), used by keyed slice diff. nil for types
        // without a key tag. See keyOff / keyed below. Set once at build.
        keyDesc *typeDesc // descriptor of the key field's type

        fields []fieldDesc // structs only

        // vecFields lists the []float32/[]float64 fields of a struct, in declaration
        // order, with the index back into fields. Non-nil only for struct types that
        // have at least one such field. Used by the batched lossy vector-column codec
        // (tagVecBatchStruct) to gather one column's vectors across all rows into a
        // single count=N block. Pure type info (option-independent), set at build.
        vecFields []vecBatchField

        // schemaFP is the structural fingerprint of this descriptor's full subtree
        // (kind + field names + recursive field/element kinds). Computed once at build
        // time (descBuild) so the delta Diff/Apply path reads it with zero runtime
        // synchronization instead of a sync.Map lookup. Safe to read concurrently:
        // set under the single-threaded build before the descriptor is published to
        // typeCache via LoadOrStore (same happens-before as encode/decode).
        schemaFP uint64

        // keyOff is the byte offset of the ,key-tagged field within the struct (see
        // keyDesc above). keyed reports its presence. Set once at build.
        keyOff uintptr // byte offset of the key field within the struct

        // kind / marshalerKind / pod are 1-byte (or kind-sized) tails kept last so
        // they share one word instead of forcing padding ahead of the 8-byte fields
        // above.
        kind reflect.Kind
        // marshalerKind: 0=none, 1=Marshaler interface, 2=encoding.TextMarshaler-ish (future)
        marshalerKind uint8
        // hasCustomDecode: the type implements Unmarshaler (the decode-half),
        // possibly WITHOUT Marshaler — the documented asymmetric case. Columnar
        // classification must reject such fields (classifyColKind): the columnar
        // scatter writes field memory directly and would silently skip the user's
        // UnmarshalQDF, diverging from the row-major path which calls it.
        hasCustomDecode bool
        // pod reports whether this type's memory is pointer-free (noPointers). Cached
        // at build for the delta diff hot path (equalSliceEV/equalArrayEV/diffElems),
        // which would otherwise call the structural noPointers walk per element.
        pod bool
        // tightPOD reports whether this type is pointer-free AND has no internal or
        // tail padding — i.e. its entire byte span is content. The delta value
        // fingerprint may bulk-hash such a span in ONE maphash.Write (collapsing N
        // per-field/per-element hashes + reflect dispatch). It must NOT bulk-hash a
        // padded type: padding bytes are indeterminate, so two logically-equal values
        // could hash differently → a false ErrPatchBaseMismatch. Computed once at
        // descBuild (single-threaded, published with the descriptor). Shares the
        // 1-byte tail with kind/marshalerKind/pod (no new padding).
        tightPOD bool
        // keyed reports whether a ,key-tagged field is present (keyOff/keyDesc valid).
        keyed bool
}

type fieldDesc struct {
        name string
        desc *typeDesc
        // preFast holds the pre-encoded fixstr/strN header for the field name in
        // Fast mode. Cheaper than emitting per-encode.
        preFast []byte
        // for Dense mode we cannot precompute the state ref bytes because IDs
        // are per-encoder. We can however precompute the bytes that get appended
        // the first time (intern record) and rely on the encoder's state table
        // to switch to refs on subsequent encodes.
        preInternStr []byte
        offset       uintptr
        isKey        bool // this field carried the ,key tag option
}

// vecBatchField identifies a []float32/[]float64 struct field for the batched
// lossy vector-column codec. fieldIdx indexes into the struct's fields slice;
// elemF32 distinguishes the element type so decode reconstructs the right slice.
type vecBatchField struct {
        offset   uintptr
        fieldIdx int
        elemF32  bool
}

var typeCache sync.Map // map[reflect.Type]*typeDesc — only fully-built entries

// buildCtx is a per-build cycle-breaking map. Goroutine A building T sees
// only its own partial entries here; goroutine B doing the same gets a
// separate ctx. Either both complete and one wins via LoadOrStore; the
// "loser" is still valid for its own caller's captured closures.
type buildCtx struct {
        inProgress map[reflect.Type]*typeDesc
}

// descOf is the top-level entry. It returns a *fully-built* descriptor and
// is safe to call from concurrent goroutines. The fast path is a single
// sync.Map.Load; the slow path constructs a builder ctx and recurses
// through descBuild.
func descOf(t reflect.Type) (*typeDesc, error) {
        if v, ok := typeCache.Load(t); ok {
                return v.(*typeDesc), nil
        }
        ctx := &buildCtx{inProgress: make(map[reflect.Type]*typeDesc)}
        if _, err := descBuild(t, ctx); err != nil {
                return nil, err
        }
        // Publish the WHOLE graph now that every descriptor in it is fully built.
        //
        // Publishing each descriptor as its own fillDesc returned (the previous
        // design) was racy for cyclic types: building cycNode requires its *cycNode
        // pointer descriptor, whose fillDesc completes — and would publish — while
        // the enclosing cycNode struct descriptor's .encode/.decode are still nil.
        // A second goroutine that Loaded the prematurely-published *cycNode then
        // read those fields while we were writing them (data race + nil-func
        // dispatch panic). Deferring publication to here guarantees a descriptor is
        // globally visible only when its entire transitive graph is complete.
        //
        // LoadOrStore still lets a racing goroutine's graph win per type; our own
        // captured closures reference our fully-built versions regardless, and both
        // graphs encode identical wire output.
        for rt, td := range ctx.inProgress {
                typeCache.LoadOrStore(rt, td)
        }
        if v, ok := typeCache.Load(t); ok {
                return v.(*typeDesc), nil
        }
        // Unreachable in practice (t was just built into ctx.inProgress), but stay
        // defensive rather than returning a nil descriptor.
        return descBuild(t, ctx)
}

// descBuild is the recursion-safe lookup used from inside fillDesc. It MUST be
// called with a non-nil ctx so type cycles can be broken. It does NOT publish
// to the global typeCache — descOf publishes the whole graph atomically-per-
// type once every descriptor is complete. Entries stay in ctx.inProgress for
// the ctx's lifetime so sibling references reuse them instead of rebuilding.
func descBuild(t reflect.Type, ctx *buildCtx) (*typeDesc, error) {
        if v, ok := typeCache.Load(t); ok {
                return v.(*typeDesc), nil
        }
        if td, ok := ctx.inProgress[t]; ok {
                return td, nil
        }
        td := &typeDesc{rType: t, kind: t.Kind()}
        // pod is a pure structural property of t (pointer-free memory). Compute it
        // once here with the UNCACHED walk so the delta diff hot path can read the
        // field instead of calling noPointers per element. Set before fillDesc so it
        // is in place by the time descOf publishes td to typeCache.
        td.pod = noPointersWalk(t)
        // tightPOD is, like pod, a pure structural property of t (pointer-free AND no
        // internal/tail padding). Compute it here with the uncached walk so the delta
        // value-fingerprint hot path can bulk-hash a tight span in one maphash.Write.
        td.tightPOD = tightPODWalk(t)
        ctx.inProgress[t] = td
        if err := fillDesc(td, t, ctx); err != nil {
                // Leave nothing half-built reachable: the failed type stays only in
                // this ctx (never published) and is discarded with it.
                return nil, err
        }
        // schemaFP is a pure function of the now-complete descriptor subtree (its
        // children were built first, so td.fields/td.elem/td.kind are populated).
        // Compute it once here under the single-threaded build so Diff/Apply read the
        // field with no sync.Map lookup. schemaFingerprintCompute does its own
        // visited-set walk from td, independent of children's cached schemaFP, so a
        // child computed before a cycle closes is still correct.
        td.schemaFP = schemaFingerprintCompute(td)
        return td, nil
}

// fillDesc populates td in place. All recursive descBuild calls happen
// under the same buildCtx so cycles in t are broken via ctx.inProgress.
// Encoders/decoders captured during this pass dereference td.encode /
// td.decode at runtime — by then fillDesc has assigned them.
func fillDesc(td *typeDesc, t reflect.Type, ctx *buildCtx) error {
        // Special-case time.Time → timestamp tag.
        if t == reflect.TypeFor[time.Time]() {
                td.encode = encodeTime
                td.decode = decodeTime
                return nil
        }
        // Marshaler interface check (pointer receiver too).
        marshalerType := reflect.TypeFor[Marshaler]()
        unmarshalerType := reflect.TypeFor[Unmarshaler]()
        if reflect.PointerTo(t).Implements(marshalerType) {
                td.marshalerKind = 1
                td.encode = encodeMarshaler(t)
        }
        if reflect.PointerTo(t).Implements(unmarshalerType) {
                td.decode = decodeUnmarshaler(t)
                td.hasCustomDecode = true
        }
        if td.encode != nil && td.decode != nil {
                return nil
        }
        // A type may implement only ONE of Marshaler/Unmarshaler (the documented
        // asymmetric case: encode structurally, decode via UnmarshalQDF, or vice
        // versa). The structural switch below unconditionally sets BOTH td.encode
        // and td.decode, so snapshot the custom codec and restore it afterward for
        // the direction it implements — otherwise the custom method is clobbered.
        customEnc, customDec := td.encode, td.decode

        switch t.Kind() {
        case reflect.Bool:
                td.encode = encodeBool
                td.decode = decodeBool
        case reflect.Int:
                // Platform-width: 8 bytes on 64-bit, 4 on 32-bit. Hardcoding 8 would read
                // (encode) and write (decode) past a 4-byte int on 32-bit — an OOB access
                // corrupting the adjacent field. Mirrors slices_fast encodeSliceInt and
                // delta_diff's platform-width handling.
                td.encode = encodeIntN(int(t.Size()))
                td.decode = decodeIntN(int(t.Size()))
        case reflect.Int8:
                td.encode = encodeIntN(1)
                td.decode = decodeIntN(1)
        case reflect.Int16:
                td.encode = encodeIntN(2)
                td.decode = decodeIntN(2)
        case reflect.Int32:
                td.encode = encodeIntN(4)
                td.decode = decodeIntN(4)
        case reflect.Int64:
                td.encode = encodeIntN(8)
                td.decode = decodeIntN(8)
        case reflect.Uint:
                td.encode = encodeUintN(int(t.Size())) // platform-width (see reflect.Int)
                td.decode = decodeUintN(int(t.Size()))
        case reflect.Uint8:
                td.encode = encodeUintN(1)
                td.decode = decodeUintN(1)
        case reflect.Uint16:
                td.encode = encodeUintN(2)
                td.decode = decodeUintN(2)
        case reflect.Uint32:
                td.encode = encodeUintN(4)
                td.decode = decodeUintN(4)
        case reflect.Uint64:
                td.encode = encodeUintN(8)
                td.decode = decodeUintN(8)
        case reflect.Uintptr:
                td.encode = encodeUintN(int(t.Size())) // platform-width (see reflect.Int)
                td.decode = decodeUintN(int(t.Size()))
        case reflect.Float32:
                td.encode = encodeF32
                td.decode = decodeF32
        case reflect.Float64:
                td.encode = encodeF64
                td.decode = decodeF64
        case reflect.String:
                td.encode = encodeString
                td.decode = decodeString
        case reflect.Slice:
                if t.Elem().Kind() == reflect.Uint8 {
                        td.encode = encodeBytes
                        td.decode = decodeBytes
                } else if enc, dec, ok := installSliceFastPath(t); ok {
                        td.encode = enc
                        td.decode = dec
                        // Resolve the element descriptor for fpHash's value-fingerprint fast
                        // path WITHOUT touching td.elem (which hashDesc folds into the
                        // on-wire schemaFP). Non-fatal: fpHash falls back to descOf if nil.
                        if el, e := descBuild(t.Elem(), ctx); e == nil {
                                td.fpElem = el
                        }
                } else {
                        elem, err := descBuild(t.Elem(), ctx)
                        if err != nil {
                                return err
                        }
                        td.elem = elem
                        // Columnar transposition replays the element's structural field
                        // layout and bypasses td.elem.encode / td.elem.decode. A struct that
                        // implements a custom codec in EITHER direction (the documented
                        // asymmetric single-direction case) keeps its structural fields
                        // populated — so without this guard a []OnlyMarshaler / []OnlyUnmarshaler
                        // would columnar-encode/decode and silently skip MarshalQDF/UnmarshalQDF.
                        // A type implementing BOTH returns early in fillDesc with empty fields,
                        // so buildColumnarPlan already yields nil there; this guard covers the
                        // single-direction case the early return misses.
                        et := t.Elem()
                        if elem.kind == reflect.Struct &&
                                !reflect.PointerTo(et).Implements(marshalerType) &&
                                !reflect.PointerTo(et).Implements(unmarshalerType) {
                                td.colPlan = buildColumnarPlan(elem)
                        }
                        td.encode = encodeSlice(elem, t.Elem().Size(), td.colPlan)
                        td.decode = decodeSlice(t, elem, t.Elem().Size(), td.colPlan)
                }
                // Nil-vs-empty preservation: the []byte and reflect encoders self-handle
                // it inline (they are field-only); the typed fast paths use their nil-aware
                // *Nil variants (installSliceFastPath) which keep the shared encodeSlice*
                // funcs nil-agnostic for their internal dense-column callers.
        case reflect.Array:
                // [N]byte fast path: encode/decode as one contiguous binary blob (flat,
                // memcpy) instead of N tagged elements — far smaller wire for real byte
                // data and zero-alloc in-place decode. Covers fixed-width IDs (trace/span
                // ids, UUIDs) and any named byte type ([N]MyByte, since Kind stays Uint8).
                if t.Elem().Kind() == reflect.Uint8 {
                        n := t.Len()
                        td.encode = encodeFixedByteArray(n)
                        td.decode = decodeFixedByteArray(n)
                        break
                }
                elem, err := descBuild(t.Elem(), ctx)
                if err != nil {
                        return err
                }
                td.elem = elem
                td.encode = encodeArray(elem, t.Elem().Size(), t.Len())
                td.decode = decodeArray(elem, t.Elem().Size(), t.Len())
        case reflect.Map:
                if enc, dec, ok := installMapFastPath(t); ok {
                        td.encode = enc
                        td.decode = dec
                        break
                }
                k, err := descBuild(t.Key(), ctx)
                if err != nil {
                        return err
                }
                v, err := descBuild(t.Elem(), ctx)
                if err != nil {
                        return err
                }
                td.elem = v
                td.encode = encodeMap(t, k, v)
                td.decode = decodeMap(t, k, v)
        case reflect.Pointer:
                elem, err := descBuild(t.Elem(), ctx)
                if err != nil {
                        return err
                }
                td.elem = elem
                td.encode = encodePtr(elem)
                td.decode = decodePtr(t, elem)
        case reflect.Interface:
                td.encode = encodeIface
                td.decode = decodeIface
        case reflect.Struct:
                fields, err := buildStructFields(t, ctx)
                if err != nil {
                        return err
                }
                td.fields = fields
                // Record []float32/[]float64 fields for the batched lossy vector codec.
                // Detect via the field's reflect type: the typed slice fast paths set the
                // encode/decode closures but leave td.elem nil, so the kind is read from
                // rType, not the descriptor's elem.
                for i := range td.fields {
                        fd := td.fields[i].desc
                        if fd == nil || fd.rType == nil {
                                continue
                        }
                        // Only the exact unnamed []float32 / []float64 types qualify. A named
                        // slice type (type Vec []float32) can carry a custom Marshaler/
                        // Unmarshaler, and batching would bypass it; unnamed slices cannot have
                        // methods, so restricting to the canonical types is both safe and the
                        // shape the typed slice fast paths already special-case.
                        switch fd.rType {
                        case sliceFloat32Type:
                                td.vecFields = append(td.vecFields, vecBatchField{offset: td.fields[i].offset, fieldIdx: i, elemF32: true})
                        case sliceFloat64Type:
                                td.vecFields = append(td.vecFields, vecBatchField{offset: td.fields[i].offset, fieldIdx: i, elemF32: false})
                        }
                }
                // Record the identity key (the ,key-tagged field), if any. Reject more
                // than one ,key field per type.
                for i := range td.fields {
                        if !td.fields[i].isKey {
                                continue
                        }
                        if td.keyed {
                                return ErrUnsupported // at most one ,key field per type
                        }
                        td.keyed = true
                        td.keyOff = td.fields[i].offset
                        td.keyDesc = td.fields[i].desc
                }
                td.encode = encodeStruct(td)
                td.decode = decodeStruct(td)
        default:
                return ErrUnsupported
        }
        // Restore a custom Marshaler/Unmarshaler for the direction it implements;
        // the structural codec above stands in only for the missing direction.
        if customEnc != nil {
                td.encode = customEnc
        }
        if customDec != nil {
                td.decode = customDec
        }
        return nil
}

func buildStructFields(t reflect.Type, ctx *buildCtx) ([]fieldDesc, error) {
        return appendStructFields(nil, t, 0, ctx)
}

// appendStructFields walks t's fields and appends them to out. base
// is the byte offset of t within the enclosing struct (0 for the
// top-level call). Anonymous embedded struct fields are flattened
// recursively into the parent's wire layout — their inner fields
// appear at the parent level just like encoding/json. A field with
// an explicit qdf / json tag of "-" is skipped at any depth.
func appendStructFields(out []fieldDesc, t reflect.Type, base uintptr, ctx *buildCtx) ([]fieldDesc, error) {
        if out == nil {
                out = make([]fieldDesc, 0, t.NumField())
        }
        for sf := range t.Fields() {
                // Anonymous embedded struct → flatten. This covers the
                // common encoding/json idiom where an unexported lower-case
                // type with exported fields is embedded; silently dropping
                // such a field loses data on round-trip. Pointer-typed
                // embedded fields fall through to the regular field path so
                // they encode as a pointer-to-struct value.
                //
                // EXCEPT a type that carries its own value codec — time.Time or
                // any Marshaler/Unmarshaler — must NOT be flattened: its fields
                // are unexported (time.Time) or its wire form is opaque, so
                // flattening yields zero fields and drops the value on round-trip.
                // Fall through to the regular field path so fillDesc applies the
                // type's own codec as one named field (matches encoding/json,
                // which never flattens an embedded time.Time / Marshaler).
                if sf.Anonymous && sf.Type.Kind() == reflect.Struct &&
                        sf.Type != reflect.TypeFor[time.Time]() &&
                        !reflect.PointerTo(sf.Type).Implements(reflect.TypeFor[Marshaler]()) &&
                        !reflect.PointerTo(sf.Type).Implements(reflect.TypeFor[Unmarshaler]()) {
                        // Tag "-" on the embedded field itself opts the whole
                        // nested layout out — matches encoding/json.
                        if tag, ok := sf.Tag.Lookup("qdf"); ok && tag == "-" {
                                continue
                        }
                        if tag, ok := sf.Tag.Lookup("json"); ok &&
                                strings.Split(tag, ",")[0] == "-" {
                                continue
                        }
                        nested, err := appendStructFields(out, sf.Type, base+sf.Offset, ctx)
                        if err != nil {
                                return nil, err
                        }
                        out = nested
                        continue
                }
                if !sf.IsExported() {
                        continue
                }
                name := sf.Name
                isKey := false
                if tag, ok := sf.Tag.Lookup("qdf"); ok {
                        if tag == "-" {
                                continue
                        }
                        parts := strings.Split(tag, ",")
                        if parts[0] != "" {
                                name = parts[0]
                        }
                        for _, opt := range parts[1:] {
                                if opt == "key" {
                                        if !sf.Type.Comparable() {
                                                return nil, ErrUnsupported // a key field must be comparable
                                        }
                                        isKey = true
                                }
                        }
                } else if tag, ok := sf.Tag.Lookup("json"); ok {
                        parts := strings.Split(tag, ",")
                        if parts[0] == "-" {
                                continue
                        }
                        if parts[0] != "" {
                                name = parts[0]
                        }
                }
                fd, err := descBuild(sf.Type, ctx)
                if err != nil {
                        return nil, err
                }
                out = append(out, fieldDesc{
                        name:         name,
                        offset:       base + sf.Offset,
                        desc:         fd,
                        preFast:      precomputeFixstrHeader(name),
                        preInternStr: precomputeInternStrHeader(name),
                        isKey:        isKey,
                })
        }
        // stable order = source order (no sort) — matches encoding/json behavior
        // for fixed-layout decode.
        return out, nil
}

func precomputeFixstrHeader(name string) []byte {
        n := len(name)
        switch {
        case n <= int(tagFixstrMask):
                out := make([]byte, 1+n)
                out[0] = tagFixstr | byte(n)
                copy(out[1:], name)
                return out
        case n <= 0xFF:
                out := make([]byte, 2+n)
                out[0] = tagStr8
                out[1] = byte(n)
                copy(out[2:], name)
                return out
        case n <= 0xFFFF:
                out := make([]byte, 3+n)
                out[0] = tagStr16
                out[1] = byte(n)
                out[2] = byte(n >> 8)
                copy(out[3:], name)
                return out
        default:
                out := make([]byte, 5+n)
                out[0] = tagStr32
                out[1] = byte(n)
                out[2] = byte(n >> 8)
                out[3] = byte(n >> 16)
                out[4] = byte(n >> 24)
                copy(out[5:], name)
                return out
        }
}

func precomputeInternStrHeader(name string) []byte {
        out := []byte{tagInternStr}
        out = appendUvarint(out, uint64(len(name)))
        out = append(out, name...)
        return out
}

package qdf

import (
        "cmp"
        "fmt"
        "math"
        "reflect"
        "slices"
        "time"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/reflectutil"
        "github.com/alex60217101990/qdf/internal/unsafestr"
)

// ----- per-kind encoders -----

func encodeBool(e *Encoder, p unsafe.Pointer) error {
        e.WriteBool(*(*bool)(p))
        return nil
}
func decodeBool(d *Decoder, p unsafe.Pointer) error {
        v, err := d.ReadBool()
        if err != nil {
                return err
        }
        *(*bool)(p) = v
        return nil
}

// encodeIntN/decodeIntN handle all Go signed int widths. The size parameter
// is the Go-level type size in bytes (1,2,4,8) so we can read through
// unsafe.Pointer without reinterpreting.
func encodeIntN(sz int) func(*Encoder, unsafe.Pointer) error {
        switch sz {
        case 1:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteInt(int64(*(*int8)(p))); return nil }
        case 2:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteInt(int64(*(*int16)(p))); return nil }
        case 4:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteInt(int64(*(*int32)(p))); return nil }
        default:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteInt(*(*int64)(p)); return nil }
        }
}
func decodeIntN(sz int) func(*Decoder, unsafe.Pointer) error {
        switch sz {
        case 1:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        *(*int8)(p) = int8(v)
                        return nil
                }
        case 2:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        *(*int16)(p) = int16(v)
                        return nil
                }
        case 4:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        *(*int32)(p) = int32(v)
                        return nil
                }
        default:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadInt()
                        if err != nil {
                                return err
                        }
                        *(*int64)(p) = v
                        return nil
                }
        }
}

func encodeUintN(sz int) func(*Encoder, unsafe.Pointer) error {
        switch sz {
        case 1:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteUint(uint64(*(*uint8)(p))); return nil }
        case 2:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteUint(uint64(*(*uint16)(p))); return nil }
        case 4:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteUint(uint64(*(*uint32)(p))); return nil }
        default:
                return func(e *Encoder, p unsafe.Pointer) error { e.WriteUint(*(*uint64)(p)); return nil }
        }
}
func decodeUintN(sz int) func(*Decoder, unsafe.Pointer) error {
        switch sz {
        case 1:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        *(*uint8)(p) = uint8(v)
                        return nil
                }
        case 2:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        *(*uint16)(p) = uint16(v)
                        return nil
                }
        case 4:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        *(*uint32)(p) = uint32(v)
                        return nil
                }
        default:
                return func(d *Decoder, p unsafe.Pointer) error {
                        v, err := d.ReadUint()
                        if err != nil {
                                return err
                        }
                        *(*uint64)(p) = v
                        return nil
                }
        }
}

func encodeF32(e *Encoder, p unsafe.Pointer) error { e.WriteFloat32(*(*float32)(p)); return nil }
func decodeF32(d *Decoder, p unsafe.Pointer) error {
        v, err := d.ReadFloat32()
        if err != nil {
                return err
        }
        *(*float32)(p) = v
        return nil
}
func encodeF64(e *Encoder, p unsafe.Pointer) error { e.WriteFloat64(*(*float64)(p)); return nil }
func decodeF64(d *Decoder, p unsafe.Pointer) error {
        v, err := d.ReadFloat64()
        if err != nil {
                return err
        }
        *(*float64)(p) = v
        return nil
}

func encodeString(e *Encoder, p unsafe.Pointer) error { e.WriteString(*(*string)(p)); return nil }
func decodeString(d *Decoder, p unsafe.Pointer) error {
        s, err := d.ReadString()
        if err != nil {
                return err
        }
        *(*string)(p) = s
        return nil
}

func encodeBytes(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) { // nil []byte → tagNil (distinct from empty)
                return nil
        }
        e.WriteBytes(*(*[]byte)(p))
        return nil
}
func decodeBytes(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        b, err := d.ReadBytes()
        if err != nil {
                return err
        }
        *(*[]byte)(p) = b
        return nil
}

func encodeSlice(elem *typeDesc, stride uintptr, colPlan *columnarPlan) func(*Encoder, unsafe.Pointer) error {
        return func(e *Encoder, p unsafe.Pointer) error {
                if e.encodeNilSlice(p) { // nil slice → tagNil (distinct from empty)
                        return nil
                }
                // Depth guard mirroring Decoder.descend in decodeSlice: count this
                // container level so the encoder refuses a payload nested deeper than the
                // decoder will accept, instead of emitting bytes that then fail to decode
                // (slice/map/array-carried recursion was previously unbounded on encode
                // while the decoder caps it at maxDepth).
                if e.maxDepth != 0 {
                        e.depth++
                        if e.depth > e.maxDepth {
                                e.depth--
                                return ErrCycleDetected
                        }
                        defer func() { e.depth-- }()
                }
                hdr := (*sliceHeader)(p)
                n := hdr.Len
                // Batched lossy vector-column path: under OptLossyVec, a []struct with an
                // equal-length []float32/[]float64 field is encoded as one count=N block
                // per vector field (tagVecBatchStruct) instead of N count=1 blocks. Gated
                // to the same contexts as columnar; falls through when nothing is batchable
                // (so non-lossy and unbatchable shapes stay byte-identical).
                if e.opts.Has(OptLossyVec) && elem != nil && len(elem.vecFields) > 0 &&
                        n >= columnarMinElems && e.state != nil && !e.stateSuspended &&
                        e.ifaceDepth == 0 &&
                        e.opts.Has(OptDense) && e.opts.Has(OptShapeIntern) {
                        if done, err := e.encodeVectorBatchStruct(elem, hdr.Data, n, stride); done {
                                return err
                        }
                }
                // Columnar / hybrid-columnar path.
                //
                // Pure plan (every field eligible): transpose under the columnarProbe
                // gate, as before — tagColStruct.
                //
                // Hybrid plan (some residual fields): auto-fire when FSST is enabled
                // (OptFSST / OptCompression), OR — under plain Balanced — when the plan has
                // a string column and the INTERN-AWARE probe predicts a win. The plain
                // per-column probe cannot see the cross-row string deduplication that
                // row-major Dense gets from the global intern table, so a low-cardinality
                // string column looks expensive row-major and the probe mispredicts a
                // columnar win that does not exist. The intern-aware probe credits that
                // dedup (a repeated value costs a 1-byte state-ref), so a low-card string
                // column is correctly cheap row-major and only a genuine columnar win
                // (a Delta/FOR numeric column, or a restricted-alphabet ID column that
                // alpha-packs) pulls the struct in. The emit picker is never-larger per
                // column, so the worst case is a small shape-header overhead — bounded by
                // the gain threshold the probe already requires. With FSST the eligible
                // string columns are compressed by the symbol table (AD/log/RTB win at
                // OptCompression). A hybrid plan with no string column and no FSST still
                // falls through to row-major, byte-identical to today.
                if colPlan != nil && n >= columnarMinElems && e.state != nil && !e.stateSuspended &&
                        e.opts.Has(OptDense) && e.opts.Has(OptShapeIntern) {
                        pure := colPlan.residual == nil
                        internAware := !pure && !e.fsst && colPlan.hasStringCol
                        if (pure || e.fsst || internAware) && columnarProbe(colPlan, hdr.Data, n, e.fsst, e.fsstDict, internAware) {
                                e.writeHeader()
                                if pure {
                                        return e.encodeColumnar(colPlan, hdr.Data, n)
                                }
                                return e.encodeHybridColumnar(colPlan, hdr.Data, n)
                        }
                }
                e.WriteArrayHeader(n)
                base := hdr.Data
                // Probe-and-grow for large slices: encode the first
                // sliceProbeSize records, measure the per-record buffer
                // growth, then pre-grow the output buffer for the rest in
                // one shot. Eliminates the log(n) doubling chain
                // (runtime.memmove + madvise) that dominated 50 k-record
                // encodes — at scale the buffer can balloon from 4 KiB to
                // 60 MiB through ~14 doublings, copying ~4× the final size.
                // The probe cost is negligible because the same elements
                // are emitted exactly once.
                const sliceProbeSize = 32
                if n <= sliceProbeSize {
                        for i := range n {
                                if err := elem.encode(e, unsafe.Add(base, uintptr(i)*stride)); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                probeStart := len(e.buf)
                for i := range sliceProbeSize {
                        if err := elem.encode(e, unsafe.Add(base, uintptr(i)*stride)); err != nil {
                                return err
                        }
                }
                probeBytes := len(e.buf) - probeStart
                if probeBytes > 0 {
                        // Project total size from probe + 25 % slack so the
                        // growslice short-circuit fires inside slices.Grow
                        // without forcing another doubling on a slight
                        // underestimate.
                        remaining := n - sliceProbeSize
                        // 64-bit intermediate: probeBytes*remaining can exceed int32 on a
                        // 32-bit build for a large slice, wrapping projected negative and
                        // panicking slices.Grow.
                        projected := int(int64(probeBytes) * int64(remaining) / int64(sliceProbeSize))
                        projected += projected >> 2
                        e.buf = slices.Grow(e.buf, projected)
                }
                for i := sliceProbeSize; i < n; i++ {
                        if err := elem.encode(e, unsafe.Add(base, uintptr(i)*stride)); err != nil {
                                return err
                        }
                }
                return nil
        }
}

func decodeSlice(t reflect.Type, elem *typeDesc, stride uintptr, colPlan *columnarPlan) func(*Decoder, unsafe.Pointer) error {
        // elemDynamic is true when the slice element is map[string]any or any, so a
        // columnar (tagColStruct) payload can be decoded dynamically into row maps
        // via decodeColumnarAny even though the static element type carries no
        // columnarPlan. This is the path UnmarshalColumns into *[]map[string]any
        // (or *[]any) takes.
        elemType := t.Elem()
        elemDynamic := elemType == reflect.TypeFor[map[string]any]() || elemType.Kind() == reflect.Interface
        elemPF := noPointers(elemType)     // gate for backing reuse (computed once per type)
        elemHasMap := typeDescHasMap(elem) // gate the map-recycle harvest (once per type)
        return func(d *Decoder, p unsafe.Pointer) error {
                if d.decodeNilSlice(p) { // tagNil → nil slice (distinct from empty)
                        return nil
                }
                if err := d.descend(); err != nil {
                        return err
                }
                defer d.ascend()
                // Batched lossy vector-column dispatch (tagVecBatchStruct). Works even when
                // colPlan is nil (a vector-only struct is not columnar-eligible).
                if elem != nil && len(elem.vecFields) > 0 {
                        if tag, err := d.peekTag(); err == nil && tag == tagVecBatchStruct {
                                if elemDynamic || d.query != nil {
                                        return ErrUnsupported // v1: no schemaless / query decode of a batched block
                                }
                                return d.decodeVectorBatchStruct(t, elem, p)
                        }
                }
                // Columnar / hybrid-columnar decode dispatch.
                if colPlan != nil {
                        if tag, err := d.peekTag(); err == nil {
                                switch tag {
                                case tagColStruct:
                                        if d.query != nil {
                                                return decodeColumnarQuery(d, t, colPlan, p)
                                        }
                                        return decodeColumnar(d, t, colPlan, p)
                                case tagHybridColStruct:
                                        if d.query != nil {
                                                return ErrUnsupported // v1: query/Select over a hybrid payload
                                        }
                                        return decodeHybridColumnar(d, t, colPlan, p)
                                }
                        }
                }
                if elemDynamic {
                        if tag, err := d.peekTag(); err == nil && (tag == tagColStruct || tag == tagHybridColStruct) {
                                var rows any
                                var err error
                                switch {
                                case tag == tagHybridColStruct:
                                        if d.query != nil {
                                                return ErrUnsupported // v1: query over a hybrid payload
                                        }
                                        rows, err = decodeHybridColumnarAny(d)
                                case d.query != nil:
                                        rows, err = decodeColumnarQueryAny(d)
                                default:
                                        rows, err = decodeColumnarAny(d)
                                }
                                if err != nil {
                                        return err
                                }
                                src := rows.([]any)
                                reflectutil.MakeSlice(t, len(src), p)
                                base := reflectutil.SliceData(t, p)
                                for i, row := range src {
                                        reflect.NewAt(elemType, unsafe.Add(base, uintptr(i)*stride)).Elem().Set(reflect.ValueOf(row))
                                }
                                return nil
                        }
                }
                // A query requires a columnar payload (tagColStruct, handled above).
                // Any non-columnar slice shape that reaches here cannot be filtered.
                if d.query != nil {
                        return &QueryError{Op: "predicate pushdown", Err: ErrUnsupported}
                }
                n, err := d.ReadArrayHeader()
                if err != nil {
                        return err
                }
                if err := d.CheckLength(n, 1); err != nil {
                        return err
                }
                // Harvest reusable maps from the elements decode-slice-reuse is about to
                // zero, so the map decoders can recycle them instead of re-allocating.
                // Gated on elemHasMap so a map-free element type pays nothing.
                if elemHasMap {
                        if old := (*sliceHeader)(p); old.Data != nil && old.Len > 0 {
                                harvestMaps(d, elem, old.Data, stride, old.Len)
                        }
                }
                // Reuse the caller's backing when pointer-free + cap suffices (decode
                // into a pre-sized/pooled slice), else fresh MakeSlice.
                base := reuseOrMakeSlice(t, n, p, stride, elemPF)
                for i := range n {
                        if err := elem.decode(d, unsafe.Add(base, uintptr(i)*stride)); err != nil {
                                return err
                        }
                }
                return nil
        }
}

// encodeFixedByteArray encodes a fixed [N]byte array as ONE contiguous binary
// blob (identical wire to a []byte of length N), not N tagged elements. Real ID
// bytes are uniformly 0..255; the generic per-element path costs ~2 bytes for
// every byte >= 128, so a [16]byte trace id bloats to ~32 wire bytes — this
// writes a flat 16 plus a tiny bin header. One memcpy, no per-element calls.
//
//go:nosplit
func encodeFixedByteArray(n int) func(*Encoder, unsafe.Pointer) error {
        return func(e *Encoder, p unsafe.Pointer) error {
                e.WriteBytes(unsafe.Slice((*byte)(p), n))
                return nil
        }
}

// decodeFixedByteArray reads the blob written by encodeFixedByteArray straight
// into the struct's inline [N]byte storage — one length-checked memcpy, zero
// allocation (the array lives in the caller's struct, never on the heap).
func decodeFixedByteArray(n int) func(*Decoder, unsafe.Pointer) error {
        return func(d *Decoder, p unsafe.Pointer) error {
                b, err := d.readStringBytes()
                if err != nil {
                        return err
                }
                if len(b) != n {
                        return ErrTypeMismatch
                }
                copy(unsafe.Slice((*byte)(p), n), b)
                return nil
        }
}

func encodeArray(elem *typeDesc, stride uintptr, n int) func(*Encoder, unsafe.Pointer) error {
        return func(e *Encoder, p unsafe.Pointer) error {
                // Depth guard mirroring Decoder.descend in decodeArray (see encodeSlice).
                if e.maxDepth != 0 {
                        e.depth++
                        if e.depth > e.maxDepth {
                                e.depth--
                                return ErrCycleDetected
                        }
                        defer func() { e.depth-- }()
                }
                e.WriteArrayHeader(n)
                for i := range n {
                        if err := elem.encode(e, unsafe.Add(p, uintptr(i)*stride)); err != nil {
                                return err
                        }
                }
                return nil
        }
}
func decodeArray(elem *typeDesc, stride uintptr, n int) func(*Decoder, unsafe.Pointer) error {
        return func(d *Decoder, p unsafe.Pointer) error {
                if err := d.descend(); err != nil {
                        return err
                }
                defer d.ascend()
                m, err := d.ReadArrayHeader()
                if err != nil {
                        return err
                }
                if m != n {
                        return ErrTypeMismatch
                }
                for i := range n {
                        if err := elem.decode(d, unsafe.Add(p, uintptr(i)*stride)); err != nil {
                                return err
                        }
                }
                return nil
        }
}

func encodeMap(t reflect.Type, k, v *typeDesc) func(*Encoder, unsafe.Pointer) error {
        keyType := t.Key()
        valType := t.Elem()
        stringKey := keyType.Kind() == reflect.String
        return func(e *Encoder, p unsafe.Pointer) error {
                // Depth guard mirroring Decoder.descend in decodeMap (see encodeSlice).
                if e.maxDepth != 0 {
                        e.depth++
                        if e.depth > e.maxDepth {
                                e.depth--
                                return ErrCycleDetected
                        }
                        defer func() { e.depth-- }()
                }
                rv := reflect.NewAt(t, p).Elem()
                if rv.IsNil() {
                        e.WriteNil()
                        return nil
                }
                n := rv.Len()
                // Canonical emit takes precedence over the OptMapShape/OptDense shape
                // branch so exactly one deterministic, sorted emit happens regardless of
                // the other shape bits. It reuses the same key/value encode closures
                // (k.encode / v.encode) as the default path — only the iteration ORDER
                // changes; the value codec, intern, and dict are untouched.
                if n > 0 && e.opts.Has(OptCanonical) {
                        return e.encodeMapCanonical(rv, keyType, valType, k, v)
                }
                if stringKey && n > 0 && e.state != nil && !e.stateSuspended && e.opts.Has(OptMapShape) && e.opts.Has(OptDense) {
                        return e.encodeStringMapShaped(rv, keyType, valType, v)
                }
                e.WriteMapHeader(n)
                // MapRange beats reflect.Value.Seq2 (Go 1.26) here by ~2x on
                // throughput and ~2x on allocations: Seq2 boxes the (k, v)
                // pair into closure arguments per yield, while MapRange
                // reuses a single *MapIter and exposes Key/Value via
                // reflect.Value (struct, no per-element heap). See
                // BenchmarkMapIter_MapRangeOriginal vs BenchmarkMapIter_Seq2.
                //
                // SetIterKey/SetIterValue (Go 1.18+) write the current map
                // iter entry into a pre-allocated addressable reflect.Value.
                // Without them, reflectValueAddr would have to materialise a
                // fresh reflect.New(T).Elem() per element — 2 allocs per map
                // entry on the previous path, O(N) total.
                //
                // The two scratch holders are pooled on encState (reused across the
                // rows of a []struct — no reflect.New per map) when a state exists;
                // re-entrancy-safe via the busy flag. Fast mode (e.state == nil) has
                // no pool, so it falls back to a local pair.
                var keyHolder, valHolder reflect.Value
                var kp, vp unsafe.Pointer
                var pooled bool
                if e.state != nil {
                        keyHolder, valHolder, vp, pooled = e.state.mapEnc.acquire(keyType, valType)
                        kp = unsafe.Pointer(keyHolder.UnsafeAddr())
                        defer e.state.mapEnc.release(pooled)
                } else {
                        keyHolder = reflect.New(keyType).Elem()
                        valHolder = reflect.New(valType).Elem()
                        kp = unsafe.Pointer(keyHolder.UnsafeAddr())
                        vp = unsafe.Pointer(valHolder.UnsafeAddr())
                }
                iter := rv.MapRange()
                for iter.Next() {
                        keyHolder.SetIterKey(iter)
                        valHolder.SetIterValue(iter)
                        if err := k.encode(e, kp); err != nil {
                                return err
                        }
                        if err := v.encode(e, vp); err != nil {
                                return err
                        }
                }
                return nil
        }
}

// encodeMapCanonical emits a map under OptCanonical: keys in sorted order so
// logically-equal maps serialize byte-identically. It writes the same plain map
// header and reuses the SAME key/value encode closures as the default path
// (k.encode / v.encode) — only the order changes.
//
// For the common string, int, uint, and bool key kinds: a single MapRange pass
// collects key scalars and copies values via SetIterValue into a pre-allocated
// typed array (reflect.ArrayOf). This avoids the per-key rv.MapIndex alloc that
// the old emit-closure design incurred for indirect value types (size > ptrSize).
//
// Float and exotic key kinds (struct / array / interface) take a slow
// reflect-comparator fallback that correctly handles NaN float keys (which are
// unfindable by MapIndex and must never be re-fetched via one).
func (e *Encoder) encodeMapCanonical(rv reflect.Value, keyType, valType reflect.Type, k, v *typeDesc) error {
        n := rv.Len()
        e.WriteMapHeader(n)
        if n == 0 {
                return nil
        }

        // Pool the key holder across map rows (one reflect.New per distinct key type
        // per Encoder lifetime). valHolder/vp are retained for the default exotic-key
        // path which still needs them for keyHolder.Set and valHolder.Set.
        var keyHolder, valHolder reflect.Value
        var vp unsafe.Pointer
        var pooled bool
        if e.state != nil {
                keyHolder, valHolder, vp, pooled = e.state.mapEnc.acquire(keyType, valType)
                defer e.state.mapEnc.release(pooled)
        } else {
                keyHolder = reflect.New(keyType).Elem()
                valHolder = reflect.New(valType).Elem()
                vp = unsafe.Pointer(valHolder.UnsafeAddr())
        }
        kp := unsafe.Pointer(keyHolder.UnsafeAddr())

        switch keyType.Kind() {
        case reflect.String:
                // Single-pass collect: SetIterKey extracts the string key into keyHolder
                // (zero alloc); SetIterValue copies the map value into the pre-allocated
                // valArr slot (zero alloc). Sort key-index pairs, then encode.
                valArr := reflect.New(reflect.ArrayOf(n, valType)).Elem()
                type sIdx struct {
                        k string
                        i int
                }
                idxs := make([]sIdx, 0, n)
                i := 0
                for it := rv.MapRange(); it.Next(); {
                        keyHolder.SetIterKey(it)
                        valArr.Index(i).SetIterValue(it)
                        idxs = append(idxs, sIdx{keyHolder.String(), i})
                        i++
                }
                slices.SortFunc(idxs, func(a, b sIdx) int { return cmp.Compare(a.k, b.k) })
                for _, p := range idxs {
                        keyHolder.SetString(p.k)
                        if err := k.encode(e, kp); err != nil {
                                return err
                        }
                        if err := v.encode(e, unsafe.Pointer(valArr.Index(p.i).UnsafeAddr())); err != nil {
                                return err
                        }
                }
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                valArr := reflect.New(reflect.ArrayOf(n, valType)).Elem()
                type iIdx struct {
                        k int64
                        i int
                }
                idxs := make([]iIdx, 0, n)
                i := 0
                for it := rv.MapRange(); it.Next(); {
                        keyHolder.SetIterKey(it)
                        valArr.Index(i).SetIterValue(it)
                        idxs = append(idxs, iIdx{keyHolder.Int(), i})
                        i++
                }
                slices.SortFunc(idxs, func(a, b iIdx) int { return cmp.Compare(a.k, b.k) })
                for _, p := range idxs {
                        keyHolder.SetInt(p.k)
                        if err := k.encode(e, kp); err != nil {
                                return err
                        }
                        if err := v.encode(e, unsafe.Pointer(valArr.Index(p.i).UnsafeAddr())); err != nil {
                                return err
                        }
                }
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                valArr := reflect.New(reflect.ArrayOf(n, valType)).Elem()
                type uIdx struct {
                        k uint64
                        i int
                }
                idxs := make([]uIdx, 0, n)
                i := 0
                for it := rv.MapRange(); it.Next(); {
                        keyHolder.SetIterKey(it)
                        valArr.Index(i).SetIterValue(it)
                        idxs = append(idxs, uIdx{keyHolder.Uint(), i})
                        i++
                }
                slices.SortFunc(idxs, func(a, b uIdx) int { return cmp.Compare(a.k, b.k) })
                for _, p := range idxs {
                        keyHolder.SetUint(p.k)
                        if err := k.encode(e, kp); err != nil {
                                return err
                        }
                        if err := v.encode(e, unsafe.Pointer(valArr.Index(p.i).UnsafeAddr())); err != nil {
                                return err
                        }
                }
        case reflect.Bool:
                // At most two keys; collect via MapRange (no MapIndex), emit false then true.
                // Pre-allocate 2 slots regardless — at most one slot stays unused.
                valArr := reflect.New(reflect.ArrayOf(2, valType)).Elem()
                var hasFalse, hasTrue bool
                for it := rv.MapRange(); it.Next(); {
                        keyHolder.SetIterKey(it)
                        if keyHolder.Bool() {
                                valArr.Index(1).SetIterValue(it)
                                hasTrue = true
                        } else {
                                valArr.Index(0).SetIterValue(it)
                                hasFalse = true
                        }
                }
                if hasFalse {
                        keyHolder.SetBool(false)
                        if err := k.encode(e, kp); err != nil {
                                return err
                        }
                        if err := v.encode(e, unsafe.Pointer(valArr.Index(0).UnsafeAddr())); err != nil {
                                return err
                        }
                }
                if hasTrue {
                        keyHolder.SetBool(true)
                        if err := k.encode(e, kp); err != nil {
                                return err
                        }
                        if err := v.encode(e, unsafe.Pointer(valArr.Index(1).UnsafeAddr())); err != nil {
                                return err
                        }
                }
        default:
                // Float and exotic comparable key kinds (struct / array / interface):
                // gather (key, value) PAIRS via MapRange and sort by key. Slow but rare;
                // correctness over speed. We must NOT re-fetch the value via MapIndex here:
                // a NaN float key (NaN != NaN) is unfindable by MapIndex, which would return
                // an invalid Value and panic on Set. MapRange yields every entry, NaN keys
                // included, so carrying the value alongside the key avoids the lookup.
                type kvPair struct{ k, v reflect.Value }
                pairs := make([]kvPair, 0, rv.Len())
                for it := rv.MapRange(); it.Next(); {
                        pairs = append(pairs, kvPair{it.Key(), it.Value()})
                }
                slices.SortFunc(pairs, func(x, y kvPair) int { return canonReflectKeyCompare(x.k, y.k) })
                for i := range pairs {
                        keyHolder.Set(pairs[i].k)
                        valHolder.Set(pairs[i].v)
                        if err := k.encode(e, kp); err != nil {
                                return err
                        }
                        if err := v.encode(e, vp); err != nil {
                                return err
                        }
                }
        }
        return nil
}

// gatherStringKeys collects the map's string keys, sorted. The returned slice is
// valid until the caller calls canonKeysRelease(pooled). pooled is true when the
// shared canonKeysStr scratch was used (the zero-alloc flat-map case); it is
// false for a nested re-entrant gather (a map whose values contain maps), which
// allocates a fresh local slice so it cannot clobber the outer map's keys still
// being iterated. The caller MUST release after it finishes iterating.
func (e *Encoder) gatherStringKeys(rv reflect.Value) (keys []string, pooled bool) {
        var buf []string
        if e.state != nil && !e.state.canonKeysBusy {
                buf = e.state.canonKeysStr[:0]
                pooled = true
        }
        it := rv.MapRange()
        kv := reflect.New(rv.Type().Key()).Elem() // one alloc total; SetIterKey reuses it in-place
        for it.Next() {
                kv.SetIterKey(it)
                buf = append(buf, kv.String())
        }
        slices.Sort(buf)
        if pooled {
                e.state.canonKeysStr = buf
                e.state.canonKeysBusy = true
        }
        return buf, pooled
}

func (e *Encoder) gatherIntKeys(rv reflect.Value) (keys []int64, pooled bool) {
        var buf []int64
        if e.state != nil && !e.state.canonKeysBusy {
                buf = e.state.canonKeysI64[:0]
                pooled = true
        }
        it := rv.MapRange()
        kv := reflect.New(rv.Type().Key()).Elem() // one alloc total; SetIterKey reuses it in-place
        for it.Next() {
                kv.SetIterKey(it)
                buf = append(buf, kv.Int())
        }
        slices.Sort(buf)
        if pooled {
                e.state.canonKeysI64 = buf
                e.state.canonKeysBusy = true
        }
        return buf, pooled
}

func (e *Encoder) gatherUintKeys(rv reflect.Value) (keys []uint64, pooled bool) {
        var buf []uint64
        if e.state != nil && !e.state.canonKeysBusy {
                buf = e.state.canonKeysU64[:0]
                pooled = true
        }
        it := rv.MapRange()
        kv := reflect.New(rv.Type().Key()).Elem() // one alloc total; SetIterKey reuses it in-place
        for it.Next() {
                kv.SetIterKey(it)
                buf = append(buf, kv.Uint())
        }
        slices.Sort(buf)
        if pooled {
                e.state.canonKeysU64 = buf
                e.state.canonKeysBusy = true
        }
        return buf, pooled
}

// canonKeysRelease clears the re-entrancy guard set by a pooled gather. A no-op
// when the gather allocated a fresh local slice (pooled false).
func (e *Encoder) canonKeysRelease(pooled bool) {
        if pooled && e.state != nil {
                e.state.canonKeysBusy = false
        }
}

// canonReflectKeyCompare is a stable total order over comparable reflect key
// values for the slow canonical fallback (float / struct / array / interface
// keys). Floats compare by canonicalized value then raw bits (so -0.0 and +0.0
// tie and distinct NaNs collapse). Structs/arrays compare field/element-wise.
// Other kinds fall back to a string rendering, which is stable within a type.
func canonReflectKeyCompare(a, b reflect.Value) int {
        // Kind-mismatch guard: interface keys (map[any]K) unwrap to differing dynamic
        // kinds, and an invalid (nil-interface) Value has Kind Invalid. Order by kind
        // ordinal so the kind-specific accessors below are never called on the wrong
        // kind (e.g. b.Int() on a float64) — that would panic.
        if a.Kind() != b.Kind() {
                return cmp.Compare(int(a.Kind()), int(b.Kind()))
        }
        switch a.Kind() {
        case reflect.Float32, reflect.Float64:
                // Order by raw bits — a strict total order even across NaN payloads (so
                // distinct NaN keys sort deterministically, not tie). -0.0 and +0.0 are the
                // SAME Go map key so never coexist. The EMITTED key bytes are normalized
                // separately by the float choke points (WriteFloat*); ordering only needs
                // to be deterministic, not numeric.
                return cmp.Compare(math.Float64bits(a.Float()), math.Float64bits(b.Float()))
        case reflect.String:
                return cmp.Compare(a.String(), b.String())
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                return cmp.Compare(a.Int(), b.Int())
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
                return cmp.Compare(a.Uint(), b.Uint())
        case reflect.Bool:
                if a.Bool() == b.Bool() {
                        return 0
                }
                if !a.Bool() {
                        return -1
                }
                return 1
        case reflect.Struct:
                for i := range a.NumField() {
                        if c := canonReflectKeyCompare(a.Field(i), b.Field(i)); c != 0 {
                                return c
                        }
                }
                return 0
        case reflect.Array:
                for i := range a.Len() {
                        if c := canonReflectKeyCompare(a.Index(i), b.Index(i)); c != 0 {
                                return c
                        }
                }
                return 0
        case reflect.Interface, reflect.Pointer:
                return canonReflectKeyCompare(a.Elem(), b.Elem())
        default:
                // Stable rendering for any residual comparable kind (complex, chan, etc.).
                return cmp.Compare(reflectRender(a), reflectRender(b))
        }
}

func reflectRender(v reflect.Value) string {
        if !v.IsValid() {
                return ""
        }
        return v.Type().String() + ":" + valueRenderString(v)
}

func valueRenderString(v reflect.Value) string {
        switch v.Kind() {
        case reflect.String:
                return v.String()
        default:
                return fmt.Sprintf("%v", v.Interface())
        }
}

// encodeStringMapShaped emits a string-keyed map via tagMapShape (OptMapShape).
// A recurring key-set declares once (keys interned); reuses emit only the shape
// ID + values in canonical (sorted) key order. Collision-safe: the set-hash
// only finds a candidate; the keys are verified present (count + membership)
// before reuse, falling back to a fresh declaration on any mismatch. This is
// the general reflect path; the common map[string]string/etc. types take the
// concrete fast path in maps_fast_generated.go.
//
// A reusable keyHolder/valHolder avoids per-key reflect.ValueOf boxing; the hot
// reuse path allocates nothing (no keys slice — values are fetched in the bound
// canonical order). The keys slice is built only on a fresh declaration (rare:
// once per distinct key-set).
func (e *Encoder) encodeStringMapShaped(rv reflect.Value, keyType, valType reflect.Type, v *typeDesc) error {
        // Ensure the stream header precedes the first tag (top-level map case; the
        // plain path emits it via WriteMapHeader). Idempotent.
        e.writeHeader()
        n := rv.Len()
        st := e.state
        // Pooled, re-entrancy-safe key holder — avoids reflect.New per map
        // (a 1000-row batch pays 1 reflect.New total, not 1 per row).
        // Values are collected into pre-allocated per-shape slots (see
        // mapShapeBinding.valSlots) via SetIterValue instead of MapIndex, so
        // valHolder and vp from acquire are not needed on this path.
        keyHolder, _, _, pooled := st.mapEnc.acquire(keyType, valType)
        defer st.mapEnc.release(pooled)

        // emitSlotsOrdered encodes st.mapShapes[sIdx].valSlots in canonical key
        // order. It uses index-based access (not a pointer) throughout: a nested
        // encode may call mapShapeRegister which appends to st.mapShapes, possibly
        // reallocating the backing array; index-based access always follows the
        // current slice header, avoiding use-after-reallocation.
        //
        // The busy flag is set for the duration so hasAllAndCollect skips this
        // binding during re-entrant nested encodes (preventing ensureValueSlots
        // from reinitialising slots that are still in use by the outer encode).
        emitSlotsOrdered := func(sIdx int) error {
                st.mapShapes[sIdx].busy = true
                defer func() { st.mapShapes[sIdx].busy = false }()
                for i := range st.mapShapes[sIdx].keys {
                        if err := v.encode(e, unsafe.Pointer(st.mapShapes[sIdx].valSlots[i].UnsafeAddr())); err != nil {
                                return err
                        }
                }
                return nil
        }

        // hasAllAndCollect does a single MapRange pass: verifies every map key is
        // in s.keys (binary search on the sorted slice) and simultaneously loads
        // each value into s.valSlots via SetIterValue — no per-key heap allocation.
        // Replaces the old two-step hasAll (MapRange) + emitValues (N MapIndex allocs)
        // flow, keeping the MapRange count to one on the hot fast path and
        // eliminating all value-copy heap allocations for struct-valued maps.
        // Callers must check !s.busy before calling (busy bindings are being
        // iterated by an outer emitSlotsOrdered and must not be re-initialised).
        hasAllAndCollect := func(s *mapShapeBinding) bool {
                s.ensureValueSlots(valType)
                it := rv.MapRange()
                for it.Next() {
                        keyHolder.SetIterKey(it) // SetIterKey reuses the holder; it.Key() would alloc
                        idx, ok := slices.BinarySearch(s.keys, keyHolder.String())
                        if !ok {
                                return false
                        }
                        s.valSlots[idx].SetIterValue(it)
                }
                return true
        }

        // Fast path: same key-set as the previous map — one MapRange pass combines
        // hasAll verification with value collection into pre-allocated slots.
        // Replaces hasAll (MapRange) + emitValues (N MapIndex) with a single sweep,
        // eliminating N per-key heap allocations for struct values.
        // !s.busy: guard against a coincidental n-match from a re-entrant nested
        // encode targeting the outer binding with a different valType.
        if st.lastMapShapeID != 0 && len(st.lastMapShapeKeys) == n {
                s := &st.mapShapes[st.lastMapShapeIdx]
                if !s.busy && hasAllAndCollect(s) {
                        e.buf = append(e.buf, tagMapShape)
                        e.buf = appendUvarint(e.buf, uint64(st.lastMapShapeID))
                        return emitSlotsOrdered(st.lastMapShapeIdx)
                }
        }

        // Key-set changed: order-independent set-hash to find an earlier shape.
        var setHash uint64
        iter := rv.MapRange()
        for iter.Next() {
                keyHolder.SetIterKey(iter) // reuse holder; iter.Key() would alloc
                setHash += internKeyHash(keyHolder.String())
        }
        // Scan EVERY shape with this (setHash, n) and verify by keys — setHash is not
        // collision-proof. Returning only the first (setHash, n) row and declaring on
        // a key mismatch would re-declare a colliding key-set on every encode: under
        // two alternating sets that collide on setHash, the already-registered second
        // set is never found again, so mapShapes grows without bound. The s.n == n
        // filter guarantees len(s.keys) == n, so hasAllAndCollect ⇒ set equality.
        // !s.busy: skip bindings currently held by an outer emitSlotsOrdered.
        for i := range st.mapShapes {
                s := &st.mapShapes[i]
                if s.setHash == setHash && s.n == n && !s.busy && hasAllAndCollect(s) {
                        st.lastMapShapeID, st.lastMapShapeKeys = s.id, s.keys
                        st.lastMapShapeIdx = i
                        e.buf = append(e.buf, tagMapShape)
                        e.buf = appendUvarint(e.buf, uint64(s.id))
                        return emitSlotsOrdered(i)
                }
        }

        // Declare path (first sight of this key-set).
        keys := make([]string, 0, n)
        it2 := rv.MapRange()
        for it2.Next() {
                keyHolder.SetIterKey(it2) // reuse holder; it2.Key() would alloc
                keys = append(keys, keyHolder.String())
        }
        slices.Sort(keys)
        id := st.shapeDeclareEnc()
        st.mapShapeRegister(setHash, n, keys, id)
        idx := len(st.mapShapes) - 1
        st.lastMapShapeID, st.lastMapShapeKeys = id, keys
        st.lastMapShapeIdx = idx
        e.buf = append(e.buf, tagMapShape)
        e.buf = appendUvarint(e.buf, 0)
        e.buf = appendUvarint(e.buf, uint64(n))
        for _, name := range keys {
                e.WriteString(name)
        }
        // Collect values into the newly declared binding's pre-allocated slots and
        // emit in canonical order. One extra MapRange pass on the declare path is
        // acceptable: declare is rare (once per distinct key-set per session).
        // s is a local pointer; no mapShapes reallocation happens in the it3 loop
        // (only in v.encode inside emitSlotsOrdered, after it3 completes).
        s := &st.mapShapes[idx]
        s.ensureValueSlots(valType)
        it3 := rv.MapRange()
        for it3.Next() {
                keyHolder.SetIterKey(it3)
                pos, _ := slices.BinarySearch(keys, keyHolder.String())
                s.valSlots[pos].SetIterValue(it3)
        }
        return emitSlotsOrdered(idx)
}

func decodeMap(t reflect.Type, k, v *typeDesc) func(*Decoder, unsafe.Pointer) error {
        keyType := t.Key()
        valType := t.Elem()
        return func(d *Decoder, p unsafe.Pointer) error {
                if err := d.descend(); err != nil {
                        return err
                }
                defer d.ascend()
                tag, err := d.peekTag()
                if err != nil {
                        return err
                }
                if tag == tagNil {
                        d.i++
                        reflect.NewAt(t, p).Elem().Set(reflect.Zero(t))
                        return nil
                }
                // tagMapShape: string-keyed map encoded via the key-set shape codec
                // (OptMapShape). Mirrors decodeStruct's shape branch; the decoder's
                // shape table is destination-agnostic (ordered names + N values).
                if tag == tagMapShape && keyType.Kind() == reflect.String {
                        names, err := decodeMapStringShapeHeader(d)
                        if err != nil {
                                return err
                        }
                        reuseOrMakeMapReflect(d, t, len(names), p)
                        mapVal := reflect.NewAt(t, p).Elem()
                        // Pooled, re-entrancy-safe holders hoisted out of the loop:
                        // SetMapIndex copies key/value into the map, so one pair is reused
                        // for every entry (and across rows) — no reflect.New per entry.
                        kh, vh, vp, pooled := d.state.mapDec.acquire(keyType, valType)
                        defer d.state.mapDec.release(pooled)
                        for _, name := range names {
                                kh.SetString(name)
                                // Reset the value holder each entry: a slice-backed value type
                                // would otherwise have its backing array reused across entries
                                // (reuseOrMakeSlice keeps a cap>=n backing), so every map value
                                // would alias the last one decoded. Zeroing forces a fresh
                                // MakeSlice per entry. Keys can't contain slices (comparable).
                                vh.SetZero()
                                if err := v.decode(d, vp); err != nil {
                                        return err
                                }
                                mapVal.SetMapIndex(kh, vh)
                        }
                        return nil
                }
                n, err := d.ReadMapHeader()
                if err != nil {
                        return err
                }
                if err := d.CheckLength(n, 2); err != nil {
                        return err
                }
                // Allocate via the swappable reflectutil backend.
                reuseOrMakeMapReflect(d, t, n, p)
                mapVal := reflect.NewAt(t, p).Elem()
                // Hoist the key/value holders out of the loop: SetMapIndex copies them
                // into the map, so one addressable pair is reused for every entry.
                // Previously this did reflect.New twice PER ENTRY (2 allocs × n);
                // now it is 2 per map. The locals stay re-entrancy-safe — a nested map
                // value decodes through its own decodeMap call with its own holders.
                kv := reflect.New(keyType).Elem()
                vv := reflect.New(valType).Elem()
                kp := unsafe.Pointer(kv.UnsafeAddr())
                vp := unsafe.Pointer(vv.UnsafeAddr())
                for range n {
                        if err := k.decode(d, kp); err != nil {
                                return err
                        }
                        // Reset the value holder each entry so a slice-backed value type does
                        // not reuse the previous entry's backing array (reuseOrMakeSlice keeps
                        // a cap>=n backing) and alias every map value onto the last one. Keys
                        // can't contain slices (map keys are comparable), so kv needs no reset.
                        vv.SetZero()
                        if err := v.decode(d, vp); err != nil {
                                return err
                        }
                        mapVal.SetMapIndex(kv, vv)
                }
                return nil
        }
}

func encodePtr(elem *typeDesc) func(*Encoder, unsafe.Pointer) error {
        return func(e *Encoder, p unsafe.Pointer) error {
                raw := *(*unsafe.Pointer)(p)
                if raw == nil {
                        e.WriteNil()
                        return nil
                }
                // Depth-based cycle guard. Cheaper than a per-pointer set and
                // catches both genuine *T->*T cycles and pathologically deep
                // payloads. maxDepth==0 disables the check for callers that
                // know their input is acyclic.
                if e.maxDepth != 0 {
                        e.depth++
                        if e.depth > e.maxDepth {
                                e.depth--
                                return ErrCycleDetected
                        }
                        defer func() { e.depth-- }()
                }
                return elem.encode(e, raw)
        }
}
func decodePtr(t reflect.Type, elem *typeDesc) func(*Decoder, unsafe.Pointer) error {
        return func(d *Decoder, p unsafe.Pointer) error {
                if err := d.descend(); err != nil {
                        return err
                }
                defer d.ascend()
                tag, err := d.peekTag()
                if err != nil {
                        return err
                }
                if tag == tagNil {
                        d.i++
                        *(*unsafe.Pointer)(p) = nil
                        return nil
                }
                // Reuse the existing heap object when the pointer field is already
                // non-nil.  Go has a non-moving GC: the raw pointer is stable; the
                // GC updates p (the pointer-to-pointer), not the pointed-to memory.
                if existing := *(*unsafe.Pointer)(p); existing != nil {
                        return elem.decode(d, existing)
                }
                // Allocate via reflect for GC-safety.
                nv := reflect.New(t.Elem())
                if err := elem.decode(d, unsafe.Pointer(nv.Elem().UnsafeAddr())); err != nil {
                        return err
                }
                reflect.NewAt(t, p).Elem().Set(nv)
                return nil
        }
}

func encodeStruct(td *typeDesc) func(*Encoder, unsafe.Pointer) error {
        fields := td.fields
        return func(e *Encoder, p unsafe.Pointer) error {
                e.writeHeader()
                n := len(fields)
                // Dense mode: route through the tagMapShape path when
                // OptShapeIntern is set. On the first emit of a struct type the
                // encoder declares the shape (writing field names through the
                // normal intern path); on every subsequent emit it writes only
                // 0xEC + shapeID + values. With OptShapeIntern off, Dense
                // falls back to the tagMap8/16/32 encoding so the rest of the
                // state stack (intern + Markov / MTF / Pair) still applies
                // per-field.
                if e.opts.Has(OptDense) && e.state != nil && !e.stateSuspended && e.opts.Has(OptShapeIntern) {
                        if id := e.state.shapeForType(td); id != 0 {
                                e.buf = append(e.buf, tagMapShape)
                                e.buf = appendUvarint(e.buf, uint64(id))
                                for i := range fields {
                                        f := &fields[i]
                                        if err := f.desc.encode(e, unsafe.Add(p, f.offset)); err != nil {
                                                return err
                                        }
                                }
                                return nil
                        }
                        // First time: declare and emit keys via the standard intern path.
                        shapeID := e.state.shapeDeclareEnc()
                        e.state.shapeBindType(td, shapeID)
                        st := e.state
                        e.buf = append(e.buf, tagMapShape)
                        e.buf = appendUvarint(e.buf, 0) // 0 ⇒ declaration follows
                        e.buf = appendUvarint(e.buf, uint64(n))
                        pairOn := e.pairPred
                        for i := range fields {
                                f := &fields[i]
                                if len(f.name) >= e.minIntern && int(st.internLoad) < e.maxStateEntries {
                                        if id, ok := st.lookupOrAssign(f.name); ok {
                                                if st.lastID == id {
                                                        e.buf = append(e.buf, tagStateRepeat)
                                                        if pairOn {
                                                                st.pairRecord(id, id)
                                                        }
                                                } else {
                                                        e.emitStateRef(id)
                                                }
                                        } else {
                                                e.buf = append(e.buf, f.preInternStr...)
                                                if st.lastID != lruInvalidID && pairOn {
                                                        st.pairRecord(st.lastID, id)
                                                }
                                                st.lastID = id
                                        }
                                } else {
                                        e.buf = append(e.buf, f.preFast...)
                                        st.lastID = lruInvalidID
                                }
                        }
                        for i := range fields {
                                f := &fields[i]
                                if err := f.desc.encode(e, unsafe.Add(p, f.offset)); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                // Dense without OptShapeIntern: tagMap8/16/32 header + per-field
                // intern via WriteString. Keys still go through the intern
                // path so state-ref / MTF / Pair codecs cover them when their
                // options are on.
                if e.opts.Has(OptDense) && e.state != nil {
                        e.WriteMapHeader(n)
                        st := e.state
                        pairOn := e.pairPred
                        for i := range fields {
                                f := &fields[i]
                                if len(f.name) >= e.minIntern && !e.stateSuspended && int(st.internLoad) < e.maxStateEntries {
                                        if id, ok := st.lookupOrAssign(f.name); ok {
                                                if st.lastID == id {
                                                        e.buf = append(e.buf, tagStateRepeat)
                                                        if pairOn {
                                                                st.pairRecord(id, id)
                                                        }
                                                } else {
                                                        e.emitStateRef(id)
                                                }
                                        } else {
                                                e.buf = append(e.buf, f.preInternStr...)
                                                if st.lastID != lruInvalidID && pairOn {
                                                        st.pairRecord(st.lastID, id)
                                                }
                                                st.lastID = id
                                        }
                                } else {
                                        e.buf = append(e.buf, f.preFast...)
                                        st.lastID = lruInvalidID
                                }
                                if err := f.desc.encode(e, unsafe.Add(p, f.offset)); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                // Fast mode (no Dense): plain tagMap8/16/32 encoding — no
                // intern, no shape, fixstr field-name headers from preFast.
                e.WriteMapHeader(n)
                for i := range fields {
                        f := &fields[i]
                        e.buf = append(e.buf, f.preFast...)
                        if err := f.desc.encode(e, unsafe.Add(p, f.offset)); err != nil {
                                return err
                        }
                }
                return nil
        }
}

func decodeStruct(td *typeDesc) func(*Decoder, unsafe.Pointer) error {
        // Build a name → field-index lookup so decode order doesn't have to
        // match encode order. Sorted lookup keeps it cache-friendly.
        type idx struct {
                f    *fieldDesc
                name string
        }
        indexed := make([]idx, len(td.fields))
        for i := range td.fields {
                indexed[i] = idx{f: &td.fields[i], name: td.fields[i].name}
        }
        // Linear scan is fine for ≤16 fields; for wide structs (rare), we use a
        // small map.
        useMap := len(indexed) > 16
        var byName map[string]*fieldDesc
        if useMap {
                byName = make(map[string]*fieldDesc, len(indexed))
                for i := range indexed {
                        byName[indexed[i].name] = indexed[i].f
                }
        }

        resolveField := func(name string) *fieldDesc {
                if useMap {
                        return byName[name]
                }
                for j := range indexed {
                        if indexed[j].name == name {
                                return indexed[j].f
                        }
                }
                return nil
        }

        return func(d *Decoder, p unsafe.Pointer) error {
                tag, err := d.peekTag()
                if err != nil {
                        return err
                }
                if tag == tagNil {
                        d.i++
                        return nil
                }
                // tagMapShape path: structs encoded via the Dense shape codec.
                if tag == tagMapShape {
                        d.i++
                        shapeID, n := readUvarint(d.buf[d.i:])
                        if n <= 0 {
                                return ErrInvalidLength
                        }
                        if shapeID > uint64(math.MaxUint32) {
                                return ErrUnknownStateID // would truncate on the uint32 cast below
                        }
                        d.i += n
                        if d.state == nil {
                                d.state = newDecState()
                        }
                        var fieldNames []string
                        if shapeID == 0 {
                                // Declaration: read count, then N keys, then N values.
                                cnt64, n := readUvarint(d.buf[d.i:])
                                if n <= 0 {
                                        return ErrInvalidLength
                                }
                                d.i += n
                                if cnt64 > uint64(math.MaxInt) { // 32-bit: int(cnt64) would wrap before CheckLength
                                        return ErrInvalidLength
                                }
                                cnt := int(cnt64)
                                if err := d.CheckLength(cnt, 1); err != nil {
                                        return err
                                }
                                sh := d.state.shapeDeclare()
                                keys := make([]string, 0, cnt)
                                for range cnt {
                                        kb, err := d.readStringBytes()
                                        if err != nil {
                                                return err
                                        }
                                        keys = append(keys, d.keyCache.Make(kb))
                                }
                                sh.names = keys
                                fieldNames = keys
                        } else {
                                sh := d.state.shapeLookup(uint32(shapeID))
                                if sh == nil {
                                        return ErrUnknownStateID
                                }
                                fieldNames = sh.names
                        }
                        cur := 0
                        for _, name := range fieldNames {
                                var fd *fieldDesc
                                if cur < len(indexed) && indexed[cur].name == name {
                                        fd = indexed[cur].f
                                        cur++
                                } else {
                                        fd = resolveField(name)
                                }
                                if fd == nil {
                                        if err := d.Skip(); err != nil {
                                                return err
                                        }
                                        continue
                                }
                                if err := fd.desc.decode(d, unsafe.Add(p, fd.offset)); err != nil {
                                        return err
                                }
                        }
                        return nil
                }
                // tagMap8/16/32 path — used by Fast mode, by Dense without
                // OptShapeIntern, and by any external encoder that does not
                // emit shape headers.
                n, err := d.ReadMapHeader()
                if err != nil {
                        return err
                }
                // Fields arrive in struct-declaration order in the common case (qdf
                // encodes them that way), so try the expected next field before the
                // map/linear lookup. An in-order hit is one string compare — no hash,
                // no map access. cur is left unchanged on a miss so a skipped unknown
                // field doesn't desync the cursor for the following in-order fields.
                cur := 0
                for range n {
                        kb, err := d.readStringBytes()
                        if err != nil {
                                return err
                        }
                        name := unsafestr.String(kb)
                        var fd *fieldDesc
                        if cur < len(indexed) && indexed[cur].name == name {
                                fd = indexed[cur].f
                                cur++
                        } else {
                                fd = resolveField(name)
                        }
                        if fd == nil {
                                if err := d.Skip(); err != nil {
                                        return err
                                }
                                continue
                        }
                        if err := fd.desc.decode(d, unsafe.Add(p, fd.offset)); err != nil {
                                return err
                        }
                }
                return nil
        }
}

// encodeIface dispatches on the dynamic type of an interface{}. Slow path.
func encodeIface(e *Encoder, p unsafe.Pointer) error {
        iv := *(*any)(p)
        if iv == nil {
                e.WriteNil()
                return nil
        }
        // Bound recursion through the dynamic (interface) dispatch. encodePtr guards
        // the static *T path, but a cycle routed through an any-typed field re-enters
        // the reflect machinery here without touching that counter; without this a
        // self-referential graph (a.Next = a, Next any) recurses unbounded into a
        // fatal stack overflow. This is the interface chokepoint — every []any /
        // map[K]any / any-field recursion funnels through it.
        if e.maxDepth != 0 {
                e.depth++
                if e.depth > e.maxDepth {
                        e.depth--
                        return ErrCycleDetected
                }
                defer func() { e.depth-- }()
        }
        // Mark the schemaless context: everything under a dynamic dispatch decodes via
        // decodeAny, which cannot read a batched vector-column block. Balanced inc/dec
        // so nested ifaces stay positive and the counter returns to 0 at the top.
        e.ifaceDepth++
        defer func() { e.ifaceDepth-- }()
        return encodeReflect(e, iv)
}

// encodeSliceAny encodes a []any without the per-value reflect.New that the
// generic encodeReflect path would take for the slice header. It is byte- and
// behavior-identical to the descOf([]any).encode closure (encodeSlice): a []any
// element carries no columnar plan, so that path reduces to nil-guard + depth
// guard + WriteArrayHeader + per-element encodeIface — exactly what this does,
// including the probe-and-grow buffer pre-sizing that avoids the log(n) realloc
// chain on large arrays. The pointer is not retained, so the caller's &tv stays
// on the stack.
func encodeSliceAny(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) { // nil slice → tagNil (distinct from empty)
                return nil
        }
        // Depth guard mirroring encodeSlice (and Decoder.descend in decodeSlice).
        if e.maxDepth != 0 {
                e.depth++
                if e.depth > e.maxDepth {
                        e.depth--
                        return ErrCycleDetected
                }
                defer func() { e.depth-- }()
        }
        s := *(*[]any)(p)
        n := len(s)
        e.WriteArrayHeader(n)
        // encodeIface applies its own per-element depth guard then dispatches via
        // encodeReflect — identical to elem.encode for an interface element.
        const probe = 32
        if n <= probe {
                for i := range s {
                        if err := encodeIface(e, unsafe.Pointer(&s[i])); err != nil {
                                return err
                        }
                }
                return nil
        }
        probeStart := len(e.buf)
        for i := range probe {
                if err := encodeIface(e, unsafe.Pointer(&s[i])); err != nil {
                        return err
                }
        }
        // Project the remaining size from the probe (+25% slack) and grow once,
        // killing the doubling chain on large dynamic arrays.
        if probeBytes := len(e.buf) - probeStart; probeBytes > 0 {
                // 64-bit intermediate: the product can exceed int32 on a 32-bit build for
                // a large slice, wrapping projected negative and panicking slices.Grow.
                projected := int(int64(probeBytes) * int64(n-probe) / int64(probe))
                projected += projected >> 2
                e.buf = slices.Grow(e.buf, projected)
        }
        for i := probe; i < n; i++ {
                if err := encodeIface(e, unsafe.Pointer(&s[i])); err != nil {
                        return err
                }
        }
        return nil
}

func decodeIface(d *Decoder, p unsafe.Pointer) error {
        v, err := decodeAny(d)
        if err != nil {
                return err
        }
        *(*any)(p) = v
        return nil
}

// decodeAny reads the next value as a generic any, mirroring encoding/json.
// isStringKeyTag reports whether tag begins a string value — i.e. a map key
// that d.readStringBytes can consume. Mirrors decodeAny's string-producing
// cases. Used to tell a string-keyed map (→ map[string]any) from a non-string-
// keyed one (→ map[any]any) when decoding a map schemalessly.
func isStringKeyTag(tag byte) bool {
        if tag >= tagFixstr && tag <= tagFixstr|tagFixstrMask {
                return true
        }
        switch tag {
        case tagStr8, tagStr16, tagStr32, tagInternStr,
                tagStateRef, tagStateRepeat, tagStateMTF, tagStatePair:
                return true
        }
        return false
}

// readStringRefAny reads a string that arrived as a back-REFERENCE to an
// already-interned value (tagStateRef / MTF / pair / repeat — the caller
// dispatches on the tag) and returns it boxed into an `any`, reusing ONE shared
// box per value: the box is cached in decState.boxValues keyed by the same
// state id, so every later reference returns it with zero allocation. This is
// the dominant win on repetitive dynamic data (map[string]any). It is called
// ONLY for reference tags — inline / first-occurrence strings box directly in
// decodeAny and never reach here, so high-cardinality data pays no overhead
// (its values never become references). Gated on !noCopy so the cached box
// matches the shared stringValues string (under noCopy ReadString returns an
// input-aliased view, boxed directly as before).
func (d *Decoder) readStringRefAny() (any, error) {
        if d.noCopy || d.state == nil {
                return d.ReadString()
        }
        s, err := d.ReadString()
        if err != nil {
                return nil, err
        }
        if box, ok := d.state.getBoxStr(d.state.lastID, d.arena); ok {
                return box, nil
        }
        return s, nil
}

func decodeAny(d *Decoder) (any, error) {
        if err := d.descend(); err != nil {
                return nil, err
        }
        defer d.ascend()
        tag, err := d.peekTag()
        if err != nil {
                return nil, err
        }
        switch {
        case tag <= tagFixintMax:
                v, err := d.ReadUint()
                return v, err
        case tag >= tagFixstr && tag <= tagFixstr|tagFixstrMask:
                return d.ReadString()
        case tag >= tagFixarr && tag <= tagFixarr|tagFixarrMask:
                n, err := d.ReadArrayHeader()
                if err != nil {
                        return nil, err
                }
                if err := d.CheckLength(n, 1); err != nil {
                        return nil, err
                }
                out := make([]any, n)
                for i := range n {
                        v, err := decodeAny(d)
                        if err != nil {
                                return nil, err
                        }
                        out[i] = v
                }
                return out, nil
        case tag >= tagNegfixint && tag <= tagNegfixint|tagNegfixintMask:
                return d.ReadInt()
        }
        switch tag {
        case tagNil:
                d.i++
                return nil, nil
        case tagTrue, tagFalse:
                return d.ReadBool()
        case tagUint8, tagUint16, tagUint32, tagUint64:
                return d.ReadUint()
        case tagInt8, tagInt16, tagInt32, tagInt64:
                return d.ReadInt()
        case tagFloat32:
                return d.ReadFloat32()
        case tagFloat64:
                return d.ReadFloat64()
        case tagStr8, tagStr16, tagStr32, tagInternStr:
                // Inline / first-occurrence string: box directly, zero cache overhead.
                return d.ReadString()
        case tagStateRef, tagStateRepeat, tagStateMTF, tagStatePair:
                // Back-reference to an interned value → return the shared cached box.
                return d.readStringRefAny()
        case tagBin8, tagBin16, tagBin32, tagInternBin:
                return d.ReadBytes()
        case tagArr16, tagArr32:
                n, err := d.ReadArrayHeader()
                if err != nil {
                        return nil, err
                }
                if err := d.CheckLength(n, 1); err != nil {
                        return nil, err
                }
                out := make([]any, n)
                for i := range n {
                        v, err := decodeAny(d)
                        if err != nil {
                                return nil, err
                        }
                        out[i] = v
                }
                return out, nil
        case tagMap8, tagMap16, tagMap32:
                n, err := d.ReadMapHeader()
                if err != nil {
                        return nil, err
                }
                if err := d.CheckLength(n, 2); err != nil {
                        return nil, err
                }
                if n == 0 {
                        return map[string]any{}, nil
                }
                // Peek the first key's tag. A string-keyed map (the common case)
                // decodes to map[string]any with interned keys. A non-string-keyed map
                // (e.g. map[int]V boxed in an any/interface, whose keys were written via
                // WriteInt/WriteUint) must NOT be read as string keys — doing so
                // returned ErrTypeMismatch and silently lost data that round-trips fine
                // into a typed destination. The wire cannot distinguish int from uint
                // for small (fixint) keys, so the only lossless schemaless form is
                // map[any]any, each key decoded via decodeAny (int64/uint64/etc.).
                // Peek EACH key's tag, not just the first: a map[any]any can carry mixed
                // key types in any order. Stay on the fast string-keyed path (→
                // map[string]any) as long as keys are strings; the moment a non-string
                // key appears, migrate the string keys decoded so far into map[any]any
                // and finish via decodeAny (which handles string and non-string keys
                // alike). map[any]any is the only lossless schemaless form — the wire
                // can't tell int from uint for small (fixint) keys.
                out := popOrMakeMap[string, any](d, n)
                for idx := 0; idx < n; idx++ {
                        ktag, err := d.peekTag()
                        if err != nil {
                                return nil, err
                        }
                        if !isStringKeyTag(ktag) {
                                anyOut := popOrMakeMap[any, any](d, n)
                                for k, v := range out {
                                        anyOut[k] = v
                                }
                                for ; idx < n; idx++ {
                                        k, err := decodeAny(d)
                                        if err != nil {
                                                return nil, err
                                        }
                                        // A valid map key is always comparable; reject a hostile wire
                                        // whose "key" decodes to a slice/map so anyOut[k] can't panic.
                                        if k != nil && !reflect.TypeOf(k).Comparable() {
                                                return nil, ErrBadTag
                                        }
                                        v, err := decodeAny(d)
                                        if err != nil {
                                                return nil, err
                                        }
                                        anyOut[k] = v
                                }
                                return anyOut, nil
                        }
                        kb, err := d.readStringBytes()
                        if err != nil {
                                return nil, err
                        }
                        v, err := decodeAny(d)
                        if err != nil {
                                return nil, err
                        }
                        out[d.keyCache.Make(kb)] = v
                }
                return out, nil
        case tagMapShape:
                d.i++
                shapeID, n := readUvarint(d.buf[d.i:])
                if n <= 0 {
                        return nil, ErrInvalidLength
                }
                if shapeID > uint64(math.MaxUint32) {
                        return nil, ErrUnknownStateID // would truncate on the uint32 cast below
                }
                d.i += n
                if d.state == nil {
                        d.state = newDecState()
                }
                var names []string
                if shapeID == 0 {
                        cnt64, n := readUvarint(d.buf[d.i:])
                        if n <= 0 {
                                return nil, ErrInvalidLength
                        }
                        d.i += n
                        if cnt64 > uint64(math.MaxInt) { // 32-bit: int(cnt64) would wrap before CheckLength
                                return nil, ErrInvalidLength
                        }
                        cnt := int(cnt64)
                        if err := d.CheckLength(cnt, 1); err != nil {
                                return nil, err
                        }
                        sh := d.state.shapeDeclare()
                        sh.names = make([]string, 0, cnt)
                        for range cnt {
                                kb, err := d.readStringBytes()
                                if err != nil {
                                        return nil, err
                                }
                                sh.names = append(sh.names, d.keyCache.Make(kb))
                        }
                        names = sh.names
                } else {
                        sh := d.state.shapeLookup(uint32(shapeID))
                        if sh == nil {
                                return nil, ErrUnknownStateID
                        }
                        names = sh.names
                }
                out := popOrMakeMap[string, any](d, len(names))
                for _, name := range names {
                        v, err := decodeAny(d)
                        if err != nil {
                                return nil, err
                        }
                        out[name] = v
                }
                return out, nil
        case tagColStruct:
                return decodeColumnarAny(d)
        case tagHybridColStruct:
                return decodeHybridColumnarAny(d)
        case tagTimestamp:
                sec, nsec, err := d.ReadTimestamp()
                if err != nil {
                        return nil, err
                }
                t := time.Unix(sec, int64(nsec)).UTC()
                // The zero time.Time (unset time fields) repeats heavily in real data
                // and boxes to a 24-byte heap value; share one immutable box for it.
                // A single IsZero() branch — no table — so non-zero timestamps (the
                // high-cardinality case) pay nothing.
                if t.IsZero() && d.state != nil {
                        if d.state.zeroTimeBox == nil {
                                d.state.zeroTimeBox = t
                        }
                        return d.state.zeroTimeBox, nil
                }
                return t, nil
        case tagPackBool:
                // A bool slice encoded under OptQPack. Decode into a typed []bool.
                var s []bool
                if err := decodeSliceBool(d, unsafe.Pointer(&s)); err != nil {
                        return nil, err
                }
                return s, nil
        case tagPackRaw, tagPackFor, tagPackDeltaFor, tagPackRLE,
                tagPackDict, tagPackPFor, tagPackGorilla, tagPackALP:
                // A numeric slice encoded under OptQPack/OptBalanced/OptCompression.
                // Without these cases decodeAny fell through to ErrBadTag, so any
                // interface{}/map[string]any value holding such a slice failed to
                // decode. Materialise into the matching typed slice (the values
                // round-trip; the int codecs widen to 64-bit on the wire).
                return decodeAnyPackedSlice(d)
        case tagPackBlock:
                // A long int/uint slice encoded with the per-block adaptive codec.
                // Emitted by writeQPackInt64/Uint64 for any []int/[]int64/[]uint/[]uint64
                // (no ifaceDepth gate), so it can reach decodeAny through an any /
                // map[K]any value / any-typed field; without this case that failed with
                // ErrBadTag. The block-kind byte after the tag selects int64 vs uint64
                // (a namespace disjoint from decodeAnyPackedSlice's qpackKind byte, so
                // it cannot share that path). decodeSliceInt64/Uint64 re-peek the tag.
                if d.i+1 >= len(d.buf) {
                        return nil, ErrShortBuffer
                }
                switch d.buf[d.i+1] {
                case blockKindInt:
                        var s []int64
                        if err := decodeSliceInt64(d, unsafe.Pointer(&s)); err != nil {
                                return nil, err
                        }
                        return s, nil
                case blockKindUint:
                        var s []uint64
                        if err := decodeSliceUint64(d, unsafe.Pointer(&s)); err != nil {
                                return nil, err
                        }
                        return s, nil
                default:
                        return nil, ErrBadTag
                }
        }
        return nil, ErrBadTag
}

// decodeAnyPackedSlice materialises a QPack-encoded numeric slice into a typed
// slice for the generic any decode path. The tag (still at d.i) is followed by a
// one-byte kind that selects the element family/width: integer codecs widen to
// 64-bit (Int64/Uint64), floats stay Float32/Float64 — the four kinds the
// encoder emits for these tags. The typed decodeSlice* helper re-peeks the tag
// and handles every pack variant for that element type.
func decodeAnyPackedSlice(d *Decoder) (any, error) {
        if d.i+1 >= len(d.buf) {
                return nil, ErrShortBuffer
        }
        switch d.buf[d.i+1] {
        case qpackKindInt64:
                var s []int64
                err := decodeSliceInt64(d, unsafe.Pointer(&s))
                return s, err
        case qpackKindInt32:
                // raw-LE preserves the native width (the bit-packing codecs widen to
                // Int64); int32 slices that don't compress land here.
                var s []int32
                err := decodeSliceInt32(d, unsafe.Pointer(&s))
                return s, err
        case qpackKindUint64:
                var s []uint64
                err := decodeSliceUint64(d, unsafe.Pointer(&s))
                return s, err
        case qpackKindUint32:
                var s []uint32
                err := decodeSliceUint32(d, unsafe.Pointer(&s))
                return s, err
        case qpackKindFloat64:
                var s []float64
                err := decodeSliceFloat64(d, unsafe.Pointer(&s))
                return s, err
        case qpackKindFloat32:
                var s []float32
                err := decodeSliceFloat32(d, unsafe.Pointer(&s))
                return s, err
        default:
                // Narrower kinds (Int8/Int16/Uint8/Uint16) never reach a pack tag — they
                // encode as a plain array (or []byte for uint8), handled by decodeAny's
                // array/bin cases — so any other kind is malformed input.
                return nil, ErrBadTag
        }
}

func encodeTime(e *Encoder, p unsafe.Pointer) error {
        t := (*time.Time)(p).UTC()
        e.WriteTimestamp(t.Unix(), uint32(t.Nanosecond()))
        return nil
}
func decodeTime(d *Decoder, p unsafe.Pointer) error {
        sec, nsec, err := d.ReadTimestamp()
        if err != nil {
                return err
        }
        *(*time.Time)(p) = time.Unix(sec, int64(nsec)).UTC()
        return nil
}

func encodeMarshaler(t reflect.Type) func(*Encoder, unsafe.Pointer) error {
        return func(e *Encoder, p unsafe.Pointer) error {
                // A Marshaler emits its own Fast-format body and ignores the
                // encoder's Options. When it is the top-level value, frame the
                // stream honestly as Fast (flag 0) instead of stamping the
                // encoder's Dense/QPack mode onto a Fast body — otherwise the
                // header lies, UnmarshalDirect takes a needless reflect fallback,
                // and the decoder allocates dense state it never uses. Nested
                // Marshaler fields (header already emitted) are unaffected: their
                // custom body is written inline as before.
                if !e.headerOut {
                        savedMode, savedQPack := e.mode, e.qpack
                        e.mode, e.qpack = Fast, false
                        e.writeHeader()
                        e.mode, e.qpack = savedMode, savedQPack
                        // The top-level Marshaler owns the framing (Fast). Mark it so the
                        // post-encode rANS pass leaves the body opts-invariant.
                        e.customFramed = true
                }
                m := reflect.NewAt(t, p).Interface().(Marshaler)
                // Thread the shared encoder when the type supports it (generated code):
                // no fresh encoder (and its state) per element, and shape/intern state is
                // shared across a slice so it can be interned. The top-level framing above
                // has already run; EncodeQDF writes the body into e directly.
                if em, ok := m.(EncoderMarshaler); ok {
                        return em.EncodeQDF(e)
                }
                out, err := m.MarshalQDF(e.buf)
                if err != nil {
                        return err
                }
                e.buf = out
                return nil
        }
}
func decodeUnmarshaler(t reflect.Type) func(*Decoder, unsafe.Pointer) error {
        return func(d *Decoder, p unsafe.Pointer) error {
                // Consume the 5-byte stream header when this is the top-level
                // decoder (no-op once headerRead is set, e.g. a nested Unmarshaler
                // field whose outer decoder already read it). Without this, a
                // top-level Unmarshal into an Unmarshaler type hands the user's
                // UnmarshalQDF the magic+flags bytes instead of the body —
                // mirroring UnmarshalDirect, which slices data[5:].
                if err := d.readHeader(); err != nil {
                        return err
                }
                u := reflect.NewAt(t, p).Interface().(Unmarshaler)
                // Thread the shared decoder when the type supports it (generated code):
                // no fresh decoder per element, and it inherits d's noCopy / arena. This
                // is what drops the per-element decoder on a []GeneratedStruct decode.
                if du, ok := u.(DecoderUnmarshaler); ok {
                        return du.DecodeQDF(d)
                }
                var n int
                var err error
                if d.arena != nil {
                        n, err = UnmarshalNestedArena(u, d.buf[d.i:], d.noCopy, d.arena)
                } else {
                        n, err = UnmarshalNested(u, d.buf[d.i:], d.noCopy)
                }
                if err != nil {
                        return err
                }
                d.i += n
                return nil
        }
}

// encodeReflect is the entry point from Marshal. v can be any.
func encodeReflect(e *Encoder, v any) error {
        if v == nil {
                e.WriteNil()
                return nil
        }
        // Fast path for common primitive top-level encodings: skip the
        // descriptor lookup and the reflect.New copy.
        switch tv := v.(type) {
        case bool:
                e.WriteBool(tv)
                return nil
        case int:
                e.WriteInt(int64(tv))
                return nil
        case int64:
                e.WriteInt(tv)
                return nil
        case uint64:
                e.WriteUint(tv)
                return nil
        case float64:
                e.WriteFloat64(tv)
                return nil
        case float32:
                e.WriteFloat32(tv)
                return nil
        case string:
                e.WriteString(tv)
                return nil
        case []byte:
                e.WriteBytes(tv)
                return nil
        case map[string]any:
                // Fast path for the dominant dynamic shapes (json.Unmarshal output):
                // take the address of the local typed copy and hand it to the concrete
                // generated encoder. This is byte-identical to the general path below
                // (descOf(map[string]any).encode IS encodeMapStringAny) but skips the
                // reflect.New + Set copy — and because encodeMapStringAny does not retain
                // the pointer, &tv stays on the stack, so it is allocation-free. Without
                // this, every nested map[string]any in an any-tree cost one reflect.New.
                return encodeMapStringAny(e, unsafe.Pointer(&tv))
        case []any:
                return encodeSliceAny(e, unsafe.Pointer(&tv))
        }
        rv := reflect.ValueOf(v)
        t := rv.Type()
        // Unwrap pointer once to match encoding/json behavior for *T. When the
        // caller passes a pointer we can skip the reflect.New copy because
        // rv.Elem() is already addressable.
        if t.Kind() == reflect.Pointer {
                if rv.IsNil() {
                        e.WriteNil()
                        return nil
                }
                elemT := t.Elem()
                td, err := descOf(elemT)
                if err != nil {
                        return err
                }
                return td.encode(e, unsafe.Pointer(rv.Pointer()))
        }
        td, err := descOf(t)
        if err != nil {
                return err
        }
        // Value passed by-value: need an addressable location for unsafe.Pointer.
        // For struct and array types larger than a pointer, the interface boxing at
        // the call site already allocated a heap copy and eface.data IS that copy.
        // We extract it directly to skip reflect.New. This removes one alloc per
        // Marshal(structVal) for all structs that are not "direct interface" types.
        //
        // Direct-interface structs (size == ptrSize with exactly one pointer field,
        // e.g. struct{M map[K]V}) store the pointer value itself in eface.data, not
        // a pointer to the struct — so the trick is unsafe for them. Size > ptrSize
        // is a sufficient condition to rule them out on all platforms.
        //
        // Safety: the data pointer is valid for the lifetime of v (the any parameter
        // stays alive until encodeReflect returns), and td.encode does not retain p.
        if k := t.Kind(); (k == reflect.Struct || k == reflect.Array) &&
                t.Size() > unsafe.Sizeof(unsafe.Pointer(nil)) {
                type eface struct {
                        _ unsafe.Pointer
                        p unsafe.Pointer
                }
                return td.encode(e, (*eface)(unsafe.Pointer(&v)).p)
        }
        // Fallback: named scalar types, small structs, and direct-interface structs.
        ptr := reflect.New(t)
        ptr.Elem().Set(rv)
        return td.encode(e, unsafe.Pointer(ptr.Pointer()))
}

func decodeReflect(d *Decoder, out any) error {
        rv := reflect.ValueOf(out)
        if rv.Kind() != reflect.Pointer || rv.IsNil() {
                return ErrTypeMismatch
        }
        t := rv.Type().Elem()
        // A query (predicate pushdown / projection) is only meaningful for a
        // columnar slice payload. A non-slice target (single struct, scalar, map)
        // can never be columnar, so reject it before decoding. Non-columnar slice
        // shapes are caught inside the slice decoder (it sees the wire tag).
        if d.query != nil && t.Kind() != reflect.Slice {
                return &QueryError{Op: "predicate pushdown", Err: ErrUnsupported}
        }
        td, err := descOf(t)
        if err != nil {
                return err
        }
        return td.decode(d, unsafe.Pointer(rv.Pointer()))
}

// encodeNilSlice emits tagNil and returns true when the slice at p is nil
// (Data==nil), distinct from an empty (non-nil, len-0) slice — the nil-vs-empty
// distinction maps and pointers already keep, and encoding/json keeps as null vs
// []. Called as the first line of the nil-aware slice FIELD encoders so a nil
// slice round-trips as nil. A small direct (inlinable) call, NOT a wrapping
// closure, so the hot slice paths pay no extra indirection; the shared
// encodeSlice* funcs stay nil-agnostic for their internal direct callers
// (nullable/columnar dense columns), which must still emit a real header.
func (e *Encoder) encodeNilSlice(p unsafe.Pointer) bool {
        if (*sliceHeader)(p).Data == nil {
                e.WriteNil()
                return true
        }
        return false
}

// decodeNilSlice consumes a tagNil nil-slice marker, zeroes the destination, and
// returns true. The first line of the nil-aware slice decoders. The header is
// read first on a top-level decode (headerRead==false); the common struct-field/
// element path is a bounds test + tag compare.
func (d *Decoder) decodeNilSlice(p unsafe.Pointer) bool {
        if !d.headerRead {
                if err := d.readHeaderSlow(); err != nil {
                        return false // the caller's normal decode re-surfaces the error
                }
        }
        if d.i < len(d.buf) && d.buf[d.i] == tagNil {
                d.i++
                *(*sliceHeader)(p) = sliceHeader{}
                return true
        }
        return false
}

package qdf

import (
        "reflect"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/reflectutil"
)

// This file groups the decode-time "reuse the caller's container" helpers:
// reuseOrMakeSlice / reuseOrMakeMap let a decode into a pre-sized (pooled)
// target reuse the existing slice backing / map instead of allocating a fresh
// one — the dominant decode allocation on a server hot path that recycles its
// decode target. sliceHeader and noPointers support the slice variant.

// sliceHeader mirrors reflect.SliceHeader using unsafe.Pointer instead of
// uintptr so the GC can see the data pointer. Required for taking pointers
// out of a slice without losing it to the GC.
type sliceHeader struct {
        Data unsafe.Pointer
        Len  int
        Cap  int
}

// noPointers reports whether t contains no pointers (so a byte-clear of its
// memory is GC-safe — no write barriers needed). Used to gate decode slice
// backing reuse: a pointer-free element can be zeroed with clear() over a raw
// []byte view before the values are decoded in place.
//
// The delta diff hot path (the per-element caller that once motivated a cache)
// now reads the precomputed typeDesc.pod field instead, so the remaining callers
// (columnar.go, reflect_encode.go) are per-slice/per-type — not hot enough to
// warrant a sync.Map. This is the uncached structural walk.
func noPointers(t reflect.Type) bool {
        return noPointersWalk(t)
}

// noPointersWalk is the uncached structural worker behind noPointers. It uses
// NumField/Field instead of the Fields() iterator so a cache miss does not
// allocate an iterator backing for every nested struct level.
func noPointersWalk(t reflect.Type) bool {
        switch t.Kind() {
        case reflect.Bool,
                reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
                reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
                reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
                return true
        case reflect.Array:
                return noPointersWalk(t.Elem())
        case reflect.Struct:
                // NumField/Field, NOT the Fields() range-over-func iterator: the iterator
                // allocates a heap closure+state per call, and this walk runs per decode
                // (columnar reuse gate) — a modernize pass reintroduced that alloc here.
                for i := range t.NumField() {
                        if !noPointersWalk(t.Field(i).Type) {
                                return false
                        }
                }
                return true
        default:
                // string, slice, map, ptr, interface, chan, func, unsafe.Pointer.
                return false
        }
}

// tightPODWalk reports whether t is pointer-free AND has no internal or tail
// padding, so its full byte span is content (and may be bulk-hashed in the delta
// value fingerprint). Scalars are tight by definition. An array is tight iff its
// element is tight (the array stride already includes the element's own padding).
// A struct is tight iff it is pointer-free, every field type is tight, and the
// fields are packed with no gaps and no tail pad: field[0] at offset 0, each
// subsequent field immediately after the previous, and the struct size equal to
// the end of the last field. An empty struct is tight (zero-length span). Any
// type with a pointer (string/slice/map/ptr/iface/chan/func) is not tight.
func tightPODWalk(t reflect.Type) bool {
        switch t.Kind() {
        case reflect.Bool,
                reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
                reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
                reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
                return true
        case reflect.Array:
                return tightPODWalk(t.Elem())
        case reflect.Struct:
                n := t.NumField()
                if n == 0 {
                        return true
                }
                var next uintptr
                for i := range n {
                        f := t.Field(i)
                        if f.Offset != next || !tightPODWalk(f.Type) {
                                return false
                        }
                        next = f.Offset + f.Type.Size()
                }
                return t.Size() == next
        default:
                return false
        }
}

// reuseOrMakeSlice sets the slice at p to length n and returns its data base.
// When the caller-provided slice already has cap >= n it reuses that backing
// instead of allocating a fresh one — eliminating the result backing
// allocation on a decode into a pre-sized (pooled) slice, the dominant decode
// allocation. The reused elements are zeroed first so a wire shape that omits
// fields (schema evolution) cannot leak stale data: pointer-free elements
// (elemPF) take a barrier-free byte clear; pointer-containing elements take
// reflect.Value.Clear (a single barrier-correct typedmemclr). With no usable
// backing it allocates fresh via MakeSlice. elemPF must be noPointers(t.Elem()).
func reuseOrMakeSlice(t reflect.Type, n int, p unsafe.Pointer, stride uintptr, elemPF bool) unsafe.Pointer {
        if hdr := (*sliceHeader)(p); hdr.Cap >= n && hdr.Data != nil {
                hdr.Len = n
                if elemPF {
                        // Pointer-free: a raw byte clear is GC-safe and zeroes any struct
                        // fields the wire shape does not set (schema evolution).
                        clear(unsafe.Slice((*byte)(hdr.Data), n*int(stride)))
                } else {
                        // Pointer-containing: a byte clear would skip write barriers and
                        // corrupt the GC. reflect.Value.Clear bulk-zeroes the slice
                        // elements with a single barrier-correct typedmemclr.
                        reflect.NewAt(t, p).Elem().Clear()
                }
                return hdr.Data
        }
        reflectutil.MakeSlice(t, n, p)
        return reflectutil.SliceData(t, p)
}

// reuseOrMakeMap returns a map[K]V to decode n entries into, reusing the
// caller's existing non-nil map at p (cleared so no stale keys survive a
// schema-evolving re-decode) instead of allocating a fresh one. clear() keeps
// the bucket capacity, so a server re-decoding into the same target skips the
// makemap + bucket-growth that dominate map-heavy decode allocation. A nil
// destination (fresh target) first tries the Decoder's per-type map free-list
// (maps harvested from a reused []struct{map} target whose elements
// decode-slice-reuse is about to zero), then allocates fresh. Mirrors
// reuseOrMakeSlice.
func reuseOrMakeMap[K comparable, V any](d *Decoder, p unsafe.Pointer, n int) map[K]V {
        if m := *(*map[K]V)(p); m != nil { // direct-target reuse (unchanged)
                clear(m)
                return m
        }
        // Only consult the harvest free-list when it actually holds something — a
        // fresh decode (no reused []struct{map} target to harvest from) skips the
        // reflect.TypeFor + map lookup entirely, so this path is zero-cost vs a plain
        // make when no recycling is in play.
        if len(d.mapFreeList) > 0 {
                t := reflect.TypeFor[map[K]V]()
                if lst := d.mapFreeList[t]; len(lst) > 0 {
                        ptr := lst[len(lst)-1]
                        d.mapFreeList[t] = lst[:len(lst)-1]
                        // Reconstruct the map header value from its harvested hmap pointer.
                        m := *(*map[K]V)(unsafe.Pointer(&ptr))
                        clear(m)
                        return m
                }
        }
        return make(map[K]V, n)
}

// popOrMakeMap returns a map[K]V to decode n entries into, recycling a harvested
// map of this type from the Decoder free-list (cleared) when one is available,
// else allocating fresh. Unlike reuseOrMakeMap it has no destination pointer
// (the caller — decodeAny — returns the map rather than writing through a *map),
// so it only consults the free-list. Zero-cost when the free-list is empty.
func popOrMakeMap[K comparable, V any](d *Decoder, n int) map[K]V {
        if len(d.mapFreeList) > 0 {
                t := reflect.TypeFor[map[K]V]()
                if lst := d.mapFreeList[t]; len(lst) > 0 {
                        ptr := lst[len(lst)-1]
                        d.mapFreeList[t] = lst[:len(lst)-1]
                        m := *(*map[K]V)(unsafe.Pointer(&ptr))
                        clear(m)
                        return m
                }
        }
        return make(map[K]V, n)
}

// reuseOrMakeMapReflect installs at p a map of type t to decode n entries into,
// reusing the caller's existing non-nil map (cleared) or a harvested free-list
// map of type t, else allocating fresh via reflectutil.MakeMap. The reflect-path
// analogue of reuseOrMakeMap, used by decodeMap. Zero-cost (just a nil check)
// when there is nothing to reuse.
func reuseOrMakeMapReflect(d *Decoder, t reflect.Type, n int, p unsafe.Pointer) {
        if *(*unsafe.Pointer)(p) != nil { // direct-target reuse: existing map at p
                reflect.NewAt(t, p).Elem().Clear() // clear keeps the map, drops entries
                return
        }
        if len(d.mapFreeList) > 0 {
                if lst := d.mapFreeList[t]; len(lst) > 0 {
                        ptr := lst[len(lst)-1]
                        d.mapFreeList[t] = lst[:len(lst)-1]
                        *(*unsafe.Pointer)(p) = ptr        // install the harvested map at p
                        reflect.NewAt(t, p).Elem().Clear() // empty its entries before refill
                        return
                }
        }
        reflectutil.MakeMap(t, n, p)
}

// maxRecycledMaps bounds how many maps of one type the harvest free-list
// retains, so a one-off huge []struct{map} decode can't pin unbounded map
// headers. Past the cap the surplus maps are simply not recycled (allocated
// fresh) — a size bound, not a correctness limit.
const maxRecycledMaps = 1 << 14

// typeDescHasMap reports whether a value of type td holds a map at any
// struct-nesting depth (a map field, or a map reachable through nested struct
// fields). decodeSlice computes this once per slice type so the per-decode
// harvest walk runs only for elements that can actually yield a recyclable map.
// Conservative: maps behind a pointer/interface/slice/array field are not
// counted (their decoders re-allocate them), matching what harvestValue walks.
func typeDescHasMap(td *typeDesc) bool {
        switch td.kind {
        case reflect.Map:
                return true
        case reflect.Struct:
                for j := range td.fields {
                        if fd := &td.fields[j]; fd.desc != nil && typeDescHasMap(fd.desc) {
                                return true
                        }
                }
        }
        return false
}

// harvestMaps pushes the non-nil maps held by the first oldLen elements (each a
// value of the slice's element type elem) onto the Decoder's per-type free-list,
// so reuseOrMakeMap can recycle them after decode-slice-reuse zeroes those
// elements. Recurses through nested struct fields, so a map at any nesting depth
// inside the element is harvested; a map element directly ([]map) is harvested
// too. (Maps behind a pointer/interface/slice/array field, and map-valued maps,
// are not walked — those are re-allocated by their own decoders.)
func harvestMaps(d *Decoder, elem *typeDesc, base unsafe.Pointer, stride uintptr, oldLen int) {
        for i := range oldLen {
                d.harvestValue(elem, unsafe.Add(base, uintptr(i)*stride))
        }
}

// harvestValue harvests every map reachable from one value of type td at p,
// recursing through struct fields.
func (d *Decoder) harvestValue(td *typeDesc, p unsafe.Pointer) {
        switch td.kind {
        case reflect.Map:
                if hmap := *(*unsafe.Pointer)(p); hmap != nil {
                        if d.mapFreeList == nil {
                                d.mapFreeList = make(map[reflect.Type][]unsafe.Pointer)
                        }
                        if len(d.mapFreeList[td.rType]) < maxRecycledMaps {
                                d.mapFreeList[td.rType] = append(d.mapFreeList[td.rType], hmap)
                        }
                }
        case reflect.Struct:
                for j := range td.fields {
                        fd := &td.fields[j]
                        if fd.desc == nil {
                                continue
                        }
                        if k := fd.desc.kind; k == reflect.Map || k == reflect.Struct {
                                d.harvestValue(fd.desc, unsafe.Add(p, fd.offset))
                        }
                }
        }
}

package qdf

import (
        "reflect"
        "unsafe"
)

// Specialized fast paths for slices of common primitive element types.
// Selected in fillDesc when the element kind matches.

var (
        sliceStringType  = reflect.TypeFor[[]string]()
        sliceIntType     = reflect.TypeFor[[]int]()
        sliceInt32Type   = reflect.TypeFor[[]int32]()
        sliceInt64Type   = reflect.TypeFor[[]int64]()
        sliceUint32Type  = reflect.TypeFor[[]uint32]()
        sliceUint64Type  = reflect.TypeFor[[]uint64]()
        sliceFloat32Type = reflect.TypeFor[[]float32]()
        sliceFloat64Type = reflect.TypeFor[[]float64]()
        sliceBoolType    = reflect.TypeFor[[]bool]()
)

// installSliceFastPath returns (encode, decode, true) if t is a recognized
// primitive slice. Returns (_, _, false) for the generic case.
func installSliceFastPath(t reflect.Type) (
        enc func(*Encoder, unsafe.Pointer) error,
        dec func(*Decoder, unsafe.Pointer) error,
        ok bool,
) {
        // The *Nil variants preserve the nil-vs-empty distinction at the FIELD level
        // (a nil slice → tagNil) while the bare encodeSlice*/decodeSlice* stay
        // nil-agnostic for their internal direct callers (dense columns). Each *Nil
        // adds an inline nil check then a DIRECT call to the bare codec — no wrapping
        // closure, so a slice field encode/decode costs no extra indirect call.
        switch t {
        case sliceStringType:
                return encodeSliceStringNil, decodeSliceStringNil, true
        case sliceIntType:
                return encodeSliceIntNil, decodeSliceIntNil, true
        case sliceInt32Type:
                return encodeSliceInt32Nil, decodeSliceInt32Nil, true
        case sliceInt64Type:
                return encodeSliceInt64Nil, decodeSliceInt64Nil, true
        case sliceUint32Type:
                return encodeSliceUint32Nil, decodeSliceUint32Nil, true
        case sliceUint64Type:
                return encodeSliceUint64Nil, decodeSliceUint64Nil, true
        case sliceFloat32Type:
                return encodeSliceFloat32Nil, decodeSliceFloat32Nil, true
        case sliceFloat64Type:
                return encodeSliceFloat64Nil, decodeSliceFloat64Nil, true
        case sliceBoolType:
                return encodeSliceBoolNil, decodeSliceBoolNil, true
        }
        return nil, nil, false
}

// Nil-aware field variants: inline nil check + DIRECT call to the bare codec.
func encodeSliceStringNil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceString(e, p)
}
func decodeSliceStringNil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceString(d, p)
}
func encodeSliceIntNil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceInt(e, p)
}
func decodeSliceIntNil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceInt(d, p)
}
func encodeSliceInt32Nil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceInt32(e, p)
}
func decodeSliceInt32Nil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceInt32(d, p)
}
func encodeSliceInt64Nil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceInt64(e, p)
}
func decodeSliceInt64Nil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceInt64(d, p)
}
func encodeSliceUint32Nil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceUint32(e, p)
}
func decodeSliceUint32Nil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceUint32(d, p)
}
func encodeSliceUint64Nil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceUint64(e, p)
}
func decodeSliceUint64Nil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceUint64(d, p)
}
func encodeSliceFloat32Nil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceFloat32(e, p)
}
func decodeSliceFloat32Nil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceFloat32(d, p)
}
func encodeSliceFloat64Nil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceFloat64(e, p)
}
func decodeSliceFloat64Nil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceFloat64(d, p)
}
func encodeSliceBoolNil(e *Encoder, p unsafe.Pointer) error {
        if e.encodeNilSlice(p) {
                return nil
        }
        return encodeSliceBool(e, p)
}
func decodeSliceBoolNil(d *Decoder, p unsafe.Pointer) error {
        if d.decodeNilSlice(p) {
                return nil
        }
        return decodeSliceBool(d, p)
}

// ----- []string -----

func encodeSliceString(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]string)(p)
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteString(s[i])
        }
        return nil
}
func decodeSliceString(d *Decoder, p unsafe.Pointer) error {
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        out := make([]string, n)
        for i := range n {
                v, err := d.ReadString()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        *(*[]string)(p) = out
        return nil
}

// ----- QPack codec helpers shared by 64-bit and widened 32-bit paths -----

// writeQPackUint64 runs the codec picker over s and writes the chosen QPack form,
// first trying the per-block adaptive codec (reusing the single pick so a
// non-firing column is picked exactly once).
func (e *Encoder) writeQPackUint64(s []uint64) {
        codec, mn, forBits, first, minDelta, deltaBits, pforBits, _ := pickU64Codec(s)
        if e.tryWriteBlockUint64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits) {
                return
        }
        e.emitQPackUint64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
}

// qpackConstantOverCap reports whether emitting n elements in the chosen codec
// would produce an empty/sub-linear body that the decoder rejects because n
// exceeds qpackMaxStandaloneCount. The decoder bounds make([]T, n) at that cap
// for the empty-body forms — a constant FOR/Delta/PFor (bits == 0), a long-run
// RLE, or a single-value Dict — since their tiny wire cannot otherwise limit
// the allocation. A proportional body (bits > 0) stays input-bounded and needs
// no cap. When this returns true the caller must emit raw instead, so the wire
// the encoder produces is one the decoder accepts (round-trip preserved) while
// the OOM cap still holds for hostile input.
func qpackConstantOverCap(n int, codec qpackCodec, forBits, deltaBits, pforBits int) bool {
        if n <= qpackMaxStandaloneCount {
                return false
        }
        switch codec {
        case qpackFor:
                return forBits == 0
        case qpackDeltaFor:
                return deltaBits == 0
        case qpackPFor:
                return pforBits == 0
        case qpackRLE, qpackDict:
                return true
        }
        return false
}

// emitQPackUint64 writes s in the already-chosen codec form (picker output passed
// in, so a caller that needs to inspect the choice does not pick twice).
func (e *Encoder) emitQPackUint64(s []uint64, codec qpackCodec, mn uint64, forBits int, first uint64, minDelta int64, deltaBits, pforBits int) {
        if qpackConstantOverCap(len(s), codec, forBits, deltaBits, pforBits) {
                e.writePackedUint64Slice(s)
                return
        }
        switch codec {
        case qpackFor:
                e.writePackedForUint64Slice(s, mn, forBits)
        case qpackDeltaFor:
                e.writePackedDeltaForUint64Slice(s, first, minDelta, deltaBits)
        case qpackRLE:
                e.writePackedRLEUint64Slice(s)
        case qpackDict:
                e.writePackedDictUint64Slice(s)
        case qpackPFor:
                e.writePackedPForUint64Slice(s, mn, pforBits)
        default:
                e.writePackedUint64Slice(s)
        }
}

// writeQPackInt64 runs the codec picker over s and writes the chosen QPack form,
// first trying the per-block adaptive codec (reusing the single pick so a
// non-firing column is picked exactly once).
func (e *Encoder) writeQPackInt64(s []int64) {
        codec, mn, forBits, first, minDelta, deltaBits, pforBits, _ := pickI64Codec(s)
        if e.tryWriteBlockInt64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits) {
                return
        }
        e.emitQPackInt64(s, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
}

// emitQPackInt64 writes s in the already-chosen codec form (picker output passed
// in, so a caller that needs to inspect the choice does not pick twice).
func (e *Encoder) emitQPackInt64(s []int64, codec qpackCodec, mn int64, forBits int, first int64, minDelta int64, deltaBits, pforBits int) {
        if qpackConstantOverCap(len(s), codec, forBits, deltaBits, pforBits) {
                e.writePackedInt64Slice(s)
                return
        }
        switch codec {
        case qpackFor:
                e.writePackedForInt64Slice(s, mn, forBits)
        case qpackDeltaFor:
                e.writePackedDeltaForInt64Slice(s, first, minDelta, deltaBits)
        case qpackRLE:
                e.writePackedRLEInt64Slice(s)
        case qpackDict:
                e.writePackedDictInt64Slice(s)
        case qpackPFor:
                e.writePackedPForInt64Slice(s, mn, pforBits)
        default:
                e.writePackedInt64Slice(s)
        }
}

// ----- []int / []int32 / []int64 / []uint32 / []uint64 -----

func encodeSliceInt(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]int)(p)
        if e.qpack {
                // []int is platform-sized. On 64-bit platforms we re-view as
                // int64 and dispatch through pickI64Codec; on 32-bit we use the
                // raw int32 fast path. unsafe.Sizeof is a compile-time constant
                // so the dead branch is eliminated.
                if unsafe.Sizeof(int(0)) == 8 {
                        s64 := unsafe.Slice((*int64)(unsafe.Pointer(unsafe.SliceData(s))), len(s))
                        e.writeQPackInt64(s64)
                        return nil
                }
                s32 := unsafe.Slice((*int32)(unsafe.Pointer(unsafe.SliceData(s))), len(s))
                e.writePackedInt32Slice(s32)
                return nil
        }
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteInt(int64(s[i]))
        }
        return nil
}
func decodeSliceInt(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch t {
        case tagPackRaw, tagPackFor, tagPackDeltaFor, tagPackRLE, tagPackDict, tagPackPFor, tagPackBlock, tagZoneChunk:
                if unsafe.Sizeof(int(0)) == 8 {
                        var dest []int64
                        if err := decodeSliceInt64(d, unsafe.Pointer(&dest)); err != nil {
                                return err
                        }
                        out := unsafe.Slice((*int)(unsafe.Pointer(unsafe.SliceData(dest))), len(dest))
                        *(*[]int)(p) = out
                        return nil
                }
                var dest []int32
                if err := decodeSliceInt32(d, unsafe.Pointer(&dest)); err != nil {
                        return err
                }
                out := unsafe.Slice((*int)(unsafe.Pointer(unsafe.SliceData(dest))), len(dest))
                *(*[]int)(p) = out
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        out := make([]int, n)
        for i := range n {
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                out[i] = int(v)
        }
        *(*[]int)(p) = out
        return nil
}

// widenI64 promotes s into the encoder's reused int64 widening scratch and
// returns the filled slice. The result is valid only until the next widenI64
// call on the same encoder; the QPack int32 path consumes it (pick + emit)
// before any such call, so a single buffer serves every []int32 field.
func (e *Encoder) widenI64(s []int32) []int64 {
        if cap(e.wideI64) < len(s) {
                e.wideI64 = make([]int64, len(s))
        }
        w := e.wideI64[:len(s)]
        for i, v := range s {
                w[i] = int64(v)
        }
        return w
}

// widenU64 is the uint32→uint64 analogue of widenI64.
func (e *Encoder) widenU64(s []uint32) []uint64 {
        if cap(e.wideU64) < len(s) {
                e.wideU64 = make([]uint64, len(s))
        }
        w := e.wideU64[:len(s)]
        for i, v := range s {
                w[i] = uint64(v)
        }
        return w
}

func encodeSliceInt32(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]int32)(p)
        if e.qpack {
                w := e.widenI64(s)
                codec, mn, forBits, first, minDelta, deltaBits, pforBits, bestCost := pickI64Codec(w)
                // Never-worse floor: native int32-raw is 4 B/elem vs the picker's 8 B/elem
                // uint64-raw baseline. Emit the widened codec only when it beats native
                // int32-raw; else native raw so incompressible 32-bit data isn't inflated.
                if bestCost >= 2+uvarintLen(uint64(len(s)))+4*len(s) {
                        e.writePackedInt32Slice(s)
                        return nil
                }
                // A constant/empty-body codec over the standalone cap makes emitQPackInt64
                // fall back to int64-raw (8 B/elem); for a 32-bit slice native int32-raw
                // (4 B/elem) is the real never-larger floor, so redirect here instead.
                if qpackConstantOverCap(len(s), codec, forBits, deltaBits, pforBits) {
                        e.writePackedInt32Slice(s)
                        return nil
                }
                e.emitQPackInt64(w, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
                return nil
        }
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteInt(int64(s[i]))
        }
        return nil
}

// readQPackUint64 consumes a QPack uint64 codec body whose tag t was peeked.
// The caller must have already peeked (not consumed) the tag; this function
// increments d.i to consume it before reading the payload.
func (d *Decoder) readQPackUint64(t byte) ([]uint64, error) {
        d.i++
        switch t {
        case tagPackRaw:
                return d.readPackedUint64Slice()
        case tagPackFor:
                return d.readPackedForUint64Slice()
        case tagPackDeltaFor:
                return d.readPackedDeltaForUint64Slice()
        case tagPackRLE:
                return d.readPackedRLEUint64Slice()
        case tagPackDict:
                return d.readPackedDictUint64Slice()
        case tagPackPFor:
                return d.readPackedPForUint64Slice()
        case tagPackBlock:
                return d.readBlockUint64()
        case tagZoneChunk:
                return d.readZoneChunkUint64()
        }
        return nil, ErrBadTag
}

// readQPackInt64 consumes a QPack int64 codec body whose tag t was peeked.
// The caller must have already peeked (not consumed) the tag; this function
// increments d.i to consume it before reading the payload.
func (d *Decoder) readQPackInt64(t byte) ([]int64, error) {
        d.i++
        switch t {
        case tagPackRaw:
                return d.readPackedInt64Slice()
        case tagPackFor:
                return d.readPackedForInt64Slice()
        case tagPackDeltaFor:
                return d.readPackedDeltaForInt64Slice()
        case tagPackRLE:
                return d.readPackedRLEInt64Slice()
        case tagPackDict:
                return d.readPackedDictInt64Slice()
        case tagPackPFor:
                return d.readPackedPForInt64Slice()
        case tagPackBlock:
                return d.readBlockInt64()
        case tagZoneChunk:
                return d.readZoneChunkInt64()
        }
        return nil, ErrBadTag
}

func decodeSliceInt32(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch t {
        case tagPackRaw:
                // Could be qpackKindInt32 (legacy raw) or qpackKindInt64 (new QPack path).
                // Peek the kind byte (one position ahead of the tag) to decide.
                if d.i+1 < len(d.buf) && d.buf[d.i+1] == qpackKindInt64 {
                        v64, err := d.readQPackInt64(t)
                        if err != nil {
                                return err
                        }
                        out := make([]int32, len(v64))
                        for i, x := range v64 {
                                out[i] = int32(x)
                        }
                        *(*[]int32)(p) = out
                        return nil
                }
                d.i++
                v, err := d.readPackedInt32Slice()
                if err != nil {
                        return err
                }
                *(*[]int32)(p) = v
                return nil
        case tagPackFor, tagPackDeltaFor, tagPackRLE, tagPackDict, tagPackPFor:
                v64, err := d.readQPackInt64(t)
                if err != nil {
                        return err
                }
                out := make([]int32, len(v64))
                for i, x := range v64 {
                        out[i] = int32(x)
                }
                *(*[]int32)(p) = out
                return nil
        case tagPackBlock, tagZoneChunk:
                // A []int/[]int64 wire (writeQPackInt64 → tagPackBlock 0xF0, columnar
                // writeZoneChunkInt64 → tagZoneChunk 0xF1) decoded into a []int32 field
                // after an int→int32 schema change. Mirror decodeSliceInt64's block/
                // zone-chunk handling (and the For/PFor case above) instead of falling
                // through to ReadArrayHeader, which rejects these tags as ErrTypeMismatch.
                d.i++
                var v64 []int64
                var err error
                if t == tagPackBlock {
                        v64, err = d.readBlockInt64()
                } else {
                        v64, err = d.readZoneChunkInt64()
                }
                if err != nil {
                        return err
                }
                out := make([]int32, len(v64))
                for i, x := range v64 {
                        out[i] = int32(x)
                }
                *(*[]int32)(p) = out
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        out := make([]int32, n)
        for i := range n {
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                out[i] = int32(v)
        }
        *(*[]int32)(p) = out
        return nil
}
func encodeSliceInt64(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]int64)(p)
        if e.qpack {
                e.writeQPackInt64(s) // tries the per-block codec, else the whole-column pick
                return nil
        }
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteInt(s[i])
        }
        return nil
}
func decodeSliceInt64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch t {
        case tagPackRaw:
                d.i++
                v, err := d.readPackedInt64Slice()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        case tagPackFor:
                d.i++
                v, err := d.readPackedForInt64Slice()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        case tagPackDeltaFor:
                d.i++
                v, err := d.readPackedDeltaForInt64Slice()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        case tagPackRLE:
                d.i++
                v, err := d.readPackedRLEInt64Slice()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        case tagPackDict:
                d.i++
                v, err := d.readPackedDictInt64Slice()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        case tagPackPFor:
                d.i++
                v, err := d.readPackedPForInt64Slice()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        case tagPackBlock:
                d.i++
                v, err := d.readBlockInt64()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        case tagZoneChunk:
                d.i++
                v, err := d.readZoneChunkInt64()
                if err != nil {
                        return err
                }
                *(*[]int64)(p) = v
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        out := make([]int64, n)
        for i := range n {
                v, err := d.ReadInt()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        *(*[]int64)(p) = out
        return nil
}
func encodeSliceUint32(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]uint32)(p)
        if e.qpack {
                w := e.widenU64(s)
                codec, mn, forBits, first, minDelta, deltaBits, pforBits, bestCost := pickU64Codec(w)
                // Never-worse floor: the picker scores codecs against the uint64-raw
                // 8 B/elem baseline, but the native form for a uint32 is 4 B/elem. Emit
                // the widened codec only when its cost beats native uint32-raw; otherwise
                // fall back to native raw so incompressible 32-bit data is never inflated.
                if bestCost >= 2+uvarintLen(uint64(len(s)))+4*len(s) {
                        e.writePackedUint32Slice(s)
                        return nil
                }
                // A constant/empty-body codec over the standalone cap makes emitQPackUint64
                // fall back to uint64-raw (8 B/elem); for a 32-bit slice native uint32-raw
                // (4 B/elem) is the real never-larger floor, so redirect here instead.
                if qpackConstantOverCap(len(s), codec, forBits, deltaBits, pforBits) {
                        e.writePackedUint32Slice(s)
                        return nil
                }
                e.emitQPackUint64(w, codec, mn, forBits, first, minDelta, deltaBits, pforBits)
                return nil
        }
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteUint(uint64(s[i]))
        }
        return nil
}
func decodeSliceUint32(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch t {
        case tagPackRaw:
                // Could be qpackKindUint32 (legacy raw) or qpackKindUint64 (new QPack path).
                // Peek the kind byte (one position ahead of the tag) to decide.
                if d.i+1 < len(d.buf) && d.buf[d.i+1] == qpackKindUint64 {
                        v64, err := d.readQPackUint64(t)
                        if err != nil {
                                return err
                        }
                        out := make([]uint32, len(v64))
                        for i, x := range v64 {
                                out[i] = uint32(x)
                        }
                        *(*[]uint32)(p) = out
                        return nil
                }
                d.i++
                v, err := d.readPackedUint32Slice()
                if err != nil {
                        return err
                }
                *(*[]uint32)(p) = v
                return nil
        case tagPackFor, tagPackDeltaFor, tagPackRLE, tagPackDict, tagPackPFor:
                v64, err := d.readQPackUint64(t)
                if err != nil {
                        return err
                }
                out := make([]uint32, len(v64))
                for i, x := range v64 {
                        out[i] = uint32(x)
                }
                *(*[]uint32)(p) = out
                return nil
        case tagPackBlock, tagZoneChunk:
                // []uint/[]uint64 wire (writeQPackUint64 → tagPackBlock, columnar
                // writeZoneChunkUint64 → tagZoneChunk) decoded into a []uint32 field
                // after a uint→uint32 schema change. Mirror decodeSliceUint64 instead of
                // falling through to ReadArrayHeader (ErrTypeMismatch on these tags).
                d.i++
                var v64 []uint64
                var err error
                if t == tagPackBlock {
                        v64, err = d.readBlockUint64()
                } else {
                        v64, err = d.readZoneChunkUint64()
                }
                if err != nil {
                        return err
                }
                out := make([]uint32, len(v64))
                for i, x := range v64 {
                        out[i] = uint32(x)
                }
                *(*[]uint32)(p) = out
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        out := make([]uint32, n)
        for i := range n {
                v, err := d.ReadUint()
                if err != nil {
                        return err
                }
                out[i] = uint32(v)
        }
        *(*[]uint32)(p) = out
        return nil
}
func encodeSliceUint64(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]uint64)(p)
        if e.qpack {
                e.writeQPackUint64(s) // tries the per-block codec, else the whole-column pick
                return nil
        }
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteUint(s[i])
        }
        return nil
}
func decodeSliceUint64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        switch t {
        case tagPackRaw:
                d.i++
                v, err := d.readPackedUint64Slice()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        case tagPackFor:
                d.i++
                v, err := d.readPackedForUint64Slice()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        case tagPackDeltaFor:
                d.i++
                v, err := d.readPackedDeltaForUint64Slice()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        case tagPackRLE:
                d.i++
                v, err := d.readPackedRLEUint64Slice()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        case tagPackDict:
                d.i++
                v, err := d.readPackedDictUint64Slice()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        case tagPackPFor:
                d.i++
                v, err := d.readPackedPForUint64Slice()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        case tagPackBlock:
                d.i++
                v, err := d.readBlockUint64()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        case tagZoneChunk:
                d.i++
                v, err := d.readZoneChunkUint64()
                if err != nil {
                        return err
                }
                *(*[]uint64)(p) = v
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        out := make([]uint64, n)
        for i := range n {
                v, err := d.ReadUint()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        *(*[]uint64)(p) = out
        return nil
}

// ----- []float32 / []float64 -----

// encodeSliceFloat32 is the lossy-eligible entry for a genuine []float32 VECTOR
// field (one slice per record). Under OptLossyVec it emits the smaller of the
// lossy 0xFD block and the lossless body (never-worse). Scalar float32 columns
// transposed by the columnar path must NOT go through here — they call
// encodeSliceFloat32Lossless directly so OptLossyVec never makes a scalar field
// lossy.
func encodeSliceFloat32(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]float32)(p)
        if e.opts.Has(OptCanonical) {
                s = e.canonicalFloat32Slice(s)
        }
        if e.opts.Has(OptLossyVec) && e.ifaceDepth == 0 && len(s) >= lossyVecMinElems {
                // Build the lossy block into scratch; emit it only if it is no larger
                // than the lossless body (never-worse). Widen into the reused e.wideF64
                // buffer so appendLossyVec's in-place NaN/Inf zeroing does not touch s.
                // ifaceDepth==0: a lossy 0xFD block has no decodeAny case, so a vector held
                // in a schemaless (any) position must stay lossless to round-trip.
                e.wideF64 = toF64Into(s, e.wideF64)
                lossy, lossyOK := appendLossyVec([][]float64{e.wideF64}, true, toBudget(e.vecBudget), &e.vecScratch)
                start := len(e.buf)
                hdrBefore, flagBefore := e.headerOut, e.headerFlagAt
                if err := encodeSliceFloat32Lossless(e, s); err != nil {
                        return err
                }
                losslessBody := len(e.buf) - start
                if !hdrBefore {
                        // Bare top-level slice: the lossless write above also emitted the stream
                        // header, which the lossy-wins branch re-emits too. Exclude it from the
                        // lossless side so the never-worse compare is body-to-body (lossy carries
                        // no header), not header-biased toward lossy.
                        losslessBody -= streamHeaderLen
                }
                if lossyOK && len(lossy) <= losslessBody {
                        // Lossy wins (or ties): roll back the lossless body and emit lossy.
                        // Restore the header latch, then re-emit the stream header if the
                        // lossless write had to roll it away (bare top-level slice: no header
                        // existed before this call). writeHeader is a no-op when the header is
                        // already present (struct-field case), so this is safe for both.
                        e.buf = e.buf[:start]
                        e.headerOut, e.headerFlagAt = hdrBefore, flagBefore
                        e.writeHeader()
                        e.buf = append(e.buf, lossy...)
                }
                return nil
        }
        return encodeSliceFloat32Lossless(e, s)
}

// encodeSliceFloat32Lossless encodes a []float32 with the lossless float codec
// (raw-LE / qpack / Gorilla / ALP). Never emits a lossy 0xFD block. Shared by
// the lossy-eligible entry's fallback and by columnar scalar-float gathers.
// Canonicalization is idempotent, so the lossy entry (which canonicalizes
// before calling here) is unaffected while direct callers still get it applied.
func encodeSliceFloat32Lossless(e *Encoder, s []float32) error {
        if e.opts.Has(OptCanonical) {
                s = e.canonicalFloat32Slice(s)
        }
        if e.qpack {
                // Under OptCompression both Gorilla and ALP are enabled. Pick the smallest
                // of {raw-LE, Gorilla projection, ALP estimate}. ALP's estimate is a
                // conservative upper bound, so it is chosen only when it strictly beats both
                // alternatives — pure-smooth floats keep Gorilla, quantized/decimal floats
                // take ALP, and nothing grows the wire. Mirrors encodeSliceFloat64.
                if e.gorillaFloat {
                        rawExact := 2 + uvarintLen(uint64(len(s))) + len(s)*4
                        plan, alpEst, alpOK := alpPlanFloat32(s) // safe upper bound
                        alpWins := alpOK && alpEst < rawExact
                        if gorCodec, _ := pickF32Codec(s); gorCodec == qpackGorilla {
                                start := len(e.buf)
                                hdrBefore, flagBefore := e.headerOut, e.headerFlagAt
                                e.writePackedGorillaFloat32Slice(s)
                                gorActual := len(e.buf) - start
                                if gorActual < rawExact && (!alpWins || alpEst >= gorActual) {
                                        return nil
                                }
                                // Gorilla did not win — roll back. writePackedGorilla* may have
                                // emitted the stream header on a top-level first write; truncating to
                                // start drops it, and writeHeader's headerOut latch would then
                                // suppress the fallback's header and produce a headerless, undecodable
                                // stream. Restore the pre-attempt header state so the raw/ALP fallback
                                // re-emits the header when it was rolled away.
                                e.buf = e.buf[:start]
                                e.headerOut, e.headerFlagAt = hdrBefore, flagBefore
                        }
                        if alpWins {
                                e.writePackedALPFloat32Slice(s, plan)
                                return nil
                        }
                }
                e.writePackedFloat32Slice(s)
                return nil
        }
        return encodeSliceFloat32Impl(e, s)
}
func decodeSliceFloat32(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagColVecLossy {
                vecs, elemF32, used, err := readLossyVec(d.buf[d.i:])
                if err != nil {
                        return err
                }
                if len(vecs) == 0 {
                        return ErrShortBuffer
                }
                if !elemF32 {
                        return ErrTypeMismatch
                }
                d.i += used
                out := make([]float32, len(vecs[0]))
                for i, v := range vecs[0] {
                        out[i] = float32(v)
                }
                *(*[]float32)(p) = out
                return nil
        }
        if t == tagPackRaw {
                d.i++
                v, err := d.readPackedFloat32Slice()
                if err != nil {
                        return err
                }
                *(*[]float32)(p) = v
                return nil
        }
        if t == tagPackGorilla {
                d.i++
                v, err := d.readPackedGorillaFloat32Slice()
                if err != nil {
                        return err
                }
                *(*[]float32)(p) = v
                return nil
        }
        if t == tagPackALP {
                d.i++
                v, err := d.readPackedALPFloat32Slice()
                if err != nil {
                        return err
                }
                *(*[]float32)(p) = v
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 5); err != nil { // each ReadFloat32 elem is 5 bytes (tag+4); bound make() by remaining/5
                return err
        }
        out := make([]float32, n)
        for i := range n {
                v, err := d.ReadFloat32()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        *(*[]float32)(p) = out
        return nil
}

// encodeSliceFloat64 is the lossy-eligible entry for a genuine []float64 VECTOR
// field (one slice per record). Under OptLossyVec it emits the smaller of the
// lossy 0xFD block and the lossless body (never-worse). Scalar float64 columns
// transposed by the columnar / delta / nullable paths must NOT go through here —
// they call encodeSliceFloat64Lossless directly so OptLossyVec never makes a
// scalar field lossy.
func encodeSliceFloat64(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]float64)(p)
        if e.opts.Has(OptCanonical) {
                s = e.canonicalFloat64Slice(s)
        }
        if e.opts.Has(OptLossyVec) && e.ifaceDepth == 0 && len(s) >= lossyVecMinElems {
                // Build the lossy block into scratch; emit it only if it is no larger
                // than the lossless body (never-worse). Copy s into the reused e.wideF64
                // buffer so appendLossyVec's in-place NaN/Inf zeroing does not touch the
                // caller's []float64 slice. toF64Into returns s unchanged for []float64,
                // ifaceDepth==0: a schemaless (any) vector must stay lossless (no 0xFD
                // block) so decodeAny can read it back.
                // so we must copy explicitly here rather than routing through toF64Into.
                e.wideF64 = append(e.wideF64[:0], s...)
                lossy, lossyOK := appendLossyVec([][]float64{e.wideF64}, false, toBudget(e.vecBudget), &e.vecScratch)
                start := len(e.buf)
                hdrBefore, flagBefore := e.headerOut, e.headerFlagAt
                if err := encodeSliceFloat64Lossless(e, s); err != nil {
                        return err
                }
                losslessBody := len(e.buf) - start
                if !hdrBefore {
                        // See encodeSliceFloat32: exclude the stream header the bare-slice
                        // lossless write emitted so the never-worse compare is body-to-body.
                        losslessBody -= streamHeaderLen
                }
                if lossyOK && len(lossy) <= losslessBody {
                        // Lossy wins (or ties): roll back the lossless body and emit lossy.
                        // Restore the header latch, then re-emit the stream header if the
                        // lossless write had to roll it away (bare top-level slice: no header
                        // existed before this call). writeHeader is a no-op when the header is
                        // already present (struct-field case), so this is safe for both.
                        e.buf = e.buf[:start]
                        e.headerOut, e.headerFlagAt = hdrBefore, flagBefore
                        e.writeHeader()
                        e.buf = append(e.buf, lossy...)
                }
                return nil
        }
        return encodeSliceFloat64Lossless(e, s)
}

// encodeSliceFloat64Lossless encodes a []float64 with the lossless float codec
// (raw-LE / qpack / Gorilla / ALP). Never emits a lossy 0xFD block. Shared by
// the lossy-eligible entry's fallback and by columnar scalar-float gathers.
// Canonicalization is idempotent, so the lossy entry (which canonicalizes
// before calling here) is unaffected while direct callers still get it applied.
func encodeSliceFloat64Lossless(e *Encoder, s []float64) error {
        if e.opts.Has(OptCanonical) {
                s = e.canonicalFloat64Slice(s)
        }
        if e.qpack {
                // Under OptCompression both Gorilla and ALP are enabled. Pick the
                // smallest of {raw-LE, Gorilla projection, ALP estimate}. ALP's
                // estimate is a conservative upper bound, so it is chosen only when it
                // strictly beats both alternatives — pure-smooth floats keep Gorilla,
                // and nothing grows the wire.
                if e.gorillaFloat {
                        // Exact raw size (tag + kind + uvarint(n) + 8n), matching what
                        // writePackedFloat64Slice emits. A looser fixed estimate (e.g.
                        // 12+8n) over-counts raw by up to ~10 bytes and could keep Gorilla
                        // when it is marginally LARGER than raw; the exact figure makes the
                        // never-larger gate tight, mirroring the float32 path.
                        rawEst := 2 + uvarintLen(uint64(len(s))) + len(s)*8
                        plan, alpEst, alpOK := alpPlanFloat64(s) // ALP estimate is a safe upper bound
                        alpWins := alpOK && alpEst < rawEst
                        // pickF64Codec only projects Gorilla from a sample prefix, which can
                        // be wildly optimistic on a smooth-prefix/high-entropy-tail slice.
                        // Emit Gorilla for real and measure it; keep it only when it is
                        // actually smaller than raw (and than ALP) — a true never-larger
                        // gate. The rollback re-emits raw/ALP only on the rare lose case, so
                        // the common smooth-data path still encodes Gorilla once.
                        if gorCodec, _ := pickF64Codec(s); gorCodec == qpackGorilla {
                                start := len(e.buf)
                                hdrBefore, flagBefore := e.headerOut, e.headerFlagAt
                                e.writePackedGorillaFloat64Slice(s)
                                gorActual := len(e.buf) - start
                                if gorActual < rawEst && (!alpWins || alpEst >= gorActual) {
                                        return nil
                                }
                                // Gorilla did not win — roll back. writePackedGorilla* may have
                                // emitted the stream header (top-level first write); truncating to
                                // `start` drops those bytes, but writeHeader's headerOut latch would
                                // then suppress the fallback's header and produce a headerless,
                                // undecodable stream. Restore the pre-attempt header state so the
                                // raw/ALP fallback re-emits the header when it was rolled away.
                                e.buf = e.buf[:start]
                                e.headerOut, e.headerFlagAt = hdrBefore, flagBefore
                        }
                        if alpWins {
                                e.writePackedALPFloat64Slice(s, plan)
                                return nil
                        }
                }
                e.writePackedFloat64Slice(s)
                return nil
        }
        return encodeSliceFloat64Impl(e, s)
}
func decodeSliceFloat64(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagColVecLossy {
                vecs, elemF32, used, err := readLossyVec(d.buf[d.i:])
                if err != nil {
                        return err
                }
                if len(vecs) == 0 {
                        return ErrShortBuffer
                }
                if elemF32 {
                        return ErrTypeMismatch
                }
                d.i += used
                *(*[]float64)(p) = vecs[0]
                return nil
        }
        if t == tagPackRaw {
                d.i++
                v, err := d.readPackedFloat64Slice()
                if err != nil {
                        return err
                }
                *(*[]float64)(p) = v
                return nil
        }
        if t == tagZoneChunk {
                d.i++
                v, err := d.readZoneChunkFloat64()
                if err != nil {
                        return err
                }
                *(*[]float64)(p) = v
                return nil
        }
        if t == tagPackGorilla {
                d.i++
                v, err := d.readPackedGorillaFloat64Slice()
                if err != nil {
                        return err
                }
                *(*[]float64)(p) = v
                return nil
        }
        if t == tagPackALP {
                d.i++
                v, err := d.readPackedALPFloat64Slice()
                if err != nil {
                        return err
                }
                *(*[]float64)(p) = v
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 5); err != nil { // each ReadFloat64 elem is >=5 bytes (tag+4); bound make() by remaining/5
                return err
        }
        out := make([]float64, n)
        for i := range n {
                v, err := d.ReadFloat64()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        *(*[]float64)(p) = out
        return nil
}

// ----- []bool -----

func encodeSliceBool(e *Encoder, p unsafe.Pointer) error {
        s := *(*[]bool)(p)
        if e.qpack {
                e.writePackedBool(s)
                return nil
        }
        e.WriteArrayHeader(len(s))
        for i := range s {
                e.WriteBool(s[i])
        }
        return nil
}
func decodeSliceBool(d *Decoder, p unsafe.Pointer) error {
        t, err := d.peekTag()
        if err != nil {
                return err
        }
        if t == tagPackBool {
                d.i++
                out, err := d.readPackedBool()
                if err != nil {
                        return err
                }
                *(*[]bool)(p) = out
                return nil
        }
        n, err := d.ReadArrayHeader()
        if err != nil {
                return err
        }
        if err := d.CheckLength(n, 1); err != nil {
                return err
        }
        out := make([]bool, n)
        for i := range n {
                v, err := d.ReadBool()
                if err != nil {
                        return err
                }
                out[i] = v
        }
        *(*[]bool)(p) = out
        return nil
}

package qdf

import (
        "hash/maphash"
        "reflect"
        "strings"
        "unsafe"

        "github.com/alex60217101990/qdf/internal/internarena"
        "github.com/alex60217101990/qdf/internal/unsafestr"
)

// internHashSeed is the shared maphash seed for the flat intern
// table. Per-process random; values are stable across goroutines
// but differ across binary invocations (no semantic impact — the
// intern table is per-encoder and reset between Marshals).
var internHashSeed = maphash.MakeSeed()

// internKeyHash hashes a single map key for the order-independent key-set hash
// used by OptMapShape. Same seed/function as the intern table so distribution
// matches; callers combine per-key results commutatively (sum) so key order
// does not affect the set identity.
func internKeyHash(s string) uint64 {
        return maphash.String(internHashSeed, s)
}

// internSlot is one entry in the flat hash table that replaces the
// old map[string]uint32 intern dictionary. Profiling on telemetry
// workloads showed Go's map[string]uint32 spending ~17 ns/op in
// mapaccess2_faststr (string hash + bucket walk + memequal).
// Open-addressing on a contiguous []internSlot with the hash
// precomputed and stored alongside lets the hot path read one
// cache line, compare the hash, then `==` the key — saving ~5 ns
// per lookup at the cost of one extra 8 B slot field.
//
// hash == 0 reserves the "empty slot" sentinel; computed hashes
// that fall on 0 are bumped to 1 before storage.
//
// The key (a pointer-bearing string) leads so the GC pointer-scan range is
// 8 B instead of 16 B for a hash-first layout; the total stays 32 B:
//
//        key 16 B  +  hash 8 B  +  id 4 B  +  pad 4 B  =  32 B  →  2 slots / cache line
type internSlot struct {
        key  string
        hash uint64
        id   uint32
        _    uint32
}

// Initial intern table size. Doubles when load > 0.5 so the
// linear-probing chain stays short. 64 covers the common
// telemetry / config payloads (a few dozen distinct values) with
// no resize at all.
const internTableInitSize = 64

// Intern table backing Dense mode. The encoder maintains a string→ID
// map; the decoder maintains the matching ID-ordered list of byte
// slices. IDs are assigned in encode order starting at 0. There is no
// eviction — callers bound lifetime by resetting or recycling.

// pairPredK is the wire bound on the rank varuint emitted after
// tagStatePair. The current predictor stores top-1 only (K=1) so a
// hit always emits rank=0 and the decoder rejects any rank ≥ 1 as
// malformed. The constant stays around so the wire-side validation
// reads the same on both sides.
const pairPredK = 1

// Pair predictor storage: []uint32 indexed by prev intern ID. The
// stored value is `succ+1` so an empty slot is zero, which lets
// reset() use the runtime `clear()` builtin (a memclr) instead of a
// per-element loop. Sentinel choice trades 1 bit of representable
// range we never use (max intern id ≤ maxStateEntries < ^uint32(0)-1)
// for a cleaner reset path.
//
// Top-1 vs the previous K=4 ring: 4 bytes per slot down from 17, so
// at default maxStateEntries=16384 the predictor's residual capacity
// drops from ~280 KiB to 64 KiB per encoder (-77 %). Hit rate on
// strictly cyclic workloads (A→{B,C,D,E,B,C,…}) falls to zero, but
// on the common stable-transition case (A→B→A→B) top-1 catches every
// hit the ring did. Real telemetry workloads sit close to the
// stable case.
const pairPredEmpty uint32 = 0

// mruRingSize is the side-cache covering recent emit history for fast
// MTF rank discovery. Scanning a contiguous 128-entry uint16 ring is
// dominated by sequential L1 hits (cache-prefetcher friendly) and
// completes in roughly N×1.5ns. The canonical LRU linked list still
// holds every ID; the ring just shortcuts the rank-walk on the hot
// path. 128 covers the full 1-byte rank space (uvarintLen(rank)==1
// for rank ≤ 127), which is exactly the range where MTF beats the
// raw 2-byte state-ref encoding most ids land in.
//
// uint16 not uint32: the encoder caps maxStateEntries at 1<<14 (=
// 16 384), well under the uint16 range. Halving the slot width packs
// the whole ring into 256 B (4 × 64 B cache lines) instead of 512 B
// — a sequential scan reads half as many lines and finishes faster
// on the same prefetcher budget. mruEmpty (0xFFFF) is reserved as
// the "no id here" sentinel; the runtime id space stays below it.
//
// Power of two so the modulo collapses to a single AND.
const (
        mruRingSize        = 128
        mruRingMask        = mruRingSize - 1
        mruEmpty    uint16 = 0xFFFF
)

type encState struct { // betteralign:ignore — hot-scalar-first layout is cache-critical; do not reorder
        // Hot scalars first so they share a single cache line with the
        // adjacent mruHead and the map header. lastID + lruHead + mruHead
        // are touched on every state-ref emit; co-locating them with
        // mruRing keeps the per-emit footprint at 1-2 cache lines.
        lastID     uint32
        lruHead    uint32
        mruHead    uint32
        internLoad uint32 // number of occupied slots in internTable

        // mruRing is a side-cache of the last mruRingSize state-ref emit
        // IDs. Each new emit pushes its ID at mruRing[mruHead&mask] and
        // bumps mruHead. Rank discovery scans backwards from mruHead: a
        // hit at offset r means r OTHER ids were emitted after this one,
        // which is exactly the LRU chain rank. Storing the ring as uint16
        // (id space fits) cuts the scan footprint to 256 B / 4 cache
        // lines.
        mruRing [mruRingSize]uint16

        // internTable is a flat open-addressed hash table replacing the
        // old map[string]uint32. See internSlot for layout. Hot —
        // touched on every WriteString. Linear probing keeps the access
        // pattern sequential and predictable; load is kept below 0.5 by
        // doubling on growth so probe chains stay short (typically 1).
        internTable []internSlot

        // Move-to-front LRU over intern IDs. lruHead is the ID at rank 0
        // (most recently emitted state-ref or freshly interned). lruLink
        // packs the previous + next ids for each chain slot into a single
        // uint32 (low 16 bits = prev, high 16 bits = next). IDs are
        // bounded by maxStateEntries (1 << 14) so they fit in 16 bits
        // with 0xFFFF reserved as the "no neighbour" sentinel. Packing
        // halves the cache lines a lruMoveFront has to touch (one array
        // instead of two) while keeping the unlink/insert update O(1).
        lruLink []uint32

        // Pair predictor: top-1 successor per prev intern ID. Slot stores
        // `succ+1` so zero = empty; this keeps Reset() on a memclr fast
        // path while preserving the ability to predict succ==0 cleanly.
        // See the pairPredEmpty / pairPredK constants and the
        // pairLookup / pairRecord methods for the lookup contract.
        pairPred []uint32

        // Shape table for tagMapShape. shapes is indexed by id-1 (id 0 is
        // reserved on the wire to mean "declare"). shapeBindings is a
        // small linear-scan registry of (typeDesc → wire ID). Typical
        // streams emit a handful of struct types, so a slice beats a map
        // on lookup cost AND keeps the race-detector instrumentation off
        // the hot path. lastShapeTd / lastShapeID memoise the
        // most-recent successful shape lookup; shapeCount is the running
        // total of declared shape IDs.
        lastShapeTd   *typeDesc
        lastShapeID   uint32
        shapeCount    uint32
        shapeBindings []shapeBinding

        // tokenShapes interns recurring struct shapes for code-generated types,
        // which have no *typeDesc — keyed on a stable per-type token address the
        // generated EncodeQDF passes (see Encoder.StructShape). Shares the shapeCount
        // ID space with shapeBindings / mapShapes so the decoder's single shape table
        // stays in lockstep. lastTokenPtr/lastTokenID memoise the most recent lookup
        // so a homogeneous slice (the common case: one token, N elements threaded
        // through one encoder) hits a single pointer compare, mirroring lastShapeTd.
        tokenShapes  []tokenShape
        lastTokenPtr *byte
        lastTokenID  uint32
        // lastMapShapeID memoises the most recently used map shape so a run of
        // homogeneous rows (the common case) verifies against it directly — no
        // set-hash recompute, no registry scan (see lastMapShapeKeys). 0 means none.
        // Packed adjacent to lastTokenID so the two cold uint32 memo IDs share one
        // 8-byte word instead of each stranding a 4-byte pad before the next slice.
        lastMapShapeID uint32

        // mapShapes interns recurring map key-sets (OptMapShape), parallel to
        // shapeBindings but keyed on the key-set rather than a *typeDesc. Shares
        // the shapeCount ID space with struct shapes (shapeDeclareEnc) so the
        // decoder's single shape table stays in lockstep.
        mapShapes []mapShapeBinding
        // lastMapShapeKeys holds the key order of the shape memoised by
        // lastMapShapeID, so a homogeneous run skips the registry scan.
        lastMapShapeKeys []string
        // lastMapShapeIdx is the index into mapShapes for the shape memoised by
        // lastMapShapeID. Gives O(1) access to the binding's pre-allocated valSlots
        // without a registry scan. Valid only when lastMapShapeID != 0.
        lastMapShapeIdx int
        // mapEnc pools reflect holders for the generic (reflect) string-keyed map
        // encode path so it does not reflect.New per map (OptMapShape).
        mapEnc mapHolderCache

        // Columnar shape table (tagColStruct). Separate from shapeBindings
        // because columnar shapes carry field kinds. Keyed by structural
        // identity (names + kinds) since the same struct type always produces
        // the same columnar shape.
        colShapeNames [][]string
        colShapeKinds [][]colKind
        // Hybrid columnar shape table (tagHybridColStruct). Separate ID space from
        // colShapeNames/colShapeKinds so hybrid and pure-columnar shapes never alias
        // within a stream. kinds carry residualKind (0xFF) for residual fields.
        hybridShapeNames [][]string
        hybridShapeKinds [][]colKind
        // Pooled transpose scratch, reused across columns and across calls.
        colScratchI64  []int64
        colScratchU64  []uint64
        colScratchF64  []float64
        colScratchBool []bool
        colScratchF32  []float32 // codegen columnar float32 gather scratch
        colScratchStr  []string  // gathered string column values
        // Canonical map-key sort scratch (OptCanonical), reused across maps; adaptive
        // retention (dropped when oversized in the encoder pool reset, like
        // colScratch*). Pointer-free numeric scratch needs no clear; canonKeysStr
        // holds caller string headers, so it is cleared on reset to drop references.
        canonKeysStr []string
        canonKeysI64 []int64
        canonKeysU64 []uint64
        // Canonical float-slice normalization scratch (OptCanonical): when a
        // []float64/[]float32 contains -0.0 or NaN, the normalized copy lands here
        // (never mutating the caller's slice). Pointer-free, adaptive retention.
        canonFloat64 []float64
        canonFloat32 []float32
        // Columnar column-level diff scratch (delta_columnar.go). deltaColBitmap is
        // the per-row changed bitmap; deltaColRows the changed-row indices for one
        // column; deltaColBuf the built tagColSlicePatch body for the never-larger
        // compare. Row-scaled, retained/dropped like the colScratch* above.
        deltaColBitmap []uint64
        deltaColRows   []int
        deltaColBuf    []byte
        // deltaColAux* hold the OLD column gathered contiguously for the arithmetic
        // delta encode (new − old), so both operands are width-hoisted gathers fed to
        // a vectorizable subtract instead of two per-element width switches.
        deltaColAuxI64 []int64
        deltaColAuxU64 []uint64
        colDictTable   []string // distinct table for the string-dict codec
        colMaskScratch []byte   // presence bitmap for nullable columns
        // FSST codec scratch, reused across columns (same lifetime as colDictTable).
        fsstScratch []byte   // compressed bytes for all rows, concatenated
        fsstLens    []int    // per-row compressed lengths
        fsstSamples [][]byte // per-column []byte views fed to the FSST trainer
        // strDictMap maps a string column's distinct values to dense indices
        // while the string-dict codec decides/encodes. Reused (cleared) per
        // column to avoid a per-column map allocation.
        strDictMap map[string]uint32

        // canonKeysBusy guards the pooled canonKeys* scratch against re-entrancy: a
        // map whose values contain maps recurses into the gather mid-iteration and
        // would clobber the outer map's sorted-key slice. When busy, the gather falls
        // back to a fresh local slice (like mapHolderCache.busy). Flat maps — the
        // common case — keep the zero-alloc pooled path. Packed next to retainStreak
        // (both cold 1-byte flags) so the two share one word instead of each
        // stranding a 7-byte pad before the following 8-byte field.
        canonKeysBusy bool

        // retainStreak counts consecutive small (sub-cap) messages for the
        // adaptive-retention policy in reset(). Cold — touched once per reset.
        retainStreak uint8

        // arenaSmallStreak is the arena's own analogue of retainStreak, driven by
        // the byte volume interned per message (internarena.DefaultRetainBytes)
        // rather than the intern-id count (maxRetainedIDs). The two signals diverge:
        // a batch can intern < maxRetainedIDs distinct keys yet still push the arena
        // past its byte cap (long keys), so reusing retainStreak would shed the
        // arena's spike slabs every steady batch in that band. Cold — touched once
        // per reset. Packs into the same word as the two flags above (no size cost).
        arenaSmallStreak uint8

        // arena owns the byte storage that backs every intern key the
        // encoder allocates — accessed only on intern miss, kept at the
        // end so the hot fields above share earlier cache lines.
        arena internarena.Arena
}

// shapeBinding is a (typeDesc → wire shape ID) pair. Stored in a
// linear-scan slice on the encoder state so the hot path adds no map
// access (and no allocation under -race).
type shapeBinding struct {
        td *typeDesc
        id uint32
}

// tokenShape is a (per-type token address → wire shape ID) pair, the
// code-generated analogue of shapeBinding (which keys on *typeDesc). The token
// is a stable package-level address the generated EncodeQDF passes; a linear
// scan keeps the hot path map-free and allocation-free under -race.
type tokenShape struct {
        token *byte
        id    uint32
}

// mapShapeBinding maps a recurring map key-set to a shared shape ID.
// setHash is an order-independent hash of the (string) keys; n is the key
// count (disambiguates a setHash collision across different sizes); keys holds
// the canonical (sorted) key order, cloned so it survives the caller's map. id
// is drawn from the same sequential space as struct shapeBindings
// (shapeDeclareEnc).
//
// valSlots is a pre-allocated []reflect.Value (one per key, type valType) used
// by hasAllAndCollect to receive values via SetIterValue — no per-key heap
// allocation. Initialised lazily on first encode of this shape.
type mapShapeBinding struct {
        valType  reflect.Type // element type of valSlots; zero if unset
        keys     []string
        valSlots []reflect.Value // pre-allocated value holders; see ensureValueSlots
        setHash  uint64
        n        int
        id       uint32
        // busy is true while emitSlotsOrdered is iterating this binding's valSlots.
        // hasAllAndCollect must not be called (and thus ensureValueSlots must not
        // reinitialise) while busy, to prevent a re-entrant nested encode from
        // corrupting the outer encode's in-progress slot array.
        busy bool
}

// ensureValueSlots initialises the pre-allocated reflect.Value holders used
// for zero-alloc MapRange value collection via SetIterValue. It is a no-op
// when slots are already allocated for the same value type; it reinitialises
// only when the type changes (rare: two maps with identical key-sets but
// different value types sharing the same binding).
func (b *mapShapeBinding) ensureValueSlots(vt reflect.Type) {
        if b.valType == vt && b.valSlots != nil {
                return
        }
        b.valType = vt
        b.valSlots = make([]reflect.Value, len(b.keys))
        for i := range b.valSlots {
                b.valSlots[i] = reflect.New(vt).Elem()
        }
}

// mapHolderCache pools the addressable reflect.Value scratch the generic
// (reflect) map encode/decode path needs — a key holder + a value holder —
// so it does not reflect.New on every map encoded/decoded. Reused across the
// rows of a []struct (same map type every row), so a 1000-row batch pays 2
// reflect.New total instead of 2 per row. A busy flag keeps it
// re-entrancy-safe: a nested map (e.g. map[string]map[string]T), or any
// acquire while the cache is already in use, falls back to a fresh local pair.
type mapHolderCache struct {
        kt, vt reflect.Type
        vp     unsafe.Pointer
        kh, vh reflect.Value
        busy   bool
}

func (c *mapHolderCache) acquire(kt, vt reflect.Type) (kh, vh reflect.Value, vp unsafe.Pointer, pooled bool) {
        if c.busy {
                vh = reflect.New(vt).Elem()
                return reflect.New(kt).Elem(), vh, unsafe.Pointer(vh.UnsafeAddr()), false
        }
        if c.kt != kt || c.vt != vt {
                c.kt, c.vt = kt, vt
                c.kh = reflect.New(kt).Elem()
                c.vh = reflect.New(vt).Elem()
                c.vp = unsafe.Pointer(c.vh.UnsafeAddr())
        }
        c.busy = true
        return c.kh, c.vh, c.vp, true
}

func (c *mapHolderCache) release(pooled bool) {
        if pooled {
                c.busy = false
        }
}

const lruInvalidID = ^uint32(0)

// maxInternEntries is the hard ceiling on the intern-table size. Intern ids are
// packed into uint16 fields in the MRU ring and the LRU prev/next links, with
// 0xFFFF reserved as the "empty / no-neighbour" sentinel (mruEmpty,
// lruLink16Invalid). The largest assignable id must therefore stay below 0xFFFF.
// The assign gate is `internLoad < maxStateEntries`, so capping maxStateEntries
// at 0xFFFF yields a max id of 0xFFFE — one below the sentinel. A larger cap
// would let id 0xFFFF collide with the sentinel and corrupt the LRU/MRU chains
// (silent wrong-string resolution on later state-refs).
const maxInternEntries = 0xFFFF

func newEncState() *encState {
        // arena is zero-value initialised here — its slab is lazily
        // allocated on first Put (see internarena.Arena.Put).
        e := &encState{
                internTable: make([]internSlot, internTableInitSize),
                lruHead:     lruInvalidID,
                lastID:      lruInvalidID,
        }
        // Prime ring with sentinels so a scan never matches id 0 by
        // accident before the ring has been written.
        for i := range e.mruRing {
                e.mruRing[i] = mruEmpty
        }
        return e
}

// Soft caps on per-encoder state retention across Reset(). A single
// payload that pushes any of these past their threshold is dropped
// rather than pinned to the pooled encoder forever — long-running
// services with bursty traffic keep a bounded resident set instead
// of growing to peak and staying there.
//
// Numbers picked so that a "typical" telemetry batch (≤ a few
// thousand interned strings, ≤ a few thousand state-ref ids) stays
// under every cap; only outlier payloads trigger the shrink.
const (
        maxRetainedIDs      = 4096
        maxRetainedLRUCap   = 4096
        maxRetainedPairCap  = 4096
        maxRetainedShapeCap = 1024

        // retainReleaseStreak governs adaptive retention (see encState.reset /
        // decState.reset). A pooled state RETAINS oversized backing arrays —
        // clearing them in place instead of dropping them to nil — while
        // consecutive messages stay large, so a steady high-cardinality /
        // large-batch workload amortizes the table allocation instead of
        // reallocating every single message (the dominant encode-alloc cost on
        // AD/log/telemetry batches). Only after this many consecutive SMALL
        // (sub-cap) messages does it conclude the burst subsided and release the
        // memory. sync.Pool's GC-driven eviction bounds idle retention regardless.
        retainReleaseStreak = 8

        // maxRetainedColScratch hard-caps the row-count-scaled columnar scratch
        // arrays (colScratch*, colDictTable, colMaskScratch, fsstScratch): a
        // backing larger than this is dropped, bounding worst-case pooled memory
        // after a one-off giant columnar batch. 1<<17 (131072 rows) covers any
        // realistic batch while capping the per-encoder pin at ~2 MB for the
        // []string scratch (16 B/elem). The intern/LRU/pair arrays need no such
        // ceiling — maxStateEntries (≤ 0xFFFF) already bounds them to ~1-4 MB.
        maxRetainedColScratch = 1 << 17
)

func (e *encState) reset() {
        // Adaptive retention. A pooled encoder that just handled a large
        // (high-cardinality / wide-batch) message would, under a fixed cap, drop
        // its grown backing arrays and reallocate them from scratch on the very
        // next message — so a STEADY large workload (AD / log / telemetry sync)
        // never amortizes the table allocation and pays a full table regrow every
        // message (the dominant encode-alloc cost measured on such batches).
        // Instead we keep the intern/LRU/pair/shape backings (clearing them in
        // place) while messages stay large, releasing only after
        // retainReleaseStreak consecutive small messages. The decision is driven
        // by internLoad — the count of strings interned THIS message, a true
        // per-message demand signal (the retained cap would stay large forever
        // and never let the streak advance). The row-scaled columnar scratch is
        // governed separately by a hard ceiling (maxRetainedColScratch): retained
        // up to it, dropped above, so a one-off giant batch can't pin unbounded
        // memory. The intern/LRU/pair arrays are already bounded by
        // maxStateEntries; sync.Pool's GC eviction caps idle retention regardless.
        if int(e.internLoad) > maxRetainedIDs {
                e.retainStreak = 0
        } else if e.retainStreak < retainReleaseStreak {
                e.retainStreak++
        }
        release := e.retainStreak >= retainReleaseStreak

        // Intern table. Clear (zero every slot — internSlot{} is the empty
        // sentinel) to reuse in place; drop the backing only when releasing.
        //
        // Order: clear / rebuild BEFORE arena.Reset. The slot.key fields alias
        // arena bytes; arena.Reset rolls cursors back and the next Put overwrites
        // the prior payload area, so any surviving aliased key would read garbage.
        if cap(e.internTable) > maxRetainedIDs*2 && release {
                e.internTable = make([]internSlot, internTableInitSize)
        } else {
                clear(e.internTable)
        }
        e.internLoad = 0
        // Adaptive arena retention. The default Reset() soft cap (256 KiB) drops the
        // spike slabs a high-cardinality AD/log batch just grew, forcing a full arena
        // regrow every batch — the dominant streaming-encode allocation (~181 KB/value
        // measured on high-card AD data, where each per-batch StreamEncoder.Reset
        // sheds and the next batch rebuilds the slabs). While a steady large-volume
        // workload keeps filling the arena past the cap, retain every slab —
        // ResetWithLimit(0) rolls the cursor back to chunks[0] and keeps the spike
        // chunks so the next same-shaped batch reuses them in place. Resident memory
        // stays bounded by the single-batch peak (the cursor resets each batch and
        // grow() walks the existing slabs before allocating).
        //
        // The trigger is the arena's OWN per-message byte demand (BytesPut, the
        // payload volume interned THIS batch — an O(1) counter, read before the
        // cursor rolls back), not the intern-id streak above: a batch can intern
        // fewer than maxRetainedIDs keys yet still exceed the byte cap with long
        // keys, so the id-based `release` would shed the arena's slabs every steady
        // batch in that band. Only after retainReleaseStreak consecutive sub-cap
        // batches (burst genuinely subsided) do we fall back to the default cap and
        // shed the spike memory, so a one-shot/bursty pool encoder still bounds its
        // resident set. Under a sustained streak the resident set is bounded by the
        // streak-window peak (plus doubling slack), reclaimed by the sub-cap shed and
        // sync.Pool's GC eviction. The intern table was already cleared above,
        // dropping every aliased slot.key, so rolling the cursor back is safe at any
        // retain limit.
        if e.arena.BytesPut() > internarena.DefaultRetainBytes {
                e.arenaSmallStreak = 0
        } else if e.arenaSmallStreak < retainReleaseStreak {
                e.arenaSmallStreak++
        }
        if e.arenaSmallStreak >= retainReleaseStreak {
                e.arena.Reset() // default 256 KiB soft cap — burst subsided, shrink.
        } else {
                e.arena.ResetWithLimit(0) // large-volume streak: keep slabs warm.
        }

        e.lastID = lruInvalidID
        e.lruHead = lruInvalidID
        if cap(e.lruLink) > maxRetainedLRUCap && release {
                e.lruLink = nil
        } else {
                e.lruLink = e.lruLink[:0]
        }

        // pairPred slice: clear in place (memclr-fast because the empty sentinel
        // is zero); drop the backing array only when releasing.
        if cap(e.pairPred) > maxRetainedPairCap && release {
                e.pairPred = nil
        } else {
                clear(e.pairPred)
        }

        e.shapeCount = 0
        if cap(e.shapeBindings) > maxRetainedShapeCap && release {
                e.shapeBindings = nil
        } else {
                e.shapeBindings = e.shapeBindings[:0]
        }
        e.lastShapeTd = nil
        e.lastShapeID = 0
        if cap(e.tokenShapes) > maxRetainedShapeCap && release {
                e.tokenShapes = nil
        } else {
                e.tokenShapes = e.tokenShapes[:0]
        }
        e.lastTokenPtr = nil
        e.lastTokenID = 0
        if cap(e.mapShapes) > maxRetainedShapeCap && release {
                e.mapShapes = nil
        } else {
                // Each binding holds keys []string aliasing caller strings; a bare [:0]
                // leaves those headers live in the backing array and pins the strings
                // until overwritten. Clear the FULL backing first (mirrors colDictTable).
                clear(e.mapShapes[:cap(e.mapShapes)])
                e.mapShapes = e.mapShapes[:0]
        }
        e.lastMapShapeID = 0
        e.lastMapShapeKeys = nil
        e.lastMapShapeIdx = 0
        e.mapEnc = mapHolderCache{}

        if cap(e.colShapeNames) > maxRetainedShapeCap && release {
                e.colShapeNames = nil
                e.colShapeKinds = nil
        } else {
                e.colShapeNames = e.colShapeNames[:0]
                e.colShapeKinds = e.colShapeKinds[:0]
        }
        if cap(e.hybridShapeNames) > maxRetainedShapeCap && release {
                e.hybridShapeNames = nil
                e.hybridShapeKinds = nil
        } else {
                e.hybridShapeNames = e.hybridShapeNames[:0]
                e.hybridShapeKinds = e.hybridShapeKinds[:0]
        }
        // Row-scaled columnar scratch: retained across batches (amortizes a steady
        // columnar workload, whose row count is independent of internLoad), dropped
        // only past the hard ceiling so a one-off giant batch can't pin unbounded
        // memory. sync.Pool's GC eviction reclaims it when the encoder goes idle.
        // Each backing grows on its own per-column-type demand (a batch of only
        // float64 columns grows colScratchF64 while colScratchI64 stays at 0), so
        // gate each independently — a single check keyed on colScratchI64 would miss
        // an oversized U64/F64/Bool backing and pin double the intended scratch (same
        // class as the deltaColAux* fix below).
        if cap(e.colScratchI64) > maxRetainedColScratch {
                e.colScratchI64 = nil
        }
        if cap(e.colScratchU64) > maxRetainedColScratch {
                e.colScratchU64 = nil
        }
        if cap(e.colScratchF64) > maxRetainedColScratch {
                e.colScratchF64 = nil
        }
        if cap(e.colScratchBool) > maxRetainedColScratch {
                e.colScratchBool = nil
        }
        if cap(e.colScratchF32) > maxRetainedColScratch {
                e.colScratchF32 = nil
        }
        // deltaColAux* are swapped with colScratchI64/U64 in encodeDeltaColumn, so
        // after a large columnar-delta batch one of the two grown backings lives here
        // and is missed by the colScratchI64 check above — cap them independently or
        // the pooled encoder retains double the intended scratch.
        if cap(e.deltaColAuxI64) > maxRetainedColScratch {
                e.deltaColAuxI64 = nil
        }
        if cap(e.deltaColAuxU64) > maxRetainedColScratch {
                e.deltaColAuxU64 = nil
        }
        // Column-diff scratch (delta_columnar.go): same row-scaled hard-ceiling
        // retention as colScratch* (pointer-free, no clear needed).
        if cap(e.deltaColBitmap) > maxRetainedColScratch {
                e.deltaColBitmap = nil
        }
        if cap(e.deltaColRows) > maxRetainedColScratch {
                e.deltaColRows = nil
        }
        if cap(e.deltaColBuf) > maxRetainedColScratch {
                e.deltaColBuf = nil
        }
        if cap(e.fsstScratch) > maxRetainedColScratch {
                e.fsstScratch = nil
                e.fsstLens = nil
        }
        // fsstSamples holds []byte views into caller strings; clear the headers to
        // drop those references across a pool recycle, drop backing past the ceiling.
        if cap(e.fsstSamples) > maxRetainedColScratch {
                e.fsstSamples = nil
        } else {
                clear(e.fsstSamples)
                e.fsstSamples = e.fsstSamples[:0]
        }
        // String-column scratch: []string slices retain header references that pin
        // the caller's string memory across a pool recycle. clear() drops those
        // headers while keeping the backing; drop the backing only past the ceiling.
        if cap(e.colScratchStr) > maxRetainedColScratch {
                e.colScratchStr = nil
        } else {
                // Clear across the FULL backing, not just len: the gather reslices via
                // [:0] per column, so a column with fewer rows than an earlier one leaves
                // a high-water tail of headers aliasing the caller's (now possibly dead)
                // struct strings, pinning them from GC across the pool recycle.
                clear(e.colScratchStr[:cap(e.colScratchStr)])
                e.colScratchStr = e.colScratchStr[:0]
        }
        // Canonical map-key sort scratch (OptCanonical): numeric scratch is
        // pointer-free (drop only past the ceiling); canonKeysStr holds caller
        // string headers, so clear them to drop references across a pool recycle.
        // canonKeysI64/U64 grow independently (a uint64-key-map workload grows
        // canonKeysU64 while canonKeysI64 stays empty), so gate each on its own cap —
        // a single check keyed on canonKeysI64 would never drop an oversized
        // canonKeysU64. Both are pointer-free: drop only past the ceiling.
        if cap(e.canonKeysI64) > maxRetainedColScratch {
                e.canonKeysI64 = nil
        }
        if cap(e.canonKeysU64) > maxRetainedColScratch {
                e.canonKeysU64 = nil
        }
        if cap(e.canonKeysStr) > maxRetainedColScratch {
                e.canonKeysStr = nil
        } else {
                // Clear across the FULL backing, not just len: a shorter key-set leaves a
                // high-water tail of headers aliasing the caller's map key strings.
                clear(e.canonKeysStr[:cap(e.canonKeysStr)])
                e.canonKeysStr = e.canonKeysStr[:0]
        }
        // Canonical float-slice scratch is pointer-free: drop only past the ceiling.
        // canonFloat64/canonFloat32 grow independently (a dirty []float32 workload
        // grows canonFloat32 while canonFloat64 stays empty), so gate each on its own
        // cap — a single check keyed on canonFloat64 would never drop an oversized
        // canonFloat32.
        if cap(e.canonFloat64) > maxRetainedColScratch {
                e.canonFloat64 = nil
        }
        if cap(e.canonFloat32) > maxRetainedColScratch {
                e.canonFloat32 = nil
        }
        if cap(e.colDictTable) > maxRetainedColScratch {
                e.colDictTable = nil
        } else {
                // []string resliced via [:0] per column, so a shorter column leaves a
                // high-water tail of headers aliasing caller strings; clear the FULL
                // backing (mirrors the decoder's colDictTableScr reset).
                clear(e.colDictTable[:cap(e.colDictTable)])
                e.colDictTable = e.colDictTable[:0]
        }
        if cap(e.colMaskScratch) > maxRetainedColScratch {
                e.colMaskScratch = nil
        } else {
                e.colMaskScratch = e.colMaskScratch[:0]
        }
        if len(e.strDictMap) > 0 {
                clear(e.strDictMap)
        }

        // Ring side-cache: re-prime with sentinels so post-reset emits
        // can't false-match a stale id 0.
        for i := range e.mruRing {
                e.mruRing[i] = mruEmpty
        }
        e.mruHead = 0
}

// shapeForType returns the wire shape ID bound to t in this encoder's
// state, or 0 if none. Pair with shapeBindType after a declaration.
//
//go:nosplit
func (e *encState) shapeForType(t *typeDesc) uint32 {
        if e.lastShapeTd == t && e.lastShapeID != 0 {
                return e.lastShapeID
        }
        for i := range e.shapeBindings {
                if e.shapeBindings[i].td == t {
                        id := e.shapeBindings[i].id
                        e.lastShapeTd = t
                        e.lastShapeID = id
                        return id
                }
        }
        return 0
}

func (e *encState) shapeBindType(t *typeDesc, id uint32) {
        e.shapeBindings = append(e.shapeBindings, shapeBinding{td: t, id: id})
        e.lastShapeTd = t
        e.lastShapeID = id
}

// shapeForToken returns the wire shape ID bound to a code-generated type's token
// address in this encoder's state, or 0 if none. Pair with shapeBindToken after
// a declaration.
func (e *encState) shapeForToken(token *byte) uint32 {
        if e.lastTokenPtr == token && e.lastTokenID != 0 {
                return e.lastTokenID
        }
        for i := range e.tokenShapes {
                if e.tokenShapes[i].token == token {
                        e.lastTokenPtr = token
                        e.lastTokenID = e.tokenShapes[i].id
                        return e.tokenShapes[i].id
                }
        }
        return 0
}

func (e *encState) shapeBindToken(token *byte, id uint32) {
        e.tokenShapes = append(e.tokenShapes, tokenShape{token: token, id: id})
        e.lastTokenPtr = token
        e.lastTokenID = id
}

// shapeDeclareEnc reserves the next sequential wire ID and returns
// it. Caller emits the keys on the wire; this side only tracks the
// count to keep IDs aligned with the decoder.
func (e *encState) shapeDeclareEnc() uint32 {
        e.shapeCount++
        return e.shapeCount
}

// mapShapeRegister binds a key-set to a shape ID. keys must be the canonical
// (sorted) order and is taken over by the binding — the caller must not reuse or
// mutate it afterward. Both callers pass a freshly-allocated per-declare slice
// (the strings alias the caller's map keys, exactly as a clone would have), so
// ownership transfer drops one []string allocation per first-sight key-set.
func (e *encState) mapShapeRegister(setHash uint64, n int, keys []string, id uint32) {
        e.mapShapes = append(e.mapShapes, mapShapeBinding{setHash: setHash, n: n, keys: keys, id: id})
}

// pairLookup reports whether the top-1 predicted successor of prev
// is curr. The wire emits a rank byte after tagStatePair that is
// always 0 in the top-1 design — callers hand-write the literal
// instead of consuming a rank return value.
//
//go:nosplit
func (e *encState) pairLookup(prev, curr uint32) bool {
        if int(prev) >= len(e.pairPred) {
                return false
        }
        return e.pairPred[prev] == curr+1
}

// pairEnsure grows the predictor slice so prev is a valid index. New
// slots default to pairPredEmpty (zero) via the runtime's append
// zero-fill — no extra initialisation needed.
//
//go:nosplit
func (e *encState) pairEnsure(prev uint32) {
        for uint32(len(e.pairPred)) <= prev {
                e.pairPred = append(e.pairPred, pairPredEmpty)
        }
}

// pairRecord installs curr as the top-1 successor of prev. Always
// overwrites — the predictor remembers the most recent transition
// only.
//
//go:nosplit
func (e *encState) pairRecord(prev, curr uint32) {
        e.pairEnsure(prev)
        e.pairPred[prev] = curr + 1
}

// mruPush records id as the newest entry in the side-cache ring.
// Overwrites the slot at mruHead and advances the head. The ring is
// power-of-two sized so the modulo collapses to an AND. Caller
// guarantees id < mruEmpty (maxStateEntries cap ensures this).
//
//go:nosplit
func (e *encState) mruPush(id uint32) {
        e.mruRing[e.mruHead&mruRingMask] = uint16(id)
        e.mruHead++
}

// mruRank scans the ring from newest to oldest looking for id. If
// found at offset r from the head, r equals the current LRU chain
// rank (since every state-ref emit is recorded in the ring in
// order). Returns (rank, true) on hit, (0, false) when id is not in
// the last mruRingSize emissions — in which case the caller falls
// back to the raw state-ref encoding (chain rank is necessarily
// ≥ mruRingSize and would need a multi-byte varuint anyway).
//
// Hand-unrolled 4-way: profiling on telemetry workloads showed the
// scalar loop at ~17 % flat (top hotspot post the May 2026
// series). The unroll amortises the back-edge branch, lets the CPU
// issue 4 independent loads per iteration, and keeps the typical
// low-rank early-exit semantics. Falls back to scalar for the
// final partial iteration when mruRingSize is not a multiple of 4
// (it is at 128, but the guard keeps the function correct under
// future ring-size changes).
//
//go:nosplit
func (e *encState) mruRank(id uint32) (uint32, bool) {
        // IDs above the uint16 representable range can never appear in
        // the uint16 ring; short-circuit so an oversized id (only
        // reachable if maxStateEntries was bumped) never false-hits the
        // mruEmpty sentinel.
        if id >= uint32(mruEmpty) {
                return 0, false
        }
        target := uint16(id)
        h := e.mruHead - 1 // newest emission lives at h after this offset
        r := uint32(0)
        for ; r+3 < mruRingSize; r += 4 {
                if e.mruRing[(h-r)&mruRingMask] == target {
                        return r, true
                }
                if e.mruRing[(h-r-1)&mruRingMask] == target {
                        return r + 1, true
                }
                if e.mruRing[(h-r-2)&mruRingMask] == target {
                        return r + 2, true
                }
                if e.mruRing[(h-r-3)&mruRingMask] == target {
                        return r + 3, true
                }
        }
        for ; r < mruRingSize; r++ {
                if e.mruRing[(h-r)&mruRingMask] == target {
                        return r, true
                }
        }
        return 0, false
}

// lruLinkInvalid encodes (prev=0xFFFF, next=0xFFFF) — an isolated
// slot with no neighbours. Used as the append default when growing
// the lruLink slice.
const lruLinkInvalid uint32 = 0xFFFF | (0xFFFF << 16)
const lruLink16Invalid uint32 = 0xFFFF // 16-bit sentinel masked into a uint32

//go:nosplit
func linkPrev(link uint32) uint32 { return link & 0xFFFF }

//go:nosplit
func linkNext(link uint32) uint32 { return link >> 16 }

//go:nosplit
func setLinkPrev(link, prev uint32) uint32 { return (link &^ 0xFFFF) | (prev & 0xFFFF) }

//go:nosplit
func setLinkNext(link, next uint32) uint32 { return (link & 0xFFFF) | ((next & 0xFFFF) << 16) }

// lruAddFresh inserts a brand-new ID (just assigned) at the head of
// the LRU. Caller must have ensured id == len(ids)-1 (i.e. ids assigns
// sequentially starting from 0). Also records the emit in the MRU
// ring so the rank-discovery side-cache reflects the new chain head.
func (e *encState) lruAddFresh(id uint32) {
        for uint32(len(e.lruLink)) <= id {
                e.lruLink = append(e.lruLink, lruLinkInvalid)
        }
        head := e.lruHead
        // id.prev = invalid, id.next = head
        if head == lruInvalidID {
                e.lruLink[id] = lruLinkInvalid
        } else {
                e.lruLink[id] = lruLink16Invalid | (head << 16)
                // head.prev = id
                e.lruLink[head] = setLinkPrev(e.lruLink[head], id)
        }
        e.lruHead = id
        e.mruPush(id)
}

// lruMoveFront performs the unlink+insert-at-head update of the LRU
// but skips the rank walk. Use when the caller does not need the
// rank (e.g. raw state-ref where MTF cannot win). Also records the
// emit in the MRU ring so the rank side-cache mirrors the chain
// head update.
//
//go:nosplit
func (e *encState) lruMoveFront(id uint32) {
        if e.lruHead == id {
                e.mruPush(id)
                return
        }
        link := e.lruLink[id]
        p := linkPrev(link)
        n := linkNext(link)
        // p is always valid here (id was not head). Patch p.next = n.
        e.lruLink[p] = setLinkNext(e.lruLink[p], n)
        if n != lruLink16Invalid {
                // Patch n.prev = p.
                e.lruLink[n] = setLinkPrev(e.lruLink[n], p)
        }
        // Insert id at head: id.prev=invalid, id.next=head.
        head := e.lruHead
        e.lruLink[id] = lruLink16Invalid | (head << 16)
        // head.prev = id (head is always valid here — id was in chain).
        e.lruLink[head] = setLinkPrev(e.lruLink[head], id)
        e.lruHead = id
        e.mruPush(id)
}

// lookupOrAssign returns (id, hit). On a miss a fresh entry is
// installed and (id, false) is returned; the caller is expected to
// emit an intern record. The key bytes are copied into the encState
// arena so the table is independent of the caller's buffer.
//
// Uses the flat open-addressed hash table (internTable) — a single
// memhash + a couple of cache-line loads instead of Go's
// mapaccess2_faststr (hash + bucket walk + tophash + memequal).
// The stored slot.key aliases the arena copy via unsafestr.String
// so the encoder owns the bytes and the caller's buffer can be
// reused immediately after the call.
//
// For payloads longer than the arena's per-string limit
// (internarena.MaxStringLen, 65 535 bytes), fall back to
// strings.Clone. Such oversized intern attempts are not expected
// on real workloads; the path exists so a hostile input cannot
// crash the encoder.
//
//go:nosplit
func (e *encState) lookupOrAssign(key string) (uint32, bool) {
        // Hot-path fast lookup: hash + one slot probe. Inlinable (no
        // loop, no allocs); the slow tail (collision probing, miss-
        // install, grow) is split out so the linear-probing loop does
        // not pollute the inline budget. Hit rate at slot 0 is high
        // because the table is kept under 0.5 load.
        h := maphash.String(internHashSeed, key)
        if h == 0 {
                h = 1 // reserve 0 as the empty-slot sentinel
        }
        i := h & uint64(len(e.internTable)-1)
        slot := &e.internTable[i]
        if slot.hash == h && slot.key == key {
                return slot.id, true
        }
        if slot.hash == 0 {
                // Empty first slot: direct install.
                return e.installInternSlot(slot, h, key), false
        }
        // Collision at first slot — fall to the probing loop.
        return e.lookupOrAssignSlow(h, key, i)
}

// lookupOrAssignSlow handles the collision case: probe past startIdx
// looking for either an empty slot (install) or a matching entry
// (hit). Separated from lookupOrAssign so the inliner keeps the
// fast path tight.
func (e *encState) lookupOrAssignSlow(h uint64, key string, startIdx uint64) (uint32, bool) {
        mask := uint64(len(e.internTable) - 1)
        for i := (startIdx + 1) & mask; ; i = (i + 1) & mask {
                slot := &e.internTable[i]
                if slot.hash == 0 {
                        return e.installInternSlot(slot, h, key), false
                }
                if slot.hash == h && slot.key == key {
                        return slot.id, true
                }
        }
}

// installInternSlot writes a fresh entry into slot, copies the key
// into the encoder arena (so it survives the caller's buffer
// lifetime), bumps the LRU + intern counters, and grows the table
// when the load crosses 3/4. The slot pointer can be invalidated
// by the grow; callers must not touch it after this returns.
func (e *encState) installInternSlot(slot *internSlot, h uint64, key string) uint32 {
        id := e.internLoad
        var stored string
        if len(key) <= internarena.MaxStringLen {
                arenaID := e.arena.Put(key)
                stored = unsafestr.String(e.arena.Get(arenaID))
        } else {
                stored = strings.Clone(key)
        }
        slot.hash = h
        slot.key = stored
        slot.id = id
        e.internLoad++
        e.lruAddFresh(id)
        // Grow at 3/4 load, not 1/2. A denser table is smaller (better cache)
        // and rehashes less often; with the well-distributed maphash the longer
        // linear-probe chains cost less than the cache + rehash savings.
        // Measured -12.6% encode on the large-payload Archive profile (thousands
        // of interned strings), neutral on small/medium payloads, wire unchanged.
        if e.internLoad*4 >= uint32(len(e.internTable))*3 {
                e.internTableGrow()
        }
        return id
}

// internTableGrow doubles the flat hash table and rehashes every
// occupied slot. Called from installInternSlot when the load factor
// reaches 3/4. Amortised insert stays O(1); the denser table trades
// slightly longer probe chains for fewer rehashes and a smaller cache
// footprint (a net encode win on large, intern-heavy payloads).
func (e *encState) internTableGrow() {
        old := e.internTable
        newSize := len(old) * 2
        if newSize == 0 {
                newSize = internTableInitSize
        }
        e.internTable = make([]internSlot, newSize)
        mask := uint64(newSize - 1)
        for i := range old {
                if old[i].hash == 0 {
                        continue
                }
                for j := old[i].hash & mask; ; j = (j + 1) & mask {
                        if e.internTable[j].hash == 0 {
                                e.internTable[j] = old[i]
                                break
                        }
                }
        }
}

// decShape is the decoder-side mirror of an encShape. names is the
// resolved Go-string for each field in declaration order — used to
// dispatch values to struct fields when the shape is re-used.
type decShape struct {
        names []string
}

// decColShape is the decoder-side descriptor for a columnar struct shape
// (tagColStruct). Parallel to encState's colShapeNames/colShapeKinds entries.
type decColShape struct {
        names []string
        kinds []colKind
}

// Field order: every pointer-bearing field (the slices and mapDec, which
// holds reflect handles) is grouped FIRST so the GC pointer-scan range stays
// tight (440 pointer bytes vs 720 for a hot-scalars-first layout); the
// non-pointer hot scalars + mruRing + retainStreak trail at the end. The
// scalars stay packed together (lastID/lruHead/mruHead/mruRing) so the
// per-emit MTF update still touches one contiguous span.
type decState struct {
        // values holds the decoded byte slices indexed by intern id —
        // each entry aliases the wire buffer (zero-copy).
        values [][]byte

        // stringValues caches a heap-allocated `string` copy per intern
        // record. Populated once at append time (one `string(b)` alloc
        // per first occurrence); subsequent state-ref / MTF / pair /
        // repeat reads return the cached string from Decoder.ReadString
        // without paying another `string(b)` copy each time. On dense
        // payloads with repeated values (telemetry, archive,
        // LargePayload) this collapses N-1 of every N reads to zero
        // alloc.
        stringValues []string

        // boxValues is the interface-boxing analogue of stringValues, indexed
        // by the same intern id and grown in lockstep (one nil placeholder per
        // record). decodeAny boxes a repeated string's `any(s)` exactly once and
        // caches it here; every later state-ref / MTF / pair / repeat occurrence
        // of that value returns the shared box with zero allocation. A unique
        // (never-referenced) string arrives inline with lastID == lruInvalidID and
        // never touches this cache, so high-cardinality payloads pay no lookup
        // overhead — the wire's own intern encoding is the adaptivity signal.
        // Sharing an immutable boxed scalar across map/slice slots is safe.
        boxValues []any

        // zeroTimeBox caches the boxed `any` of the zero time.Time (the value
        // unset time fields — DeletedAt, ExpiresAt, … — decode to en masse). It is
        // a universal, immutable constant, so it is boxed once and shared across
        // every occurrence AND across decodes on this pooled state (never reset):
        // one alloc replaces N. decodeAny gates on time.IsZero(), a single cheap
        // branch, so non-zero timestamps pay nothing.
        zeroTimeBox any

        // LRU mirror of encState's. Decoder maintains the same MTF chain
        // the encoder did so tagStateMTF + rank resolves to the same ID.
        // See encState.lruLink for the packing layout.
        lruLink []uint32

        // Pair predictor mirror. See encState.pairPred for the storage
        // layout (succ+1 packed into a uint32, 0 = empty). The decoder
        // only ever reads rank 0; any rank ≥ pairPredK on the wire is
        // rejected upstream as malformed.
        pairPred []uint32

        // Shape table mirror. shapes[i] is the shape with wire-ID i+1.
        shapes []decShape

        // Columnar shape table (tagColStruct). colShapes[i] is the columnar
        // shape with wire-ID i+1. Parallel to encState's colShapeNames/colShapeKinds.
        colShapes []decColShape
        // Hybrid columnar shape table (tagHybridColStruct), separate ID space from
        // colShapes. kinds carry residualKind (0xFF) for residual fields.
        hybridShapes   []decColShape
        colScratchI64  []int64
        colScratchU64  []uint64
        colScratchF64  []float64
        colScratchBool []bool
        colScratchF32  []float32 // codegen columnar float32 scatter scratch
        colScratchStr  []string  // codegen columnar plain string scatter scratch
        colStrHandles  []Str     // string column handle scratch (mirrors colScratchI64 pattern)
        // String-dict column scratch, reused across columns. Both are transient: the
        // dispatcher copies table[idx[i]] into the column result before reading the
        // next column, so neither is aliased past one column read.
        colDictTableScr []string // distinct-value table for a string-dict column
        colDictIdxScr   []uint32 // per-row dictionary index for a string-dict column

        // deltaColRows is reused storage for a sparse column's decoded ascending row
        // indices during a tagColSlicePatch apply (delta_columnar.go).
        deltaColRows []int

        // colLenScratch is reused storage for the column-length index parsed from
        // a FlagColIndex columnar payload (one uint32 byte-length per column).
        colLenScratch []uint32

        // mapDec pools reflect holders for the generic (reflect) string-keyed map
        // decode path so it does not reflect.New per map entry (OptMapShape).
        mapDec mapHolderCache

        // Hot scalars — touched on every tagState* read. Packing them with the
        // mruRing/head update keeps the per-emit footprint contiguous.
        lastID  uint32
        lruHead uint32
        mruHead uint32
        _       uint32 // align mruRing on 8-byte boundary

        // mruRing mirrors the encoder's side-cache: the last mruRingSize
        // state-ref ids in emission order, stored as uint16 (the id
        // space is < 2^14 by encoder cap). For tagStateMTF the wire
        // carries rank — direct index into the ring resolves the id in
        // O(1) (mruRing[(mruHead-1-rank)&mask]) instead of walking the
        // LRU chain. Pure decoder-side optimization; the wire format is
        // unchanged.
        mruRing [mruRingSize]uint16

        // retainStreak counts consecutive small (sub-cap) messages for the
        // adaptive-retention policy in reset(). Cold — touched once per reset.
        retainStreak uint8
}

func newDecState() *decState {
        d := &decState{
                values:       make([][]byte, 0, 64),
                stringValues: make([]string, 0, 64),
                boxValues:    make([]any, 0, 64),
                lruHead:      lruInvalidID,
                lastID:       lruInvalidID,
        }
        for i := range d.mruRing {
                d.mruRing[i] = mruEmpty
        }
        return d
}

func (d *decState) reset() {
        // Symmetric to encState.reset's adaptive-retention policy: retain grown
        // backing arrays (reuse in place) while messages stay large so a steady
        // large-batch decode workload amortizes the table allocation, releasing
        // only after retainReleaseStreak consecutive small messages. The decision
        // is driven by len(d.values) — the count of intern records decoded THIS
        // message, a true per-message demand signal (the retained cap would stay
        // large forever and never let the streak advance). Row-scaled columnar
        // scratch is governed by the hard ceiling; the id-bounded tables are
        // already bounded by maxStateEntries. See encState.reset for rationale.
        if len(d.values) > maxRetainedIDs {
                d.retainStreak = 0
        } else if d.retainStreak < retainReleaseStreak {
                d.retainStreak++
        }
        release := d.retainStreak >= retainReleaseStreak

        if cap(d.values) > maxRetainedIDs && release {
                d.values = nil
                d.stringValues = nil
                d.boxValues = nil
        } else {
                // Retain the backing, but clear() first: d.values entries are []byte
                // aliasing the previous message's input (wire) buffer, and
                // stringValues entries are prior decoded-string headers. A bare [:0]
                // keeps those headers live in the tail, pinning the previous caller's
                // input buffer (and decoded strings) from GC for the lifetime of the
                // pooled decoder. clear() drops the headers before reuse — mirrors the
                // encoder's colScratchStr treatment. memclr only, no allocation.
                clear(d.values)
                clear(d.stringValues)
                clear(d.boxValues)
                d.values = d.values[:0]
                d.stringValues = d.stringValues[:0]
                d.boxValues = d.boxValues[:0]
        }
        d.lastID = lruInvalidID
        d.lruHead = lruInvalidID
        if cap(d.lruLink) > maxRetainedLRUCap && release {
                d.lruLink = nil
        } else {
                d.lruLink = d.lruLink[:0]
        }
        if cap(d.pairPred) > maxRetainedPairCap && release {
                d.pairPred = nil
        } else {
                clear(d.pairPred)
        }
        if cap(d.shapes) > maxRetainedShapeCap && release {
                d.shapes = nil
        } else {
                d.shapes = d.shapes[:0]
        }
        if cap(d.colShapes) > maxRetainedShapeCap && release {
                d.colShapes = nil
        } else {
                d.colShapes = d.colShapes[:0]
        }
        if cap(d.hybridShapes) > maxRetainedShapeCap && release {
                d.hybridShapes = nil
        } else {
                d.hybridShapes = d.hybridShapes[:0]
        }
        // Row-scaled columnar scratch: ceiling-only (independent of the intern
        // streak), reclaimed when idle by sync.Pool GC. Each backing grows on its own
        // per-column-type demand, so gate each independently — a single check keyed on
        // colScratchI64 would miss an oversized U64/F64/Bool backing (mirrors the
        // encode-side fix).
        if cap(d.colScratchI64) > maxRetainedColScratch {
                d.colScratchI64 = nil
        }
        if cap(d.colScratchU64) > maxRetainedColScratch {
                d.colScratchU64 = nil
        }
        if cap(d.colScratchF64) > maxRetainedColScratch {
                d.colScratchF64 = nil
        }
        if cap(d.colScratchBool) > maxRetainedColScratch {
                d.colScratchBool = nil
        }
        if cap(d.colScratchF32) > maxRetainedColScratch {
                d.colScratchF32 = nil
        }
        if cap(d.colScratchStr) > maxRetainedColScratch {
                d.colScratchStr = nil
        } else {
                // Drop retained string headers across the FULL backing, not just len:
                // colScratchStr is resliced via [:n] (ReadStringColumn) / [:0] (gather),
                // so nested columnar structs of differing row counts leave a high-water
                // tail of live headers that would pin prior-message decoded strings from
                // GC for the pooled decoder's lifetime. Sibling to d.stringValues' clear.
                clear(d.colScratchStr[:cap(d.colScratchStr)])
                d.colScratchStr = d.colScratchStr[:0]
        }
        if cap(d.colStrHandles) > maxRetainedColScratch { // pointer-free (Str={off,len uint32}), no clear
                d.colStrHandles = nil
        } else {
                d.colStrHandles = d.colStrHandles[:0]
        }
        if cap(d.colDictTableScr) > maxRetainedColScratch {
                d.colDictTableScr = nil
        } else {
                // Drop retained string headers across the full backing (sibling to
                // colScratchStr) so a prior message's dict values cannot be pinned.
                clear(d.colDictTableScr[:cap(d.colDictTableScr)])
                d.colDictTableScr = d.colDictTableScr[:0]
        }
        if cap(d.colDictIdxScr) > maxRetainedColScratch { // pointer-free, no clear
                d.colDictIdxScr = nil
        }
        if cap(d.colLenScratch) > maxRetainedColScratch {
                d.colLenScratch = nil
        }
        // Column-diff sparse-row scratch (delta_columnar.go): same ceiling policy.
        if cap(d.deltaColRows) > maxRetainedColScratch {
                d.deltaColRows = nil
        }
        for i := range d.mruRing {
                d.mruRing[i] = mruEmpty
        }
        d.mruHead = 0
        d.mapDec = mapHolderCache{}
}

// pairAtRank returns the predicted successor of prev. With top-1
// storage the only valid wire rank is 0; any rank ≥ pairPredK was
// already rejected upstream. ok=false marks an empty slot.
//
//go:nosplit
func (d *decState) pairAtRank(prev uint32, rank uint8) (uint32, bool) {
        if int(prev) >= len(d.pairPred) || rank != 0 {
                return 0, false
        }
        v := d.pairPred[prev]
        if v == pairPredEmpty {
                return 0, false
        }
        return v - 1, true
}

//go:nosplit
func (d *decState) pairEnsure(prev uint32) {
        for uint32(len(d.pairPred)) <= prev {
                d.pairPred = append(d.pairPred, pairPredEmpty)
        }
}

// pairRecord mirrors encState.pairRecord exactly. Overwrites the
// stored successor — top-1 keeps only the most recent transition.
//
//go:nosplit
func (d *decState) pairRecord(prev, curr uint32) {
        d.pairEnsure(prev)
        d.pairPred[prev] = curr + 1
}

// shapeDeclare appends a new shape with the next sequential wire ID
// and returns a pointer to its slot. The wire ID equals
// len(d.shapes) after the append; callers do not need it returned
// because the encoder hands shape IDs out in the same order.
func (d *decState) shapeDeclare() *decShape {
        d.shapes = append(d.shapes, decShape{})
        return &d.shapes[len(d.shapes)-1]
}

// shapeLookup returns the shape with the given wire ID (≥ 1). nil
// means an unknown ID — the stream is malformed.
func (d *decState) shapeLookup(id uint32) *decShape {
        if id == 0 || id > uint32(len(d.shapes)) {
                return nil
        }
        return &d.shapes[id-1]
}

// mruPush records id as the newest entry in the decoder side-cache.
// Mirrors encState.mruPush so the ring sees every emit the encoder
// recorded. id must fit in uint16 (id space is < 2^14 by encoder
// cap).
//
//go:nosplit
func (d *decState) mruPush(id uint32) {
        d.mruRing[d.mruHead&mruRingMask] = uint16(id)
        d.mruHead++
}

// mruIDAtRank returns the id stored rank positions back from the head
// of the ring. Provided the encoder emitted tagStateMTF only for
// ranks ≤ mruRingSize-1 (which is the only range where MTF beats raw
// for the common 2-byte id wire form), this resolves the id in O(1)
// without walking the LRU chain.
//
//go:nosplit
func (d *decState) mruIDAtRank(rank uint32) (uint32, bool) {
        if rank >= mruRingSize {
                return 0, false
        }
        id := d.mruRing[(d.mruHead-1-rank)&mruRingMask]
        if id == mruEmpty {
                return 0, false
        }
        return uint32(id), true
}

func (d *decState) lruAddFresh(id uint32) {
        for uint32(len(d.lruLink)) <= id {
                d.lruLink = append(d.lruLink, lruLinkInvalid)
        }
        head := d.lruHead
        if head == lruInvalidID {
                d.lruLink[id] = lruLinkInvalid
        } else {
                d.lruLink[id] = lruLink16Invalid | (head << 16)
                d.lruLink[head] = setLinkPrev(d.lruLink[head], id)
        }
        d.lruHead = id
        d.mruPush(id)
}

// lruIDAtRank returns the ID currently at the given MTF rank (head
// = 0) by walking the linked-list chain. Used as a fallback when the
// MRU ring side-cache misses (rank ≥ mruRingSize, which the encoder
// never emits today but the decoder must still handle for forward
// compatibility with larger ring sizes on the encoder side).
func (d *decState) lruIDAtRank(rank uint32) (uint32, bool) {
        cur := d.lruHead
        for range rank {
                if cur == lruInvalidID {
                        return 0, false
                }
                cur = linkNext(d.lruLink[cur])
                if cur == lruLink16Invalid {
                        cur = lruInvalidID
                }
        }
        if cur == lruInvalidID {
                return 0, false
        }
        return cur, true
}

func (d *decState) lruMoveToFront(id uint32) {
        if d.lruHead == id {
                d.mruPush(id)
                return
        }
        link := d.lruLink[id]
        p := linkPrev(link)
        n := linkNext(link)
        d.lruLink[p] = setLinkNext(d.lruLink[p], n)
        if n != lruLink16Invalid {
                d.lruLink[n] = setLinkPrev(d.lruLink[n], p)
        }
        head := d.lruHead
        d.lruLink[id] = lruLink16Invalid | (head << 16)
        d.lruLink[head] = setLinkPrev(d.lruLink[head], id)
        d.lruHead = id
        d.mruPush(id)
}

// append registers a fresh intern record with the decoder. b
// aliases the wire buffer; the cached string slot is left empty so
// the first ReadString of this record pays the string(b) copy
// exactly once, and every later state-ref / MTF / pair / repeat
// read returns the cached value without alloc.
//
// Eager materialisation would punish single-shot decodes (Config-
// shaped workloads, ~10 distinct interns, each read once) where
// the cache slot never gets re-read; lazy population matches what
// the old direct-string(b) path did on first sight and adds zero
// alloc on subsequent reads.
func (d *decState) append(b []byte) uint32 {
        id := uint32(len(d.values))
        d.values = append(d.values, b)
        d.stringValues = append(d.stringValues, "")
        d.boxValues = append(d.boxValues, nil)
        d.lruAddFresh(id)
        return id
}

func (d *decState) get(id uint32) ([]byte, bool) {
        if id >= uint32(len(d.values)) {
                return nil, false
        }
        return d.values[id], true
}

// getString returns the cached string copy of the intern record at
// id, populating the slot on first call. Empty interned bytes
// resolve to "" without an extra alloc — the `len(b) == 0` branch
// short-circuits before the materialisation. Used on the state-ref
// / MTF / pair / repeat decode paths so ReadString skips the
// string(b) heap copy after the first sight.
//
// When arena is non-nil the first-sight materialisation is packed into the
// bump arena instead of its own heap allocation, so a Dense/intern decode of a
// high-cardinality string column amortises its copies the same way the plain
// (Speed-mode) string path already does. Each interned id is still copied at
// most once — the cached aliasing string is reused on every later reference.
//
//go:nosplit
func (d *decState) getString(id uint32, arena *Arena) (string, bool) {
        if id >= uint32(len(d.stringValues)) {
                return "", false
        }
        s := d.stringValues[id]
        if s != "" {
                return s, true
        }
        b := d.values[id]
        if len(b) == 0 {
                return "", true
        }
        if arena != nil {
                s = arena.appendStr(b)
        } else {
                s = string(b)
        }
        d.stringValues[id] = s
        return s, true
}

// getBoxStr returns the string at id boxed into an `any`, cached so repeated
// occurrences of the same interned value share ONE box (zero alloc after the
// first). Called only when the string arrived as an intern/state reference
// (lastID valid) — a unique inline string never reaches here, so the boxing
// cache never adds overhead on high-cardinality data. The shared box is
// immutable, so handing it to many map/slice slots is safe.
func (d *decState) getBoxStr(id uint32, arena *Arena) (any, bool) {
        if id >= uint32(len(d.boxValues)) {
                return nil, false
        }
        if b := d.boxValues[id]; b != nil {
                return b, true
        }
        s, ok := d.getString(id, arena)
        if !ok {
                return nil, false
        }
        box := any(s)
        d.boxValues[id] = box
        return box, true
}

package qdf

import (
        "errors"
        "io"

        "github.com/alex60217101990/qdf/internal/bufpool"
)

// StreamEncoder writes a sequence of values into an io.Writer. The header
// is emitted once before the first value; in Dense mode the intern table
// survives across Encode calls so back-references span the whole stream.
type StreamEncoder struct {
        w   io.Writer
        enc *Encoder
        buf *[]byte
        // broken is set when a mid-message encode fails. The body bytes are rolled
        // back, but the encoder's cross-message state (intern table, declared
        // struct/map shapes, LRU, predictors) may have advanced past what the
        // decoder saw — so every later frame's back-refs would desync. Once broken,
        // further Encode is refused; the already-buffered valid prefix can still be
        // Flushed.
        broken bool
}

// ErrStreamBroken is returned by StreamEncoder.Encode after a previous Encode
// failed mid-message: the cross-message encoder state can no longer be trusted,
// so the stream must be abandoned (the valid prefix may still be flushed).
var ErrStreamBroken = errors.New("qdf: stream encoder broken by a prior mid-message error")

// ErrStreamDecoderBroken is returned by StreamDecoder.Decode after a previous
// Decode failed mid-message: the read cursor and the shared dense state can no
// longer be trusted (subsequent frames would misparse), so the stream must be
// abandoned.
var ErrStreamDecoderBroken = errors.New("qdf: stream decoder broken by a prior mid-message error")

// NewStreamEncoder returns a stream encoder backed by w. Dense mode
// activates the full balanced codec set (OptBalanced); Fast mode
// emits raw tagged bytes (OptSpeed). For finer per-stream tuning,
// build the StreamEncoder via NewStreamEncoderWith.
func NewStreamEncoder(w io.Writer, mode Mode) *StreamEncoder {
        opts := OptSpeed
        if mode == Dense {
                opts = OptBalanced
        }
        return NewStreamEncoderWith(w, opts)
}

// NewStreamEncoderWith returns a stream encoder with the given
// Options bit-mask. The intern table (when OptDense is set) survives
// across Encode calls so back-references span the whole stream.
func NewStreamEncoderWith(w io.Writer, opts Options) *StreamEncoder {
        buf := bufpool.Get(4096)
        enc := &Encoder{buf: (*buf)[:0], minIntern: 4, maxStateEntries: maxInternEntries, maxDepth: DefaultMaxDepth}
        enc.applyOpts(opts)
        // The column index is a single-message feature: it backpatches the header
        // flag at a fixed offset, which a stream's shared/reused buffer invalidates
        // after the first Flush. Like the whole-body rANS pass, it is not emitted in
        // streaming mode.
        enc.colIndex = false
        if opts.Has(OptDense) {
                enc.state = newEncState()
        }
        return &StreamEncoder{w: w, enc: enc, buf: buf}
}

// Encode writes v as the next value in the stream. Each message is framed with
// a uvarint byte-length prefix so the decoder can buffer a whole message — of
// any size — before decoding it. The 5-byte stream header is written once,
// before the first frame, and is not itself framed. The encoder flushes
// internally when its buffer crosses 16 KiB; call Flush to push earlier.
func (s *StreamEncoder) Encode(v any) error {
        if s.enc == nil {
                return io.ErrClosedPipe
        }
        if s.broken {
                return ErrStreamBroken
        }
        // One-per-stream header preamble, outside the frames.
        if !s.enc.headerOut {
                s.enc.writeHeader()
        }
        // Reserve one byte for the length prefix, then encode the body after it.
        // The common case (a message under 128 bytes) needs exactly that one byte,
        // so it is written in place with no memmove; only larger messages shift.
        start := len(s.enc.buf)
        s.enc.buf = append(s.enc.buf, 0)
        bodyStart := len(s.enc.buf)
        if err := encodeReflect(s.enc, v); err != nil {
                s.enc.buf = s.enc.buf[:start] // drop the reservation + any partial body
                // The buffer is rolled back, but encoder state (intern IDs, shape
                // declarations, predictors) may have advanced — desyncing every later
                // frame. Poison the stream so no further frame is emitted against it.
                s.broken = true
                return err
        }
        n := len(s.enc.buf) - bodyStart
        if n < 0x80 {
                s.enc.buf[start] = byte(n) // single-byte uvarint, body already in place
        } else {
                // Need an m-byte prefix; one byte is reserved, so make room for m-1 more
                // and shift the body right by m-1 (overlap-safe copy).
                var lb [10]byte
                pre := appendUvarint(lb[:0], uint64(n))
                m := len(pre)
                s.enc.buf = append(s.enc.buf, make([]byte, m-1)...)
                copy(s.enc.buf[start+m:], s.enc.buf[bodyStart:bodyStart+n])
                copy(s.enc.buf[start:start+m], pre)
        }
        if len(s.enc.buf) >= 1<<14 {
                return s.Flush()
        }
        return nil
}

// Flush writes any buffered bytes to the underlying writer.
func (s *StreamEncoder) Flush() error {
        if s.enc == nil {
                return io.ErrClosedPipe
        }
        if len(s.enc.buf) == 0 {
                return nil
        }
        // Write the whole framed buffer, honoring short writes. An io.Writer may
        // stop early (n < len with a non-nil error) or, misbehaving, return n < len
        // with a nil error; a single Write that discards n would either silently
        // truncate the stream or, on a retry, re-send the already-written prefix and
        // duplicate it. Loop on the remainder and, on error, compact the unwritten
        // tail to the front so the next Flush resumes exactly where this one stopped.
        buf := s.enc.buf
        for len(buf) > 0 {
                n, err := s.w.Write(buf)
                buf = buf[n:]
                if err != nil {
                        s.enc.buf = s.enc.buf[:copy(s.enc.buf, buf)]
                        return err
                }
                if n == 0 {
                        s.enc.buf = s.enc.buf[:copy(s.enc.buf, buf)]
                        return io.ErrShortWrite
                }
        }
        s.enc.buf = s.enc.buf[:0]
        return nil
}

// Close flushes pending data and releases the scratch buffer to the
// pool. The underlying writer is not closed. Idempotent: subsequent
// calls are a safe no-op.
func (s *StreamEncoder) Close() error {
        if s.enc == nil {
                return nil
        }
        if err := s.Flush(); err != nil {
                return err
        }
        *s.buf = s.enc.buf
        bufpool.Put(s.buf)
        s.buf = nil
        s.enc = nil
        return nil
}

// Reset prepares the encoder to write a NEW, independent stream to w, reusing
// the encoder, its grown intern / shape state, and the scratch buffer instead
// of allocating a fresh one. The configured Options (Dense, codecs) are kept;
// the cross-message state is cleared so the new stream's back-references start
// from scratch and the next Encode re-emits the one-per-stream header. A no-op
// after Close.
//
// This is the streaming counterpart of the encoder pool behind Marshal: rather
// than construct (and re-allocate the intern table for) a StreamEncoder per
// independent batch, construct one and Reset it between batches — the heavy
// newEncState() is paid once.
func (s *StreamEncoder) Reset(w io.Writer) {
        if s.enc == nil {
                return // closed: the buffer was returned to the pool
        }
        s.enc.resetForReuse()
        s.w = w
        s.broken = false
}

// StreamDecoder reads a sequence of values from an io.Reader. The intern
// table is preserved across Decode calls to match StreamEncoder.
//
// The decoder grows its window buffer as needed. State-table entries
// alias the buffer, so the window is never compacted during a stream;
// total memory tracks the stream length. For unbounded streams partition
// into envelopes: decode one envelope, then Reset this decoder for the next
// (Reset reuses the decoder, its dense state, and the window backing — only the
// contents are cleared, so the window's high-water memory is reused, not
// re-grown, and the per-envelope footprint stays bounded). Pair Reset with
// SetArena to also bound the decoded-value memory across envelopes.
type StreamDecoder struct {
        r   io.Reader
        dec *Decoder
        buf *[]byte
        // broken mirrors StreamEncoder.broken: a mid-frame decode error (or a frame
        // whose body the value did not consume exactly) leaves the read cursor inside
        // the failed frame and the shared dense state (intern table, LRU, shapes)
        // partially advanced — every later frame would misparse. Once broken, further
        // Decode is refused rather than returning silently wrong values.
        broken bool
}

// NewStreamDecoder returns a stream decoder reading from r.
func NewStreamDecoder(r io.Reader) *StreamDecoder {
        buf := bufpool.Get(4096)
        return &StreamDecoder{r: r, dec: &Decoder{buf: (*buf)[:0]}, buf: buf}
}

// SetNoCopy makes Decode return string / []byte values that alias the
// decoder's internal window buffer instead of copying them — eliminating the
// per-value copy that dominates decode allocation.
//
// Unlike the one-shot Decoder.SetNoCopy (which aliases the caller's input
// buffer and is unsafe the moment that buffer is reused — the typical server
// recycles its read buffer right after the handler returns), the stream owns
// its window buffer, so aliasing is safe for the lifetime of the stream:
//
//   - A decoded value stays valid until Close. Close returns the window to a
//     pool, after which any retained aliased value is undefined — copy
//     (strings.Clone / append) anything you need past Close.
//   - The window grows but is never compacted mid-stream (the Dense intern
//     table already aliases it for cross-message back-references), so a growth
//     does not invalidate earlier values, and noCopy adds no extra memory: the
//     bytes are retained either way.
//
// Because the window is retained for the whole stream regardless, this is the
// safe home for zero-copy decode — process or copy each value before Close and
// you pay zero per-value allocation across every message. No-op after Close.
func (s *StreamDecoder) SetNoCopy(v bool) {
        if s.dec != nil {
                s.dec.noCopy = v
        }
}

// SetArena directs Decode to pack copied string / []byte-as-string bodies into
// the caller-owned Arena (bump-allocated, amortized to ~zero alloc) instead of
// one heap allocation per value. It is the streaming counterpart of
// Unmarshal(..., WithArena(a)).
//
// Why both this AND SetNoCopy: they trade off lifetime vs copies.
//
//   - SetNoCopy aliases the window: zero copies, but a decoded value lives only
//     as long as the window — i.e. until Close (or until a Reset for a new
//     stream). The window only ever grows (it is never compacted mid-stream), so
//     for a long stream its memory tracks the whole stream length.
//   - SetArena copies once into the arena: the decoded values then live as long
//     as the Arena (independent of the window), and you control that — keep the
//     arena across batches, Reset it once a batch's values are dead to reuse its
//     blocks for the next batch at zero allocation, or drop it and let the GC
//     free it. This is the right choice when you process a stream in batches and
//     want each batch's decoded memory released without holding the growing
//     window, or when values must outlive a Reset/new stream.
//
// SetArena is ignored while SetNoCopy(true) is set (no-copy already avoids the
// copy the arena would pack). The setting is preserved across Reset; pass nil to
// clear it. Arena strings ALIAS the arena — see Arena for the use-after-free
// contract around Arena.Reset. No-op after Close. One Arena per goroutine.
//
// Memory note: SetArena bounds the DECODED-VALUE memory (via Arena.Reset), not
// the read WINDOW. To bound a long stream's total footprint, partition it into
// envelopes and reuse this decoder across them with Reset (see Reset) — that is
// what caps the window; the arena caps the values.
func (s *StreamDecoder) SetArena(a *Arena) {
        if s.dec != nil {
                s.dec.arena = a
        }
}

// maxStreamMsg bounds a single framed message so a hostile length prefix can't
// drive an unbounded buffer read. 2 GiB is far above any realistic message.
const maxStreamMsg = 1 << 31

// Decode reads the next value into out. out must be a pointer. Returns io.EOF
// at a clean end of stream. Each message is length-framed, so a message of any
// size is buffered in full before decoding — no 4 KiB window limit.
func (s *StreamDecoder) Decode(out any) error {
        if s.dec == nil {
                return io.ErrClosedPipe
        }
        if s.broken {
                return ErrStreamDecoderBroken
        }
        // Consume the one-per-stream header before the first frame.
        if !s.dec.headerRead {
                if err := s.fill(5, true); err != nil {
                        return err // io.EOF here means an empty stream
                }
                // A stream is a sequence of length-prefixed frames. The whole-payload
                // rANS post-pass (FlagRANS) and the columnar column-index (FlagColIndex)
                // do not apply to it; honoring FlagRANS in readHeaderSlow would replace
                // the shared window buffer with a decoded body mid-stream and desync all
                // later framing. A conforming StreamEncoder never sets them — reject the
                // hostile header up front and latch broken before readHeaderSlow can
                // touch the buffer. (FlagDense is legitimate for a Dense stream.)
                if s.dec.buf[s.dec.i+4]&(FlagRANS|FlagColIndex) != 0 {
                        s.broken = true
                        return ErrStreamBadFlags
                }
                if err := s.dec.readHeader(); err != nil {
                        // A corrupt header may have half-advanced the decoder (readHeaderSlow
                        // sets headerRead/cursor before a failing rANS-body pass), so latch
                        // the stream broken — like a mid-frame decode error below — to uphold
                        // the "once broken, further Decode is refused" invariant instead of
                        // silently resuming at the next byte.
                        s.broken = true
                        return err
                }
        }
        // Read the frame length (io.EOF at a clean frame boundary = end of stream).
        framelen, err := s.readFrameLen()
        if err != nil {
                return err
        }
        // Buffer the whole message body, then decode it in a single pass so a
        // partial decode never mutates the shared dense state. Past the length
        // prefix a short read is a truncated frame, never a clean end.
        if err := s.fill(framelen, false); err != nil {
                // readFrameLen already advanced the cursor past the length prefix, so the
                // stream is parked mid-frame with a known-unreadable body. Latch broken —
                // like the header and decode-error paths — so a caller that ignores this
                // error and calls Decode again cannot re-parse the buffered partial body
                // bytes as a fresh frame and silently misparse.
                s.broken = true
                return err // ErrShortBuffer on a truncated final frame
        }
        // Each frame is an independent target: drop any maps the previous frame
        // harvested but did not reuse, so a recycled map never crosses frames. Runs
        // BEFORE decode, so this frame's own harvest (into the same out — the common
        // streaming reuse) still recycles normally.
        clear(s.dec.mapFreeList)
        start := s.dec.i
        if err := decodeReflect(s.dec, out); err != nil {
                // The cursor is left partway through the frame and the dense state is
                // half-advanced; poison the stream so the next Decode fails cleanly
                // instead of misparsing the rest of this frame as new frames.
                s.broken = true
                return err
        }
        // The value must consume exactly the framed length; a mismatch means a
        // corrupt or hostile frame, so reject it instead of desyncing the stream.
        if s.dec.i-start != framelen {
                s.broken = true
                return ErrInvalidLength
        }
        return nil
}

// readFrameLen parses the uvarint length prefix of the next message, advancing
// the cursor past it. Returns io.EOF when no more frames remain.
func (s *StreamDecoder) readFrameLen() (int, error) {
        for {
                if v, k := readUvarint(s.dec.buf[s.dec.i:]); k > 0 {
                        // >= so a frame length of exactly maxStreamMsg (1<<31) is rejected: on a
                        // 32-bit build int(v) would otherwise overflow to a negative length.
                        if v >= maxStreamMsg {
                                return 0, ErrInvalidLength
                        }
                        s.dec.i += k
                        return int(v), nil
                }
                had := len(s.dec.buf) - s.dec.i
                if had >= 10 { // a uvarint is at most 10 bytes; longer is malformed
                        return 0, ErrInvalidLength
                }
                if err := s.fill(had+1, true); err != nil {
                        return 0, err // io.EOF (boundary) = clean end; ErrShortBuffer = partial length
                }
        }
}

// fill reads from the underlying reader until at least need unread bytes are
// buffered. When boundary is true (reading at a message boundary — the header
// or a frame length) an immediate clean end of stream returns io.EOF; in every
// other case a stream that ends before need bytes returns ErrShortBuffer.
func (s *StreamDecoder) fill(need int, boundary bool) error {
        for len(s.dec.buf)-s.dec.i < need {
                if cap(*s.buf)-len(s.dec.buf) == 0 {
                        // Grow geometrically. We deliberately do NOT pre-size to s.dec.i+need:
                        // readFrameLen accepts any frame length below maxStreamMsg (~2 GiB), so
                        // a hostile ~5-byte length prefix could otherwise force a ~2 GiB
                        // allocation here before a single body byte is read or validated.
                        // Doubling caps the peak allocation at ~2x the bytes actually received,
                        // so a truncated or lying stream errors (ErrShortBuffer / io.EOF) after
                        // reading little, instead of OOM-ing up front. A legitimately large
                        // frame still converges in O(log need) geometric reallocations.
                        newCap := cap(*s.buf)*2 + 4096
                        grown := make([]byte, len(s.dec.buf), newCap)
                        copy(grown, s.dec.buf)
                        *s.buf = grown
                        s.dec.buf = grown
                }
                end := len(s.dec.buf)
                (*s.buf) = (*s.buf)[:cap(*s.buf)]
                s.dec.buf = *s.buf
                n, rerr := s.r.Read(s.dec.buf[end:])
                s.dec.buf = s.dec.buf[:end+n]
                (*s.buf) = s.dec.buf
                if n == 0 {
                        if rerr == nil {
                                continue // no progress but no error: retry
                        }
                        if errors.Is(rerr, io.EOF) {
                                if boundary && len(s.dec.buf)-s.dec.i == 0 {
                                        return io.EOF // clean end of stream at a message boundary
                                }
                                return ErrShortBuffer // truncated mid-message
                        }
                        return rerr
                }
                if rerr != nil && !errors.Is(rerr, io.EOF) {
                        return rerr
                }
        }
        return nil
}

// Reset prepares the decoder to read a NEW, independent stream from r, reusing
// the decoder, its dense state, and the window buffer instead of allocating a
// fresh one. The noCopy setting is preserved; the cross-message state is cleared
// so the new stream is decoded from scratch. A no-op after Close.
func (s *StreamDecoder) Reset(r io.Reader) {
        if s.dec == nil {
                return // closed: the buffer was returned to the pool
        }
        s.dec.buf = (*s.buf)[:0]
        s.dec.i = 0
        s.dec.depth = 0
        s.dec.headerRead = false
        s.dec.mode = Fast
        s.dec.colIndex = false
        s.dec.colMaxLen = 0
        clear(s.dec.mapFreeList)
        // keyIdx keys alias the prior stream's base data (keyTokenAt); clear so a
        // reused StreamDecoder does not GC-pin it across Reset (mirrors mapFreeList).
        clear(s.dec.keyIdx)
        if s.dec.state != nil {
                s.dec.state.reset()
        }
        s.r = r
        s.broken = false
}

// Close releases the scratch buffer to the pool. The underlying reader
// is not closed. Idempotent: subsequent calls are a safe no-op.
func (s *StreamDecoder) Close() error {
        if s.dec == nil {
                return nil
        }
        *s.buf = s.dec.buf
        bufpool.Put(s.buf)
        s.buf = nil
        s.dec = nil
        return nil
}

package qdf

// Wire format constants. The 5-byte file header is 'QDF' + 1-byte version
// + 1-byte flags. Tag space layout is documented inline below.

const (
        Magic0   byte = 'Q'
        Magic1   byte = 'D'
        Magic2   byte = 'F'
        Version1 byte = 0x01
)

// Flag bits in the 5th header byte.
const (
        FlagDense byte = 1 << 0
        // FlagQPack signals that the encoder may emit QPack codec tags
        // (0xE3..0xEF). A decoder that does not recognize the tags will fail
        // with ErrBadTag on the first packed slice; the flag is an early hint
        // so callers can refuse the buffer up front.
        FlagQPack byte = 1 << 1

        // FlagRANS signals that the body (everything after the 5-byte header) is
        // rANS-compressed: varuint(origLen) + 256-entry frequency table +
        // rANS stream. The decoder reverses it before reading tags. Set only
        // when the rANS form is strictly smaller than the plain body.
        FlagRANS byte = 1 << 2

        // FlagColIndex marks a tagColStruct (columnar []struct) payload that
        // carries a fixed-width column-length table right after the shape
        // declaration and before the column bodies. The table is K little-endian
        // uint32 entries (one per column, K = number of columns), each the byte
        // length of the corresponding column body. It lets a reader skip columns
        // it does not need without decoding them. Opt-in (~4 bytes per column);
        // only meaningful when the value is columnar — for any other top-level
        // shape the bit is set but no index block exists.
        FlagColIndex byte = 1 << 3
)

// Tag bytes.
const (
        // 0x00..0x7F  positive fixint
        tagFixintMax = 0x7F

        // 0x80..0x9F  fixstr (len 0..31)
        tagFixstr     = 0x80
        tagFixstrMask = 0x1F

        // 0xA0..0xBF  fixarr (len 0..31)
        tagFixarr     = 0xA0
        tagFixarrMask = 0x1F

        tagNil     = 0xC0
        tagFalse   = 0xC1
        tagTrue    = 0xC2
        tagUint8   = 0xC3
        tagUint16  = 0xC4
        tagUint32  = 0xC5
        tagUint64  = 0xC6
        tagInt8    = 0xC7
        tagInt16   = 0xC8
        tagInt32   = 0xC9
        tagInt64   = 0xCA
        tagFloat32 = 0xCB
        tagFloat64 = 0xCC
        tagStr8    = 0xCD
        tagStr16   = 0xCE
        tagStr32   = 0xCF
        tagBin8    = 0xD0
        tagBin16   = 0xD1
        tagBin32   = 0xD2
        tagArr16   = 0xD3
        tagArr32   = 0xD4
        tagMap8    = 0xD5
        tagMap16   = 0xD6
        tagMap32   = 0xD7

        // 0xD8..0xDF  negfixint (-1..-8) — 3-bit body
        tagNegfixint     = 0xD8
        tagNegfixintMask = 0x07
        negfixintMaxAbs  = 8 // values -1..-8 fit

        tagInternStr = 0xE0
        tagStateRef  = 0xE1
        tagInternBin = 0xE2

        // QPack codec tags (0xE3..0xEF). Each opens a self-described payload
        // that replaces the per-element tag stream for a single slice.
        tagPackBool = 0xE3 // bitpacked []bool: tag, varuint(n), ceil(n/8) bytes (LSB-first)
        tagPackRaw  = 0xE4 // raw-LE numeric: tag, kind byte, varuint(n), n*width bytes (LE)
        tagPackFor  = 0xE5 // Frame-of-Reference bitpacked integer slice:
        //                    tag, kind, bits (0..56), min varuint (zigzag for signed),
        //                    varuint(n), ceil(n*bits/8) bytes (LSB-first).
        tagStateRepeat = 0xE8 // Markov-0 predictor for Dense state-refs:
        //                    a state-ref whose ID equals the immediately
        //                    previous state-ref emission is encoded as a
        //                    single byte (no varuint payload).
        tagStateMTF = 0xE9 // Move-To-Front coding for Dense state-refs:
        //                    encodes the MTF rank (position in the LRU
        //                    list of recently-emitted IDs) instead of the
        //                    raw intern ID. Used when uvarintLen(rank) <
        //                    uvarintLen(id) so the wire never grows over
        //                    the plain tagStateRef encoding.
        tagPackGorilla = 0xE7 // Gorilla XOR-coded float slice:
        //                    tag, kind (qpackKindFloat32/64), varuint(n),
        //                    first value (4 or 8 LE bytes), varuint(numBits),
        //                    ceil(numBits/8) bytes of MSB-first XOR-delta bit-stream.
        tagPackDeltaFor = 0xE6 // Delta + zigzag + Frame-of-Reference integer slice:
        //                    tag, kind, bits (0..56),
        //                    first value (varuint or zigzag varuint),
        //                    minDelta (zigzag varuint),
        //                    varuint(n),
        //                    ceil((n-1)*bits/8) bytes (LSB-first) of (Δᵢ - minDelta).
        tagPackRLE = 0xEB // Run-length encoded integer slice. Wins on
        //                    high-repeat telemetry columns (Status, Level
        //                    enum-likes, sparse counters). Wire form:
        //                       tag, kind (qpackKindUint64/qpackKindInt64),
        //                       varuint(n),
        //                       runs: pairs of (value-varuint, runLen-varuint).
        //                    Signed kinds zigzag-encode the value. Total of
        //                    runLens equals n; the decoder unrolls until n
        //                    elements are produced.
        tagPackDict = 0xED // Dictionary-coded integer slice. Wins on
        //                    enum-like columns where distinct cardinality
        //                    is small (≤ 16) but values are spread out
        //                    enough that Frame-of-Reference can't bit-pack
        //                    them cheaply. Wire form:
        //                       tag, kind, varuint(distinct),
        //                       distinct values (each varuint, zigzag for
        //                         signed kinds),
        //                       varuint(n),
        //                       ceil(n * ceil(log2(distinct)) / 8) bytes
        //                         (LSB-first bit stream of indices).
        //                    bitsPer is implicit (derivable from distinct);
        //                    distinct == 1 emits a zero-width body and the
        //                    decoder broadcasts the single value.
        tagPackPFor = 0xEE // Patched Frame-of-Reference integer slice. The FOR
        //                    body is bit-packed at a reduced width b chosen so the
        //                    few values that don't fit (outliers) go to an
        //                    exception list instead of widening every slot. Wire:
        //                       tag, kind, varuint(n), b(1 byte), <min>,
        //                       body (n*b bits, LSB-first, (delta & mask)),
        //                       varuint(excN),
        //                       excN x ( varuint(dPos), varuint(delta) ).
        //                    <min> is varuint (unsigned kinds) or zigzag-varuint
        //                    (signed). Selected only when strictly smaller than
        //                    every other codec, so the wire never grows.
        tagStatePair = 0xEA // Markov-1 predictor for Dense state-refs.
        //                    Conditioned on the previously emitted state-ref ID
        //                    (lastID), the encoder maintains a small ring of the
        //                    most recent successors per prev. If the current ID
        //                    is in the ring, emit tagStatePair + varuint(rank).
        //                    Ring size pairPredK; rank ∈ [0, pairPredK). The
        //                    encoder picks tagStatePair only when its byte cost
        //                    is strictly smaller than every other state-ref
        //                    variant (Repeat, MTF, raw), so the wire never grows.
        tagMapShape = 0xEC // Struct shape interning (OptShapeIntern) and, for
        //                    string-keyed maps, map key-set interning
        //                    (OptMapShape). Shared sequential shape-ID space.
        //                    Wire form:
        //                       0xEC + varuint(shapeID)
        //                    shapeID == 0 declares a new shape inline:
        //                       0xEC, 0, varuint(N), [N x key-emit], [N x value]
        //                    The decoder assigns the new shape the next
        //                    sequential ID (starting at 1) for reuse.
        //                    shapeID > 0 reuses a previously declared shape:
        //                       0xEC, varuint(id), [N x value]
        //                    N is recovered from the shape table.
        tagColStruct = 0xEF // Columnar container for a slice of homogeneous flat
        //                    structs. Wire: 0xEF, varuint(M), varuint(shapeID)
        //                    (shapeID==0 declares: varuint(K) + K×(name, kind
        //                    byte)), then K columns in field order. Numeric/bool
        //                    columns are a QPack slice payload; string/[]byte
        //                    columns are M values emitted consecutively through the
        //                    intern path. Chosen per-array by a probe; the encoder
        //                    falls back to row-major when columnar would not win.
        //                    A nullable column (kind byte has bit 0x80 set) emits
        //                    ceil(M/8) LSB-first presence-mask bytes followed by
        //                    the dense non-nil values encoded with the base kind's
        //                    codec.

        tagPackBlock = 0xF0 // Per-block adaptive integer slice. A long int/uint
        //                    column is split into fixed-size blocks; each block is an
        //                    independent QPack sub-slice (FOR/Delta/RLE/dict/PFOR/raw)
        //                    so a column whose statistics shift along its length packs
        //                    each region optimally instead of one codec for the whole.
        //                    Wire:
        //                       tag, kind(1), blkLog(1; 8=>256, 10=>1024),
        //                       varuint(n),
        //                       nBlocks x uint32 LE  (byte offset of each block body,
        //                                             relative to the first body),
        //                       nBlocks x <QPack int sub-slice (tag+header+body)>.
        //                    nBlocks = ceil(n / (1<<blkLog)); the last block holds the
        //                    remainder. The offset table enables O(1) block seek, so a
        //                    predicate query can decode only the blocks covering its
        //                    matched rows. Selected only when strictly smaller than the
        //                    whole-column codec (never-larger), so the wire never grows.
        tagZoneChunk = 0xF1 // Zone-chunked integer column with a min/max zonemap
        //                    (OptZoneMap). A column is split into 256-row zones, each an
        //                    independent QPack int sub-slice, prefixed by a uint32 offset
        //                    table and a per-zone [min,max] zonemap. Wire:
        //                       tag, kind(1), zoneLog(1; 8=>256), varuint(n),
        //                       zoneCount x uint32 LE  (byte offset of each zone body),
        //                       zoneCount x (min,max)  (zigzag/var-int per kind),
        //                       zoneCount x <QPack int sub-slice (tag+header+body)>.
        //                    A bound-carrying predicate (WhereRange/GE/LE/Eq) skips
        //                    zones whose [min,max] cannot match, without decoding them.
        //                    Opt-in size-for-query-speed trade; without OptZoneMap the
        //                    column uses the normal codec and the wire is unchanged.
        // 0xF2 unassigned (reserved for a future MessagePack-style ext type).
        tagTimestamp = 0xF3

        // ALP (Adaptive Lossless floating-Point, CWI 2023), decimal path,
        // for []float64 under OptCompression. Self-describing:
        //   0xF4, qpackKindFloat64, varuint(n),
        //   d(1), zigzag-varuint(forMin), width(1),
        //   ceil(n*width/8) LSB-first body (absent when width==0),
        //   varuint(excN), excN×(varuint(pos), 8 LE raw float64).
        // Chosen by the float64 picker only when strictly smaller than raw
        // and the Gorilla projection, so it never grows the wire.
        tagPackALP = 0xF4

        // Dictionary-coded string column inside a tagColStruct payload. Emitted
        // as the first byte where a string column's M values would otherwise be
        // written consecutively, so it is self-describing: a decoder peeks the
        // byte and either takes this path or reads M per-value strings. Wire:
        //   0xF5, varuint(d), d×(varuint(len), len bytes),  // distinct table
        //   varuint(M), ceil(M*ceil(log2 d)/8) LSB-first index body (absent
        //   when d==1). Chosen by the column emitter only when the bitpacked
        //   index body beats the per-value run cost, so it never grows the wire.
        tagColStrDict = 0xF5
        tagColStrFSST = 0xF6 // FSST-coded string column (inside tagColStruct)

        // tagHybridColStruct is a columnar container for a slice of MIXED structs:
        // the columnar-eligible scalar/string fields are transposed into columns
        // (same per-column encoding as tagColStruct) while the remaining fields
        // (maps, non-[]byte slices, nested structs, interfaces) are kept row-major
        // in a per-row residual block. Wire:
        //   0xF7, varuint(N),
        //   varuint(shapeID)  // 0 = declare inline: varuint(K), K×(WriteString(name),
        //                     //   byte(colKind | 0xFF=residual)); else reuse by ID
        //   [K_eligible column bodies, same layout as tagColStruct],
        //   N × (K_residual values in field order, existing per-value tags).
        // A decoder that does not implement it sees an unknown tag → ErrBadTag.
        tagHybridColStruct = 0xF7

        // tagColStrRaw is a bulk-materialized string column inside a tagColStruct
        // payload: the high-cardinality counterpart to tagColStrDict. A mostly-
        // distinct column (IDs, GUIDs, emails, free text) cannot dict-compress and,
        // written per value, costs one heap allocation per row on decode. tagColStrRaw
        // instead lays every value down once, length-prefixed, so the decoder
        // materializes the whole column into ONE backing slab (every row a sub-slice)
        // — distinct strings still allocate their bytes once (wire-neutral vs the
        // per-value form, which also stores each distinct value once) but the decode
        // drops from n allocations to one. Chosen automatically (no option) by the
        // column emitter for high-cardinality columns where dict does not fire and
        // intern dedup cannot help. Wire:
        //   0xF8, varuint(n), varuint(total),
        //   n × (varuint(len), len bytes)            // values, interleaved
        // total is the summed value-byte count, so the decoder pre-sizes the slab in
        // one allocation. A decoder that does not implement it sees an unknown tag →
        // ErrBadTag.
        tagColStrRaw = 0xF8

        // tagColStrConst is a single-distinct (constant) string column inside a
        // tagColStruct payload: every row holds the SAME string. The dict codec
        // rejects this (it requires count >= 2 for a bounded index body) and the
        // per-value form stores the value n times (Dense state-repeats it, but the
        // codegen/Fast path does not), so a constant column is both wire-bloated and
        // decodes to n allocations there. tagColStrConst stores the value once plus
        // the row count; the decoder fills n shares of the single owned string (one
        // allocation). Chosen automatically when every value is identical. Wire:
        //   0xF9, varuint(len), len bytes, varuint(n)
        // n is checked against the columnar header's row count, which bounds it; a
        // decoder that does not implement it sees an unknown tag → ErrBadTag.
        tagColStrConst = 0xF9

        // tagColStrDictFC is a front-coded variant of tagColStrDict: the distinct
        // table is SORTED and incrementally (front-) coded — each entry stores the
        // length of the prefix it shares with the previous entry plus only its
        // suffix. The table order is the encoder's free choice (the per-row indices
        // point into it), so this needs no sorted input and is never larger: the
        // encoder emits it only when the front-coded table is strictly smaller than
        // the plain table (the index body is byte-identical between the two forms).
        // Big on prefix-shared medium-cardinality columns (SIDs, DNs, paths, URLs).
        // Wire (indices identical to tagColStrDict):
        //   0xFA, varuint(d),
        //        d×(varuint(sharedPrefixLen), varuint(suffixLen), suffix bytes),  // sorted
        //        varuint(n),
        //        ⌈n·ceil(log2 d)/8⌉ LSB-first bitpacked indices into the sorted table
        // sharedPrefixLen <= len(previous entry); entry 0 has prefixLen 0.
        tagColStrDictFC = 0xFA

        // tagColStrAlpha is an alphabet-aware bit-packed string column inside a
        // tagColStruct payload: the high-cardinality, restricted-alphabet counterpart
        // to tagColStrRaw. When every byte of every value is drawn from a small
        // alphabet (|A| <= 64 — hex, base32, base64, decimal IDs: trace/span/request
        // IDs, hashes, GUIDs), each character is stored in ceil(log2 |A|) bits via
        // pure positional notation instead of 8: hex (|A|=16) halves the body. This
        // is the one class dict (high-card), front-coding (no shared prefix) and FSST
        // (high entropy, few shared substrings) all miss, and it is captured WITHOUT
        // the rANS CPU cost, so it wins on the Balanced (rANS-off) tier.
        //
        // Wire:
        //   0xFB,
        //        varuint(a), a alphabet bytes,            // code k -> alphabet[k], a in [2,64]
        //        varuint(n),                              // row count (cross-checked)
        //        flags byte (bit0 = fixed length),
        //        if fixedLen: varuint(L)  else: n varuint lengths,
        //        ⌈(sum of lengths)·ceil(log2 a)/8⌉ LSB-first bitpacked char codes.
        // Never-larger: emitted only when the packed body + header beats the raw
        // per-value floor. a >= 2 keeps ceil(log2 a) >= 1 so the packed body is
        // non-empty and the decode allocation is buffer-bounded.
        tagColStrAlpha = 0xFB

        // tagColStrDictQ is a plain dictionary string column (identical distinct
        // table to tagColStrDict) whose per-row index is QPack-coded (RLE / Dict /
        // FOR / DeltaFor — the integer-slice picker) instead of a flat ceil(log2 d)-
        // bit pack. A skewed (Zipf-like) low-cardinality column packs its index far
        // below the flat width via run-length / dictionary coding, without the rANS
        // CPU cost, so it wins on the Balanced (rANS-off) tier. The front-coded table
        // variant (tagColStrDictFC) is never paired with a QPack index: front-coding
        // fires on high-cardinality sorted-prefix data whose index is near-uniform,
        // where the picker cannot beat the flat pack.
        //
        // Wire:
        //   0xFC, varuint(d), d×(varuint(len), len bytes),  // distinct table (plain)
        //        <QPack uint64 block>                        // per-row index, kind uint64
        // Never-larger: emitted only when the picker's chosen-codec byte cost is
        // strictly below the flat ceil(log2 d)-bit index body; otherwise tagColStrDict
        // (flat) is written. The QPack block carries its own row count, cross-checked
        // against the columnar header and buffer-bounded before allocation.
        tagColStrDictQ = 0xFC

        tagColVecLossy = 0xFD // lossy float-vector block (Hadamard+quant+rANS)

        // tagVecBatchStruct is a container for a []struct whose []float32/[]float64
        // vector field(s) are batched into ONE count=N lossy block each (amortizing
        // the per-vector block header + rANS frequency framing that a per-row count=1
        // block pays). Only emitted under OptLossyVec when at least one vector field
        // is batchable (equal length across all rows, >= lossyVecMinElems, and the
        // batched block beats raw). Layout:
        //
        //   0xFE, varuint(n), byte(numVecFields), byte(batchedMask), byte(polarMask),
        //     varuint(numStructFields),
        //     for each set batchedMask bit (vector-field order):
        //       if polarMask bit set: <norm stream: f64 logMin, f64 logMax, n×uint16>,
        //       <0xFD count=n block>,
        //     per row: each NON-batched field encoded in declaration order via its
        //              own (row-major) field codec.
        //
        // polarMask (a subset of batchedMask) marks fields stored in polar-split form:
        // each vector's L2 norm separately (log-domain 16-bit) plus the quantized unit
        // direction in the block, which the decoder rescales.
        //
        // numStructFields is the total struct field count, so Skip() (schema
        // evolution) can walk the n×(numStructFields-popcount(batchedMask)) per-row
        // non-batched fields without a typeDesc, replaying their intern/shape state.
        //
        // Non-batched vector fields and all scalar/string fields stay row-major, so
        // decode reconstructs each row by decoding those fields and scattering the
        // batched vectors. Forward-compat: Skip() walks an unknown 0xFE field cleanly.
        tagVecBatchStruct = 0xFE
)

// isStringColumnBlockTag reports whether b begins a self-describing string
// column block (dict / dictFC / FSST / raw / const) rather than a per-value run.
// The columnar decoders peek this to choose the block reader (readStringColumn)
// over a ReadString loop.
func isStringColumnBlockTag(b byte) bool {
        return b == tagColStrDict || b == tagColStrDictFC || b == tagColStrFSST ||
                b == tagColStrRaw || b == tagColStrConst || b == tagColStrAlpha ||
                b == tagColStrDictQ
}

// Varint (ULEB128) helpers. Used for state-table IDs and intern-payload
// lengths. The encoder always appends; the decoder returns the consumed
// length so the caller can advance its cursor.

//go:nosplit
func appendUvarint(b []byte, x uint64) []byte {
        // 3-byte unrolled fast path. Covers values up to 2^21 = 2 097 151
        // which is well past the default maxStateEntries (16 384) and
        // covers every state-ref / shape ID / fixstr length in practical
        // payloads. Multi-byte values fall through to the loop.
        if x < 0x80 {
                return append(b, byte(x))
        }
        if x < 0x4000 {
                return append(b, byte(x)|0x80, byte(x>>7))
        }
        if x < 0x200000 {
                return append(b, byte(x)|0x80, byte(x>>7)|0x80, byte(x>>14))
        }
        for x >= 0x80 {
                b = append(b, byte(x)|0x80)
                x >>= 7
        }
        return append(b, byte(x))
}

// readUvarint decodes a ULEB128 and returns value, bytes-consumed. n==0 means
// not enough input; n<0 means overflow (>10 bytes).
//
// The 1-byte branch stays inline (cost ≤ 80) — most varints in
// practice are state-ref ranks / shape IDs / small lengths < 128.
// Multi-byte values fall through to the loop below, which is
// extracted into a non-inlinable slow path on purpose so the hot
// path keeps its inline budget.
//
//go:nosplit
func readUvarint(b []byte) (uint64, int) {
        if len(b) > 0 && b[0] < 0x80 {
                return uint64(b[0]), 1
        }
        var x uint64
        var shift uint
        for i, c := range b {
                if i >= 10 {
                        return 0, -1
                }
                if c < 0x80 {
                        // Canonical-form guard, matching encoding/binary.Uvarint: the 10th
                        // byte (i==9, shift==63) may only carry bit 63, so c>1 would set
                        // bits above 63 — reject instead of silently truncating.
                        if i == 9 && c > 1 {
                                return 0, -1
                        }
                        return x | uint64(c)<<shift, i + 1
                }
                x |= uint64(c&0x7F) << shift
                shift += 7
        }
        return 0, 0
}

// uvarintLen returns the number of ULEB128 bytes needed to encode v.
// Inlined by the compiler; used in QPack size estimators and the columnar probe.
func uvarintLen(v uint64) int {
        n := 1
        for v >= 0x80 {
                v >>= 7
                n++
        }
        return n
}

// zigzagEncode64 maps a signed int64 to an unsigned int64 with
// magnitude-preserving low-bit cost: |v| small => result small.
// Used by the FOR / Delta-FOR codecs and the columnar probe.
func zigzagEncode64(v int64) uint64 {
        return uint64((v << 1) ^ (v >> 63))
}

// zigzagDecode64 reverses zigzagEncode64.
func zigzagDecode64(u uint64) int64 {
        return int64((u >> 1) ^ -(u & 1))
}