Skip to main content

quack_rs/aggregate/
state.rs

1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! Generic `FfiState<T>` wrapper for safe aggregate state management.
7//!
8//! # Problem solved
9//!
10//! Writing the state init/destroy lifecycle using raw pointers is error-prone.
11//! The canonical pattern is:
12//!
13//! ```rust,no_run
14//! use libduckdb_sys::{duckdb_function_info, duckdb_aggregate_state, idx_t};
15//!
16//! #[derive(Default)]
17//! struct MyState { count: u64 }
18//!
19//! #[repr(C)]
20//! struct FfiState { inner: *mut MyState }
21//!
22//! unsafe extern "C" fn state_init(_: duckdb_function_info, state: duckdb_aggregate_state) {
23//!     let ffi = &mut *(state as *mut FfiState);
24//!     ffi.inner = Box::into_raw(Box::new(MyState::default()));
25//! }
26//!
27//! unsafe extern "C" fn state_destroy(states: *mut duckdb_aggregate_state, count: idx_t) {
28//!     for i in 0..count as usize {
29//!         let state_ptr = *states.add(i);
30//!         let ffi = &mut *(state_ptr as *mut FfiState);
31//!         if !ffi.inner.is_null() {
32//!             drop(Box::from_raw(ffi.inner));
33//!             ffi.inner = std::ptr::null_mut();
34//!         }
35//!     }
36//! }
37//! ```
38//!
39//! [`FfiState<T>`] encapsulates this pattern. Your type `T` only needs to
40//! implement [`AggregateState`] (which requires `Default`), and you call the
41//! provided helper methods instead of writing raw pointer code.
42//!
43//! # Pitfalls prevented
44//!
45//! - **L1**: Combine propagates all fields because your type `T`'s `combine`
46//!   method is responsible — the `FfiState` wrapper ensures `T`'s method is called.
47//! - **L2**: No double-free — `destroy_callback` clears the state's tag before
48//!   dropping `T`, and drops only a state whose tag matches.
49//! - **L3**: No panic across FFI — `with_state_mut` returns an `Option`, not a panic.
50
51use libduckdb_sys::{duckdb_aggregate_state, duckdb_function_info, idx_t};
52
53/// Trait for types that can be used as `DuckDB` aggregate state.
54///
55/// Implement this for your state struct. It requires `Default` (used to
56/// create the initial state in `state_init`), `Send` (`DuckDB` moves states
57/// between worker threads) and `Sync`: a window's segment tree keeps its
58/// states in shared, unlocked storage (`WindowSegmentTreeGlobalState`,
59/// `window_segment_tree.cpp`), and every thread evaluating a frame that covers
60/// one reads it as a `combine` source, so two threads can hold `&T` to the
61/// same state at once. A `T` with a `Cell` or `RefCell` would race there.
62///
63/// **Breaking** in 0.18.0: the `Sync` bound is new.
64///
65/// # Example
66///
67/// ```rust
68/// use quack_rs::aggregate::AggregateState;
69///
70/// #[derive(Default)]
71/// struct WordCount {
72///     count: u64,
73/// }
74///
75/// impl AggregateState for WordCount {}
76///
77/// // FfiState::<WordCount>::size_callback and other methods are now available.
78/// ```
79pub trait AggregateState: Default + Send + Sync + 'static {}
80
81/// 64-bit FNV-1a, the hash behind `FfiState`'s per-type salt: deterministic
82/// in every build, unlike `std`'s randomly keyed hashers.
83struct Fnv1a(u64);
84
85impl Fnv1a {
86    const fn new() -> Self {
87        Self(0xCBF2_9CE4_8422_2325)
88    }
89}
90
91impl core::hash::Hasher for Fnv1a {
92    fn finish(&self) -> u64 {
93        self.0
94    }
95    fn write(&mut self, bytes: &[u8]) {
96        for &byte in bytes {
97            self.0 = (self.0 ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01B3);
98        }
99    }
100}
101
102/// A generic FFI-compatible state wrapper for use with `DuckDB` aggregate functions.
103///
104/// `FfiState<T>` describes the bytes `DuckDB` allocates for each aggregate
105/// group: [`size_callback`][FfiState::size_callback] tells `DuckDB` how many,
106/// [`init_callback`][FfiState::init_callback] puts a `T::default()` in them,
107/// [`with_state`][FfiState::with_state] and
108/// [`with_state_mut`][FfiState::with_state_mut] hand out the `T`, and
109/// [`destroy_callback`][FfiState::destroy_callback] drops it. The type is
110/// never constructed; it only names the layout and its callbacks.
111///
112/// # Known `DuckDB` limitation
113///
114/// **Before `DuckDB` 2.0.0, two query shapes make every C-API aggregate read
115/// out of bounds.** In them `DuckDB` calls `update` with a state array holding
116/// **one** state while
117/// passing `count > 1` rows, so the callback reads `states[1..count]` past the
118/// end of the array — undefined behaviour in *any* C-API aggregate, whether
119/// built with quack-rs or by hand:
120///
121/// - **Window aggregates whose frame is the whole partition**, e.g.
122///   `agg(x) OVER ()` or an explicit `ROWS BETWEEN UNBOUNDED PRECEDING AND
123///   UNBOUNDED FOLLOWING` frame (`OVER (PARTITION BY p)` without `ORDER BY`
124///   crashed on 1.4.4 only in the shapes tried) — `WindowConstantAggregator`
125///   (`src/function/window/window_constant_aggregator.cpp`, ~lines 106 and
126///   296–299 in `DuckDB` 1.5.5).
127/// - **Ordered aggregates**, e.g. `agg(x ORDER BY y)` —
128///   `src/function/aggregate/sorted_aggregate_function.cpp`, ~lines 630–633.
129///
130/// Both pass a `CONSTANT_VECTOR` of states because the function has no
131/// `simple_update` (the C API cannot set one), and `CAPIAggregateUpdate`
132/// (`src/main/capi/aggregate_function-c.cpp`, ~lines 92–110) hands the
133/// vector's data pointer to the extension without flattening it. This is a
134/// defect in `DuckDB`'s C API, not in quack-rs, and it cannot be detected
135/// from inside the callback — reading `states[1]` to check is itself the
136/// out-of-bounds read. Reported upstream as
137/// [duckdb/duckdb#26109](https://github.com/duckdb/duckdb/issues/26109).
138/// On `DuckDB` 1.4.x and 1.5.x, do not use C-API aggregates in those two
139/// query shapes.
140///
141/// **Fixed in `DuckDB` 2.0.0:** `CAPIAggregateUpdate` flattens a non-flat
142/// state vector into one state pointer per row before calling `update`
143/// (`src/main/capi/v1/aggregate_function-c.cpp`, lines 104–108 at v2.0.0).
144/// Both shapes then give the right answer, and a plain C aggregate runs them
145/// clean under Valgrind; on 1.5.6 the same program segfaults.
146///
147/// # Memory layout
148///
149/// ```text
150/// [ tag: usize ][ T, padded to a multiple of usize ]   T stored inline
151/// [ tag: usize ][ *mut T                          ]   T boxed
152/// ```
153///
154/// `T` is stored in the state bytes themselves when its alignment is at most
155/// `usize`'s and it is at most 256 bytes; otherwise the slot holds a
156/// `Box<T>`. The size is a multiple of `usize` either way, because some
157/// `DuckDB` operators lay states out back to back at a stride of exactly
158/// `state_size` (the window aggregators), and every state must stay aligned.
159/// `DuckDB` aligns each state to 8 bytes (`AlignValue` in `AggregateObject`
160/// and `TupleDataLayout`; `new[]`-allocated buffers elsewhere), which covers
161/// both forms on 64- and 32-bit targets.
162///
163/// # States `DuckDB` never destroys
164///
165/// `DuckDB` 1.4.4 to 1.5.5 (not re-measured on 1.5.6) destroys a grouped
166/// aggregate's states as its result
167/// scan passes them. When the scan stops early — a `LIMIT` above the
168/// aggregate, an error raised above it, an interrupt — the states it did not
169/// reach are never destroyed (`docs/upstream-duckdb-reports.md`, item 20):
170/// under `LIMIT 10`, 2,048 of 300,000 on one thread. Their `T` is never
171/// dropped. A `T` stored inline costs nothing more: its bytes belong to
172/// `DuckDB`'s hash table, which `DuckDB` frees. What leaks is whatever `T`
173/// itself owns on the heap (a `Vec`, a `String`, a `HashMap`), and, for a
174/// boxed `T`, the box. `DuckDB` 2.0.0 still behaves this way (pinned by
175/// `tests/aggregate_leaks.rs`). Before 2.0.0 the same holds for a window
176/// frame with an `EXCLUDE` clause, whose segment tree initialises one extra
177/// state per row and never destroys it (item 35; 5000 `T`s undropped over a
178/// 5000-row window); 2.0.0 destroys those states
179/// (`WindowSegmentTreeLocalState::Evaluate`, `window_segment_tree.cpp`
180/// 506–510). Until `DuckDB` fixes the stopped scan, prefer a state that owns
181/// no heap memory and is small enough to be stored inline.
182///
183/// # States `DuckDB` never initialised
184///
185/// When a `state_init` call fails — a panicking `T::default()`, or an
186/// allocation failure — `DuckDB` 1.4.4 to 1.5.5 (not re-measured on 1.5.6)
187/// still calls the destructor
188/// on every state row it had created, initialised or not
189/// (`RowOperations::InitializeStates` is not exception-safe; see
190/// `docs/upstream-duckdb-reports.md`). That includes the states of *other*
191/// aggregates in the same query whose `state_init` never ran. Those bytes
192/// are whatever the allocation held, and dropping them as a `T` would run a
193/// destructor over garbage.
194///
195/// So [`init_callback`][Self::init_callback] also stores `tag`, a value
196/// derived from `T` (and, for a boxed `T`, from the box's address), and
197/// [`destroy_callback`][Self::destroy_callback] drops only a slot whose tag
198/// matches, clearing it first. A slot the destructor has already processed no
199/// longer matches; a slot another aggregate's type initialised does not match
200/// either. What is left is chance: uninitialised bytes that happen to hold
201/// the tag, one in 2<sup>64</sup> on a 64-bit target. This is a mitigation of
202/// a `DuckDB` defect, not a guarantee. Reading bytes `DuckDB` may never have
203/// written is, in Rust's abstract machine, a read of uninitialised memory
204/// whatever the load: the destructor uses volatile loads so the compiler
205/// cannot reason from them, but no Rust construct makes such a read defined.
206/// It is the least bad of the options this defect leaves.
207///
208/// The tag depends on the slot's contents, not its address, because `DuckDB`
209/// moves states: radix repartitioning of a hash aggregate copies each state's
210/// bytes to a new row (`radix_partitioned_hashtable.cpp`) and destroys it
211/// there. The fourth audit's tag used the address, so every moved state was
212/// skipped and its `T` leaked — millions per query on eight threads. Moving
213/// a Rust value by copying its bytes is always valid; the copy left behind is
214/// never destroyed.
215///
216/// # Usage
217///
218/// ```rust
219/// use quack_rs::aggregate::{AggregateState, FfiState};
220/// use libduckdb_sys::{duckdb_function_info, duckdb_aggregate_state, idx_t};
221///
222/// #[derive(Default)]
223/// struct MyState { sum: i64 }
224/// impl AggregateState for MyState {}
225///
226/// // In your registration code:
227/// // .ffi_state::<MyState>()
228/// ```
229///
230/// **Breaking** in 0.18.0: `FfiState<T>` was a one-word `#[repr(C)]` struct
231/// with a public `inner: *mut T` field that always boxed `T`. The layout is
232/// now private.
233pub struct FfiState<T: AggregateState> {
234    _state: core::marker::PhantomData<fn() -> T>,
235}
236
237impl<T: AggregateState> FfiState<T> {
238    /// The tag's width, and the offset of the payload.
239    const WORD: usize = core::mem::size_of::<usize>();
240
241    /// Whether `T` is stored in the state bytes rather than boxed.
242    const INLINE: bool =
243        core::mem::align_of::<T>() <= Self::WORD && core::mem::size_of::<T>() <= INLINE_LIMIT;
244
245    /// Bytes per state: the tag, then `T` (rounded up to a whole number of
246    /// words) or a `Box<T>`.
247    const SIZE: usize = Self::WORD
248        + if Self::INLINE {
249            core::mem::size_of::<T>().div_ceil(Self::WORD) * Self::WORD
250        } else {
251            Self::WORD
252        };
253
254    /// Mixed into the tag: an arbitrary odd constant, so that a zeroed slot
255    /// does not match.
256    #[cfg(target_pointer_width = "64")]
257    const TAG_KEY: usize = 0x9E37_79B9_7F4A_7C15;
258    #[cfg(not(target_pointer_width = "64"))]
259    const TAG_KEY: usize = 0x9E37_79B9;
260
261    /// The tag a slot carries once initialised for this `T`: `boxed` is the
262    /// box's address for a boxed `T`, and zero for an inline one.
263    ///
264    /// `TAG_KEY` is salted with a hash of `T`'s [`TypeId`](core::any::TypeId),
265    /// which differs between types, so a slot initialised for one aggregate's
266    /// state type is not accepted by another's destructor. The salt's low bit
267    /// is cleared, so the key stays odd and never zero: a zeroed slot never
268    /// matches.
269    ///
270    /// The salt must be the same in every callback. An address is not: the
271    /// salt was once the address of `type_name::<T>()`, and rustc gives each
272    /// codegen unit that uses a constant its own copy, so under Cargo's
273    /// default release profile (16 codegen units, no fat LTO) `init`,
274    /// `with_state` and `destroy` could each see a different address.
275    fn tag_for(boxed: *const T) -> usize {
276        (boxed as usize) ^ Self::TAG_KEY ^ Self::salt(Self::type_hash())
277    }
278
279    /// FNV-1a over `T`'s `TypeId`, as its `Hash` impl feeds it: the same
280    /// value wherever it is computed, and a constant once optimised.
281    fn type_hash() -> usize {
282        let mut hasher = Fnv1a::new();
283        core::hash::Hash::hash(&core::any::TypeId::of::<T>(), &mut hasher);
284        // Truncation on 32-bit targets keeps the low half, which is as good a
285        // salt as any.
286        #[allow(clippy::cast_possible_truncation)]
287        let hash = core::hash::Hasher::finish(&hasher) as usize;
288        hash
289    }
290
291    /// `hash` with its low bit cleared, so that `TAG_KEY ^ salt` stays odd.
292    const fn salt(hash: usize) -> usize {
293        hash & !1
294    }
295
296    /// The payload: `T` itself, or the `Box<T>`'s pointer.
297    ///
298    /// Derived as the word after the tag, so it keeps the tag's `usize`
299    /// alignment, which is all either form needs.
300    const fn payload(state: duckdb_aggregate_state) -> *mut usize {
301        state.cast::<usize>().wrapping_add(1)
302    }
303
304    /// Returns the number of bytes `DuckDB` allocates per state.
305    ///
306    /// # Example
307    ///
308    /// ```rust
309    /// use quack_rs::aggregate::{AggregateState, FfiState};
310    ///
311    /// #[derive(Default)]
312    /// struct MyState { val: i64 }
313    /// impl AggregateState for MyState {}
314    ///
315    /// // A tag, then the `i64` itself.
316    /// assert_eq!(FfiState::<MyState>::size(), 2 * std::mem::size_of::<usize>());
317    /// ```
318    #[inline]
319    #[must_use]
320    pub const fn size() -> usize {
321        Self::SIZE
322    }
323
324    /// The `state_size` callback function for use in the builder.
325    ///
326    /// Returns [`size`][Self::size]: the number of bytes `DuckDB` must
327    /// allocate per aggregate group.
328    ///
329    /// # Safety
330    ///
331    /// This is an `unsafe extern "C"` function pointer. It is safe to pass to
332    /// [`AggregateFunctionBuilder::state_size`][crate::aggregate::AggregateFunctionBuilder::state_size].
333    pub const unsafe extern "C" fn size_callback(_info: duckdb_function_info) -> idx_t {
334        Self::SIZE as idx_t
335    }
336
337    /// The `state_init` callback function for use in the builder.
338    ///
339    /// Stores a `T::default()` in the state at `state` (or a box holding one).
340    ///
341    /// # Pitfall L3: `T::default()` is user code
342    ///
343    /// `DuckDB` invokes this through an `extern "C"` function pointer, so a
344    /// panic escaping `T::default()` would abort the process (Rust 1.81+ turns
345    /// an unwind across `extern "C"` into `panic_cannot_unwind`). The call is
346    /// therefore run under
347    /// [`catch_ffi_panic`][crate::callback::catch_ffi_panic] and a panic is
348    /// reported through `duckdb_aggregate_function_set_error` — which
349    /// `DuckDB`'s `CAPIAggregateStateInit` checks, turning it into an ordinary
350    /// SQL error — instead of taking the session down.
351    ///
352    /// # A failing initialiser in a window frame still crashes the host
353    ///
354    /// That error report is itself the hazard when the aggregate runs as a
355    /// *window* function (a `ROWS`/`RANGE` frame). `DuckDB`'s
356    /// `WindowAggregateStates` fills its table of state pointers one entry per
357    /// *successful* initialisation and then, when one fails, destroys the whole
358    /// table, unwritten entries included: the destructor is handed pointers
359    /// that never pointed at a state. The tag check below cannot help, because
360    /// there is no slot to read a tag from. Reproduced with a plain C
361    /// aggregate, no Rust involved, on `DuckDB` 1.5.6 (55 of 68 pointers
362    /// garbage) and 2.0.0 (65 of 68); the host dies with `SIGSEGV` or `SIGBUS`.
363    /// Ungrouped, grouped, `DISTINCT` and `FILTER` aggregates report a clean
364    /// error. So: **`T::default()` must not panic** for an aggregate that can
365    /// be used in a window.
366    ///
367    /// # Uninitialised memory
368    ///
369    /// `DuckDB` does not promise the state allocation is zeroed, so the tag is
370    /// cleared (and a boxed slot's pointer nulled) *before* any user code
371    /// runs. A slot whose `T::default()` panicked therefore never matches its
372    /// tag: [`with_state`][Self::with_state] returns `None` for it and
373    /// [`destroy_callback`][Self::destroy_callback] skips it.
374    ///
375    /// # Safety
376    ///
377    /// - `state` must point to [`size`][Self::size] writable bytes allocated
378    ///   by `DuckDB`, aligned to `usize`.
379    /// - This function must only be called once per state allocation.
380    pub unsafe extern "C" fn init_callback(
381        info: duckdb_function_info,
382        state: duckdb_aggregate_state,
383    ) {
384        let payload = Self::payload(state);
385        // SAFETY: `state` is `size()` writable bytes aligned to `usize`, so
386        // the tag word and (for a boxed `T`) the pointer word after it are in
387        // bounds and aligned. `ptr::write` does not read what was there.
388        unsafe {
389            core::ptr::write(state.cast::<usize>(), 0);
390            if !Self::INLINE {
391                core::ptr::write(payload.cast::<*mut T>(), core::ptr::null_mut());
392            }
393        }
394        let made = if Self::INLINE {
395            crate::callback::catch_ffi_panic(T::default).map(|value| {
396                // SAFETY: the payload is `size_of::<T>()` bytes inside the
397                // state, at offset `usize`, which `INLINE` guarantees is a
398                // multiple of `align_of::<T>()`.
399                unsafe { core::ptr::write(payload.cast::<T>(), value) };
400                Self::tag_for(core::ptr::null())
401            })
402        } else {
403            crate::callback::catch_ffi_panic(Box::<T>::default).map(|boxed| {
404                let inner = Box::into_raw(boxed);
405                // SAFETY: the pointer word was written above.
406                unsafe { core::ptr::write(payload.cast::<*mut T>(), inner) };
407                Self::tag_for(inner)
408            })
409        };
410        match made {
411            // SAFETY: the tag word was written above.
412            Ok(tag) => unsafe { core::ptr::write(state.cast::<usize>(), tag) },
413            Err(message) => {
414                let c_msg = crate::callback::message_to_c_string(&format!(
415                    "quack-rs: aggregate state initialiser panicked: {message}"
416                ));
417                // SAFETY: `info` is the handle DuckDB passed in, and DuckDB
418                // checks the error flag on return from the state-init callback.
419                unsafe {
420                    libduckdb_sys::duckdb_aggregate_function_set_error(info, c_msg.as_ptr());
421                }
422            }
423        }
424    }
425
426    /// The `state_destroy` callback function for use in the builder.
427    ///
428    /// Drops the `T` in each state in `states[0..count]` (freeing its box, if
429    /// boxed), clearing the tag first so a second call is a no-op.
430    ///
431    /// # Count conversion
432    ///
433    /// If `count` (an `idx_t`) cannot be converted to `usize`, the loop iterates
434    /// zero times — no states are freed. This is a defensive choice to avoid
435    /// panicking across FFI. In practice, `idx_t` is `u64` and `usize` is at
436    /// least 64 bits on all `DuckDB`-supported platforms, so this path is
437    /// unreachable on supported targets.
438    ///
439    /// # Pitfall L3: `T::drop` is user code
440    ///
441    /// `DuckDB` calls the destructor as `info.destroy(states, count)` — no info
442    /// handle, no return value, and therefore **no error channel at all**
443    /// (verified in `CAPIAggregateDestructor`). A panic escaping `T::drop`
444    /// would abort the process, so each drop runs under
445    /// [`catch_ffi_panic`][crate::callback::catch_ffi_panic] and the message is
446    /// discarded. Every remaining state is still dropped: one bad destructor
447    /// does not leak the rest of the group.
448    ///
449    /// One case cannot be contained: a `T::drop` that panics while one of
450    /// `T`'s fields also panics in its own `drop` during that unwind. Rust
451    /// aborts the process for a panic raised while unwinding ("panic in a
452    /// destructor during cleanup") before any `catch_unwind` sees it.
453    ///
454    /// # Safety
455    ///
456    /// - `states` must point to an array of `count` valid `duckdb_aggregate_state`
457    ///   pointers, each to [`size`][Self::size] bytes, aligned to `usize`, that
458    ///   `DuckDB` allocated for this aggregate. A slot
459    ///   [`init_callback`][Self::init_callback] never initialised is skipped
460    ///   (see "States `DuckDB` never initialised" on the type).
461    pub unsafe extern "C" fn destroy_callback(states: *mut duckdb_aggregate_state, count: idx_t) {
462        for i in 0..usize::try_from(count).unwrap_or(0) {
463            // SAFETY: `states` is a valid array of `count` pointers.
464            let state = unsafe { *states.add(i) };
465            let payload = Self::payload(state);
466            // SAFETY: the slot is `size()` bytes DuckDB allocated for this
467            // aggregate, aligned to `usize`, so the tag word (and a boxed
468            // slot's pointer word) are in bounds. They may never have been
469            // written by `init_callback`, so they are read as plain words with
470            // volatile loads, which the compiler must perform as written and
471            // cannot reason about, and only trusted once the tag matches.
472            let tag = unsafe { core::ptr::read_volatile(state.cast::<usize>()) };
473            let inner = if Self::INLINE {
474                core::ptr::null_mut()
475            } else {
476                // SAFETY: as above.
477                unsafe { core::ptr::read_volatile(payload.cast::<*mut T>()) }
478            };
479            if tag != Self::tag_for(inner) {
480                continue;
481            }
482            // Clear the tag (and the pointer) *before* dropping: if `T::drop`
483            // panics, the value is still released by the unwind, so leaving
484            // the slot live would make a second destructor call a double drop.
485            // SAFETY: the tag matched, so `init_callback` initialised the slot.
486            unsafe {
487                core::ptr::write(state.cast::<usize>(), 0);
488                if !Self::INLINE {
489                    core::ptr::write(payload.cast::<*mut T>(), core::ptr::null_mut());
490                }
491            }
492            // The drop runs arbitrary user code, so it must not be allowed to
493            // unwind out of this `extern "C"` function.
494            drop(crate::callback::catch_ffi_panic(|| {
495                if Self::INLINE {
496                    // SAFETY: the tag matched, so the payload holds a `T` that
497                    // `init_callback` wrote and nothing has dropped; the tag is
498                    // cleared, so nothing will drop it again.
499                    unsafe { core::ptr::drop_in_place(payload.cast::<T>()) };
500                } else if !inner.is_null() {
501                    // SAFETY: `inner` came from `Box::into_raw` in
502                    // `init_callback`, and the slot no longer refers to it.
503                    drop(unsafe { Box::from_raw(inner) });
504                }
505            }));
506        }
507    }
508
509    /// The `T` in an initialised slot, or `None` if the slot's tag does not
510    /// match (its `T::default()` panicked).
511    ///
512    /// # Safety
513    ///
514    /// `state` must be a slot [`init_callback`][Self::init_callback] has run on.
515    unsafe fn value(state: duckdb_aggregate_state) -> Option<*mut T> {
516        let payload = Self::payload(state);
517        // SAFETY: `init_callback` wrote the tag word, and for a boxed `T` the
518        // pointer word.
519        let tag = unsafe { core::ptr::read(state.cast::<usize>()) };
520        if Self::INLINE {
521            (tag == Self::tag_for(core::ptr::null())).then(|| payload.cast::<T>())
522        } else {
523            // SAFETY: as above.
524            let inner = unsafe { core::ptr::read(payload.cast::<*mut T>()) };
525            (!inner.is_null() && tag == Self::tag_for(inner)).then_some(inner)
526        }
527    }
528
529    /// Provides safe mutable access to the inner `T` value.
530    ///
531    /// Returns `None` if the slot holds no `T` — its `T::default()` panicked
532    /// (and `DuckDB` is about to fail the query), or it was destroyed.
533    ///
534    /// # Pitfall L3: No panic across FFI
535    ///
536    /// This method returns `Option<&mut T>` rather than unwrapping, so callers
537    /// can use `if let Some(state) = ...` patterns without panicking.
538    ///
539    /// # Safety
540    ///
541    /// - `state` must point to a state `DuckDB` allocated for this aggregate
542    ///   and [`init_callback`][Self::init_callback] initialised.
543    /// - No other reference to the same `T` must exist simultaneously.
544    /// - The returned reference must not outlive the callback that received
545    ///   `state`. The lifetime `'a` is the caller's to choose and nothing ties
546    ///   it to `state`; `DuckDB` destroys the state — and
547    ///   [`destroy_callback`][Self::destroy_callback] drops the `T` — once it
548    ///   is finalized or merged, and may move its bytes between callbacks.
549    ///
550    /// # Example
551    ///
552    /// ```rust,no_run
553    /// use quack_rs::aggregate::{AggregateState, FfiState};
554    ///
555    /// #[derive(Default)]
556    /// struct Counter { n: u64 }
557    /// impl AggregateState for Counter {}
558    ///
559    /// // Inside your update callback:
560    /// // let ffi_state: duckdb_aggregate_state = ...;
561    /// // if let Some(state) = unsafe { FfiState::<Counter>::with_state_mut(ffi_state) } {
562    /// //     state.n += 1;
563    /// // }
564    /// ```
565    pub unsafe fn with_state_mut<'a>(state: duckdb_aggregate_state) -> Option<&'a mut T> {
566        // SAFETY: the caller guarantees `state` was initialised, and that no
567        // other reference to its `T` exists.
568        unsafe { Self::value(state).map(|value| &mut *value) }
569    }
570
571    /// Provides safe immutable access to the inner `T` value.
572    ///
573    /// See [`with_state_mut`][Self::with_state_mut] for safety requirements.
574    ///
575    /// # Safety
576    ///
577    /// Same as `with_state_mut`, but only borrows immutably: no mutable
578    /// reference to the same `T` may exist while this one does.
579    pub unsafe fn with_state<'a>(state: duckdb_aggregate_state) -> Option<&'a T> {
580        // SAFETY: the caller guarantees `state` was initialised, and that no
581        // mutable reference to its `T` exists.
582        unsafe { Self::value(state).map(|value| &*value) }
583    }
584}
585
586/// The largest `T`, in bytes, that [`FfiState`] stores inline. A larger `T`
587/// is boxed. Every aggregate state of a group shares one hash-table row, and
588/// `DuckDB` allocates rows in blocks of 256 KiB by default
589/// (`DEFAULT_BLOCK_ALLOC_SIZE`, `TupleDataAllocator::Build`); boxing a large
590/// `T` keeps the row narrow.
591const INLINE_LIMIT: usize = 256;
592
593impl<T: AggregateState> core::fmt::Debug for FfiState<T> {
594    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
595        f.debug_struct("FfiState")
596            .field("state", &core::any::type_name::<T>())
597            .field("inline", &Self::INLINE)
598            .field("size", &Self::SIZE)
599            .finish()
600    }
601}
602
603#[cfg(test)]
604mod tests {
605    use super::*;
606    use std::sync::atomic::{AtomicIsize, Ordering};
607
608    const WORD: usize = core::mem::size_of::<usize>();
609
610    /// Word-sized and word-aligned, so inline on every target.
611    #[derive(Default, Debug, PartialEq)]
612    struct Counter {
613        value: usize,
614    }
615    impl AggregateState for Counter {}
616
617    /// Too large to store inline.
618    #[derive(Debug, PartialEq)]
619    struct Big {
620        values: [u64; 64],
621    }
622    impl Default for Big {
623        fn default() -> Self {
624            Self { values: [0; 64] }
625        }
626    }
627    impl AggregateState for Big {}
628
629    /// Aligned more strictly than `usize`.
630    #[allow(dead_code)] // only its layout matters
631    #[derive(Default)]
632    #[repr(align(32))]
633    struct Aligned(u8);
634    impl AggregateState for Aligned {}
635
636    #[derive(Default)]
637    struct Zst;
638    impl AggregateState for Zst {}
639
640    #[allow(dead_code)] // only its layout matters
641    #[derive(Default)]
642    struct Odd([u8; 3]);
643    impl AggregateState for Odd {}
644
645    #[allow(dead_code)] // only its layout matters
646    struct AtLimit([u8; INLINE_LIMIT]);
647    impl Default for AtLimit {
648        fn default() -> Self {
649            Self([0; INLINE_LIMIT])
650        }
651    }
652    impl AggregateState for AtLimit {}
653
654    #[allow(dead_code)] // only its layout matters
655    struct OverLimit([u8; INLINE_LIMIT + 1]);
656    impl Default for OverLimit {
657        fn default() -> Self {
658            Self([0; INLINE_LIMIT + 1])
659        }
660    }
661    impl AggregateState for OverLimit {}
662
663    /// Live `Tracked` / `TrackedBig` values, to see whether a destructor ran.
664    static TRACKED_LIVE: AtomicIsize = AtomicIsize::new(0);
665
666    /// Held by every test that counts `TRACKED_LIVE`: the tests run in
667    /// parallel, and one test's live value would otherwise show in another's
668    /// count.
669    static TRACKED: std::sync::Mutex<()> = std::sync::Mutex::new(());
670
671    fn tracked() -> std::sync::MutexGuard<'static, ()> {
672        TRACKED
673            .lock()
674            .unwrap_or_else(std::sync::PoisonError::into_inner)
675    }
676
677    struct Tracked {
678        value: u64,
679    }
680    impl Default for Tracked {
681        fn default() -> Self {
682            TRACKED_LIVE.fetch_add(1, Ordering::SeqCst);
683            Self { value: 7 }
684        }
685    }
686    impl Drop for Tracked {
687        fn drop(&mut self) {
688            TRACKED_LIVE.fetch_sub(1, Ordering::SeqCst);
689        }
690    }
691    impl AggregateState for Tracked {}
692
693    struct TrackedBig {
694        values: [u64; 64],
695    }
696    impl Default for TrackedBig {
697        fn default() -> Self {
698            TRACKED_LIVE.fetch_add(1, Ordering::SeqCst);
699            Self { values: [7; 64] }
700        }
701    }
702    impl Drop for TrackedBig {
703        fn drop(&mut self) {
704            TRACKED_LIVE.fetch_sub(1, Ordering::SeqCst);
705        }
706    }
707    impl AggregateState for TrackedBig {}
708
709    /// `count` states of `T`'s size, back to back like `DuckDB`'s window
710    /// arrays, each word filled with `fill`.
711    fn slots<T: AggregateState>(count: usize, fill: usize) -> Vec<usize> {
712        assert_eq!(FfiState::<T>::size() % WORD, 0);
713        vec![fill; count * FfiState::<T>::size() / WORD]
714    }
715
716    /// State `index` of the array starting at `base`. Every state of one
717    /// array is derived from the same `base`, so none invalidates another
718    /// (Miri, Stacked Borrows).
719    fn state_at<T: AggregateState>(base: *mut usize, index: usize) -> duckdb_aggregate_state {
720        base.wrapping_add(index * FfiState::<T>::size() / WORD)
721            .cast()
722    }
723
724    fn init<T: AggregateState>(state: duckdb_aggregate_state) {
725        // SAFETY: `state` is `size()` writable, word-aligned bytes; a
726        // non-panicking `T::default()` never touches `info`.
727        unsafe { FfiState::<T>::init_callback(core::ptr::null_mut(), state) };
728    }
729
730    fn destroy<T: AggregateState>(states: &mut [duckdb_aggregate_state]) {
731        // SAFETY: each state is `size()` word-aligned bytes, initialised or
732        // not, which is what the destructor must cope with.
733        unsafe { FfiState::<T>::destroy_callback(states.as_mut_ptr(), states.len() as idx_t) };
734    }
735
736    /// A tag is the box's address XOR a per-type key, so two tags of one
737    /// type differ exactly where the addresses do, and an inline state's tag
738    /// (no box) is the key itself.
739    #[test]
740    fn a_tag_is_the_box_address_xor_the_types_key() {
741        // Pin the whole structure `addr ^ TAG_KEY ^ salt` against `TAG_KEY`
742        // and the salt directly. Asserting only `tag ^ key == addr` is weaker:
743        // it also holds when the second XOR is an AND and `addr` is a submask
744        // of the salt.
745        let salt = FfiState::<Counter>::salt(FfiState::<Counter>::type_hash());
746        let key = FfiState::<Counter>::TAG_KEY ^ salt;
747        // An inline state (null box) carries the key itself.
748        assert_eq!(FfiState::<Counter>::tag_for(core::ptr::null()), key);
749        for addr in [0x10_usize, 0x1000, 0xFFFF_FFF0, usize::MAX & !0xF] {
750            let boxed = core::ptr::without_provenance::<Counter>(addr);
751            let tag = FfiState::<Counter>::tag_for(boxed);
752            assert_eq!(tag, addr ^ FfiState::<Counter>::TAG_KEY ^ salt, "{addr:#x}");
753            // And two tags of one type differ exactly where the addresses do.
754            assert_eq!(tag ^ key, addr, "{addr:#x}");
755        }
756    }
757
758    /// The key is odd, whatever the type hash, so no zeroed slot and no
759    /// aligned box address can produce a tag of zero.
760    #[test]
761    fn the_salt_clears_the_low_bit_so_the_key_stays_odd() {
762        for hash in [0x1000_usize, 0x1001, 0x7FFF_FFFF, usize::MAX] {
763            let salt = FfiState::<Counter>::salt(hash);
764            assert_eq!(salt & 1, 0, "{hash:#x}");
765            assert_eq!(salt | 1, hash | 1, "{hash:#x}");
766            assert_eq!((FfiState::<Counter>::TAG_KEY ^ salt) & 1, 1);
767        }
768    }
769
770    /// `Fnv1a` is FNV-1a: pinned to the published 64-bit test vectors
771    /// (<http://www.isthe.com/chongo/src/fnv/test_fnv.c>), so a slip in the
772    /// mixing step (an OR for the XOR, say) cannot pass as "still a hash".
773    #[test]
774    fn fnv1a_matches_the_reference_vectors() {
775        use core::hash::Hasher;
776        for (input, want) in [
777            (&b""[..], 0xCBF2_9CE4_8422_2325_u64),
778            (b"a", 0xAF63_DC4C_8601_EC8C),
779            (b"foobar", 0x8594_4171_F739_67E8),
780        ] {
781            let mut h = Fnv1a::new();
782            h.write(input);
783            assert_eq!(h.finish(), want, "{input:?}");
784        }
785    }
786
787    /// Different state types get different salts, so one aggregate's
788    /// destructor does not accept another's slots.
789    #[test]
790    fn distinct_types_have_distinct_salts() {
791        let salts = [
792            FfiState::<Counter>::salt(FfiState::<Counter>::type_hash()),
793            FfiState::<Big>::salt(FfiState::<Big>::type_hash()),
794            FfiState::<Zst>::salt(FfiState::<Zst>::type_hash()),
795            FfiState::<Odd>::salt(FfiState::<Odd>::type_hash()),
796        ];
797        for (i, a) in salts.iter().enumerate() {
798            for b in &salts[i + 1..] {
799                assert_ne!(a, b);
800            }
801        }
802    }
803
804    /// Regression: the salt was the address of `type_name::<T>()`, and rustc
805    /// gives each codegen unit its own copy of a constant, so callbacks in
806    /// different codegen units computed different tags for one type and
807    /// `with_state` / `destroy` rejected every state. Each module below tends
808    /// to land in its own codegen unit; with the address salt, a release build
809    /// with 16 codegen units and no fat LTO got 6 distinct tags from 8 of them.
810    /// Only such a build can fail here: CI runs this test that way.
811    #[test]
812    fn the_tag_is_the_same_in_every_codegen_unit() {
813        let tags = [
814            cgu1::tag(),
815            cgu2::tag(),
816            cgu3::tag(),
817            cgu4::tag(),
818            cgu5::tag(),
819            cgu6::tag(),
820            cgu7::tag(),
821            cgu8::tag(),
822        ];
823        assert!(tags.iter().all(|&t| t == tags[0]), "{tags:x?}");
824    }
825
826    macro_rules! tag_in_own_module {
827        ($($m:ident),*) => {$(
828            mod $m {
829                #[inline(never)]
830                pub fn tag() -> usize {
831                    super::FfiState::<super::Counter>::tag_for(core::ptr::null())
832                }
833            }
834        )*};
835    }
836    tag_in_own_module!(cgu1, cgu2, cgu3, cgu4, cgu5, cgu6, cgu7, cgu8);
837
838    #[test]
839    fn a_small_state_is_stored_inline_after_a_tag() {
840        const { assert!(FfiState::<Counter>::INLINE) };
841        assert_eq!(FfiState::<Counter>::size(), 2 * WORD);
842        const { assert!(FfiState::<Zst>::INLINE) };
843        assert_eq!(FfiState::<Zst>::size(), WORD);
844        // Padded to whole words, so back-to-back states stay aligned.
845        const { assert!(FfiState::<Odd>::INLINE) };
846        assert_eq!(FfiState::<Odd>::size(), 2 * WORD);
847        const { assert!(FfiState::<AtLimit>::INLINE) };
848        assert_eq!(FfiState::<AtLimit>::size(), WORD + INLINE_LIMIT);
849        // A `u64` is 8-aligned on wasm32 too, where `usize` is 4 bytes, so a
850        // state holding one is boxed there and inline on 64-bit targets.
851        assert_eq!(
852            FfiState::<Tracked>::INLINE,
853            core::mem::align_of::<u64>() <= WORD
854        );
855    }
856
857    #[test]
858    fn a_large_or_over_aligned_state_is_boxed() {
859        for (inline, size) in [
860            (FfiState::<Big>::INLINE, FfiState::<Big>::size()),
861            (FfiState::<OverLimit>::INLINE, FfiState::<OverLimit>::size()),
862            (FfiState::<Aligned>::INLINE, FfiState::<Aligned>::size()),
863        ] {
864            assert!(!inline);
865            assert_eq!(size, 2 * WORD);
866        }
867    }
868
869    #[test]
870    fn size_callback_returns_size() {
871        // SAFETY: size_callback does not read its argument.
872        let size = unsafe { FfiState::<Odd>::size_callback(core::ptr::null_mut()) };
873        assert_eq!(usize::try_from(size).unwrap(), FfiState::<Odd>::size());
874        // SAFETY: as above.
875        let size = unsafe { FfiState::<Big>::size_callback(core::ptr::null_mut()) };
876        assert_eq!(usize::try_from(size).unwrap(), FfiState::<Big>::size());
877    }
878
879    #[test]
880    fn debug_names_the_type_and_the_storage() {
881        let text = format!(
882            "{:?}",
883            FfiState::<Counter> {
884                _state: core::marker::PhantomData
885            }
886        );
887        assert!(text.contains("Counter"), "{text}");
888        assert!(text.contains("inline: true"), "{text}");
889        assert!(text.contains(&format!("size: {}", 2 * WORD)), "{text}");
890    }
891
892    fn lifecycle<T: AggregateState, V: PartialEq + core::fmt::Debug>(
893        read: fn(&T) -> V,
894        write: fn(&mut T),
895        written: V,
896    ) {
897        let mut words = slots::<T>(3, 0xA5A5_A5A5);
898        let base = words.as_mut_ptr();
899        let mut states: Vec<_> = (0..3).map(|i| state_at::<T>(base, i)).collect();
900        for &state in &states {
901            init::<T>(state);
902        }
903        // SAFETY: initialised above; no other reference exists.
904        let value = unsafe { FfiState::<T>::with_state_mut(states[1]) }.expect("initialised");
905        write(value);
906        // SAFETY: initialised; the mutable borrow above has ended.
907        let read_back = unsafe { FfiState::<T>::with_state(states[1]) }.map(read);
908        assert_eq!(read_back, Some(written));
909        destroy::<T>(&mut states);
910        for &state in &states {
911            // SAFETY: the slots stay allocated; destroyed ones hold no `T`.
912            assert!(unsafe { FfiState::<T>::value(state) }.is_none());
913        }
914    }
915
916    #[test]
917    fn init_access_and_destroy_an_inline_state() {
918        lifecycle::<Counter, usize>(|c| c.value, |c| c.value = 42, 42);
919    }
920
921    #[test]
922    fn init_access_and_destroy_a_boxed_state() {
923        lifecycle::<Big, u64>(|b| b.values[63], |b| b.values[63] = 42, 42);
924    }
925
926    /// The fourth audit's F3. `DuckDB` destroys states whose `state_init`
927    /// never ran when another init fails, so a slot can hold any bytes. A
928    /// slot whose tag does not match is skipped, not dropped. Under Miri,
929    /// dropping a boxed slot's garbage pointer would be reported as
930    /// undefined behaviour.
931    fn skips_a_slot_init_never_initialised<T: AggregateState>() {
932        let _serial = tracked();
933        for fill in [0, 0x5a5a_5a5a, usize::MAX] {
934            let mut words = slots::<T>(1, fill);
935            let base = words.as_mut_ptr();
936            let before = words.clone();
937            let mut states = [state_at::<T>(base, 0)];
938            destroy::<T>(&mut states);
939            assert_eq!(words, before, "an untagged slot is left alone");
940            // SAFETY: the slot is allocated; a mismatched tag reads as no `T`.
941            assert!(unsafe { FfiState::<T>::value(states[0]) }.is_none());
942        }
943    }
944
945    #[test]
946    fn destroy_skips_an_uninitialised_inline_slot() {
947        skips_a_slot_init_never_initialised::<Tracked>();
948    }
949
950    #[test]
951    fn destroy_skips_an_uninitialised_boxed_slot() {
952        skips_a_slot_init_never_initialised::<TrackedBig>();
953    }
954
955    /// `DuckDB` moves states: radix repartitioning copies each state's bytes
956    /// to another row and destroys it there. A tag derived from the slot's
957    /// address (the fourth audit's) made the destructor skip every moved
958    /// state, leaking its `T`. The tag follows the contents, so a moved state
959    /// is destroyed.
960    fn a_moved_state_is_destroyed<T: AggregateState>() {
961        let _serial = tracked();
962        let before = TRACKED_LIVE.load(Ordering::SeqCst);
963        let mut words = slots::<T>(2, 0);
964        let base = words.as_mut_ptr();
965        let from = state_at::<T>(base, 0);
966        let to = state_at::<T>(base, 1);
967        init::<T>(from);
968        // SAFETY: both slots are `size()` bytes of one allocation; the copy
969        // is what `DuckDB` does when it moves a row.
970        unsafe {
971            core::ptr::copy_nonoverlapping(
972                from.cast::<u8>(),
973                to.cast::<u8>(),
974                FfiState::<T>::size(),
975            );
976        }
977        assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before + 1);
978        destroy::<T>(&mut [to]);
979        assert_eq!(
980            TRACKED_LIVE.load(Ordering::SeqCst),
981            before,
982            "the moved state's T was dropped"
983        );
984    }
985
986    #[test]
987    fn a_moved_inline_state_is_destroyed() {
988        a_moved_state_is_destroyed::<Tracked>();
989    }
990
991    #[test]
992    fn a_moved_boxed_state_is_destroyed() {
993        a_moved_state_is_destroyed::<TrackedBig>();
994    }
995
996    /// A slot initialised for one state type is not dropped by another type's
997    /// destructor: that would drop an `A` as a `B` (the soundness audit's F4,
998    /// reachable when `DuckDB` reuses the bytes of a state it never destroyed).
999    #[test]
1000    fn a_slot_initialised_for_another_type_is_skipped() {
1001        let _serial = tracked();
1002        let before = TRACKED_LIVE.load(Ordering::SeqCst);
1003        let mut words = slots::<Tracked>(1, 0);
1004        let base = words.as_mut_ptr();
1005        let mut states = [state_at::<Tracked>(base, 0)];
1006        init::<Counter>(states[0]);
1007        let initialised = words.clone();
1008        destroy::<Tracked>(&mut states);
1009        assert_eq!(words, initialised, "another type's slot is left alone");
1010        assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before);
1011        destroy::<Counter>(&mut states);
1012        assert_eq!(words[0], 0, "its own destructor clears the tag");
1013
1014        let mut words = slots::<TrackedBig>(1, 0);
1015        let base = words.as_mut_ptr();
1016        let mut states = [state_at::<TrackedBig>(base, 0)];
1017        init::<Big>(states[0]);
1018        let initialised = words.clone();
1019        destroy::<TrackedBig>(&mut states);
1020        assert_eq!(words, initialised, "another type's slot is left alone");
1021        destroy::<Big>(&mut states);
1022        assert_eq!(words, [0, 0]);
1023    }
1024
1025    /// A destroyed slot's tag is cleared, so destroying it again is a no-op
1026    /// even though its other bytes are unchanged.
1027    fn destroying_twice_drops_once<T: AggregateState>() {
1028        let _serial = tracked();
1029        let before = TRACKED_LIVE.load(Ordering::SeqCst);
1030        let mut words = slots::<T>(1, 0);
1031        let base = words.as_mut_ptr();
1032        let mut states = [state_at::<T>(base, 0)];
1033        init::<T>(states[0]);
1034        assert_ne!(words[0], 0);
1035        destroy::<T>(&mut states);
1036        assert_eq!(words[0], 0);
1037        assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before);
1038        destroy::<T>(&mut states);
1039        assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before);
1040    }
1041
1042    #[test]
1043    fn destroying_an_inline_state_twice_drops_it_once() {
1044        destroying_twice_drops_once::<Tracked>();
1045    }
1046
1047    #[test]
1048    fn destroying_a_boxed_state_twice_drops_it_once() {
1049        destroying_twice_drops_once::<TrackedBig>();
1050    }
1051
1052    #[test]
1053    fn the_initial_value_is_t_default() {
1054        let _serial = tracked();
1055        let mut words = slots::<Tracked>(1, usize::MAX);
1056        let base = words.as_mut_ptr();
1057        let mut states = [state_at::<Tracked>(base, 0)];
1058        init::<Tracked>(states[0]);
1059        // SAFETY: initialised above.
1060        assert_eq!(
1061            unsafe { FfiState::<Tracked>::with_state(states[0]) }.map(|t| t.value),
1062            Some(7)
1063        );
1064        destroy::<Tracked>(&mut states);
1065        let mut words = slots::<TrackedBig>(1, usize::MAX);
1066        let base = words.as_mut_ptr();
1067        let mut states = [state_at::<TrackedBig>(base, 0)];
1068        init::<TrackedBig>(states[0]);
1069        // SAFETY: initialised above.
1070        let first = unsafe { FfiState::<TrackedBig>::with_state(states[0]) }.map(|t| t.values[0]);
1071        assert_eq!(first, Some(7));
1072        destroy::<TrackedBig>(&mut states);
1073    }
1074}