quack_rs/aggregate/state.rs
1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! Generic `FfiState<T>` wrapper for safe aggregate state management.
7//!
8//! # Problem solved
9//!
10//! Writing the state init/destroy lifecycle using raw pointers is error-prone.
11//! The canonical pattern is:
12//!
13//! ```rust,no_run
14//! use libduckdb_sys::{duckdb_function_info, duckdb_aggregate_state, idx_t};
15//!
16//! #[derive(Default)]
17//! struct MyState { count: u64 }
18//!
19//! #[repr(C)]
20//! struct FfiState { inner: *mut MyState }
21//!
22//! unsafe extern "C" fn state_init(_: duckdb_function_info, state: duckdb_aggregate_state) {
23//! let ffi = &mut *(state as *mut FfiState);
24//! ffi.inner = Box::into_raw(Box::new(MyState::default()));
25//! }
26//!
27//! unsafe extern "C" fn state_destroy(states: *mut duckdb_aggregate_state, count: idx_t) {
28//! for i in 0..count as usize {
29//! let state_ptr = *states.add(i);
30//! let ffi = &mut *(state_ptr as *mut FfiState);
31//! if !ffi.inner.is_null() {
32//! drop(Box::from_raw(ffi.inner));
33//! ffi.inner = std::ptr::null_mut();
34//! }
35//! }
36//! }
37//! ```
38//!
39//! [`FfiState<T>`] encapsulates this pattern. Your type `T` only needs to
40//! implement [`AggregateState`] (which requires `Default`), and you call the
41//! provided helper methods instead of writing raw pointer code.
42//!
43//! # Pitfalls prevented
44//!
45//! - **L1**: Combine propagates all fields because your type `T`'s `combine`
46//! method is responsible — the `FfiState` wrapper ensures `T`'s method is called.
47//! - **L2**: No double-free — `destroy_callback` clears the state's tag before
48//! dropping `T`, and drops only a state whose tag matches.
49//! - **L3**: No panic across FFI — `with_state_mut` returns an `Option`, not a panic.
50
51use libduckdb_sys::{duckdb_aggregate_state, duckdb_function_info, idx_t};
52
53/// Trait for types that can be used as `DuckDB` aggregate state.
54///
55/// Implement this for your state struct. It requires `Default` (used to
56/// create the initial state in `state_init`), `Send` (`DuckDB` moves states
57/// between worker threads) and `Sync`: a window's segment tree keeps its
58/// states in shared, unlocked storage (`WindowSegmentTreeGlobalState`,
59/// `window_segment_tree.cpp`), and every thread evaluating a frame that covers
60/// one reads it as a `combine` source, so two threads can hold `&T` to the
61/// same state at once. A `T` with a `Cell` or `RefCell` would race there.
62///
63/// **Breaking** in 0.18.0: the `Sync` bound is new.
64///
65/// # Example
66///
67/// ```rust
68/// use quack_rs::aggregate::AggregateState;
69///
70/// #[derive(Default)]
71/// struct WordCount {
72/// count: u64,
73/// }
74///
75/// impl AggregateState for WordCount {}
76///
77/// // FfiState::<WordCount>::size_callback and other methods are now available.
78/// ```
79pub trait AggregateState: Default + Send + Sync + 'static {}
80
81/// 64-bit FNV-1a, the hash behind `FfiState`'s per-type salt: deterministic
82/// in every build, unlike `std`'s randomly keyed hashers.
83struct Fnv1a(u64);
84
85impl Fnv1a {
86 const fn new() -> Self {
87 Self(0xCBF2_9CE4_8422_2325)
88 }
89}
90
91impl core::hash::Hasher for Fnv1a {
92 fn finish(&self) -> u64 {
93 self.0
94 }
95 fn write(&mut self, bytes: &[u8]) {
96 for &byte in bytes {
97 self.0 = (self.0 ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01B3);
98 }
99 }
100}
101
102/// A generic FFI-compatible state wrapper for use with `DuckDB` aggregate functions.
103///
104/// `FfiState<T>` describes the bytes `DuckDB` allocates for each aggregate
105/// group: [`size_callback`][FfiState::size_callback] tells `DuckDB` how many,
106/// [`init_callback`][FfiState::init_callback] puts a `T::default()` in them,
107/// [`with_state`][FfiState::with_state] and
108/// [`with_state_mut`][FfiState::with_state_mut] hand out the `T`, and
109/// [`destroy_callback`][FfiState::destroy_callback] drops it. The type is
110/// never constructed; it only names the layout and its callbacks.
111///
112/// # Known `DuckDB` limitation
113///
114/// **Before `DuckDB` 2.0.0, two query shapes make every C-API aggregate read
115/// out of bounds.** In them `DuckDB` calls `update` with a state array holding
116/// **one** state while
117/// passing `count > 1` rows, so the callback reads `states[1..count]` past the
118/// end of the array — undefined behaviour in *any* C-API aggregate, whether
119/// built with quack-rs or by hand:
120///
121/// - **Window aggregates whose frame is the whole partition**, e.g.
122/// `agg(x) OVER ()` or an explicit `ROWS BETWEEN UNBOUNDED PRECEDING AND
123/// UNBOUNDED FOLLOWING` frame (`OVER (PARTITION BY p)` without `ORDER BY`
124/// crashed on 1.4.4 only in the shapes tried) — `WindowConstantAggregator`
125/// (`src/function/window/window_constant_aggregator.cpp`, ~lines 106 and
126/// 296–299 in `DuckDB` 1.5.5).
127/// - **Ordered aggregates**, e.g. `agg(x ORDER BY y)` —
128/// `src/function/aggregate/sorted_aggregate_function.cpp`, ~lines 630–633.
129///
130/// Both pass a `CONSTANT_VECTOR` of states because the function has no
131/// `simple_update` (the C API cannot set one), and `CAPIAggregateUpdate`
132/// (`src/main/capi/aggregate_function-c.cpp`, ~lines 92–110) hands the
133/// vector's data pointer to the extension without flattening it. This is a
134/// defect in `DuckDB`'s C API, not in quack-rs, and it cannot be detected
135/// from inside the callback — reading `states[1]` to check is itself the
136/// out-of-bounds read. Reported upstream as
137/// [duckdb/duckdb#26109](https://github.com/duckdb/duckdb/issues/26109).
138/// On `DuckDB` 1.4.x and 1.5.x, do not use C-API aggregates in those two
139/// query shapes.
140///
141/// **Fixed in `DuckDB` 2.0.0:** `CAPIAggregateUpdate` flattens a non-flat
142/// state vector into one state pointer per row before calling `update`
143/// (`src/main/capi/v1/aggregate_function-c.cpp`, lines 104–108 at v2.0.0).
144/// Both shapes then give the right answer, and a plain C aggregate runs them
145/// clean under Valgrind; on 1.5.6 the same program segfaults.
146///
147/// # Memory layout
148///
149/// ```text
150/// [ tag: usize ][ T, padded to a multiple of usize ] T stored inline
151/// [ tag: usize ][ *mut T ] T boxed
152/// ```
153///
154/// `T` is stored in the state bytes themselves when its alignment is at most
155/// `usize`'s and it is at most 256 bytes; otherwise the slot holds a
156/// `Box<T>`. The size is a multiple of `usize` either way, because some
157/// `DuckDB` operators lay states out back to back at a stride of exactly
158/// `state_size` (the window aggregators), and every state must stay aligned.
159/// `DuckDB` aligns each state to 8 bytes (`AlignValue` in `AggregateObject`
160/// and `TupleDataLayout`; `new[]`-allocated buffers elsewhere), which covers
161/// both forms on 64- and 32-bit targets.
162///
163/// # States `DuckDB` never destroys
164///
165/// `DuckDB` 1.4.4 to 1.5.5 (not re-measured on 1.5.6) destroys a grouped
166/// aggregate's states as its result
167/// scan passes them. When the scan stops early — a `LIMIT` above the
168/// aggregate, an error raised above it, an interrupt — the states it did not
169/// reach are never destroyed (`docs/upstream-duckdb-reports.md`, item 20):
170/// under `LIMIT 10`, 2,048 of 300,000 on one thread. Their `T` is never
171/// dropped. A `T` stored inline costs nothing more: its bytes belong to
172/// `DuckDB`'s hash table, which `DuckDB` frees. What leaks is whatever `T`
173/// itself owns on the heap (a `Vec`, a `String`, a `HashMap`), and, for a
174/// boxed `T`, the box. `DuckDB` 2.0.0 still behaves this way (pinned by
175/// `tests/aggregate_leaks.rs`). Before 2.0.0 the same holds for a window
176/// frame with an `EXCLUDE` clause, whose segment tree initialises one extra
177/// state per row and never destroys it (item 35; 5000 `T`s undropped over a
178/// 5000-row window); 2.0.0 destroys those states
179/// (`WindowSegmentTreeLocalState::Evaluate`, `window_segment_tree.cpp`
180/// 506–510). Until `DuckDB` fixes the stopped scan, prefer a state that owns
181/// no heap memory and is small enough to be stored inline.
182///
183/// # States `DuckDB` never initialised
184///
185/// When a `state_init` call fails — a panicking `T::default()`, or an
186/// allocation failure — `DuckDB` 1.4.4 to 1.5.5 (not re-measured on 1.5.6)
187/// still calls the destructor
188/// on every state row it had created, initialised or not
189/// (`RowOperations::InitializeStates` is not exception-safe; see
190/// `docs/upstream-duckdb-reports.md`). That includes the states of *other*
191/// aggregates in the same query whose `state_init` never ran. Those bytes
192/// are whatever the allocation held, and dropping them as a `T` would run a
193/// destructor over garbage.
194///
195/// So [`init_callback`][Self::init_callback] also stores `tag`, a value
196/// derived from `T` (and, for a boxed `T`, from the box's address), and
197/// [`destroy_callback`][Self::destroy_callback] drops only a slot whose tag
198/// matches, clearing it first. A slot the destructor has already processed no
199/// longer matches; a slot another aggregate's type initialised does not match
200/// either. What is left is chance: uninitialised bytes that happen to hold
201/// the tag, one in 2<sup>64</sup> on a 64-bit target. This is a mitigation of
202/// a `DuckDB` defect, not a guarantee. Reading bytes `DuckDB` may never have
203/// written is, in Rust's abstract machine, a read of uninitialised memory
204/// whatever the load: the destructor uses volatile loads so the compiler
205/// cannot reason from them, but no Rust construct makes such a read defined.
206/// It is the least bad of the options this defect leaves.
207///
208/// The tag depends on the slot's contents, not its address, because `DuckDB`
209/// moves states: radix repartitioning of a hash aggregate copies each state's
210/// bytes to a new row (`radix_partitioned_hashtable.cpp`) and destroys it
211/// there. The fourth audit's tag used the address, so every moved state was
212/// skipped and its `T` leaked — millions per query on eight threads. Moving
213/// a Rust value by copying its bytes is always valid; the copy left behind is
214/// never destroyed.
215///
216/// # Usage
217///
218/// ```rust
219/// use quack_rs::aggregate::{AggregateState, FfiState};
220/// use libduckdb_sys::{duckdb_function_info, duckdb_aggregate_state, idx_t};
221///
222/// #[derive(Default)]
223/// struct MyState { sum: i64 }
224/// impl AggregateState for MyState {}
225///
226/// // In your registration code:
227/// // .ffi_state::<MyState>()
228/// ```
229///
230/// **Breaking** in 0.18.0: `FfiState<T>` was a one-word `#[repr(C)]` struct
231/// with a public `inner: *mut T` field that always boxed `T`. The layout is
232/// now private.
233pub struct FfiState<T: AggregateState> {
234 _state: core::marker::PhantomData<fn() -> T>,
235}
236
237impl<T: AggregateState> FfiState<T> {
238 /// The tag's width, and the offset of the payload.
239 const WORD: usize = core::mem::size_of::<usize>();
240
241 /// Whether `T` is stored in the state bytes rather than boxed.
242 const INLINE: bool =
243 core::mem::align_of::<T>() <= Self::WORD && core::mem::size_of::<T>() <= INLINE_LIMIT;
244
245 /// Bytes per state: the tag, then `T` (rounded up to a whole number of
246 /// words) or a `Box<T>`.
247 const SIZE: usize = Self::WORD
248 + if Self::INLINE {
249 core::mem::size_of::<T>().div_ceil(Self::WORD) * Self::WORD
250 } else {
251 Self::WORD
252 };
253
254 /// Mixed into the tag: an arbitrary odd constant, so that a zeroed slot
255 /// does not match.
256 #[cfg(target_pointer_width = "64")]
257 const TAG_KEY: usize = 0x9E37_79B9_7F4A_7C15;
258 #[cfg(not(target_pointer_width = "64"))]
259 const TAG_KEY: usize = 0x9E37_79B9;
260
261 /// The tag a slot carries once initialised for this `T`: `boxed` is the
262 /// box's address for a boxed `T`, and zero for an inline one.
263 ///
264 /// `TAG_KEY` is salted with a hash of `T`'s [`TypeId`](core::any::TypeId),
265 /// which differs between types, so a slot initialised for one aggregate's
266 /// state type is not accepted by another's destructor. The salt's low bit
267 /// is cleared, so the key stays odd and never zero: a zeroed slot never
268 /// matches.
269 ///
270 /// The salt must be the same in every callback. An address is not: the
271 /// salt was once the address of `type_name::<T>()`, and rustc gives each
272 /// codegen unit that uses a constant its own copy, so under Cargo's
273 /// default release profile (16 codegen units, no fat LTO) `init`,
274 /// `with_state` and `destroy` could each see a different address.
275 fn tag_for(boxed: *const T) -> usize {
276 (boxed as usize) ^ Self::TAG_KEY ^ Self::salt(Self::type_hash())
277 }
278
279 /// FNV-1a over `T`'s `TypeId`, as its `Hash` impl feeds it: the same
280 /// value wherever it is computed, and a constant once optimised.
281 fn type_hash() -> usize {
282 let mut hasher = Fnv1a::new();
283 core::hash::Hash::hash(&core::any::TypeId::of::<T>(), &mut hasher);
284 // Truncation on 32-bit targets keeps the low half, which is as good a
285 // salt as any.
286 #[allow(clippy::cast_possible_truncation)]
287 let hash = core::hash::Hasher::finish(&hasher) as usize;
288 hash
289 }
290
291 /// `hash` with its low bit cleared, so that `TAG_KEY ^ salt` stays odd.
292 const fn salt(hash: usize) -> usize {
293 hash & !1
294 }
295
296 /// The payload: `T` itself, or the `Box<T>`'s pointer.
297 ///
298 /// Derived as the word after the tag, so it keeps the tag's `usize`
299 /// alignment, which is all either form needs.
300 const fn payload(state: duckdb_aggregate_state) -> *mut usize {
301 state.cast::<usize>().wrapping_add(1)
302 }
303
304 /// Returns the number of bytes `DuckDB` allocates per state.
305 ///
306 /// # Example
307 ///
308 /// ```rust
309 /// use quack_rs::aggregate::{AggregateState, FfiState};
310 ///
311 /// #[derive(Default)]
312 /// struct MyState { val: i64 }
313 /// impl AggregateState for MyState {}
314 ///
315 /// // A tag, then the `i64` itself.
316 /// assert_eq!(FfiState::<MyState>::size(), 2 * std::mem::size_of::<usize>());
317 /// ```
318 #[inline]
319 #[must_use]
320 pub const fn size() -> usize {
321 Self::SIZE
322 }
323
324 /// The `state_size` callback function for use in the builder.
325 ///
326 /// Returns [`size`][Self::size]: the number of bytes `DuckDB` must
327 /// allocate per aggregate group.
328 ///
329 /// # Safety
330 ///
331 /// This is an `unsafe extern "C"` function pointer. It is safe to pass to
332 /// [`AggregateFunctionBuilder::state_size`][crate::aggregate::AggregateFunctionBuilder::state_size].
333 pub const unsafe extern "C" fn size_callback(_info: duckdb_function_info) -> idx_t {
334 Self::SIZE as idx_t
335 }
336
337 /// The `state_init` callback function for use in the builder.
338 ///
339 /// Stores a `T::default()` in the state at `state` (or a box holding one).
340 ///
341 /// # Pitfall L3: `T::default()` is user code
342 ///
343 /// `DuckDB` invokes this through an `extern "C"` function pointer, so a
344 /// panic escaping `T::default()` would abort the process (Rust 1.81+ turns
345 /// an unwind across `extern "C"` into `panic_cannot_unwind`). The call is
346 /// therefore run under
347 /// [`catch_ffi_panic`][crate::callback::catch_ffi_panic] and a panic is
348 /// reported through `duckdb_aggregate_function_set_error` — which
349 /// `DuckDB`'s `CAPIAggregateStateInit` checks, turning it into an ordinary
350 /// SQL error — instead of taking the session down.
351 ///
352 /// # A failing initialiser in a window frame still crashes the host
353 ///
354 /// That error report is itself the hazard when the aggregate runs as a
355 /// *window* function (a `ROWS`/`RANGE` frame). `DuckDB`'s
356 /// `WindowAggregateStates` fills its table of state pointers one entry per
357 /// *successful* initialisation and then, when one fails, destroys the whole
358 /// table, unwritten entries included: the destructor is handed pointers
359 /// that never pointed at a state. The tag check below cannot help, because
360 /// there is no slot to read a tag from. Reproduced with a plain C
361 /// aggregate, no Rust involved, on `DuckDB` 1.5.6 (55 of 68 pointers
362 /// garbage) and 2.0.0 (65 of 68); the host dies with `SIGSEGV` or `SIGBUS`.
363 /// Ungrouped, grouped, `DISTINCT` and `FILTER` aggregates report a clean
364 /// error. So: **`T::default()` must not panic** for an aggregate that can
365 /// be used in a window.
366 ///
367 /// # Uninitialised memory
368 ///
369 /// `DuckDB` does not promise the state allocation is zeroed, so the tag is
370 /// cleared (and a boxed slot's pointer nulled) *before* any user code
371 /// runs. A slot whose `T::default()` panicked therefore never matches its
372 /// tag: [`with_state`][Self::with_state] returns `None` for it and
373 /// [`destroy_callback`][Self::destroy_callback] skips it.
374 ///
375 /// # Safety
376 ///
377 /// - `state` must point to [`size`][Self::size] writable bytes allocated
378 /// by `DuckDB`, aligned to `usize`.
379 /// - This function must only be called once per state allocation.
380 pub unsafe extern "C" fn init_callback(
381 info: duckdb_function_info,
382 state: duckdb_aggregate_state,
383 ) {
384 let payload = Self::payload(state);
385 // SAFETY: `state` is `size()` writable bytes aligned to `usize`, so
386 // the tag word and (for a boxed `T`) the pointer word after it are in
387 // bounds and aligned. `ptr::write` does not read what was there.
388 unsafe {
389 core::ptr::write(state.cast::<usize>(), 0);
390 if !Self::INLINE {
391 core::ptr::write(payload.cast::<*mut T>(), core::ptr::null_mut());
392 }
393 }
394 let made = if Self::INLINE {
395 crate::callback::catch_ffi_panic(T::default).map(|value| {
396 // SAFETY: the payload is `size_of::<T>()` bytes inside the
397 // state, at offset `usize`, which `INLINE` guarantees is a
398 // multiple of `align_of::<T>()`.
399 unsafe { core::ptr::write(payload.cast::<T>(), value) };
400 Self::tag_for(core::ptr::null())
401 })
402 } else {
403 crate::callback::catch_ffi_panic(Box::<T>::default).map(|boxed| {
404 let inner = Box::into_raw(boxed);
405 // SAFETY: the pointer word was written above.
406 unsafe { core::ptr::write(payload.cast::<*mut T>(), inner) };
407 Self::tag_for(inner)
408 })
409 };
410 match made {
411 // SAFETY: the tag word was written above.
412 Ok(tag) => unsafe { core::ptr::write(state.cast::<usize>(), tag) },
413 Err(message) => {
414 let c_msg = crate::callback::message_to_c_string(&format!(
415 "quack-rs: aggregate state initialiser panicked: {message}"
416 ));
417 // SAFETY: `info` is the handle DuckDB passed in, and DuckDB
418 // checks the error flag on return from the state-init callback.
419 unsafe {
420 libduckdb_sys::duckdb_aggregate_function_set_error(info, c_msg.as_ptr());
421 }
422 }
423 }
424 }
425
426 /// The `state_destroy` callback function for use in the builder.
427 ///
428 /// Drops the `T` in each state in `states[0..count]` (freeing its box, if
429 /// boxed), clearing the tag first so a second call is a no-op.
430 ///
431 /// # Count conversion
432 ///
433 /// If `count` (an `idx_t`) cannot be converted to `usize`, the loop iterates
434 /// zero times — no states are freed. This is a defensive choice to avoid
435 /// panicking across FFI. In practice, `idx_t` is `u64` and `usize` is at
436 /// least 64 bits on all `DuckDB`-supported platforms, so this path is
437 /// unreachable on supported targets.
438 ///
439 /// # Pitfall L3: `T::drop` is user code
440 ///
441 /// `DuckDB` calls the destructor as `info.destroy(states, count)` — no info
442 /// handle, no return value, and therefore **no error channel at all**
443 /// (verified in `CAPIAggregateDestructor`). A panic escaping `T::drop`
444 /// would abort the process, so each drop runs under
445 /// [`catch_ffi_panic`][crate::callback::catch_ffi_panic] and the message is
446 /// discarded. Every remaining state is still dropped: one bad destructor
447 /// does not leak the rest of the group.
448 ///
449 /// One case cannot be contained: a `T::drop` that panics while one of
450 /// `T`'s fields also panics in its own `drop` during that unwind. Rust
451 /// aborts the process for a panic raised while unwinding ("panic in a
452 /// destructor during cleanup") before any `catch_unwind` sees it.
453 ///
454 /// # Safety
455 ///
456 /// - `states` must point to an array of `count` valid `duckdb_aggregate_state`
457 /// pointers, each to [`size`][Self::size] bytes, aligned to `usize`, that
458 /// `DuckDB` allocated for this aggregate. A slot
459 /// [`init_callback`][Self::init_callback] never initialised is skipped
460 /// (see "States `DuckDB` never initialised" on the type).
461 pub unsafe extern "C" fn destroy_callback(states: *mut duckdb_aggregate_state, count: idx_t) {
462 for i in 0..usize::try_from(count).unwrap_or(0) {
463 // SAFETY: `states` is a valid array of `count` pointers.
464 let state = unsafe { *states.add(i) };
465 let payload = Self::payload(state);
466 // SAFETY: the slot is `size()` bytes DuckDB allocated for this
467 // aggregate, aligned to `usize`, so the tag word (and a boxed
468 // slot's pointer word) are in bounds. They may never have been
469 // written by `init_callback`, so they are read as plain words with
470 // volatile loads, which the compiler must perform as written and
471 // cannot reason about, and only trusted once the tag matches.
472 let tag = unsafe { core::ptr::read_volatile(state.cast::<usize>()) };
473 let inner = if Self::INLINE {
474 core::ptr::null_mut()
475 } else {
476 // SAFETY: as above.
477 unsafe { core::ptr::read_volatile(payload.cast::<*mut T>()) }
478 };
479 if tag != Self::tag_for(inner) {
480 continue;
481 }
482 // Clear the tag (and the pointer) *before* dropping: if `T::drop`
483 // panics, the value is still released by the unwind, so leaving
484 // the slot live would make a second destructor call a double drop.
485 // SAFETY: the tag matched, so `init_callback` initialised the slot.
486 unsafe {
487 core::ptr::write(state.cast::<usize>(), 0);
488 if !Self::INLINE {
489 core::ptr::write(payload.cast::<*mut T>(), core::ptr::null_mut());
490 }
491 }
492 // The drop runs arbitrary user code, so it must not be allowed to
493 // unwind out of this `extern "C"` function.
494 drop(crate::callback::catch_ffi_panic(|| {
495 if Self::INLINE {
496 // SAFETY: the tag matched, so the payload holds a `T` that
497 // `init_callback` wrote and nothing has dropped; the tag is
498 // cleared, so nothing will drop it again.
499 unsafe { core::ptr::drop_in_place(payload.cast::<T>()) };
500 } else if !inner.is_null() {
501 // SAFETY: `inner` came from `Box::into_raw` in
502 // `init_callback`, and the slot no longer refers to it.
503 drop(unsafe { Box::from_raw(inner) });
504 }
505 }));
506 }
507 }
508
509 /// The `T` in an initialised slot, or `None` if the slot's tag does not
510 /// match (its `T::default()` panicked).
511 ///
512 /// # Safety
513 ///
514 /// `state` must be a slot [`init_callback`][Self::init_callback] has run on.
515 unsafe fn value(state: duckdb_aggregate_state) -> Option<*mut T> {
516 let payload = Self::payload(state);
517 // SAFETY: `init_callback` wrote the tag word, and for a boxed `T` the
518 // pointer word.
519 let tag = unsafe { core::ptr::read(state.cast::<usize>()) };
520 if Self::INLINE {
521 (tag == Self::tag_for(core::ptr::null())).then(|| payload.cast::<T>())
522 } else {
523 // SAFETY: as above.
524 let inner = unsafe { core::ptr::read(payload.cast::<*mut T>()) };
525 (!inner.is_null() && tag == Self::tag_for(inner)).then_some(inner)
526 }
527 }
528
529 /// Provides safe mutable access to the inner `T` value.
530 ///
531 /// Returns `None` if the slot holds no `T` — its `T::default()` panicked
532 /// (and `DuckDB` is about to fail the query), or it was destroyed.
533 ///
534 /// # Pitfall L3: No panic across FFI
535 ///
536 /// This method returns `Option<&mut T>` rather than unwrapping, so callers
537 /// can use `if let Some(state) = ...` patterns without panicking.
538 ///
539 /// # Safety
540 ///
541 /// - `state` must point to a state `DuckDB` allocated for this aggregate
542 /// and [`init_callback`][Self::init_callback] initialised.
543 /// - No other reference to the same `T` must exist simultaneously.
544 /// - The returned reference must not outlive the callback that received
545 /// `state`. The lifetime `'a` is the caller's to choose and nothing ties
546 /// it to `state`; `DuckDB` destroys the state — and
547 /// [`destroy_callback`][Self::destroy_callback] drops the `T` — once it
548 /// is finalized or merged, and may move its bytes between callbacks.
549 ///
550 /// # Example
551 ///
552 /// ```rust,no_run
553 /// use quack_rs::aggregate::{AggregateState, FfiState};
554 ///
555 /// #[derive(Default)]
556 /// struct Counter { n: u64 }
557 /// impl AggregateState for Counter {}
558 ///
559 /// // Inside your update callback:
560 /// // let ffi_state: duckdb_aggregate_state = ...;
561 /// // if let Some(state) = unsafe { FfiState::<Counter>::with_state_mut(ffi_state) } {
562 /// // state.n += 1;
563 /// // }
564 /// ```
565 pub unsafe fn with_state_mut<'a>(state: duckdb_aggregate_state) -> Option<&'a mut T> {
566 // SAFETY: the caller guarantees `state` was initialised, and that no
567 // other reference to its `T` exists.
568 unsafe { Self::value(state).map(|value| &mut *value) }
569 }
570
571 /// Provides safe immutable access to the inner `T` value.
572 ///
573 /// See [`with_state_mut`][Self::with_state_mut] for safety requirements.
574 ///
575 /// # Safety
576 ///
577 /// Same as `with_state_mut`, but only borrows immutably: no mutable
578 /// reference to the same `T` may exist while this one does.
579 pub unsafe fn with_state<'a>(state: duckdb_aggregate_state) -> Option<&'a T> {
580 // SAFETY: the caller guarantees `state` was initialised, and that no
581 // mutable reference to its `T` exists.
582 unsafe { Self::value(state).map(|value| &*value) }
583 }
584}
585
586/// The largest `T`, in bytes, that [`FfiState`] stores inline. A larger `T`
587/// is boxed. Every aggregate state of a group shares one hash-table row, and
588/// `DuckDB` allocates rows in blocks of 256 KiB by default
589/// (`DEFAULT_BLOCK_ALLOC_SIZE`, `TupleDataAllocator::Build`); boxing a large
590/// `T` keeps the row narrow.
591const INLINE_LIMIT: usize = 256;
592
593impl<T: AggregateState> core::fmt::Debug for FfiState<T> {
594 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
595 f.debug_struct("FfiState")
596 .field("state", &core::any::type_name::<T>())
597 .field("inline", &Self::INLINE)
598 .field("size", &Self::SIZE)
599 .finish()
600 }
601}
602
603#[cfg(test)]
604mod tests {
605 use super::*;
606 use std::sync::atomic::{AtomicIsize, Ordering};
607
608 const WORD: usize = core::mem::size_of::<usize>();
609
610 /// Word-sized and word-aligned, so inline on every target.
611 #[derive(Default, Debug, PartialEq)]
612 struct Counter {
613 value: usize,
614 }
615 impl AggregateState for Counter {}
616
617 /// Too large to store inline.
618 #[derive(Debug, PartialEq)]
619 struct Big {
620 values: [u64; 64],
621 }
622 impl Default for Big {
623 fn default() -> Self {
624 Self { values: [0; 64] }
625 }
626 }
627 impl AggregateState for Big {}
628
629 /// Aligned more strictly than `usize`.
630 #[allow(dead_code)] // only its layout matters
631 #[derive(Default)]
632 #[repr(align(32))]
633 struct Aligned(u8);
634 impl AggregateState for Aligned {}
635
636 #[derive(Default)]
637 struct Zst;
638 impl AggregateState for Zst {}
639
640 #[allow(dead_code)] // only its layout matters
641 #[derive(Default)]
642 struct Odd([u8; 3]);
643 impl AggregateState for Odd {}
644
645 #[allow(dead_code)] // only its layout matters
646 struct AtLimit([u8; INLINE_LIMIT]);
647 impl Default for AtLimit {
648 fn default() -> Self {
649 Self([0; INLINE_LIMIT])
650 }
651 }
652 impl AggregateState for AtLimit {}
653
654 #[allow(dead_code)] // only its layout matters
655 struct OverLimit([u8; INLINE_LIMIT + 1]);
656 impl Default for OverLimit {
657 fn default() -> Self {
658 Self([0; INLINE_LIMIT + 1])
659 }
660 }
661 impl AggregateState for OverLimit {}
662
663 /// Live `Tracked` / `TrackedBig` values, to see whether a destructor ran.
664 static TRACKED_LIVE: AtomicIsize = AtomicIsize::new(0);
665
666 /// Held by every test that counts `TRACKED_LIVE`: the tests run in
667 /// parallel, and one test's live value would otherwise show in another's
668 /// count.
669 static TRACKED: std::sync::Mutex<()> = std::sync::Mutex::new(());
670
671 fn tracked() -> std::sync::MutexGuard<'static, ()> {
672 TRACKED
673 .lock()
674 .unwrap_or_else(std::sync::PoisonError::into_inner)
675 }
676
677 struct Tracked {
678 value: u64,
679 }
680 impl Default for Tracked {
681 fn default() -> Self {
682 TRACKED_LIVE.fetch_add(1, Ordering::SeqCst);
683 Self { value: 7 }
684 }
685 }
686 impl Drop for Tracked {
687 fn drop(&mut self) {
688 TRACKED_LIVE.fetch_sub(1, Ordering::SeqCst);
689 }
690 }
691 impl AggregateState for Tracked {}
692
693 struct TrackedBig {
694 values: [u64; 64],
695 }
696 impl Default for TrackedBig {
697 fn default() -> Self {
698 TRACKED_LIVE.fetch_add(1, Ordering::SeqCst);
699 Self { values: [7; 64] }
700 }
701 }
702 impl Drop for TrackedBig {
703 fn drop(&mut self) {
704 TRACKED_LIVE.fetch_sub(1, Ordering::SeqCst);
705 }
706 }
707 impl AggregateState for TrackedBig {}
708
709 /// `count` states of `T`'s size, back to back like `DuckDB`'s window
710 /// arrays, each word filled with `fill`.
711 fn slots<T: AggregateState>(count: usize, fill: usize) -> Vec<usize> {
712 assert_eq!(FfiState::<T>::size() % WORD, 0);
713 vec![fill; count * FfiState::<T>::size() / WORD]
714 }
715
716 /// State `index` of the array starting at `base`. Every state of one
717 /// array is derived from the same `base`, so none invalidates another
718 /// (Miri, Stacked Borrows).
719 fn state_at<T: AggregateState>(base: *mut usize, index: usize) -> duckdb_aggregate_state {
720 base.wrapping_add(index * FfiState::<T>::size() / WORD)
721 .cast()
722 }
723
724 fn init<T: AggregateState>(state: duckdb_aggregate_state) {
725 // SAFETY: `state` is `size()` writable, word-aligned bytes; a
726 // non-panicking `T::default()` never touches `info`.
727 unsafe { FfiState::<T>::init_callback(core::ptr::null_mut(), state) };
728 }
729
730 fn destroy<T: AggregateState>(states: &mut [duckdb_aggregate_state]) {
731 // SAFETY: each state is `size()` word-aligned bytes, initialised or
732 // not, which is what the destructor must cope with.
733 unsafe { FfiState::<T>::destroy_callback(states.as_mut_ptr(), states.len() as idx_t) };
734 }
735
736 /// A tag is the box's address XOR a per-type key, so two tags of one
737 /// type differ exactly where the addresses do, and an inline state's tag
738 /// (no box) is the key itself.
739 #[test]
740 fn a_tag_is_the_box_address_xor_the_types_key() {
741 // Pin the whole structure `addr ^ TAG_KEY ^ salt` against `TAG_KEY`
742 // and the salt directly. Asserting only `tag ^ key == addr` is weaker:
743 // it also holds when the second XOR is an AND and `addr` is a submask
744 // of the salt.
745 let salt = FfiState::<Counter>::salt(FfiState::<Counter>::type_hash());
746 let key = FfiState::<Counter>::TAG_KEY ^ salt;
747 // An inline state (null box) carries the key itself.
748 assert_eq!(FfiState::<Counter>::tag_for(core::ptr::null()), key);
749 for addr in [0x10_usize, 0x1000, 0xFFFF_FFF0, usize::MAX & !0xF] {
750 let boxed = core::ptr::without_provenance::<Counter>(addr);
751 let tag = FfiState::<Counter>::tag_for(boxed);
752 assert_eq!(tag, addr ^ FfiState::<Counter>::TAG_KEY ^ salt, "{addr:#x}");
753 // And two tags of one type differ exactly where the addresses do.
754 assert_eq!(tag ^ key, addr, "{addr:#x}");
755 }
756 }
757
758 /// The key is odd, whatever the type hash, so no zeroed slot and no
759 /// aligned box address can produce a tag of zero.
760 #[test]
761 fn the_salt_clears_the_low_bit_so_the_key_stays_odd() {
762 for hash in [0x1000_usize, 0x1001, 0x7FFF_FFFF, usize::MAX] {
763 let salt = FfiState::<Counter>::salt(hash);
764 assert_eq!(salt & 1, 0, "{hash:#x}");
765 assert_eq!(salt | 1, hash | 1, "{hash:#x}");
766 assert_eq!((FfiState::<Counter>::TAG_KEY ^ salt) & 1, 1);
767 }
768 }
769
770 /// `Fnv1a` is FNV-1a: pinned to the published 64-bit test vectors
771 /// (<http://www.isthe.com/chongo/src/fnv/test_fnv.c>), so a slip in the
772 /// mixing step (an OR for the XOR, say) cannot pass as "still a hash".
773 #[test]
774 fn fnv1a_matches_the_reference_vectors() {
775 use core::hash::Hasher;
776 for (input, want) in [
777 (&b""[..], 0xCBF2_9CE4_8422_2325_u64),
778 (b"a", 0xAF63_DC4C_8601_EC8C),
779 (b"foobar", 0x8594_4171_F739_67E8),
780 ] {
781 let mut h = Fnv1a::new();
782 h.write(input);
783 assert_eq!(h.finish(), want, "{input:?}");
784 }
785 }
786
787 /// Different state types get different salts, so one aggregate's
788 /// destructor does not accept another's slots.
789 #[test]
790 fn distinct_types_have_distinct_salts() {
791 let salts = [
792 FfiState::<Counter>::salt(FfiState::<Counter>::type_hash()),
793 FfiState::<Big>::salt(FfiState::<Big>::type_hash()),
794 FfiState::<Zst>::salt(FfiState::<Zst>::type_hash()),
795 FfiState::<Odd>::salt(FfiState::<Odd>::type_hash()),
796 ];
797 for (i, a) in salts.iter().enumerate() {
798 for b in &salts[i + 1..] {
799 assert_ne!(a, b);
800 }
801 }
802 }
803
804 /// Regression: the salt was the address of `type_name::<T>()`, and rustc
805 /// gives each codegen unit its own copy of a constant, so callbacks in
806 /// different codegen units computed different tags for one type and
807 /// `with_state` / `destroy` rejected every state. Each module below tends
808 /// to land in its own codegen unit; with the address salt, a release build
809 /// with 16 codegen units and no fat LTO got 6 distinct tags from 8 of them.
810 /// Only such a build can fail here: CI runs this test that way.
811 #[test]
812 fn the_tag_is_the_same_in_every_codegen_unit() {
813 let tags = [
814 cgu1::tag(),
815 cgu2::tag(),
816 cgu3::tag(),
817 cgu4::tag(),
818 cgu5::tag(),
819 cgu6::tag(),
820 cgu7::tag(),
821 cgu8::tag(),
822 ];
823 assert!(tags.iter().all(|&t| t == tags[0]), "{tags:x?}");
824 }
825
826 macro_rules! tag_in_own_module {
827 ($($m:ident),*) => {$(
828 mod $m {
829 #[inline(never)]
830 pub fn tag() -> usize {
831 super::FfiState::<super::Counter>::tag_for(core::ptr::null())
832 }
833 }
834 )*};
835 }
836 tag_in_own_module!(cgu1, cgu2, cgu3, cgu4, cgu5, cgu6, cgu7, cgu8);
837
838 #[test]
839 fn a_small_state_is_stored_inline_after_a_tag() {
840 const { assert!(FfiState::<Counter>::INLINE) };
841 assert_eq!(FfiState::<Counter>::size(), 2 * WORD);
842 const { assert!(FfiState::<Zst>::INLINE) };
843 assert_eq!(FfiState::<Zst>::size(), WORD);
844 // Padded to whole words, so back-to-back states stay aligned.
845 const { assert!(FfiState::<Odd>::INLINE) };
846 assert_eq!(FfiState::<Odd>::size(), 2 * WORD);
847 const { assert!(FfiState::<AtLimit>::INLINE) };
848 assert_eq!(FfiState::<AtLimit>::size(), WORD + INLINE_LIMIT);
849 // A `u64` is 8-aligned on wasm32 too, where `usize` is 4 bytes, so a
850 // state holding one is boxed there and inline on 64-bit targets.
851 assert_eq!(
852 FfiState::<Tracked>::INLINE,
853 core::mem::align_of::<u64>() <= WORD
854 );
855 }
856
857 #[test]
858 fn a_large_or_over_aligned_state_is_boxed() {
859 for (inline, size) in [
860 (FfiState::<Big>::INLINE, FfiState::<Big>::size()),
861 (FfiState::<OverLimit>::INLINE, FfiState::<OverLimit>::size()),
862 (FfiState::<Aligned>::INLINE, FfiState::<Aligned>::size()),
863 ] {
864 assert!(!inline);
865 assert_eq!(size, 2 * WORD);
866 }
867 }
868
869 #[test]
870 fn size_callback_returns_size() {
871 // SAFETY: size_callback does not read its argument.
872 let size = unsafe { FfiState::<Odd>::size_callback(core::ptr::null_mut()) };
873 assert_eq!(usize::try_from(size).unwrap(), FfiState::<Odd>::size());
874 // SAFETY: as above.
875 let size = unsafe { FfiState::<Big>::size_callback(core::ptr::null_mut()) };
876 assert_eq!(usize::try_from(size).unwrap(), FfiState::<Big>::size());
877 }
878
879 #[test]
880 fn debug_names_the_type_and_the_storage() {
881 let text = format!(
882 "{:?}",
883 FfiState::<Counter> {
884 _state: core::marker::PhantomData
885 }
886 );
887 assert!(text.contains("Counter"), "{text}");
888 assert!(text.contains("inline: true"), "{text}");
889 assert!(text.contains(&format!("size: {}", 2 * WORD)), "{text}");
890 }
891
892 fn lifecycle<T: AggregateState, V: PartialEq + core::fmt::Debug>(
893 read: fn(&T) -> V,
894 write: fn(&mut T),
895 written: V,
896 ) {
897 let mut words = slots::<T>(3, 0xA5A5_A5A5);
898 let base = words.as_mut_ptr();
899 let mut states: Vec<_> = (0..3).map(|i| state_at::<T>(base, i)).collect();
900 for &state in &states {
901 init::<T>(state);
902 }
903 // SAFETY: initialised above; no other reference exists.
904 let value = unsafe { FfiState::<T>::with_state_mut(states[1]) }.expect("initialised");
905 write(value);
906 // SAFETY: initialised; the mutable borrow above has ended.
907 let read_back = unsafe { FfiState::<T>::with_state(states[1]) }.map(read);
908 assert_eq!(read_back, Some(written));
909 destroy::<T>(&mut states);
910 for &state in &states {
911 // SAFETY: the slots stay allocated; destroyed ones hold no `T`.
912 assert!(unsafe { FfiState::<T>::value(state) }.is_none());
913 }
914 }
915
916 #[test]
917 fn init_access_and_destroy_an_inline_state() {
918 lifecycle::<Counter, usize>(|c| c.value, |c| c.value = 42, 42);
919 }
920
921 #[test]
922 fn init_access_and_destroy_a_boxed_state() {
923 lifecycle::<Big, u64>(|b| b.values[63], |b| b.values[63] = 42, 42);
924 }
925
926 /// The fourth audit's F3. `DuckDB` destroys states whose `state_init`
927 /// never ran when another init fails, so a slot can hold any bytes. A
928 /// slot whose tag does not match is skipped, not dropped. Under Miri,
929 /// dropping a boxed slot's garbage pointer would be reported as
930 /// undefined behaviour.
931 fn skips_a_slot_init_never_initialised<T: AggregateState>() {
932 let _serial = tracked();
933 for fill in [0, 0x5a5a_5a5a, usize::MAX] {
934 let mut words = slots::<T>(1, fill);
935 let base = words.as_mut_ptr();
936 let before = words.clone();
937 let mut states = [state_at::<T>(base, 0)];
938 destroy::<T>(&mut states);
939 assert_eq!(words, before, "an untagged slot is left alone");
940 // SAFETY: the slot is allocated; a mismatched tag reads as no `T`.
941 assert!(unsafe { FfiState::<T>::value(states[0]) }.is_none());
942 }
943 }
944
945 #[test]
946 fn destroy_skips_an_uninitialised_inline_slot() {
947 skips_a_slot_init_never_initialised::<Tracked>();
948 }
949
950 #[test]
951 fn destroy_skips_an_uninitialised_boxed_slot() {
952 skips_a_slot_init_never_initialised::<TrackedBig>();
953 }
954
955 /// `DuckDB` moves states: radix repartitioning copies each state's bytes
956 /// to another row and destroys it there. A tag derived from the slot's
957 /// address (the fourth audit's) made the destructor skip every moved
958 /// state, leaking its `T`. The tag follows the contents, so a moved state
959 /// is destroyed.
960 fn a_moved_state_is_destroyed<T: AggregateState>() {
961 let _serial = tracked();
962 let before = TRACKED_LIVE.load(Ordering::SeqCst);
963 let mut words = slots::<T>(2, 0);
964 let base = words.as_mut_ptr();
965 let from = state_at::<T>(base, 0);
966 let to = state_at::<T>(base, 1);
967 init::<T>(from);
968 // SAFETY: both slots are `size()` bytes of one allocation; the copy
969 // is what `DuckDB` does when it moves a row.
970 unsafe {
971 core::ptr::copy_nonoverlapping(
972 from.cast::<u8>(),
973 to.cast::<u8>(),
974 FfiState::<T>::size(),
975 );
976 }
977 assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before + 1);
978 destroy::<T>(&mut [to]);
979 assert_eq!(
980 TRACKED_LIVE.load(Ordering::SeqCst),
981 before,
982 "the moved state's T was dropped"
983 );
984 }
985
986 #[test]
987 fn a_moved_inline_state_is_destroyed() {
988 a_moved_state_is_destroyed::<Tracked>();
989 }
990
991 #[test]
992 fn a_moved_boxed_state_is_destroyed() {
993 a_moved_state_is_destroyed::<TrackedBig>();
994 }
995
996 /// A slot initialised for one state type is not dropped by another type's
997 /// destructor: that would drop an `A` as a `B` (the soundness audit's F4,
998 /// reachable when `DuckDB` reuses the bytes of a state it never destroyed).
999 #[test]
1000 fn a_slot_initialised_for_another_type_is_skipped() {
1001 let _serial = tracked();
1002 let before = TRACKED_LIVE.load(Ordering::SeqCst);
1003 let mut words = slots::<Tracked>(1, 0);
1004 let base = words.as_mut_ptr();
1005 let mut states = [state_at::<Tracked>(base, 0)];
1006 init::<Counter>(states[0]);
1007 let initialised = words.clone();
1008 destroy::<Tracked>(&mut states);
1009 assert_eq!(words, initialised, "another type's slot is left alone");
1010 assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before);
1011 destroy::<Counter>(&mut states);
1012 assert_eq!(words[0], 0, "its own destructor clears the tag");
1013
1014 let mut words = slots::<TrackedBig>(1, 0);
1015 let base = words.as_mut_ptr();
1016 let mut states = [state_at::<TrackedBig>(base, 0)];
1017 init::<Big>(states[0]);
1018 let initialised = words.clone();
1019 destroy::<TrackedBig>(&mut states);
1020 assert_eq!(words, initialised, "another type's slot is left alone");
1021 destroy::<Big>(&mut states);
1022 assert_eq!(words, [0, 0]);
1023 }
1024
1025 /// A destroyed slot's tag is cleared, so destroying it again is a no-op
1026 /// even though its other bytes are unchanged.
1027 fn destroying_twice_drops_once<T: AggregateState>() {
1028 let _serial = tracked();
1029 let before = TRACKED_LIVE.load(Ordering::SeqCst);
1030 let mut words = slots::<T>(1, 0);
1031 let base = words.as_mut_ptr();
1032 let mut states = [state_at::<T>(base, 0)];
1033 init::<T>(states[0]);
1034 assert_ne!(words[0], 0);
1035 destroy::<T>(&mut states);
1036 assert_eq!(words[0], 0);
1037 assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before);
1038 destroy::<T>(&mut states);
1039 assert_eq!(TRACKED_LIVE.load(Ordering::SeqCst), before);
1040 }
1041
1042 #[test]
1043 fn destroying_an_inline_state_twice_drops_it_once() {
1044 destroying_twice_drops_once::<Tracked>();
1045 }
1046
1047 #[test]
1048 fn destroying_a_boxed_state_twice_drops_it_once() {
1049 destroying_twice_drops_once::<TrackedBig>();
1050 }
1051
1052 #[test]
1053 fn the_initial_value_is_t_default() {
1054 let _serial = tracked();
1055 let mut words = slots::<Tracked>(1, usize::MAX);
1056 let base = words.as_mut_ptr();
1057 let mut states = [state_at::<Tracked>(base, 0)];
1058 init::<Tracked>(states[0]);
1059 // SAFETY: initialised above.
1060 assert_eq!(
1061 unsafe { FfiState::<Tracked>::with_state(states[0]) }.map(|t| t.value),
1062 Some(7)
1063 );
1064 destroy::<Tracked>(&mut states);
1065 let mut words = slots::<TrackedBig>(1, usize::MAX);
1066 let base = words.as_mut_ptr();
1067 let mut states = [state_at::<TrackedBig>(base, 0)];
1068 init::<TrackedBig>(states[0]);
1069 // SAFETY: initialised above.
1070 let first = unsafe { FfiState::<TrackedBig>::with_state(states[0]) }.map(|t| t.values[0]);
1071 assert_eq!(first, Some(7));
1072 destroy::<TrackedBig>(&mut states);
1073 }
1074}