Skip to main content

quack_rs/
chunk_writer.rs

1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! Auto-sizing chunk writer for table function scan callbacks.
7//!
8//! [`ChunkWriter`] wraps a `duckdb_data_chunk` and counts the rows its scan
9//! claims. On drop it calls `duckdb_data_chunk_set_size` with that count, which
10//! removes the error-prone manual `set_size` after writing rows — a common
11//! source of bugs in table functions.
12//!
13//! # Claim a row, then write it or give it back
14//!
15//! [`next_row`][ChunkWriter::next_row] **claims** the next row: it returns the
16//! row's index and counts it, and the drop publishes every row counted. The
17//! writer cannot see whether a claimed row was then written, because the
18//! writes go through [`VectorWriter`]s that do not report back. A claimed row
19//! that is never written is published anyway, holding whatever the output
20//! vector last held: `DuckDB` reuses one output chunk for every scan call, so
21//! that is a value from an earlier call (a wrong answer), and in a `LIST` or
22//! `MAP` column written by hand it is an earlier call's `{offset, length}`
23//! entry, pointing into child data this call did not write.
24//!
25//! So write every row you claim, or give it back with
26//! [`release_row`][ChunkWriter::release_row] before the writer drops. The
27//! natural loop over a source of unknown length claims one row more than the
28//! source fills:
29//!
30//! ```rust,no_run
31//! use quack_rs::chunk_writer::ChunkWriter;
32//! use libduckdb_sys::{duckdb_function_info, duckdb_data_chunk};
33//!
34//! unsafe extern "C" fn my_scan(info: duckdb_function_info, output: duckdb_data_chunk) {
35//!     let mut source = [("a", 1_i64), ("b", 2)].into_iter();
36//!     let mut cw = unsafe { ChunkWriter::new(output) };
37//!     // One writer per column, fetched once: building a writer costs several
38//!     // C API calls, so doing it for every row is several times slower.
39//!     let mut names = unsafe { cw.writer(0) };
40//!     let mut values = unsafe { cw.writer(1) };
41//!     while let Some(row) = cw.next_row() {
42//!         let Some((name, value)) = source.next() else {
43//!             // The source ran out after `row` was claimed: give it back, or
44//!             // it is published with stale contents.
45//!             cw.release_row(row);
46//!             break;
47//!         };
48//!         unsafe { names.write_varchar(row, name) };
49//!         unsafe { values.write_i64(row, value) };
50//!     }
51//!     // set_size is called automatically when `cw` is dropped
52//! }
53//! ```
54//!
55//! Asking the source first (`for item in source { let Some(row) =
56//! cw.next_row() else { break }; ... }`) never claims a row it cannot fill,
57//! but takes the item that found the chunk full out of the source: keep it for
58//! the next call if the source cannot be rewound.
59//!
60//! # Estimated impact
61//!
62//! Eliminates ~15 manual `set_size` calls and prevents off-by-one errors.
63
64use libduckdb_sys::{duckdb_data_chunk, duckdb_data_chunk_set_size, idx_t};
65
66use crate::vector::{StructWriter, VectorWriter};
67
68/// `DuckDB`'s compile-time default vector size.
69///
70/// This is only a fallback for the (unreachable in practice) case where
71/// `duckdb_vector_size()` reports `0`. **Never hardcode this value** — `DuckDB`
72/// can be built with `-DSTANDARD_VECTOR_SIZE=<n>` and its `duckdb.h` documents
73/// `duckdb_vector_size()` as the authoritative "amount of tuples that will fit
74/// into a data chunk". Writing past the real capacity is a heap overflow.
75pub const DEFAULT_STANDARD_VECTOR_SIZE: usize = 2048;
76
77/// Queries the running `DuckDB` for its actual vector size.
78///
79/// Falls back to [`DEFAULT_STANDARD_VECTOR_SIZE`] only if the engine reports a
80/// nonsensical `0`.
81fn runtime_vector_size() -> usize {
82    let size = usize::try_from(crate::vector::vector_size()).unwrap_or(0);
83    if size == 0 {
84        DEFAULT_STANDARD_VECTOR_SIZE
85    } else {
86        size
87    }
88}
89
90/// A row-counting writer for a `DuckDB` output data chunk.
91///
92/// Counts the rows claimed with [`next_row`][Self::next_row] and not given
93/// back with [`release_row`][Self::release_row], and calls
94/// `duckdb_data_chunk_set_size` with that count on drop. Every claimed row
95/// must be written or released: one that is neither is published with stale
96/// contents (see the [module docs][self]).
97///
98/// The drop does not check that, because it cannot: the writes go through
99/// [`VectorWriter`]s, which do not report back to this writer, so nothing here
100/// knows whether the last claimed row was written.
101#[derive(Debug)]
102pub struct ChunkWriter {
103    raw: duckdb_data_chunk,
104    row_count: usize,
105    capacity: usize,
106}
107
108impl ChunkWriter {
109    /// Creates a new `ChunkWriter` for the given output data chunk.
110    ///
111    /// The capacity is read from `duckdb_vector_size()`, so it tracks the
112    /// running engine even when `DuckDB` was built with a non-default
113    /// `STANDARD_VECTOR_SIZE`. Do not assume 2048.
114    ///
115    /// # Safety
116    ///
117    /// - `chunk` must be a valid, writable `duckdb_data_chunk` obtained from a
118    ///   `DuckDB` table function scan callback.
119    /// - The `DuckDB` C API dispatch table must be initialised (it always is
120    ///   inside a callback).
121    #[inline]
122    #[must_use]
123    pub unsafe fn new(chunk: duckdb_data_chunk) -> Self {
124        Self {
125            raw: chunk,
126            row_count: 0,
127            capacity: runtime_vector_size(),
128        }
129    }
130
131    /// Creates a new `ChunkWriter` with a custom capacity.
132    ///
133    /// # Safety
134    ///
135    /// Same as [`new`][Self::new]. `capacity` must not exceed the chunk's
136    /// actual capacity.
137    #[inline]
138    #[must_use]
139    pub const unsafe fn with_capacity(chunk: duckdb_data_chunk, capacity: usize) -> Self {
140        Self {
141            raw: chunk,
142            row_count: 0,
143            capacity,
144        }
145    }
146
147    /// Returns `true` if the chunk has reached its capacity.
148    #[inline]
149    #[must_use]
150    pub const fn is_full(&self) -> bool {
151        self.row_count >= self.capacity
152    }
153
154    /// Claims the next row and returns its index, or `None` if the chunk is
155    /// full.
156    ///
157    /// The row counts towards the size the drop publishes from this call on,
158    /// whether or not it is then written. Write it, or give it back with
159    /// [`release_row`][Self::release_row]: a claimed row that is neither is
160    /// published holding whatever the reused output vector last held (see the
161    /// [module docs][self]).
162    #[inline]
163    pub const fn next_row(&mut self) -> Option<usize> {
164        if self.is_full() {
165            return None;
166        }
167        let row = self.row_count;
168        self.row_count += 1;
169        Some(row)
170    }
171
172    /// Gives back `row`, the row the last [`next_row`][Self::next_row] claimed,
173    /// so that the drop does not publish it.
174    ///
175    /// For a scan that claims a row before it knows it has data for it: the
176    /// `while let Some(row) = cw.next_row()` loop whose source runs out (see
177    /// the [module docs][self]). The next `next_row` returns the same index
178    /// again. Rows are given back one at a time, most recent first; whatever
179    /// was written to a row given back stays in the vector but is not
180    /// published.
181    ///
182    /// The row is named, rather than implied, so that giving back the wrong
183    /// one is caught instead of silently dropping a written row.
184    ///
185    /// # Panics
186    ///
187    /// If `row` is not the most recently claimed row still counted: below it,
188    /// the rows claimed after it would have to move down; above it, the row
189    /// was never claimed. Either is a bug in the scan. The callback macros
190    /// ([`table_scan_callback!`](crate::table_scan_callback) and the others)
191    /// report the panic as the query's error.
192    #[inline]
193    pub fn release_row(&mut self, row: usize) {
194        assert!(
195            row.checked_add(1) == Some(self.row_count),
196            "ChunkWriter::release_row({row}): only the most recently claimed row can be \
197             given back, and that is {}",
198            self.row_count
199                .checked_sub(1)
200                .map_or_else(|| "none: no row is claimed".to_owned(), |r| r.to_string())
201        );
202        self.row_count = row;
203    }
204
205    /// Returns the number of rows claimed and not given back: the size the
206    /// drop publishes.
207    #[inline]
208    #[must_use]
209    pub const fn row_count(&self) -> usize {
210        self.row_count
211    }
212
213    /// Returns the chunk's capacity (maximum rows).
214    #[inline]
215    #[must_use]
216    pub const fn capacity(&self) -> usize {
217        self.capacity
218    }
219
220    /// Returns the number of columns in this data chunk.
221    #[inline]
222    #[must_use]
223    pub fn column_count(&self) -> usize {
224        // SAFETY: `self.raw` is a valid data chunk for this wrapper's lifetime.
225        usize::try_from(unsafe { libduckdb_sys::duckdb_data_chunk_get_column_count(self.raw) })
226            .unwrap_or(0)
227    }
228
229    /// Returns the raw `duckdb_vector` handle for the given column index.
230    ///
231    /// Use this when a column has a complex type (LIST, MAP, ARRAY) that requires
232    /// operations beyond simple scalar writes — for example, calling
233    /// [`ListVector::set_entry`][crate::vector::complex::ListVector::set_entry].
234    ///
235    /// # Safety
236    ///
237    /// `col_idx` must be less than the chunk's column count.
238    #[must_use]
239    pub unsafe fn vector(&self, col_idx: usize) -> libduckdb_sys::duckdb_vector {
240        // SAFETY: `self.raw` is the valid output chunk of the scan callback in progress,
241        // per the first `# Safety` clause of `ChunkWriter::new` / `with_capacity`.
242        // `duckdb_data_chunk_get_vector` bounds-checks `col_idx` itself (null when out of
243        // range, data_chunk-c.cpp); this function's `# Safety` clause keeps it in range, so
244        // the result is the column's `Vector`, owned by the chunk.
245        unsafe { libduckdb_sys::duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) }
246    }
247
248    /// Creates a [`VectorWriter`] for the given column index.
249    ///
250    /// # Safety
251    ///
252    /// - `col_idx` must be less than the chunk's column count.
253    /// - The returned writer borrows nothing and caches pointers into the
254    ///   chunk's buffers: it must not be used after the scan callback returns.
255    ///
256    /// Fetch each column's writer once per callback and keep it: building one
257    /// costs several C API calls, which a per-row `cw.writer(i)` pays on every
258    /// row (about 5× the cost of the write itself).
259    pub unsafe fn writer(&self, col_idx: usize) -> VectorWriter {
260        // SAFETY: self.raw is valid per constructor. col_idx is in bounds per caller.
261        let vec =
262            unsafe { libduckdb_sys::duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) };
263        // SAFETY: vec is a valid writable vector from the output chunk.
264        unsafe { VectorWriter::from_vector(vec) }
265    }
266
267    /// Creates a [`StructWriter`] for a STRUCT column at the given index.
268    ///
269    /// # Safety
270    ///
271    /// - `col_idx` must be less than the chunk's column count.
272    /// - The column at `col_idx` must have a STRUCT type with `field_count` fields.
273    /// - The returned writer borrows nothing and caches pointers into the
274    ///   chunk's buffers: it must not be used after the scan callback returns.
275    pub unsafe fn struct_writer(&self, col_idx: usize, field_count: usize) -> StructWriter {
276        // SAFETY: `self.raw` is the valid output chunk of the scan callback in progress,
277        // per the first `# Safety` clause of `ChunkWriter::new` / `with_capacity`.
278        // `col_idx` is in range per this function's first `# Safety` clause, so
279        // `duckdb_data_chunk_get_vector` (which returns null otherwise, data_chunk-c.cpp)
280        // yields the chunk-owned, writable column vector.
281        let vec =
282            unsafe { libduckdb_sys::duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) };
283        // SAFETY: `vec` is the non-null, writable column vector fetched above, owned by the
284        // output chunk and so live for the scan callback; this function's second `# Safety`
285        // clause is exactly `StructWriter::new`'s requirement that it be a STRUCT with
286        // `field_count` fields.
287        unsafe { StructWriter::new(vec, field_count) }
288    }
289
290    /// Returns the raw `duckdb_data_chunk` handle.
291    #[inline]
292    #[must_use]
293    pub const fn as_raw(&self) -> duckdb_data_chunk {
294        self.raw
295    }
296
297    /// Manually sets the chunk size and consumes the writer without auto-setting
298    /// size on drop. Use this if you need to override the auto-calculated size.
299    ///
300    /// # Safety
301    ///
302    /// - `size` must not exceed the chunk's capacity (typically 2048).
303    /// - `size` must match the actual number of rows written.
304    pub unsafe fn finish_with_size(self, size: usize) {
305        // SAFETY: self.raw is valid per constructor.
306        unsafe { duckdb_data_chunk_set_size(self.raw, size as idx_t) };
307        // Prevent Drop from setting size again.
308        std::mem::forget(self);
309    }
310}
311
312impl Drop for ChunkWriter {
313    fn drop(&mut self) {
314        // SAFETY: self.raw is valid per constructor's contract.
315        // Publish the rows claimed and not given back; that each was written
316        // is the scan's to ensure (see the struct docs).
317        unsafe { duckdb_data_chunk_set_size(self.raw, self.row_count as idx_t) };
318    }
319}
320
321#[cfg(test)]
322mod tests {
323    use super::*;
324
325    #[test]
326    fn size_of_chunk_writer() {
327        assert_eq!(
328            std::mem::size_of::<ChunkWriter>(),
329            std::mem::size_of::<usize>() * 3 // pointer + row_count + capacity
330        );
331    }
332
333    #[test]
334    fn next_row_increments() {
335        // We can't call Drop safely without a real chunk, so we use forget.
336        let mut cw = ChunkWriter {
337            raw: std::ptr::null_mut(),
338            row_count: 0,
339            capacity: 3,
340        };
341        assert_eq!(cw.next_row(), Some(0));
342        assert_eq!(cw.next_row(), Some(1));
343        assert_eq!(cw.next_row(), Some(2));
344        assert_eq!(cw.next_row(), None);
345        assert!(cw.is_full());
346        assert_eq!(cw.row_count(), 3);
347        // Forget to avoid calling Drop with a null pointer in FFI.
348        std::mem::forget(cw);
349    }
350
351    /// A writer over no chunk, for the counting tests; never dropped, since
352    /// the drop calls into `DuckDB`.
353    const fn detached(capacity: usize) -> ChunkWriter {
354        ChunkWriter {
355            raw: std::ptr::null_mut(),
356            row_count: 0,
357            capacity,
358        }
359    }
360
361    /// The source-runs-out loop: the row claimed last is given back, the
362    /// count drops to the rows written, and the same index comes back next.
363    #[test]
364    fn release_row_gives_back_the_last_claimed_row() {
365        let mut cw = detached(4);
366        let mut source = [10, 20].into_iter();
367        let mut written = Vec::new();
368        while let Some(row) = cw.next_row() {
369            let Some(v) = source.next() else {
370                cw.release_row(row);
371                break;
372            };
373            written.push((row, v));
374        }
375        assert_eq!(written, [(0, 10), (1, 20)]);
376        assert_eq!(cw.row_count(), 2, "only the written rows are published");
377        assert_eq!(
378            cw.next_row(),
379            Some(2),
380            "the released index is claimed again"
381        );
382        // Most recent first, down to none, and a full writer can give back.
383        cw.release_row(2);
384        cw.release_row(1);
385        cw.release_row(0);
386        assert_eq!(cw.row_count(), 0);
387        let mut full = detached(1);
388        assert_eq!(full.next_row(), Some(0));
389        assert!(full.is_full());
390        full.release_row(0);
391        assert!(!full.is_full());
392        std::mem::forget(full);
393        std::mem::forget(cw);
394    }
395
396    /// Giving back any row but the last claimed one is a bug in the scan, and
397    /// says which row could have been given back. The panic is caught here
398    /// so that the writer, whose drop calls into `DuckDB`, can be forgotten.
399    #[test]
400    fn release_row_refuses_any_row_but_the_last_claimed_one() {
401        let release = |cw: &mut ChunkWriter, row: usize| {
402            std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cw.release_row(row))).map_err(
403                |payload| {
404                    payload
405                        .downcast_ref::<String>()
406                        .cloned()
407                        .unwrap_or_default()
408                },
409            )
410        };
411        let mut cw = detached(4);
412        let err = release(&mut cw, 0).expect_err("nothing is claimed");
413        assert!(
414            err.contains("release_row(0)") && err.contains("none"),
415            "{err}"
416        );
417        assert_eq!(cw.next_row(), Some(0));
418        assert_eq!(cw.next_row(), Some(1));
419        for row in [0, 2, usize::MAX] {
420            let err = release(&mut cw, row).expect_err("not the last claimed row");
421            assert!(err.contains("and that is 1"), "{row}: {err}");
422        }
423        assert_eq!(cw.row_count(), 2, "a refused release changes nothing");
424        std::mem::forget(cw);
425    }
426
427    #[test]
428    fn default_vector_size_constant_matches_duckdb_default() {
429        // DuckDB's `DEFAULT_STANDARD_VECTOR_SIZE` in
430        // `duckdb/common/vector_size.hpp`. This is a fallback only — `new()`
431        // asks the engine.
432        assert_eq!(super::DEFAULT_STANDARD_VECTOR_SIZE, 2048);
433    }
434
435    #[test]
436    fn is_full_at_zero_capacity() {
437        let cw = ChunkWriter {
438            raw: std::ptr::null_mut(),
439            row_count: 0,
440            capacity: 0,
441        };
442        assert!(cw.is_full());
443        std::mem::forget(cw);
444    }
445}