Skip to main content

quack_rs/
data_chunk.rs

1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! Ergonomic wrapper around `DuckDB` data chunks.
7//!
8//! [`DataChunk`] provides safe, convenient access to the vectors and metadata
9//! of a `duckdb_data_chunk`, eliminating the raw FFI calls that extension
10//! authors currently need to write in every scan callback.
11//!
12//! # Example
13//!
14//! ```rust,no_run
15//! use quack_rs::data_chunk::DataChunk;
16//! use quack_rs::vector::{VectorWriter, VectorReader};
17//! use libduckdb_sys::{duckdb_function_info, duckdb_data_chunk};
18//!
19//! unsafe extern "C" fn my_scan(info: duckdb_function_info, output: duckdb_data_chunk) {
20//!     let chunk = unsafe { DataChunk::from_raw(output) };
21//!     let mut writer = unsafe { chunk.writer(0) };
22//!     unsafe { writer.write_i64(0, 42) };
23//!     unsafe { chunk.set_size(1) };
24//! }
25//! ```
26
27use libduckdb_sys::{
28    duckdb_data_chunk, duckdb_data_chunk_get_column_count, duckdb_data_chunk_get_size,
29    duckdb_data_chunk_get_vector, duckdb_data_chunk_set_size, duckdb_vector, idx_t,
30};
31
32use crate::chunk_writer::ChunkWriter;
33use crate::vector::complex::StructVector;
34use crate::vector::{StructReader, StructWriter, VectorReader, VectorWriter};
35
36/// A non-owning wrapper around a `duckdb_data_chunk`.
37///
38/// This wrapper does **not** destroy the chunk on drop — `DuckDB` owns the
39/// chunk and manages its lifetime. `DataChunk` simply provides ergonomic
40/// methods for accessing vectors and metadata within callback functions.
41#[derive(Debug)]
42pub struct DataChunk {
43    raw: duckdb_data_chunk,
44}
45
46impl DataChunk {
47    /// Wraps a raw `duckdb_data_chunk` handle.
48    ///
49    /// # Safety
50    ///
51    /// `raw` must be a valid `duckdb_data_chunk` obtained from a `DuckDB`
52    /// callback (e.g., a scan callback's `output` parameter or an aggregate
53    /// `update` callback's `input` chunk). The chunk must remain valid for
54    /// the lifetime of this wrapper.
55    #[inline]
56    #[must_use]
57    pub const unsafe fn from_raw(raw: duckdb_data_chunk) -> Self {
58        Self { raw }
59    }
60
61    /// Returns the number of rows in this data chunk.
62    #[inline]
63    #[must_use]
64    // Nothing but `duckdb_data_chunk_get_size`, and a chunk cannot be built
65    // without a live engine, so the `--lib` mutation run cannot observe this
66    // returning anything. The end-to-end suite reads it on every scan.
67    #[cfg_attr(test, mutants::skip)]
68    pub fn size(&self) -> usize {
69        // SAFETY: self.raw is valid per constructor contract.
70        usize::try_from(unsafe { duckdb_data_chunk_get_size(self.raw) }).unwrap_or(0)
71    }
72
73    /// Sets the number of rows in this data chunk.
74    ///
75    /// Call this in scan callbacks after writing output rows. Set to `0` to
76    /// signal end of stream.
77    ///
78    /// # Safety
79    ///
80    /// `size` must not exceed the chunk's capacity (typically 2048).
81    #[inline]
82    pub unsafe fn set_size(&self, size: usize) {
83        // SAFETY: self.raw is valid per constructor contract.
84        unsafe { duckdb_data_chunk_set_size(self.raw, size as idx_t) };
85    }
86
87    /// Returns the number of columns in this data chunk.
88    #[inline]
89    #[must_use]
90    // As `size`: a bare `duckdb_data_chunk_get_column_count`.
91    #[cfg_attr(test, mutants::skip)]
92    pub fn column_count(&self) -> usize {
93        // SAFETY: self.raw is valid per constructor contract.
94        usize::try_from(unsafe { duckdb_data_chunk_get_column_count(self.raw) }).unwrap_or(0)
95    }
96
97    /// Returns `true` when **any** column of this chunk is NULL at `row`.
98    ///
99    /// This is the SQL-standard NULL-propagation predicate for a scalar
100    /// function: if any argument is NULL, the result is NULL.
101    ///
102    /// # Why you have to ask
103    ///
104    /// [`NullHandling::DefaultNullHandling`][crate::types::NullHandling::DefaultNullHandling]
105    /// does **not** make `DuckDB` propagate NULLs for a C API scalar function at
106    /// run time — see
107    /// [`propagate_nulls`][Self::propagate_nulls] for the details and the
108    /// evidence.
109    ///
110    /// For a hot loop over many columns, prefer
111    /// [`propagate_nulls`][Self::propagate_nulls], which resolves each column's
112    /// validity pointer once instead of once per row.
113    ///
114    /// # Safety
115    ///
116    /// - `row` must be less than [`size`][Self::size].
117    /// - The chunk must be a valid input chunk for the current callback.
118    #[must_use]
119    pub unsafe fn any_null(&self, row: usize) -> bool {
120        for col in 0..self.column_count() {
121            // SAFETY: `col` is in range by construction, and `row` by contract.
122            let reader = unsafe { self.reader(col) };
123            // SAFETY: `row` is in range per this function's contract.
124            if !unsafe { reader.is_valid(row) } {
125                return true;
126            }
127        }
128        false
129    }
130
131    /// Marks `output` NULL at every row where any column of this chunk is NULL.
132    ///
133    /// Each row is nulled through [`VectorWriter::set_null`], so a `STRUCT` or
134    /// `ARRAY` output has its fields / elements nulled at that row too, and
135    /// `(f(x)).a` on a NULL row reads NULL rather than the value the callback
136    /// wrote there.
137    ///
138    /// # What `DuckDB` actually does with NULLs
139    ///
140    /// It is natural to read
141    /// [`NullHandling::DefaultNullHandling`][crate::types::NullHandling::DefaultNullHandling]
142    /// as "`DuckDB` returns NULL for me when an argument is NULL". For a C API
143    /// scalar function that is **not** what happens at run time:
144    ///
145    /// - `CAPIScalarFunction` calls the extension's callback for the whole
146    ///   flattened chunk, NULL rows included, and never inspects the result's
147    ///   validity.
148    /// - `ExpressionExecutor::Execute` then calls `VerifyNullHandling`, which only
149    ///   *asserts* that the function already produced NULL wherever an input was
150    ///   NULL. Through `DuckDB` 1.5.6 it is compiled **only under `#ifdef DEBUG`**; in
151    ///   2.0.0 it is compiled in but runs only after
152    ///   `SET debug_verification_mode = 'VERIFY_FUNCTIONS'`, so a default session
153    ///   still believes the callback.
154    ///
155    /// So in any release build of `DuckDB` — which is every build a user runs —
156    /// a callback that ignores input validity returns a non-NULL answer for a
157    /// NULL input, silently. `SELECT f(NULL)` still looks right, because a
158    /// literal NULL is constant-folded before the function is ever called;
159    /// the wrong answers only appear once the argument comes from a column.
160    /// (Verified against `DuckDB` 1.5.4: a function returning `999` for every
161    /// row returned `999`, valid, for a NULL row of a real table.)
162    ///
163    /// Call this after writing the output, and the SQL-standard behaviour is
164    /// restored. Functions that mean to see NULLs — `coalesce`-likes,
165    /// `is_null`-likes — should not call it, and should register with
166    /// [`NullHandling::SpecialNullHandling`][crate::types::NullHandling::SpecialNullHandling]
167    /// so `DuckDB`'s debug assertion does not fire.
168    ///
169    /// # Example
170    ///
171    /// ```rust,no_run
172    /// use quack_rs::data_chunk::DataChunk;
173    /// use quack_rs::vector::VectorWriter;
174    ///
175    /// quack_rs::scalar_callback!(double_it, |_info, input, output| {
176    ///     let chunk = unsafe { DataChunk::from_raw(input) };
177    ///     let reader = unsafe { chunk.reader(0) };
178    ///     let mut writer = unsafe { VectorWriter::from_vector(output) };
179    ///     for row in 0..chunk.size() {
180    ///         unsafe { writer.write_i64(row, reader.read_i64(row) * 2) };
181    ///     }
182    ///     // Without this, `double_it(NULL)` is 0, not NULL.
183    ///     unsafe { chunk.propagate_nulls(&mut writer) };
184    /// });
185    /// ```
186    ///
187    /// # Safety
188    ///
189    /// - `output` must write to a vector with at least [`size`][Self::size]
190    ///   rows — the output vector of the callback this chunk was passed to.
191    /// - The chunk must be a valid input chunk for the current callback.
192    pub unsafe fn propagate_nulls(&self, output: &mut VectorWriter) {
193        let rows = self.size();
194        if rows == 0 {
195            return;
196        }
197        // Resolve every column's validity pointer once. A column with no
198        // validity mask has no NULLs at all and is skipped entirely, which is
199        // the common case and costs nothing.
200        let readers: Vec<VectorReader> = (0..self.column_count())
201            // SAFETY: `col` is in range by construction.
202            .map(|col| unsafe { self.reader(col) })
203            .collect();
204        for row in 0..rows {
205            // SAFETY: `row < rows`, and each reader covers the same chunk.
206            if readers.iter().any(|r| !unsafe { r.is_valid(row) }) {
207                // SAFETY: `row < rows <= output vector length` per contract.
208                unsafe { output.set_null(row) };
209            }
210        }
211    }
212
213    /// Returns the raw `duckdb_vector` handle for the given column index.
214    ///
215    /// # Safety
216    ///
217    /// `col_idx` must be less than [`column_count`][DataChunk::column_count].
218    #[inline]
219    #[must_use]
220    pub unsafe fn vector(&self, col_idx: usize) -> duckdb_vector {
221        // SAFETY: self.raw is valid and col_idx is in bounds per caller's contract.
222        unsafe { duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) }
223    }
224
225    /// Creates a [`VectorWriter`] for the given column index.
226    ///
227    /// # Safety
228    ///
229    /// - `col_idx` must be less than [`column_count`][DataChunk::column_count].
230    /// - The chunk must be a writable output chunk (not a read-only input chunk).
231    /// - The returned value borrows nothing: it must not be used after the
232    ///   underlying `duckdb_data_chunk` is destroyed — for an
233    ///   [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
234    ///   in a callback, after the callback returns. It caches pointers into the
235    ///   chunk's buffers, so a read afterwards is a use-after-free.
236    pub unsafe fn writer(&self, col_idx: usize) -> VectorWriter {
237        // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
238        // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
239        // `DataChunk::from_raw`'s contract.
240        let vec = unsafe { self.vector(col_idx) };
241        // SAFETY: vec is a valid writable vector from the output chunk.
242        unsafe { VectorWriter::from_vector(vec) }
243    }
244
245    /// Creates a [`VectorReader`] for the given column index.
246    ///
247    /// The reader's row count is set to this chunk's current [`size`][DataChunk::size].
248    ///
249    /// # Safety
250    ///
251    /// - `col_idx` must be less than [`column_count`][DataChunk::column_count].
252    /// - The returned value borrows nothing: it must not be used after the
253    ///   underlying `duckdb_data_chunk` is destroyed — for an
254    ///   [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
255    ///   in a callback, after the callback returns. It caches pointers into the
256    ///   chunk's buffers, so a read afterwards is a use-after-free.
257    ///
258    /// Build the reader once per chunk and read every row through it: this
259    /// call makes several C API calls, which a `chunk.reader(0).read_i64(row)`
260    /// inside the row loop pays on every row (measured at about 6× the cost of
261    /// the read itself).
262    pub unsafe fn reader(&self, col_idx: usize) -> VectorReader {
263        // SAFETY: self.raw is valid; col_idx is in bounds per caller's contract.
264        unsafe { VectorReader::new(self.raw, col_idx) }
265    }
266
267    /// Creates a [`StructReader`] for a STRUCT column at the given index.
268    ///
269    /// This is a convenience method that combines [`vector`][Self::vector] with
270    /// [`StructReader::new`].
271    ///
272    /// # Safety
273    ///
274    /// - `col_idx` must be less than [`column_count`][Self::column_count].
275    /// - The column at `col_idx` must have a STRUCT type with `field_count` fields.
276    /// - The returned value borrows nothing: it must not be used after the
277    ///   underlying `duckdb_data_chunk` is destroyed — for an
278    ///   [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
279    ///   in a callback, after the callback returns. It caches pointers into the
280    ///   chunk's buffers, so a read afterwards is a use-after-free.
281    pub unsafe fn struct_reader(&self, col_idx: usize, field_count: usize) -> StructReader {
282        // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
283        // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
284        // `DataChunk::from_raw`'s contract.
285        let vec = unsafe { self.vector(col_idx) };
286        // SAFETY: vec is a valid STRUCT vector per caller's contract.
287        unsafe { StructReader::new(vec, field_count, self.size()) }
288    }
289
290    /// Creates a [`VectorReader`] for a field of a STRUCT column.
291    ///
292    /// Convenience for accessing a specific field in a STRUCT input column.
293    ///
294    /// # Safety
295    ///
296    /// - `col_idx` must be less than [`column_count`][Self::column_count].
297    /// - The column at `col_idx` must have a STRUCT type.
298    /// - `field_idx` must be a valid field index within the STRUCT.
299    /// - The returned value borrows nothing: it must not be used after the
300    ///   underlying `duckdb_data_chunk` is destroyed — for an
301    ///   [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
302    ///   in a callback, after the callback returns. It caches pointers into the
303    ///   chunk's buffers, so a read afterwards is a use-after-free.
304    pub unsafe fn struct_field_reader(&self, col_idx: usize, field_idx: usize) -> VectorReader {
305        // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
306        // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
307        // `DataChunk::from_raw`'s contract.
308        let vec = unsafe { self.vector(col_idx) };
309        // SAFETY: vec is a valid STRUCT vector per caller's contract.
310        unsafe { StructVector::field_reader(vec, field_idx, self.size()) }
311    }
312
313    /// Creates a [`StructWriter`] for a STRUCT column at the given index.
314    ///
315    /// This is a convenience method that combines [`vector`][Self::vector] with
316    /// [`StructWriter::new`].
317    ///
318    /// # Safety
319    ///
320    /// - `col_idx` must be less than [`column_count`][Self::column_count].
321    /// - The column at `col_idx` must have a STRUCT type with `field_count` fields.
322    /// - The chunk must be a writable output chunk.
323    /// - The returned value borrows nothing: it must not be used after the
324    ///   underlying `duckdb_data_chunk` is destroyed — for an
325    ///   [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
326    ///   in a callback, after the callback returns. It caches pointers into the
327    ///   chunk's buffers, so a read afterwards is a use-after-free.
328    pub unsafe fn struct_writer(&self, col_idx: usize, field_count: usize) -> StructWriter {
329        // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
330        // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
331        // `DataChunk::from_raw`'s contract.
332        let vec = unsafe { self.vector(col_idx) };
333        // SAFETY: vec is a valid STRUCT vector per caller's contract.
334        unsafe { StructWriter::new(vec, field_count) }
335    }
336
337    /// Creates a [`ChunkWriter`] for this output data chunk.
338    ///
339    /// The [`ChunkWriter`] counts the rows claimed with
340    /// [`next_row()`][ChunkWriter::next_row] and not given back with
341    /// [`release_row()`][ChunkWriter::release_row], and calls `set_size` with
342    /// that count on drop: every row claimed must be written or given back.
343    ///
344    /// # Safety
345    ///
346    /// This chunk must be a valid, writable output chunk from a table function
347    /// scan callback.
348    pub unsafe fn into_chunk_writer(self) -> ChunkWriter {
349        // SAFETY: self.raw is valid per constructor's contract.
350        unsafe { ChunkWriter::new(self.raw) }
351    }
352
353    /// Returns the raw `duckdb_data_chunk` handle.
354    #[inline]
355    #[must_use]
356    pub const fn as_raw(&self) -> duckdb_data_chunk {
357        self.raw
358    }
359}
360
361#[cfg(test)]
362mod tests {
363    use super::*;
364
365    #[test]
366    fn size_of_data_chunk() {
367        assert_eq!(
368            std::mem::size_of::<DataChunk>(),
369            std::mem::size_of::<usize>()
370        );
371    }
372}