quack_rs/data_chunk.rs
1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! Ergonomic wrapper around `DuckDB` data chunks.
7//!
8//! [`DataChunk`] provides safe, convenient access to the vectors and metadata
9//! of a `duckdb_data_chunk`, eliminating the raw FFI calls that extension
10//! authors currently need to write in every scan callback.
11//!
12//! # Example
13//!
14//! ```rust,no_run
15//! use quack_rs::data_chunk::DataChunk;
16//! use quack_rs::vector::{VectorWriter, VectorReader};
17//! use libduckdb_sys::{duckdb_function_info, duckdb_data_chunk};
18//!
19//! unsafe extern "C" fn my_scan(info: duckdb_function_info, output: duckdb_data_chunk) {
20//! let chunk = unsafe { DataChunk::from_raw(output) };
21//! let mut writer = unsafe { chunk.writer(0) };
22//! unsafe { writer.write_i64(0, 42) };
23//! unsafe { chunk.set_size(1) };
24//! }
25//! ```
26
27use libduckdb_sys::{
28 duckdb_data_chunk, duckdb_data_chunk_get_column_count, duckdb_data_chunk_get_size,
29 duckdb_data_chunk_get_vector, duckdb_data_chunk_set_size, duckdb_vector, idx_t,
30};
31
32use crate::chunk_writer::ChunkWriter;
33use crate::vector::complex::StructVector;
34use crate::vector::{StructReader, StructWriter, VectorReader, VectorWriter};
35
36/// A non-owning wrapper around a `duckdb_data_chunk`.
37///
38/// This wrapper does **not** destroy the chunk on drop — `DuckDB` owns the
39/// chunk and manages its lifetime. `DataChunk` simply provides ergonomic
40/// methods for accessing vectors and metadata within callback functions.
41#[derive(Debug)]
42pub struct DataChunk {
43 raw: duckdb_data_chunk,
44}
45
46impl DataChunk {
47 /// Wraps a raw `duckdb_data_chunk` handle.
48 ///
49 /// # Safety
50 ///
51 /// `raw` must be a valid `duckdb_data_chunk` obtained from a `DuckDB`
52 /// callback (e.g., a scan callback's `output` parameter or an aggregate
53 /// `update` callback's `input` chunk). The chunk must remain valid for
54 /// the lifetime of this wrapper.
55 #[inline]
56 #[must_use]
57 pub const unsafe fn from_raw(raw: duckdb_data_chunk) -> Self {
58 Self { raw }
59 }
60
61 /// Returns the number of rows in this data chunk.
62 #[inline]
63 #[must_use]
64 // Nothing but `duckdb_data_chunk_get_size`, and a chunk cannot be built
65 // without a live engine, so the `--lib` mutation run cannot observe this
66 // returning anything. The end-to-end suite reads it on every scan.
67 #[cfg_attr(test, mutants::skip)]
68 pub fn size(&self) -> usize {
69 // SAFETY: self.raw is valid per constructor contract.
70 usize::try_from(unsafe { duckdb_data_chunk_get_size(self.raw) }).unwrap_or(0)
71 }
72
73 /// Sets the number of rows in this data chunk.
74 ///
75 /// Call this in scan callbacks after writing output rows. Set to `0` to
76 /// signal end of stream.
77 ///
78 /// # Safety
79 ///
80 /// `size` must not exceed the chunk's capacity (typically 2048).
81 #[inline]
82 pub unsafe fn set_size(&self, size: usize) {
83 // SAFETY: self.raw is valid per constructor contract.
84 unsafe { duckdb_data_chunk_set_size(self.raw, size as idx_t) };
85 }
86
87 /// Returns the number of columns in this data chunk.
88 #[inline]
89 #[must_use]
90 // As `size`: a bare `duckdb_data_chunk_get_column_count`.
91 #[cfg_attr(test, mutants::skip)]
92 pub fn column_count(&self) -> usize {
93 // SAFETY: self.raw is valid per constructor contract.
94 usize::try_from(unsafe { duckdb_data_chunk_get_column_count(self.raw) }).unwrap_or(0)
95 }
96
97 /// Returns `true` when **any** column of this chunk is NULL at `row`.
98 ///
99 /// This is the SQL-standard NULL-propagation predicate for a scalar
100 /// function: if any argument is NULL, the result is NULL.
101 ///
102 /// # Why you have to ask
103 ///
104 /// [`NullHandling::DefaultNullHandling`][crate::types::NullHandling::DefaultNullHandling]
105 /// does **not** make `DuckDB` propagate NULLs for a C API scalar function at
106 /// run time — see
107 /// [`propagate_nulls`][Self::propagate_nulls] for the details and the
108 /// evidence.
109 ///
110 /// For a hot loop over many columns, prefer
111 /// [`propagate_nulls`][Self::propagate_nulls], which resolves each column's
112 /// validity pointer once instead of once per row.
113 ///
114 /// # Safety
115 ///
116 /// - `row` must be less than [`size`][Self::size].
117 /// - The chunk must be a valid input chunk for the current callback.
118 #[must_use]
119 pub unsafe fn any_null(&self, row: usize) -> bool {
120 for col in 0..self.column_count() {
121 // SAFETY: `col` is in range by construction, and `row` by contract.
122 let reader = unsafe { self.reader(col) };
123 // SAFETY: `row` is in range per this function's contract.
124 if !unsafe { reader.is_valid(row) } {
125 return true;
126 }
127 }
128 false
129 }
130
131 /// Marks `output` NULL at every row where any column of this chunk is NULL.
132 ///
133 /// Each row is nulled through [`VectorWriter::set_null`], so a `STRUCT` or
134 /// `ARRAY` output has its fields / elements nulled at that row too, and
135 /// `(f(x)).a` on a NULL row reads NULL rather than the value the callback
136 /// wrote there.
137 ///
138 /// # What `DuckDB` actually does with NULLs
139 ///
140 /// It is natural to read
141 /// [`NullHandling::DefaultNullHandling`][crate::types::NullHandling::DefaultNullHandling]
142 /// as "`DuckDB` returns NULL for me when an argument is NULL". For a C API
143 /// scalar function that is **not** what happens at run time:
144 ///
145 /// - `CAPIScalarFunction` calls the extension's callback for the whole
146 /// flattened chunk, NULL rows included, and never inspects the result's
147 /// validity.
148 /// - `ExpressionExecutor::Execute` then calls `VerifyNullHandling`, which only
149 /// *asserts* that the function already produced NULL wherever an input was
150 /// NULL. Through `DuckDB` 1.5.6 it is compiled **only under `#ifdef DEBUG`**; in
151 /// 2.0.0 it is compiled in but runs only after
152 /// `SET debug_verification_mode = 'VERIFY_FUNCTIONS'`, so a default session
153 /// still believes the callback.
154 ///
155 /// So in any release build of `DuckDB` — which is every build a user runs —
156 /// a callback that ignores input validity returns a non-NULL answer for a
157 /// NULL input, silently. `SELECT f(NULL)` still looks right, because a
158 /// literal NULL is constant-folded before the function is ever called;
159 /// the wrong answers only appear once the argument comes from a column.
160 /// (Verified against `DuckDB` 1.5.4: a function returning `999` for every
161 /// row returned `999`, valid, for a NULL row of a real table.)
162 ///
163 /// Call this after writing the output, and the SQL-standard behaviour is
164 /// restored. Functions that mean to see NULLs — `coalesce`-likes,
165 /// `is_null`-likes — should not call it, and should register with
166 /// [`NullHandling::SpecialNullHandling`][crate::types::NullHandling::SpecialNullHandling]
167 /// so `DuckDB`'s debug assertion does not fire.
168 ///
169 /// # Example
170 ///
171 /// ```rust,no_run
172 /// use quack_rs::data_chunk::DataChunk;
173 /// use quack_rs::vector::VectorWriter;
174 ///
175 /// quack_rs::scalar_callback!(double_it, |_info, input, output| {
176 /// let chunk = unsafe { DataChunk::from_raw(input) };
177 /// let reader = unsafe { chunk.reader(0) };
178 /// let mut writer = unsafe { VectorWriter::from_vector(output) };
179 /// for row in 0..chunk.size() {
180 /// unsafe { writer.write_i64(row, reader.read_i64(row) * 2) };
181 /// }
182 /// // Without this, `double_it(NULL)` is 0, not NULL.
183 /// unsafe { chunk.propagate_nulls(&mut writer) };
184 /// });
185 /// ```
186 ///
187 /// # Safety
188 ///
189 /// - `output` must write to a vector with at least [`size`][Self::size]
190 /// rows — the output vector of the callback this chunk was passed to.
191 /// - The chunk must be a valid input chunk for the current callback.
192 pub unsafe fn propagate_nulls(&self, output: &mut VectorWriter) {
193 let rows = self.size();
194 if rows == 0 {
195 return;
196 }
197 // Resolve every column's validity pointer once. A column with no
198 // validity mask has no NULLs at all and is skipped entirely, which is
199 // the common case and costs nothing.
200 let readers: Vec<VectorReader> = (0..self.column_count())
201 // SAFETY: `col` is in range by construction.
202 .map(|col| unsafe { self.reader(col) })
203 .collect();
204 for row in 0..rows {
205 // SAFETY: `row < rows`, and each reader covers the same chunk.
206 if readers.iter().any(|r| !unsafe { r.is_valid(row) }) {
207 // SAFETY: `row < rows <= output vector length` per contract.
208 unsafe { output.set_null(row) };
209 }
210 }
211 }
212
213 /// Returns the raw `duckdb_vector` handle for the given column index.
214 ///
215 /// # Safety
216 ///
217 /// `col_idx` must be less than [`column_count`][DataChunk::column_count].
218 #[inline]
219 #[must_use]
220 pub unsafe fn vector(&self, col_idx: usize) -> duckdb_vector {
221 // SAFETY: self.raw is valid and col_idx is in bounds per caller's contract.
222 unsafe { duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) }
223 }
224
225 /// Creates a [`VectorWriter`] for the given column index.
226 ///
227 /// # Safety
228 ///
229 /// - `col_idx` must be less than [`column_count`][DataChunk::column_count].
230 /// - The chunk must be a writable output chunk (not a read-only input chunk).
231 /// - The returned value borrows nothing: it must not be used after the
232 /// underlying `duckdb_data_chunk` is destroyed — for an
233 /// [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
234 /// in a callback, after the callback returns. It caches pointers into the
235 /// chunk's buffers, so a read afterwards is a use-after-free.
236 pub unsafe fn writer(&self, col_idx: usize) -> VectorWriter {
237 // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
238 // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
239 // `DataChunk::from_raw`'s contract.
240 let vec = unsafe { self.vector(col_idx) };
241 // SAFETY: vec is a valid writable vector from the output chunk.
242 unsafe { VectorWriter::from_vector(vec) }
243 }
244
245 /// Creates a [`VectorReader`] for the given column index.
246 ///
247 /// The reader's row count is set to this chunk's current [`size`][DataChunk::size].
248 ///
249 /// # Safety
250 ///
251 /// - `col_idx` must be less than [`column_count`][DataChunk::column_count].
252 /// - The returned value borrows nothing: it must not be used after the
253 /// underlying `duckdb_data_chunk` is destroyed — for an
254 /// [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
255 /// in a callback, after the callback returns. It caches pointers into the
256 /// chunk's buffers, so a read afterwards is a use-after-free.
257 ///
258 /// Build the reader once per chunk and read every row through it: this
259 /// call makes several C API calls, which a `chunk.reader(0).read_i64(row)`
260 /// inside the row loop pays on every row (measured at about 6× the cost of
261 /// the read itself).
262 pub unsafe fn reader(&self, col_idx: usize) -> VectorReader {
263 // SAFETY: self.raw is valid; col_idx is in bounds per caller's contract.
264 unsafe { VectorReader::new(self.raw, col_idx) }
265 }
266
267 /// Creates a [`StructReader`] for a STRUCT column at the given index.
268 ///
269 /// This is a convenience method that combines [`vector`][Self::vector] with
270 /// [`StructReader::new`].
271 ///
272 /// # Safety
273 ///
274 /// - `col_idx` must be less than [`column_count`][Self::column_count].
275 /// - The column at `col_idx` must have a STRUCT type with `field_count` fields.
276 /// - The returned value borrows nothing: it must not be used after the
277 /// underlying `duckdb_data_chunk` is destroyed — for an
278 /// [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
279 /// in a callback, after the callback returns. It caches pointers into the
280 /// chunk's buffers, so a read afterwards is a use-after-free.
281 pub unsafe fn struct_reader(&self, col_idx: usize, field_count: usize) -> StructReader {
282 // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
283 // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
284 // `DataChunk::from_raw`'s contract.
285 let vec = unsafe { self.vector(col_idx) };
286 // SAFETY: vec is a valid STRUCT vector per caller's contract.
287 unsafe { StructReader::new(vec, field_count, self.size()) }
288 }
289
290 /// Creates a [`VectorReader`] for a field of a STRUCT column.
291 ///
292 /// Convenience for accessing a specific field in a STRUCT input column.
293 ///
294 /// # Safety
295 ///
296 /// - `col_idx` must be less than [`column_count`][Self::column_count].
297 /// - The column at `col_idx` must have a STRUCT type.
298 /// - `field_idx` must be a valid field index within the STRUCT.
299 /// - The returned value borrows nothing: it must not be used after the
300 /// underlying `duckdb_data_chunk` is destroyed — for an
301 /// [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
302 /// in a callback, after the callback returns. It caches pointers into the
303 /// chunk's buffers, so a read afterwards is a use-after-free.
304 pub unsafe fn struct_field_reader(&self, col_idx: usize, field_idx: usize) -> VectorReader {
305 // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
306 // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
307 // `DataChunk::from_raw`'s contract.
308 let vec = unsafe { self.vector(col_idx) };
309 // SAFETY: vec is a valid STRUCT vector per caller's contract.
310 unsafe { StructVector::field_reader(vec, field_idx, self.size()) }
311 }
312
313 /// Creates a [`StructWriter`] for a STRUCT column at the given index.
314 ///
315 /// This is a convenience method that combines [`vector`][Self::vector] with
316 /// [`StructWriter::new`].
317 ///
318 /// # Safety
319 ///
320 /// - `col_idx` must be less than [`column_count`][Self::column_count].
321 /// - The column at `col_idx` must have a STRUCT type with `field_count` fields.
322 /// - The chunk must be a writable output chunk.
323 /// - The returned value borrows nothing: it must not be used after the
324 /// underlying `duckdb_data_chunk` is destroyed — for an
325 /// [`OwnedDataChunk`](crate::query::OwnedDataChunk), after it is dropped;
326 /// in a callback, after the callback returns. It caches pointers into the
327 /// chunk's buffers, so a read afterwards is a use-after-free.
328 pub unsafe fn struct_writer(&self, col_idx: usize, field_count: usize) -> StructWriter {
329 // SAFETY: `self.vector` needs `col_idx < column_count`, which is this function's
330 // first `# Safety` clause; `self.raw` is live for this wrapper's lifetime per
331 // `DataChunk::from_raw`'s contract.
332 let vec = unsafe { self.vector(col_idx) };
333 // SAFETY: vec is a valid STRUCT vector per caller's contract.
334 unsafe { StructWriter::new(vec, field_count) }
335 }
336
337 /// Creates a [`ChunkWriter`] for this output data chunk.
338 ///
339 /// The [`ChunkWriter`] counts the rows claimed with
340 /// [`next_row()`][ChunkWriter::next_row] and not given back with
341 /// [`release_row()`][ChunkWriter::release_row], and calls `set_size` with
342 /// that count on drop: every row claimed must be written or given back.
343 ///
344 /// # Safety
345 ///
346 /// This chunk must be a valid, writable output chunk from a table function
347 /// scan callback.
348 pub unsafe fn into_chunk_writer(self) -> ChunkWriter {
349 // SAFETY: self.raw is valid per constructor's contract.
350 unsafe { ChunkWriter::new(self.raw) }
351 }
352
353 /// Returns the raw `duckdb_data_chunk` handle.
354 #[inline]
355 #[must_use]
356 pub const fn as_raw(&self) -> duckdb_data_chunk {
357 self.raw
358 }
359}
360
361#[cfg(test)]
362mod tests {
363 use super::*;
364
365 #[test]
366 fn size_of_data_chunk() {
367 assert_eq!(
368 std::mem::size_of::<DataChunk>(),
369 std::mem::size_of::<usize>()
370 );
371 }
372}