quack_rs/chunk_writer.rs
1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! Auto-sizing chunk writer for table function scan callbacks.
7//!
8//! [`ChunkWriter`] wraps a `duckdb_data_chunk` and counts the rows its scan
9//! claims. On drop it calls `duckdb_data_chunk_set_size` with that count, which
10//! removes the error-prone manual `set_size` after writing rows — a common
11//! source of bugs in table functions.
12//!
13//! # Claim a row, then write it or give it back
14//!
15//! [`next_row`][ChunkWriter::next_row] **claims** the next row: it returns the
16//! row's index and counts it, and the drop publishes every row counted. The
17//! writer cannot see whether a claimed row was then written, because the
18//! writes go through [`VectorWriter`]s that do not report back. A claimed row
19//! that is never written is published anyway, holding whatever the output
20//! vector last held: `DuckDB` reuses one output chunk for every scan call, so
21//! that is a value from an earlier call (a wrong answer), and in a `LIST` or
22//! `MAP` column written by hand it is an earlier call's `{offset, length}`
23//! entry, pointing into child data this call did not write.
24//!
25//! So write every row you claim, or give it back with
26//! [`release_row`][ChunkWriter::release_row] before the writer drops. The
27//! natural loop over a source of unknown length claims one row more than the
28//! source fills:
29//!
30//! ```rust,no_run
31//! use quack_rs::chunk_writer::ChunkWriter;
32//! use libduckdb_sys::{duckdb_function_info, duckdb_data_chunk};
33//!
34//! unsafe extern "C" fn my_scan(info: duckdb_function_info, output: duckdb_data_chunk) {
35//! let mut source = [("a", 1_i64), ("b", 2)].into_iter();
36//! let mut cw = unsafe { ChunkWriter::new(output) };
37//! // One writer per column, fetched once: building a writer costs several
38//! // C API calls, so doing it for every row is several times slower.
39//! let mut names = unsafe { cw.writer(0) };
40//! let mut values = unsafe { cw.writer(1) };
41//! while let Some(row) = cw.next_row() {
42//! let Some((name, value)) = source.next() else {
43//! // The source ran out after `row` was claimed: give it back, or
44//! // it is published with stale contents.
45//! cw.release_row(row);
46//! break;
47//! };
48//! unsafe { names.write_varchar(row, name) };
49//! unsafe { values.write_i64(row, value) };
50//! }
51//! // set_size is called automatically when `cw` is dropped
52//! }
53//! ```
54//!
55//! Asking the source first (`for item in source { let Some(row) =
56//! cw.next_row() else { break }; ... }`) never claims a row it cannot fill,
57//! but takes the item that found the chunk full out of the source: keep it for
58//! the next call if the source cannot be rewound.
59//!
60//! # Estimated impact
61//!
62//! Eliminates ~15 manual `set_size` calls and prevents off-by-one errors.
63
64use libduckdb_sys::{duckdb_data_chunk, duckdb_data_chunk_set_size, idx_t};
65
66use crate::vector::{StructWriter, VectorWriter};
67
68/// `DuckDB`'s compile-time default vector size.
69///
70/// This is only a fallback for the (unreachable in practice) case where
71/// `duckdb_vector_size()` reports `0`. **Never hardcode this value** — `DuckDB`
72/// can be built with `-DSTANDARD_VECTOR_SIZE=<n>` and its `duckdb.h` documents
73/// `duckdb_vector_size()` as the authoritative "amount of tuples that will fit
74/// into a data chunk". Writing past the real capacity is a heap overflow.
75pub const DEFAULT_STANDARD_VECTOR_SIZE: usize = 2048;
76
77/// Queries the running `DuckDB` for its actual vector size.
78///
79/// Falls back to [`DEFAULT_STANDARD_VECTOR_SIZE`] only if the engine reports a
80/// nonsensical `0`.
81fn runtime_vector_size() -> usize {
82 let size = usize::try_from(crate::vector::vector_size()).unwrap_or(0);
83 if size == 0 {
84 DEFAULT_STANDARD_VECTOR_SIZE
85 } else {
86 size
87 }
88}
89
90/// A row-counting writer for a `DuckDB` output data chunk.
91///
92/// Counts the rows claimed with [`next_row`][Self::next_row] and not given
93/// back with [`release_row`][Self::release_row], and calls
94/// `duckdb_data_chunk_set_size` with that count on drop. Every claimed row
95/// must be written or released: one that is neither is published with stale
96/// contents (see the [module docs][self]).
97///
98/// The drop does not check that, because it cannot: the writes go through
99/// [`VectorWriter`]s, which do not report back to this writer, so nothing here
100/// knows whether the last claimed row was written.
101#[derive(Debug)]
102pub struct ChunkWriter {
103 raw: duckdb_data_chunk,
104 row_count: usize,
105 capacity: usize,
106}
107
108impl ChunkWriter {
109 /// Creates a new `ChunkWriter` for the given output data chunk.
110 ///
111 /// The capacity is read from `duckdb_vector_size()`, so it tracks the
112 /// running engine even when `DuckDB` was built with a non-default
113 /// `STANDARD_VECTOR_SIZE`. Do not assume 2048.
114 ///
115 /// # Safety
116 ///
117 /// - `chunk` must be a valid, writable `duckdb_data_chunk` obtained from a
118 /// `DuckDB` table function scan callback.
119 /// - The `DuckDB` C API dispatch table must be initialised (it always is
120 /// inside a callback).
121 #[inline]
122 #[must_use]
123 pub unsafe fn new(chunk: duckdb_data_chunk) -> Self {
124 Self {
125 raw: chunk,
126 row_count: 0,
127 capacity: runtime_vector_size(),
128 }
129 }
130
131 /// Creates a new `ChunkWriter` with a custom capacity.
132 ///
133 /// # Safety
134 ///
135 /// Same as [`new`][Self::new]. `capacity` must not exceed the chunk's
136 /// actual capacity.
137 #[inline]
138 #[must_use]
139 pub const unsafe fn with_capacity(chunk: duckdb_data_chunk, capacity: usize) -> Self {
140 Self {
141 raw: chunk,
142 row_count: 0,
143 capacity,
144 }
145 }
146
147 /// Returns `true` if the chunk has reached its capacity.
148 #[inline]
149 #[must_use]
150 pub const fn is_full(&self) -> bool {
151 self.row_count >= self.capacity
152 }
153
154 /// Claims the next row and returns its index, or `None` if the chunk is
155 /// full.
156 ///
157 /// The row counts towards the size the drop publishes from this call on,
158 /// whether or not it is then written. Write it, or give it back with
159 /// [`release_row`][Self::release_row]: a claimed row that is neither is
160 /// published holding whatever the reused output vector last held (see the
161 /// [module docs][self]).
162 #[inline]
163 pub const fn next_row(&mut self) -> Option<usize> {
164 if self.is_full() {
165 return None;
166 }
167 let row = self.row_count;
168 self.row_count += 1;
169 Some(row)
170 }
171
172 /// Gives back `row`, the row the last [`next_row`][Self::next_row] claimed,
173 /// so that the drop does not publish it.
174 ///
175 /// For a scan that claims a row before it knows it has data for it: the
176 /// `while let Some(row) = cw.next_row()` loop whose source runs out (see
177 /// the [module docs][self]). The next `next_row` returns the same index
178 /// again. Rows are given back one at a time, most recent first; whatever
179 /// was written to a row given back stays in the vector but is not
180 /// published.
181 ///
182 /// The row is named, rather than implied, so that giving back the wrong
183 /// one is caught instead of silently dropping a written row.
184 ///
185 /// # Panics
186 ///
187 /// If `row` is not the most recently claimed row still counted: below it,
188 /// the rows claimed after it would have to move down; above it, the row
189 /// was never claimed. Either is a bug in the scan. The callback macros
190 /// ([`table_scan_callback!`](crate::table_scan_callback) and the others)
191 /// report the panic as the query's error.
192 #[inline]
193 pub fn release_row(&mut self, row: usize) {
194 assert!(
195 row.checked_add(1) == Some(self.row_count),
196 "ChunkWriter::release_row({row}): only the most recently claimed row can be \
197 given back, and that is {}",
198 self.row_count
199 .checked_sub(1)
200 .map_or_else(|| "none: no row is claimed".to_owned(), |r| r.to_string())
201 );
202 self.row_count = row;
203 }
204
205 /// Returns the number of rows claimed and not given back: the size the
206 /// drop publishes.
207 #[inline]
208 #[must_use]
209 pub const fn row_count(&self) -> usize {
210 self.row_count
211 }
212
213 /// Returns the chunk's capacity (maximum rows).
214 #[inline]
215 #[must_use]
216 pub const fn capacity(&self) -> usize {
217 self.capacity
218 }
219
220 /// Returns the number of columns in this data chunk.
221 #[inline]
222 #[must_use]
223 pub fn column_count(&self) -> usize {
224 // SAFETY: `self.raw` is a valid data chunk for this wrapper's lifetime.
225 usize::try_from(unsafe { libduckdb_sys::duckdb_data_chunk_get_column_count(self.raw) })
226 .unwrap_or(0)
227 }
228
229 /// Returns the raw `duckdb_vector` handle for the given column index.
230 ///
231 /// Use this when a column has a complex type (LIST, MAP, ARRAY) that requires
232 /// operations beyond simple scalar writes — for example, calling
233 /// [`ListVector::set_entry`][crate::vector::complex::ListVector::set_entry].
234 ///
235 /// # Safety
236 ///
237 /// `col_idx` must be less than the chunk's column count.
238 #[must_use]
239 pub unsafe fn vector(&self, col_idx: usize) -> libduckdb_sys::duckdb_vector {
240 // SAFETY: `self.raw` is the valid output chunk of the scan callback in progress,
241 // per the first `# Safety` clause of `ChunkWriter::new` / `with_capacity`.
242 // `duckdb_data_chunk_get_vector` bounds-checks `col_idx` itself (null when out of
243 // range, data_chunk-c.cpp); this function's `# Safety` clause keeps it in range, so
244 // the result is the column's `Vector`, owned by the chunk.
245 unsafe { libduckdb_sys::duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) }
246 }
247
248 /// Creates a [`VectorWriter`] for the given column index.
249 ///
250 /// # Safety
251 ///
252 /// - `col_idx` must be less than the chunk's column count.
253 /// - The returned writer borrows nothing and caches pointers into the
254 /// chunk's buffers: it must not be used after the scan callback returns.
255 ///
256 /// Fetch each column's writer once per callback and keep it: building one
257 /// costs several C API calls, which a per-row `cw.writer(i)` pays on every
258 /// row (about 5× the cost of the write itself).
259 pub unsafe fn writer(&self, col_idx: usize) -> VectorWriter {
260 // SAFETY: self.raw is valid per constructor. col_idx is in bounds per caller.
261 let vec =
262 unsafe { libduckdb_sys::duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) };
263 // SAFETY: vec is a valid writable vector from the output chunk.
264 unsafe { VectorWriter::from_vector(vec) }
265 }
266
267 /// Creates a [`StructWriter`] for a STRUCT column at the given index.
268 ///
269 /// # Safety
270 ///
271 /// - `col_idx` must be less than the chunk's column count.
272 /// - The column at `col_idx` must have a STRUCT type with `field_count` fields.
273 /// - The returned writer borrows nothing and caches pointers into the
274 /// chunk's buffers: it must not be used after the scan callback returns.
275 pub unsafe fn struct_writer(&self, col_idx: usize, field_count: usize) -> StructWriter {
276 // SAFETY: `self.raw` is the valid output chunk of the scan callback in progress,
277 // per the first `# Safety` clause of `ChunkWriter::new` / `with_capacity`.
278 // `col_idx` is in range per this function's first `# Safety` clause, so
279 // `duckdb_data_chunk_get_vector` (which returns null otherwise, data_chunk-c.cpp)
280 // yields the chunk-owned, writable column vector.
281 let vec =
282 unsafe { libduckdb_sys::duckdb_data_chunk_get_vector(self.raw, col_idx as idx_t) };
283 // SAFETY: `vec` is the non-null, writable column vector fetched above, owned by the
284 // output chunk and so live for the scan callback; this function's second `# Safety`
285 // clause is exactly `StructWriter::new`'s requirement that it be a STRUCT with
286 // `field_count` fields.
287 unsafe { StructWriter::new(vec, field_count) }
288 }
289
290 /// Returns the raw `duckdb_data_chunk` handle.
291 #[inline]
292 #[must_use]
293 pub const fn as_raw(&self) -> duckdb_data_chunk {
294 self.raw
295 }
296
297 /// Manually sets the chunk size and consumes the writer without auto-setting
298 /// size on drop. Use this if you need to override the auto-calculated size.
299 ///
300 /// # Safety
301 ///
302 /// - `size` must not exceed the chunk's capacity (typically 2048).
303 /// - `size` must match the actual number of rows written.
304 pub unsafe fn finish_with_size(self, size: usize) {
305 // SAFETY: self.raw is valid per constructor.
306 unsafe { duckdb_data_chunk_set_size(self.raw, size as idx_t) };
307 // Prevent Drop from setting size again.
308 std::mem::forget(self);
309 }
310}
311
312impl Drop for ChunkWriter {
313 fn drop(&mut self) {
314 // SAFETY: self.raw is valid per constructor's contract.
315 // Publish the rows claimed and not given back; that each was written
316 // is the scan's to ensure (see the struct docs).
317 unsafe { duckdb_data_chunk_set_size(self.raw, self.row_count as idx_t) };
318 }
319}
320
321#[cfg(test)]
322mod tests {
323 use super::*;
324
325 #[test]
326 fn size_of_chunk_writer() {
327 assert_eq!(
328 std::mem::size_of::<ChunkWriter>(),
329 std::mem::size_of::<usize>() * 3 // pointer + row_count + capacity
330 );
331 }
332
333 #[test]
334 fn next_row_increments() {
335 // We can't call Drop safely without a real chunk, so we use forget.
336 let mut cw = ChunkWriter {
337 raw: std::ptr::null_mut(),
338 row_count: 0,
339 capacity: 3,
340 };
341 assert_eq!(cw.next_row(), Some(0));
342 assert_eq!(cw.next_row(), Some(1));
343 assert_eq!(cw.next_row(), Some(2));
344 assert_eq!(cw.next_row(), None);
345 assert!(cw.is_full());
346 assert_eq!(cw.row_count(), 3);
347 // Forget to avoid calling Drop with a null pointer in FFI.
348 std::mem::forget(cw);
349 }
350
351 /// A writer over no chunk, for the counting tests; never dropped, since
352 /// the drop calls into `DuckDB`.
353 const fn detached(capacity: usize) -> ChunkWriter {
354 ChunkWriter {
355 raw: std::ptr::null_mut(),
356 row_count: 0,
357 capacity,
358 }
359 }
360
361 /// The source-runs-out loop: the row claimed last is given back, the
362 /// count drops to the rows written, and the same index comes back next.
363 #[test]
364 fn release_row_gives_back_the_last_claimed_row() {
365 let mut cw = detached(4);
366 let mut source = [10, 20].into_iter();
367 let mut written = Vec::new();
368 while let Some(row) = cw.next_row() {
369 let Some(v) = source.next() else {
370 cw.release_row(row);
371 break;
372 };
373 written.push((row, v));
374 }
375 assert_eq!(written, [(0, 10), (1, 20)]);
376 assert_eq!(cw.row_count(), 2, "only the written rows are published");
377 assert_eq!(
378 cw.next_row(),
379 Some(2),
380 "the released index is claimed again"
381 );
382 // Most recent first, down to none, and a full writer can give back.
383 cw.release_row(2);
384 cw.release_row(1);
385 cw.release_row(0);
386 assert_eq!(cw.row_count(), 0);
387 let mut full = detached(1);
388 assert_eq!(full.next_row(), Some(0));
389 assert!(full.is_full());
390 full.release_row(0);
391 assert!(!full.is_full());
392 std::mem::forget(full);
393 std::mem::forget(cw);
394 }
395
396 /// Giving back any row but the last claimed one is a bug in the scan, and
397 /// says which row could have been given back. The panic is caught here
398 /// so that the writer, whose drop calls into `DuckDB`, can be forgotten.
399 #[test]
400 fn release_row_refuses_any_row_but_the_last_claimed_one() {
401 let release = |cw: &mut ChunkWriter, row: usize| {
402 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cw.release_row(row))).map_err(
403 |payload| {
404 payload
405 .downcast_ref::<String>()
406 .cloned()
407 .unwrap_or_default()
408 },
409 )
410 };
411 let mut cw = detached(4);
412 let err = release(&mut cw, 0).expect_err("nothing is claimed");
413 assert!(
414 err.contains("release_row(0)") && err.contains("none"),
415 "{err}"
416 );
417 assert_eq!(cw.next_row(), Some(0));
418 assert_eq!(cw.next_row(), Some(1));
419 for row in [0, 2, usize::MAX] {
420 let err = release(&mut cw, row).expect_err("not the last claimed row");
421 assert!(err.contains("and that is 1"), "{row}: {err}");
422 }
423 assert_eq!(cw.row_count(), 2, "a refused release changes nothing");
424 std::mem::forget(cw);
425 }
426
427 #[test]
428 fn default_vector_size_constant_matches_duckdb_default() {
429 // DuckDB's `DEFAULT_STANDARD_VECTOR_SIZE` in
430 // `duckdb/common/vector_size.hpp`. This is a fallback only — `new()`
431 // asks the engine.
432 assert_eq!(super::DEFAULT_STANDARD_VECTOR_SIZE, 2048);
433 }
434
435 #[test]
436 fn is_full_at_zero_capacity() {
437 let cw = ChunkWriter {
438 raw: std::ptr::null_mut(),
439 row_count: 0,
440 capacity: 0,
441 };
442 assert!(cw.is_full());
443 std::mem::forget(cw);
444 }
445}