quack_rs/arrow/convert.rs
1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! The four conversions between `DuckDB` data chunks and the Arrow C Data Interface.
7
8use std::ffi::CString;
9use std::os::raw::c_char;
10use std::ptr;
11
12use libduckdb_sys::{
13 duckdb_arrow_converted_schema, duckdb_connection, duckdb_data_chunk,
14 duckdb_data_chunk_from_arrow, duckdb_data_chunk_to_arrow, duckdb_logical_type,
15 duckdb_schema_from_arrow, duckdb_to_arrow_schema, idx_t,
16};
17
18use super::{
19 ArrowArray, ArrowConvertedSchema, ArrowOptions, ArrowSchema, RawArrowArray, RawArrowSchema,
20};
21use crate::data_chunk::DataChunk;
22use crate::error_data::{DuckDbErrorType, ErrorData};
23use crate::query::OwnedDataChunk;
24use crate::types::LogicalType;
25
26// ─── Conversions ─────────────────────────────────────────────────────────────
27
28/// Renders a `DuckDB` schema as an Arrow schema
29/// (`duckdb_to_arrow_schema`).
30///
31/// `columns` pairs each column's name with its logical type, in order. The
32/// result is a struct schema (`"+s"`) with one child per column — the shape
33/// every Arrow consumer expects for a record batch.
34///
35/// # Errors
36///
37/// - [`DuckDbErrorType::InvalidInput`] if a name contains an interior NUL byte,
38/// since `DuckDB` reads names as C strings.
39/// - [`DuckDbErrorType::OutOfRange`] if there are more columns than `idx_t` can
40/// hold.
41/// - Whatever `DuckDB` reports for a type it cannot render as Arrow.
42#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
43pub fn to_arrow_schema(
44 options: &ArrowOptions<'_>,
45 columns: &[(&str, &LogicalType)],
46) -> Result<ArrowSchema, ErrorData> {
47 let column_count = idx_t::try_from(columns.len()).map_err(|_| {
48 ErrorData::new(
49 DuckDbErrorType::OutOfRange,
50 "to_arrow_schema: more columns than idx_t can represent",
51 )
52 })?;
53
54 let mut names: Vec<CString> = Vec::with_capacity(columns.len());
55 for (name, _) in columns {
56 let owned = CString::new(*name).map_err(|_| {
57 ErrorData::new(
58 DuckDbErrorType::InvalidInput,
59 &format!(
60 "to_arrow_schema: column name {name:?} contains an interior NUL byte, but \
61 DuckDB reads schema names as C strings"
62 ),
63 )
64 })?;
65 names.push(owned);
66 }
67 let mut name_ptrs: Vec<*const c_char> = names.iter().map(|n| n.as_ptr()).collect();
68 let mut types: Vec<duckdb_logical_type> = columns.iter().map(|(_, ty)| ty.as_raw()).collect();
69
70 let mut out = RawArrowSchema::empty();
71 // SAFETY: `options` owns a live handle; `types` and `name_ptrs` are each
72 // valid for `column_count` elements and outlive the call; `out` is a fresh
73 // record DuckDB may fill. DuckDB copies the names and the types.
74 let raw_err = unsafe {
75 duckdb_to_arrow_schema(
76 options.as_raw(),
77 types.as_mut_ptr(),
78 name_ptrs.as_mut_ptr(),
79 column_count,
80 &raw mut out,
81 )
82 };
83 // SAFETY: the return value is an owned `duckdb_error_data`, or null.
84 let err = unsafe { ErrorData::from_raw(raw_err) };
85 if err.has_error() {
86 // `ArrowConverter::ToArrowSchema` installs `out.release` as its last
87 // statement, after everything that can throw, so a failed call leaves
88 // `release == NULL` and there is nothing here to free. Dropping the
89 // record without releasing it is therefore correct, not a leak.
90 return Err(err);
91 }
92 // SAFETY: DuckDB filled the record and installed its release callback.
93 Ok(unsafe { ArrowSchema::from_raw(out) })
94}
95
96/// Exports a `DuckDB` data chunk as an Arrow struct array
97/// (`duckdb_data_chunk_to_arrow`).
98///
99/// The chunk is only read; it stays valid and owned by the caller. The returned
100/// array owns copies of the data, so it outlives the chunk.
101///
102/// Pair it with the schema [`to_arrow_schema`] produces from the same column
103/// types and the same [`ArrowOptions`] — an Arrow consumer needs both.
104///
105/// # Values `DuckDB` would export wrongly are refused
106///
107/// `DuckDB` exports these as different values, with no error (checked on
108/// 1.4.4, 1.5.0 and 1.5.5; `docs/upstream-duckdb-reports.md`, item 16), so
109/// the chunk is checked first and refused if it holds one, at any depth:
110///
111/// - `INTERVAL`: Arrow's `month_day_nano` counts nanoseconds in an `i64`, and
112/// `DuckDB` multiplies the microseconds by 1000 with no overflow check. An
113/// interval whose microsecond field is beyond ±`i64::MAX / 1000` (about
114/// ±106,751 days, or 2,562,047 hours) would wrap: `INTERVAL 2562048 HOUR`
115/// would export as a negative interval.
116/// - `UHUGEINT`, and `HUGEINT` unless `arrow_lossless_conversion` is set:
117/// exported as `decimal128(38, 0)`, which holds 38 digits. A `UHUGEINT` of
118/// 2^127 or more would come back negative (`2^128 - 1` as `-1`), and a
119/// 39-digit `HUGEINT` would not survive either.
120///
121/// The check copies each column whose type contains one of these types (a
122/// chunk's vectors may be dictionary or constant vectors), so it costs one
123/// copy of those columns.
124///
125/// # Errors
126///
127/// - [`DuckDbErrorType::InvalidInput`] naming the column and value, for a
128/// value `DuckDB` would export wrongly (see above).
129/// - Whatever `DuckDB` reports for a type it cannot render as Arrow.
130#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
131pub fn data_chunk_to_arrow(
132 options: &ArrowOptions<'_>,
133 chunk: &DataChunk,
134) -> Result<ArrowArray, ErrorData> {
135 super::export_check::check_chunk(options, chunk)?;
136 let mut out = RawArrowArray::empty();
137 // SAFETY: `options` owns a live handle, `chunk` is valid per `DataChunk`'s
138 // constructor contract, and `out` is a fresh record DuckDB may fill.
139 let raw_err =
140 unsafe { duckdb_data_chunk_to_arrow(options.as_raw(), chunk.as_raw(), &raw mut out) };
141 // SAFETY: the return value is an owned `duckdb_error_data`, or null.
142 let err = unsafe { ErrorData::from_raw(raw_err) };
143 if err.has_error() {
144 // `ArrowConverter::ToArrowArray` assigns `*out_array` as its last
145 // statement, so a failed call never wrote to `out` and there is nothing
146 // to release.
147 return Err(err);
148 }
149 // SAFETY: DuckDB filled the record and installed its release callback.
150 let array = unsafe { ArrowArray::from_raw(out) };
151 // SAFETY: `array` is what DuckDB just exported from `chunk` under
152 // `options`. On an error it is dropped here, which releases it.
153 unsafe { super::export_check::check_layout(options, chunk, &array) }?;
154 Ok(array)
155}
156
157/// Translates an Arrow schema into `DuckDB` type descriptors
158/// (`duckdb_schema_from_arrow`).
159///
160/// `schema` is read, not consumed: the caller still owns it and it is released
161/// when its [`ArrowSchema`] drops. It is taken by `&mut` because the C signature
162/// is `struct ArrowSchema *`, even though `PopulateArrowTableSchema` receives it
163/// as `const ArrowSchema &`.
164///
165/// `schema` must be the struct schema at the root of a record batch: `DuckDB`
166/// makes one column per child.
167///
168/// # Errors
169///
170/// - [`DuckDbErrorType::InvalidInput`] if `schema` has already been released,
171/// which `DuckDB` would dereference rather than diagnose.
172/// - Whatever `DuckDB` reports for an Arrow type it cannot map — a released
173/// child schema, or an unsupported format string.
174///
175/// # Safety
176///
177/// `connection` must be a live `duckdb_connection`.
178#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
179pub unsafe fn schema_from_arrow(
180 connection: duckdb_connection,
181 schema: &mut ArrowSchema,
182) -> Result<ArrowConvertedSchema, ErrorData> {
183 if schema.is_released() {
184 return Err(ErrorData::new(
185 DuckDbErrorType::InvalidInput,
186 "schema_from_arrow: the Arrow schema has already been released, so its children \
187 pointer no longer refers to live memory",
188 ));
189 }
190 let mut out: duckdb_arrow_converted_schema = ptr::null_mut();
191 // SAFETY: `connection` is live per this function's contract, `schema` points
192 // at a live record, and `out` is a valid out-parameter.
193 let raw_err =
194 unsafe { duckdb_schema_from_arrow(connection, schema.as_mut_ptr(), &raw mut out) };
195 // SAFETY: the return value is an owned `duckdb_error_data`, or null.
196 let err = unsafe { ErrorData::from_raw(raw_err) };
197 if err.has_error() {
198 return Err(err);
199 }
200 if out.is_null() {
201 return Err(ErrorData::new(
202 DuckDbErrorType::Internal,
203 "duckdb_schema_from_arrow reported success but produced no converted schema",
204 ));
205 }
206 // SAFETY: `out` is a fresh handle this value now owns, built from `schema`
207 // (`PopulateArrowTableSchema` makes one column per child of it).
208 Ok(unsafe { ArrowConvertedSchema::from_raw(out, schema) })
209}
210
211/// Imports an Arrow struct array as a `DuckDB` data chunk
212/// (`duckdb_data_chunk_from_arrow`).
213///
214/// # Why `array` is taken by value
215///
216/// `DuckDB` claims the array: it copies the record into the chunk's owned data
217/// and then sets `arrow_array->release = nullptr` — *before* the conversion
218/// loop body, so the transfer happens on the error path too. Taking it by value
219/// makes that visible in the signature, and the by-value binding is dropped on
220/// the way out, which releases the array in the one case where `DuckDB` before
221/// 2.0.0 does not claim it (a zero-column converted schema, where the loop
222/// never runs). `DuckDB` 2.0.0 claims it before the loop, so it releases that
223/// array itself; the binding's `Drop` then finds `release` null and does
224/// nothing.
225///
226/// The resulting chunk keeps the Arrow buffers alive, so the data is shared, not
227/// copied — with two exceptions, which are copied so that the chunk is flat
228/// and every [`VectorReader`](crate::vector::VectorReader) reads it
229/// correctly. `DuckDB` imports a column that is dictionary-encoded at any
230/// depth as a *dictionary* vector, and an Arrow null-type column as a
231/// *constant* vector; a flat read of either returns wrong values (and, for a
232/// dictionary, reads past the end of its buffer). Those columns are copied
233/// into fresh flat vectors before this function returns. Run-end-encoded
234/// children are expanded by `DuckDB` itself.
235///
236/// Before `DuckDB` 2.0.0 the claim is held by the chunk's **first** column:
237/// `DuckDB` copies the record into every column's state but nulls `release`
238/// after the first (`src/main/capi/arrow-c.cpp`, lines 110–120 at v1.5.6), so
239/// the producer's buffers are released when column 0's vector is. A vector
240/// made to reference another column
241/// ([`reference_vector`][crate::vector::ops::reference_vector]) does not keep
242/// them alive there. From 2.0.0 every column shares one owner
243/// (`src/main/capi/v1/arrow-c.cpp`, lines 141–152), so a vector referencing
244/// any column keeps them. Either way `reference_vector`'s contract already
245/// requires the chunk to outlive the referencing vector.
246///
247/// # Errors
248///
249/// [`DuckDbErrorType::InvalidInput`], checked here because `DuckDB` does not
250/// check them and would read out of bounds or through a null pointer:
251///
252/// - `array` has already been released, has a negative `length`, has a
253/// nonzero `offset` (`DuckDB` ignores the top-level offset and imports the
254/// wrong rows, see `docs/upstream-duckdb-reports.md`), or has no rows (see
255/// below);
256/// - its child count does not match `converted`'s column count;
257/// - its `children` pointer, or one of the child pointers, is null;
258/// - a child is shorter than `length` (the Arrow specification requires
259/// every child of a struct array to hold `offset + length` rows);
260/// - a node anywhere in `array` has a valid Arrow layout that `DuckDB`
261/// imports from the wrong rows or out of bounds: an offset below the top
262/// level that `DuckDB` misapplies, a dictionary under a list or with more
263/// than [`duckdb_vector_size`](libduckdb_sys::duckdb_vector_size) (2048)
264/// rows that can hold NULLs, a nested dictionary, overlapping or gapped
265/// list views, a sparse union with recoded type ids, or a run-end-encoded
266/// array where `DuckDB` reads a plain one. The message names the node; the
267/// module docs of `src/arrow/import_layout.rs` and
268/// `docs/upstream-duckdb-reports.md` (items 9 and 24 to 28) give the
269/// details;
270/// - a node anywhere in `array` breaks a bound the Arrow specification sets
271/// and `DuckDB` takes on trust, reading past it once the chunk is read (a
272/// SIGSEGV on 1.5.6 and 2.0.0 for a list offset past its child): a list,
273/// large list or map offset that is negative, decreases, or lies past the
274/// child; a negative list-view offset or size; a struct field, list or
275/// fixed-size-list child, or union member shorter than the rows its parent
276/// reads from it; a dictionary index outside the dictionary in a row its own
277/// validity does not mark NULL (a row NULL only in an enclosing struct must
278/// still hold a valid index, as the specification requires), or indices of
279/// a non-integer type;
280/// - the array and its schema disagree on a node's child count or
281/// dictionary encoding.
282///
283/// [`DuckDbErrorType::Internal`] if copying a non-flat column into a flat
284/// vector fails (see above), which needs an allocation failure.
285///
286/// Every error `DuckDB` itself reports from the conversion also arrives as
287/// [`DuckDbErrorType::InvalidInput`]: `duckdb_data_chunk_from_arrow` maps all
288/// of them to `DUCKDB_ERROR_INVALID_INPUT`.
289///
290/// # Safety
291///
292/// - `connection` must be a live `duckdb_connection`, and `converted` must
293/// have been produced from it (or from another connection of the same
294/// database).
295/// - `array` must be a valid Arrow struct array (see
296/// [`ArrowArray::from_raw`]) that **conforms to `converted`'s schema**: each
297/// child's physical layout must be the one its column's Arrow format
298/// describes. `DuckDB` reads a child's buffers as that format with no check
299/// — an `int32` child imported under a `utf8` schema has its values read as
300/// string offsets into a buffer that does not exist (a crash, or worse). An
301/// array exported with [`data_chunk_to_arrow`] under the schema that
302/// `converted` was made from conforms.
303/// - `length` must be the array's true row count. `DuckDB` allocates a chunk
304/// of that capacity before its error handling starts
305/// (`dchunk->Initialize(…, length)` in `arrow-c.cpp`), so a length too large
306/// to allocate throws through the C API and aborts the process. A length,
307/// or a nested row count, above
308/// [`MAX_CAPACITY`](crate::vector::ops::MAX_CAPACITY) is refused, which on
309/// a 32-bit target also keeps every buffer's byte size within what `malloc`
310/// receives unnarrowed; below it an allocation the system cannot satisfy
311/// still aborts. The length is not otherwise checkable: a run-end-encoded
312/// column declares any length with a few bytes of buffers.
313/// - Every validity bitmap that `DuckDB` reads (one with a nonzero
314/// `null_count`) must be readable for **one byte past** the last byte that
315/// holds its rows' bits. When a node's effective bit offset is not a
316/// multiple of 8, `GetValidityMask` (`arrow_conversion.cpp`) copies
317/// `ceil(rows / 8) + 1` bytes from the first byte it needs, which can be one
318/// more than the rows occupy: an `int32` column at offset 1 of length 7 with
319/// a 1-byte bitmap is read as 2 bytes (an invalid read under valgrind on
320/// 1.5.5). That byte only supplies bits for rows past the end, so the
321/// values imported are right; the read itself is the hazard. Buffers padded to a multiple of 8 or 64 bytes, as
322/// the Arrow columnar format recommends, satisfy this; the allocation size
323/// is invisible through the C Data Interface, so it cannot be checked here.
324/// - A dictionary whose values are a fixed-width type (integers, floats,
325/// `DATE` in days, `TIMESTAMP`, `DECIMAL`, ...) and whose indices can be
326/// NULL (a nonzero `null_count`, or NULL rows in an enclosing struct) must
327/// have its values buffer readable for **one element past**
328/// `offset + length`. `DuckDB` points every NULL index at a sentinel entry
329/// one past the dictionary, but imports those types without copying, so
330/// the sentinel lies in the producer's buffer; copying the column, as this
331/// function does for every dictionary-encoded column, reads it (an invalid
332/// read under valgrind on 1.4.4 to 1.5.5,
333/// `docs/upstream-duckdb-reports.md`, item 29). The value read is
334/// discarded: those rows are NULL. A buffer padded to a multiple of 64
335/// bytes satisfies this unless the values fill it exactly.
336#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
337pub unsafe fn data_chunk_from_arrow(
338 connection: duckdb_connection,
339 mut array: ArrowArray,
340 converted: &ArrowConvertedSchema,
341) -> Result<OwnedDataChunk, ErrorData> {
342 if array.is_released() {
343 return Err(ErrorData::new(
344 DuckDbErrorType::InvalidInput,
345 "data_chunk_from_arrow: the Arrow array has already been released",
346 ));
347 }
348 let children = array.child_count();
349 if children != converted.column_count() {
350 return Err(ErrorData::new(
351 DuckDbErrorType::InvalidInput,
352 &format!(
353 "data_chunk_from_arrow: the Arrow array has {children} child array(s) but the \
354 converted schema describes {} column(s). DuckDB indexes \
355 `arrow_array->children[i]` once per schema column without a bounds check, so \
356 this is refused here rather than read out of bounds.",
357 converted.column_count(),
358 ),
359 ));
360 }
361 if let Err(message) = super::import_check::check_struct_children(&array) {
362 return Err(ErrorData::new(
363 DuckDbErrorType::InvalidInput,
364 &format!("data_chunk_from_arrow: {message}"),
365 ));
366 }
367 // SAFETY: takes no arguments and reads a compile-time constant.
368 let limit = unsafe { libduckdb_sys::duckdb_vector_size() };
369 // SAFETY: `array` is a valid, unreleased array that conforms to
370 // `converted`'s schema (this function's contract), and
371 // `check_struct_children` confirmed one live child per column.
372 let layout = unsafe { super::import_layout::check(&array.0, converted.shapes(), limit) };
373 if let Err(message) = layout {
374 return Err(ErrorData::new(
375 DuckDbErrorType::InvalidInput,
376 &format!("data_chunk_from_arrow: {message}"),
377 ));
378 }
379 // Which columns come back non-flat, decided before `DuckDB` claims the
380 // array (its buffers stay alive in the chunk, but the record is `DuckDB`'s
381 // afterwards).
382 let dictionary_columns: Vec<bool> = (0..children)
383 .map(|index| {
384 // SAFETY: `check_struct_children` confirmed `children` holds
385 // `children` non-null pointers to live records.
386 unsafe { super::import_check::has_dictionary(&**array.0.children.add(index)) }
387 })
388 .collect();
389 if array.is_empty() {
390 return Err(ErrorData::new(
391 DuckDbErrorType::InvalidInput,
392 "data_chunk_from_arrow: DuckDB cannot import a zero-row Arrow array. It passes \
393 `arrow_array->length` straight through as the chunk's *capacity* \
394 (`dchunk->Initialize(alloc, types, length)`), and a capacity of zero reaches \
395 `Allocator::AllocateData(0)`, whose `D_ASSERT(size > 0)` aborts a debug build of \
396 DuckDB. A release build allocates nothing and carries on, so this is refused here \
397 rather than left to depend on how the engine was compiled. Skip empty batches, or \
398 build the empty chunk directly with `OwnedDataChunk::new`.",
399 ));
400 }
401
402 let mut out: duckdb_data_chunk = ptr::null_mut();
403 // SAFETY: `connection` is live per this function's contract, `array` points
404 // at a live record with as many children as `converted` has columns, and
405 // `out` is a valid out-parameter. DuckDB takes ownership of the array's
406 // data; `array`'s own `release` is nulled by DuckDB, so the drop below is a
407 // no-op on that path.
408 let raw_err = unsafe {
409 duckdb_data_chunk_from_arrow(
410 connection,
411 array.as_mut_ptr(),
412 converted.as_raw(),
413 &raw mut out,
414 )
415 };
416 // SAFETY: the return value is an owned `duckdb_error_data`, or null.
417 let err = unsafe { ErrorData::from_raw(raw_err) };
418 if err.has_error() {
419 return Err(err);
420 }
421 if out.is_null() {
422 return Err(ErrorData::new(
423 DuckDbErrorType::Internal,
424 "duckdb_data_chunk_from_arrow reported success but produced no data chunk",
425 ));
426 }
427 // SAFETY: `out` is a fresh chunk this value now owns.
428 let chunk = unsafe { OwnedDataChunk::from_raw(out) };
429 for (column, &dictionary) in dictionary_columns.iter().enumerate() {
430 // SAFETY: `column` is below the chunk's column count, which is
431 // `converted`'s, which matched the array's child count above.
432 unsafe {
433 super::import_check::flatten_if_needed(chunk.vector(column), chunk.size(), dictionary)
434 }
435 .map_err(|e| {
436 ErrorData::new(
437 DuckDbErrorType::Internal,
438 &format!("data_chunk_from_arrow: flattening column {column}: {e}"),
439 )
440 })?;
441 }
442 Ok(chunk)
443}