Skip to main content

quack_rs/arrow/
convert.rs

1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6//! The four conversions between `DuckDB` data chunks and the Arrow C Data Interface.
7
8use std::ffi::CString;
9use std::os::raw::c_char;
10use std::ptr;
11
12use libduckdb_sys::{
13    duckdb_arrow_converted_schema, duckdb_connection, duckdb_data_chunk,
14    duckdb_data_chunk_from_arrow, duckdb_data_chunk_to_arrow, duckdb_logical_type,
15    duckdb_schema_from_arrow, duckdb_to_arrow_schema, idx_t,
16};
17
18use super::{
19    ArrowArray, ArrowConvertedSchema, ArrowOptions, ArrowSchema, RawArrowArray, RawArrowSchema,
20};
21use crate::data_chunk::DataChunk;
22use crate::error_data::{DuckDbErrorType, ErrorData};
23use crate::query::OwnedDataChunk;
24use crate::types::LogicalType;
25
26// ─── Conversions ─────────────────────────────────────────────────────────────
27
28/// Renders a `DuckDB` schema as an Arrow schema
29/// (`duckdb_to_arrow_schema`).
30///
31/// `columns` pairs each column's name with its logical type, in order. The
32/// result is a struct schema (`"+s"`) with one child per column — the shape
33/// every Arrow consumer expects for a record batch.
34///
35/// # Errors
36///
37/// - [`DuckDbErrorType::InvalidInput`] if a name contains an interior NUL byte,
38///   since `DuckDB` reads names as C strings.
39/// - [`DuckDbErrorType::OutOfRange`] if there are more columns than `idx_t` can
40///   hold.
41/// - Whatever `DuckDB` reports for a type it cannot render as Arrow.
42#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
43pub fn to_arrow_schema(
44    options: &ArrowOptions<'_>,
45    columns: &[(&str, &LogicalType)],
46) -> Result<ArrowSchema, ErrorData> {
47    let column_count = idx_t::try_from(columns.len()).map_err(|_| {
48        ErrorData::new(
49            DuckDbErrorType::OutOfRange,
50            "to_arrow_schema: more columns than idx_t can represent",
51        )
52    })?;
53
54    let mut names: Vec<CString> = Vec::with_capacity(columns.len());
55    for (name, _) in columns {
56        let owned = CString::new(*name).map_err(|_| {
57            ErrorData::new(
58                DuckDbErrorType::InvalidInput,
59                &format!(
60                    "to_arrow_schema: column name {name:?} contains an interior NUL byte, but \
61                     DuckDB reads schema names as C strings"
62                ),
63            )
64        })?;
65        names.push(owned);
66    }
67    let mut name_ptrs: Vec<*const c_char> = names.iter().map(|n| n.as_ptr()).collect();
68    let mut types: Vec<duckdb_logical_type> = columns.iter().map(|(_, ty)| ty.as_raw()).collect();
69
70    let mut out = RawArrowSchema::empty();
71    // SAFETY: `options` owns a live handle; `types` and `name_ptrs` are each
72    // valid for `column_count` elements and outlive the call; `out` is a fresh
73    // record DuckDB may fill. DuckDB copies the names and the types.
74    let raw_err = unsafe {
75        duckdb_to_arrow_schema(
76            options.as_raw(),
77            types.as_mut_ptr(),
78            name_ptrs.as_mut_ptr(),
79            column_count,
80            &raw mut out,
81        )
82    };
83    // SAFETY: the return value is an owned `duckdb_error_data`, or null.
84    let err = unsafe { ErrorData::from_raw(raw_err) };
85    if err.has_error() {
86        // `ArrowConverter::ToArrowSchema` installs `out.release` as its last
87        // statement, after everything that can throw, so a failed call leaves
88        // `release == NULL` and there is nothing here to free. Dropping the
89        // record without releasing it is therefore correct, not a leak.
90        return Err(err);
91    }
92    // SAFETY: DuckDB filled the record and installed its release callback.
93    Ok(unsafe { ArrowSchema::from_raw(out) })
94}
95
96/// Exports a `DuckDB` data chunk as an Arrow struct array
97/// (`duckdb_data_chunk_to_arrow`).
98///
99/// The chunk is only read; it stays valid and owned by the caller. The returned
100/// array owns copies of the data, so it outlives the chunk.
101///
102/// Pair it with the schema [`to_arrow_schema`] produces from the same column
103/// types and the same [`ArrowOptions`] — an Arrow consumer needs both.
104///
105/// # Values `DuckDB` would export wrongly are refused
106///
107/// `DuckDB` exports these as different values, with no error (checked on
108/// 1.4.4, 1.5.0 and 1.5.5; `docs/upstream-duckdb-reports.md`, item 16), so
109/// the chunk is checked first and refused if it holds one, at any depth:
110///
111/// - `INTERVAL`: Arrow's `month_day_nano` counts nanoseconds in an `i64`, and
112///   `DuckDB` multiplies the microseconds by 1000 with no overflow check. An
113///   interval whose microsecond field is beyond ±`i64::MAX / 1000` (about
114///   ±106,751 days, or 2,562,047 hours) would wrap: `INTERVAL 2562048 HOUR`
115///   would export as a negative interval.
116/// - `UHUGEINT`, and `HUGEINT` unless `arrow_lossless_conversion` is set:
117///   exported as `decimal128(38, 0)`, which holds 38 digits. A `UHUGEINT` of
118///   2^127 or more would come back negative (`2^128 - 1` as `-1`), and a
119///   39-digit `HUGEINT` would not survive either.
120///
121/// The check copies each column whose type contains one of these types (a
122/// chunk's vectors may be dictionary or constant vectors), so it costs one
123/// copy of those columns.
124///
125/// # Errors
126///
127/// - [`DuckDbErrorType::InvalidInput`] naming the column and value, for a
128///   value `DuckDB` would export wrongly (see above).
129/// - Whatever `DuckDB` reports for a type it cannot render as Arrow.
130#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
131pub fn data_chunk_to_arrow(
132    options: &ArrowOptions<'_>,
133    chunk: &DataChunk,
134) -> Result<ArrowArray, ErrorData> {
135    super::export_check::check_chunk(options, chunk)?;
136    let mut out = RawArrowArray::empty();
137    // SAFETY: `options` owns a live handle, `chunk` is valid per `DataChunk`'s
138    // constructor contract, and `out` is a fresh record DuckDB may fill.
139    let raw_err =
140        unsafe { duckdb_data_chunk_to_arrow(options.as_raw(), chunk.as_raw(), &raw mut out) };
141    // SAFETY: the return value is an owned `duckdb_error_data`, or null.
142    let err = unsafe { ErrorData::from_raw(raw_err) };
143    if err.has_error() {
144        // `ArrowConverter::ToArrowArray` assigns `*out_array` as its last
145        // statement, so a failed call never wrote to `out` and there is nothing
146        // to release.
147        return Err(err);
148    }
149    // SAFETY: DuckDB filled the record and installed its release callback.
150    let array = unsafe { ArrowArray::from_raw(out) };
151    // SAFETY: `array` is what DuckDB just exported from `chunk` under
152    // `options`. On an error it is dropped here, which releases it.
153    unsafe { super::export_check::check_layout(options, chunk, &array) }?;
154    Ok(array)
155}
156
157/// Translates an Arrow schema into `DuckDB` type descriptors
158/// (`duckdb_schema_from_arrow`).
159///
160/// `schema` is read, not consumed: the caller still owns it and it is released
161/// when its [`ArrowSchema`] drops. It is taken by `&mut` because the C signature
162/// is `struct ArrowSchema *`, even though `PopulateArrowTableSchema` receives it
163/// as `const ArrowSchema &`.
164///
165/// `schema` must be the struct schema at the root of a record batch: `DuckDB`
166/// makes one column per child.
167///
168/// # Errors
169///
170/// - [`DuckDbErrorType::InvalidInput`] if `schema` has already been released,
171///   which `DuckDB` would dereference rather than diagnose.
172/// - Whatever `DuckDB` reports for an Arrow type it cannot map — a released
173///   child schema, or an unsupported format string.
174///
175/// # Safety
176///
177/// `connection` must be a live `duckdb_connection`.
178#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
179pub unsafe fn schema_from_arrow(
180    connection: duckdb_connection,
181    schema: &mut ArrowSchema,
182) -> Result<ArrowConvertedSchema, ErrorData> {
183    if schema.is_released() {
184        return Err(ErrorData::new(
185            DuckDbErrorType::InvalidInput,
186            "schema_from_arrow: the Arrow schema has already been released, so its children \
187             pointer no longer refers to live memory",
188        ));
189    }
190    let mut out: duckdb_arrow_converted_schema = ptr::null_mut();
191    // SAFETY: `connection` is live per this function's contract, `schema` points
192    // at a live record, and `out` is a valid out-parameter.
193    let raw_err =
194        unsafe { duckdb_schema_from_arrow(connection, schema.as_mut_ptr(), &raw mut out) };
195    // SAFETY: the return value is an owned `duckdb_error_data`, or null.
196    let err = unsafe { ErrorData::from_raw(raw_err) };
197    if err.has_error() {
198        return Err(err);
199    }
200    if out.is_null() {
201        return Err(ErrorData::new(
202            DuckDbErrorType::Internal,
203            "duckdb_schema_from_arrow reported success but produced no converted schema",
204        ));
205    }
206    // SAFETY: `out` is a fresh handle this value now owns, built from `schema`
207    // (`PopulateArrowTableSchema` makes one column per child of it).
208    Ok(unsafe { ArrowConvertedSchema::from_raw(out, schema) })
209}
210
211/// Imports an Arrow struct array as a `DuckDB` data chunk
212/// (`duckdb_data_chunk_from_arrow`).
213///
214/// # Why `array` is taken by value
215///
216/// `DuckDB` claims the array: it copies the record into the chunk's owned data
217/// and then sets `arrow_array->release = nullptr` — *before* the conversion
218/// loop body, so the transfer happens on the error path too. Taking it by value
219/// makes that visible in the signature, and the by-value binding is dropped on
220/// the way out, which releases the array in the one case where `DuckDB` before
221/// 2.0.0 does not claim it (a zero-column converted schema, where the loop
222/// never runs). `DuckDB` 2.0.0 claims it before the loop, so it releases that
223/// array itself; the binding's `Drop` then finds `release` null and does
224/// nothing.
225///
226/// The resulting chunk keeps the Arrow buffers alive, so the data is shared, not
227/// copied — with two exceptions, which are copied so that the chunk is flat
228/// and every [`VectorReader`](crate::vector::VectorReader) reads it
229/// correctly. `DuckDB` imports a column that is dictionary-encoded at any
230/// depth as a *dictionary* vector, and an Arrow null-type column as a
231/// *constant* vector; a flat read of either returns wrong values (and, for a
232/// dictionary, reads past the end of its buffer). Those columns are copied
233/// into fresh flat vectors before this function returns. Run-end-encoded
234/// children are expanded by `DuckDB` itself.
235///
236/// Before `DuckDB` 2.0.0 the claim is held by the chunk's **first** column:
237/// `DuckDB` copies the record into every column's state but nulls `release`
238/// after the first (`src/main/capi/arrow-c.cpp`, lines 110–120 at v1.5.6), so
239/// the producer's buffers are released when column 0's vector is. A vector
240/// made to reference another column
241/// ([`reference_vector`][crate::vector::ops::reference_vector]) does not keep
242/// them alive there. From 2.0.0 every column shares one owner
243/// (`src/main/capi/v1/arrow-c.cpp`, lines 141–152), so a vector referencing
244/// any column keeps them. Either way `reference_vector`'s contract already
245/// requires the chunk to outlive the referencing vector.
246///
247/// # Errors
248///
249/// [`DuckDbErrorType::InvalidInput`], checked here because `DuckDB` does not
250/// check them and would read out of bounds or through a null pointer:
251///
252/// - `array` has already been released, has a negative `length`, has a
253///   nonzero `offset` (`DuckDB` ignores the top-level offset and imports the
254///   wrong rows, see `docs/upstream-duckdb-reports.md`), or has no rows (see
255///   below);
256/// - its child count does not match `converted`'s column count;
257/// - its `children` pointer, or one of the child pointers, is null;
258/// - a child is shorter than `length` (the Arrow specification requires
259///   every child of a struct array to hold `offset + length` rows);
260/// - a node anywhere in `array` has a valid Arrow layout that `DuckDB`
261///   imports from the wrong rows or out of bounds: an offset below the top
262///   level that `DuckDB` misapplies, a dictionary under a list or with more
263///   than [`duckdb_vector_size`](libduckdb_sys::duckdb_vector_size) (2048)
264///   rows that can hold NULLs, a nested dictionary, overlapping or gapped
265///   list views, a sparse union with recoded type ids, or a run-end-encoded
266///   array where `DuckDB` reads a plain one. The message names the node; the
267///   module docs of `src/arrow/import_layout.rs` and
268///   `docs/upstream-duckdb-reports.md` (items 9 and 24 to 28) give the
269///   details;
270/// - a node anywhere in `array` breaks a bound the Arrow specification sets
271///   and `DuckDB` takes on trust, reading past it once the chunk is read (a
272///   SIGSEGV on 1.5.6 and 2.0.0 for a list offset past its child): a list,
273///   large list or map offset that is negative, decreases, or lies past the
274///   child; a negative list-view offset or size; a struct field, list or
275///   fixed-size-list child, or union member shorter than the rows its parent
276///   reads from it; a dictionary index outside the dictionary in a row its own
277///   validity does not mark NULL (a row NULL only in an enclosing struct must
278///   still hold a valid index, as the specification requires), or indices of
279///   a non-integer type;
280/// - the array and its schema disagree on a node's child count or
281///   dictionary encoding.
282///
283/// [`DuckDbErrorType::Internal`] if copying a non-flat column into a flat
284/// vector fails (see above), which needs an allocation failure.
285///
286/// Every error `DuckDB` itself reports from the conversion also arrives as
287/// [`DuckDbErrorType::InvalidInput`]: `duckdb_data_chunk_from_arrow` maps all
288/// of them to `DUCKDB_ERROR_INVALID_INPUT`.
289///
290/// # Safety
291///
292/// - `connection` must be a live `duckdb_connection`, and `converted` must
293///   have been produced from it (or from another connection of the same
294///   database).
295/// - `array` must be a valid Arrow struct array (see
296///   [`ArrowArray::from_raw`]) that **conforms to `converted`'s schema**: each
297///   child's physical layout must be the one its column's Arrow format
298///   describes. `DuckDB` reads a child's buffers as that format with no check
299///   — an `int32` child imported under a `utf8` schema has its values read as
300///   string offsets into a buffer that does not exist (a crash, or worse). An
301///   array exported with [`data_chunk_to_arrow`] under the schema that
302///   `converted` was made from conforms.
303/// - `length` must be the array's true row count. `DuckDB` allocates a chunk
304///   of that capacity before its error handling starts
305///   (`dchunk->Initialize(…, length)` in `arrow-c.cpp`), so a length too large
306///   to allocate throws through the C API and aborts the process. A length,
307///   or a nested row count, above
308///   [`MAX_CAPACITY`](crate::vector::ops::MAX_CAPACITY) is refused, which on
309///   a 32-bit target also keeps every buffer's byte size within what `malloc`
310///   receives unnarrowed; below it an allocation the system cannot satisfy
311///   still aborts. The length is not otherwise checkable: a run-end-encoded
312///   column declares any length with a few bytes of buffers.
313/// - Every validity bitmap that `DuckDB` reads (one with a nonzero
314///   `null_count`) must be readable for **one byte past** the last byte that
315///   holds its rows' bits. When a node's effective bit offset is not a
316///   multiple of 8, `GetValidityMask` (`arrow_conversion.cpp`) copies
317///   `ceil(rows / 8) + 1` bytes from the first byte it needs, which can be one
318///   more than the rows occupy: an `int32` column at offset 1 of length 7 with
319///   a 1-byte bitmap is read as 2 bytes (an invalid read under valgrind on
320///   1.5.5). That byte only supplies bits for rows past the end, so the
321///   values imported are right; the read itself is the hazard. Buffers padded to a multiple of 8 or 64 bytes, as
322///   the Arrow columnar format recommends, satisfy this; the allocation size
323///   is invisible through the C Data Interface, so it cannot be checked here.
324/// - A dictionary whose values are a fixed-width type (integers, floats,
325///   `DATE` in days, `TIMESTAMP`, `DECIMAL`, ...) and whose indices can be
326///   NULL (a nonzero `null_count`, or NULL rows in an enclosing struct) must
327///   have its values buffer readable for **one element past**
328///   `offset + length`. `DuckDB` points every NULL index at a sentinel entry
329///   one past the dictionary, but imports those types without copying, so
330///   the sentinel lies in the producer's buffer; copying the column, as this
331///   function does for every dictionary-encoded column, reads it (an invalid
332///   read under valgrind on 1.4.4 to 1.5.5,
333///   `docs/upstream-duckdb-reports.md`, item 29). The value read is
334///   discarded: those rows are NULL. A buffer padded to a multiple of 64
335///   bytes satisfies this unless the values fill it exactly.
336#[cfg_attr(test, mutants::skip)] // FFI conversion — covered by tests/ffi_roundtrip.rs, which `--lib` does not run
337pub unsafe fn data_chunk_from_arrow(
338    connection: duckdb_connection,
339    mut array: ArrowArray,
340    converted: &ArrowConvertedSchema,
341) -> Result<OwnedDataChunk, ErrorData> {
342    if array.is_released() {
343        return Err(ErrorData::new(
344            DuckDbErrorType::InvalidInput,
345            "data_chunk_from_arrow: the Arrow array has already been released",
346        ));
347    }
348    let children = array.child_count();
349    if children != converted.column_count() {
350        return Err(ErrorData::new(
351            DuckDbErrorType::InvalidInput,
352            &format!(
353                "data_chunk_from_arrow: the Arrow array has {children} child array(s) but the \
354                 converted schema describes {} column(s). DuckDB indexes \
355                 `arrow_array->children[i]` once per schema column without a bounds check, so \
356                 this is refused here rather than read out of bounds.",
357                converted.column_count(),
358            ),
359        ));
360    }
361    if let Err(message) = super::import_check::check_struct_children(&array) {
362        return Err(ErrorData::new(
363            DuckDbErrorType::InvalidInput,
364            &format!("data_chunk_from_arrow: {message}"),
365        ));
366    }
367    // SAFETY: takes no arguments and reads a compile-time constant.
368    let limit = unsafe { libduckdb_sys::duckdb_vector_size() };
369    // SAFETY: `array` is a valid, unreleased array that conforms to
370    // `converted`'s schema (this function's contract), and
371    // `check_struct_children` confirmed one live child per column.
372    let layout = unsafe { super::import_layout::check(&array.0, converted.shapes(), limit) };
373    if let Err(message) = layout {
374        return Err(ErrorData::new(
375            DuckDbErrorType::InvalidInput,
376            &format!("data_chunk_from_arrow: {message}"),
377        ));
378    }
379    // Which columns come back non-flat, decided before `DuckDB` claims the
380    // array (its buffers stay alive in the chunk, but the record is `DuckDB`'s
381    // afterwards).
382    let dictionary_columns: Vec<bool> = (0..children)
383        .map(|index| {
384            // SAFETY: `check_struct_children` confirmed `children` holds
385            // `children` non-null pointers to live records.
386            unsafe { super::import_check::has_dictionary(&**array.0.children.add(index)) }
387        })
388        .collect();
389    if array.is_empty() {
390        return Err(ErrorData::new(
391            DuckDbErrorType::InvalidInput,
392            "data_chunk_from_arrow: DuckDB cannot import a zero-row Arrow array. It passes \
393             `arrow_array->length` straight through as the chunk's *capacity* \
394             (`dchunk->Initialize(alloc, types, length)`), and a capacity of zero reaches \
395             `Allocator::AllocateData(0)`, whose `D_ASSERT(size > 0)` aborts a debug build of \
396             DuckDB. A release build allocates nothing and carries on, so this is refused here \
397             rather than left to depend on how the engine was compiled. Skip empty batches, or \
398             build the empty chunk directly with `OwnedDataChunk::new`.",
399        ));
400    }
401
402    let mut out: duckdb_data_chunk = ptr::null_mut();
403    // SAFETY: `connection` is live per this function's contract, `array` points
404    // at a live record with as many children as `converted` has columns, and
405    // `out` is a valid out-parameter. DuckDB takes ownership of the array's
406    // data; `array`'s own `release` is nulled by DuckDB, so the drop below is a
407    // no-op on that path.
408    let raw_err = unsafe {
409        duckdb_data_chunk_from_arrow(
410            connection,
411            array.as_mut_ptr(),
412            converted.as_raw(),
413            &raw mut out,
414        )
415    };
416    // SAFETY: the return value is an owned `duckdb_error_data`, or null.
417    let err = unsafe { ErrorData::from_raw(raw_err) };
418    if err.has_error() {
419        return Err(err);
420    }
421    if out.is_null() {
422        return Err(ErrorData::new(
423            DuckDbErrorType::Internal,
424            "duckdb_data_chunk_from_arrow reported success but produced no data chunk",
425        ));
426    }
427    // SAFETY: `out` is a fresh chunk this value now owns.
428    let chunk = unsafe { OwnedDataChunk::from_raw(out) };
429    for (column, &dictionary) in dictionary_columns.iter().enumerate() {
430        // SAFETY: `column` is below the chunk's column count, which is
431        // `converted`'s, which matched the array's child count above.
432        unsafe {
433            super::import_check::flatten_if_needed(chunk.vector(column), chunk.size(), dictionary)
434        }
435        .map_err(|e| {
436            ErrorData::new(
437                DuckDbErrorType::Internal,
438                &format!("data_chunk_from_arrow: flattening column {column}: {e}"),
439            )
440        })?;
441    }
442    Ok(chunk)
443}