Skip to main content

quack_rs/validate/description_yml/
parser.rs

1// SPDX-License-Identifier: MIT
2// Copyright 2026 Tom F. <https://github.com/tomtom215/>
3// My way of giving something small back to the open source community
4// and encouraging more Rust development!
5
6use crate::error::ExtensionError;
7use crate::validate::platform::{
8    is_opt_in_platform, platforms_in_group, DUCKDB_OPT_IN_PLATFORMS, DUCKDB_PLATFORM_GROUPS,
9    DUCKDB_RETIRED_PLATFORMS,
10};
11use crate::validate::{
12    validate_community_extension_name, validate_excluded_platforms_str, validate_extension_version,
13    validate_platform, validate_spdx_license,
14};
15
16use super::model::DescriptionYml;
17use super::yaml::{read_sections, Entry, Section, Value};
18use super::yaml11;
19
20/// Parses and validates a `description.yml` string.
21///
22/// Returns a validated [`DescriptionYml`] if all required fields are present and correct.
23///
24/// # What is validated
25///
26/// - `extension.name` — must pass [`validate_community_extension_name`]: well
27///   formed, and not a `DuckDB` alias (`s3`) or core extension name (`json`)
28/// - `extension.description` — non-empty
29/// - `extension.version` — optional; if present, must pass [`validate_extension_version`]
30/// - `extension.language`, `extension.build` — non-empty
31/// - `extension.license` — non-empty (`licence:` is accepted, with a warning).
32///   A value [`validate_spdx_license`] does not accept is a **warning**, not an
33///   error: the community build does not read the field, and published
34///   extensions use `BSL 1.1`, `GPL-3.0` and free text
35/// - `extension.excluded_platforms` — if present, must pass
36///   [`validate_excluded_platforms_str`]; a YAML list is validated too, and warned about.
37///   A platform *group* name (`wasm`, `windows`, …) passes, with a warning: the
38///   community build compares exact platform names, so a group excludes nothing
39/// - `extension.opt_in_platforms` — if present, a warning for each name the
40///   build would ignore: a group, an unknown name, or a platform that is built
41///   without opting in
42/// - `extension.maintainers` — at least one name
43/// - `repo.github` — `owner/repo`, in the characters GitHub allows (not a URL)
44/// - `repo.ref` — non-empty; anything but a commit hash is a warning
45/// - `repo.variegata`, `repo.andium` — if present, non-empty (the revisions the
46///   community build checks out for `DuckDB` v1.5.6 and v1.4.5); anything but a
47///   commit hash is a warning
48/// - any other `repo:` key but `ref_next` and `canonical_name` is a warning:
49///   the community build ignores it
50/// - no key may appear twice in a section
51///
52/// # Errors
53///
54/// Returns [`ExtensionError`] on the first validation failure with a descriptive message.
55///
56/// # Note on parsing
57///
58/// quack-rs has no YAML dependency, so this reads `description.yml` with a
59/// small hand-written reader, held to what `PyYAML`'s `yaml.safe_load` — the
60/// first thing the community build runs on the file — accepts: a file it
61/// refuses is refused here too, `docs:` and every other section included.
62/// That covers the plain-scalar rules (no `: `, no leading `@`, `` ` ``, `%`
63/// or `- `, no tab), tabs, control characters, block-scalar headers and
64/// indentation, quoted and flow scalars across lines, anchors and aliases,
65/// and dates `PyYAML` cannot construct. Checked against the 395 published
66/// descriptors (all accepted, every field as `PyYAML` reads it but block
67/// scalars, which are trimmed) and 41,423 mutants of them (none accepted that
68/// `PyYAML` refuses).
69///
70/// It is stricter than `PyYAML` in a few places no published file needs:
71/// quoted or repeated keys in `extension:` and `repo:`, and anywhere tags,
72/// explicit `? key` entries, directives, anchors on keys and a second
73/// document.
74///
75/// # Example
76///
77/// ```rust
78/// use quack_rs::validate::description_yml::parse_description_yml;
79///
80/// let yml = "extension:
81///   name: my_ext
82///   description: My extension.
83///   version: 0.1.0
84///   language: Rust
85///   build: cargo
86///   license: MIT
87///   requires_toolchains: rust;python3
88///   maintainers:
89///     - Jane Doe
90///
91/// repo:
92///   github: janedoe/duckdb-my-ext
93///   ref: 4f1c2e9a7b3d5c8e1f0a2b4c6d8e0f1a3b5c7d9e
94/// ";
95///
96/// let desc = parse_description_yml(yml).unwrap();
97/// assert_eq!(desc.name, "my_ext");
98/// assert_eq!(desc.version.as_deref(), Some("0.1.0"));
99/// assert_eq!(desc.license, "MIT");
100/// assert_eq!(desc.github, "janedoe/duckdb-my-ext");
101/// assert_eq!(desc.maintainers, vec!["Jane Doe"]);
102/// assert!(desc.warnings.is_empty());
103/// ```
104///
105/// [`validate_community_extension_name`]: crate::validate::validate_community_extension_name
106/// [`validate_extension_version`]: crate::validate::validate_extension_version
107/// [`validate_spdx_license`]: crate::validate::validate_spdx_license
108/// [`validate_excluded_platforms_str`]: crate::validate::validate_excluded_platforms_str
109pub fn parse_description_yml(content: &str) -> Result<DescriptionYml, ExtensionError> {
110    let sections = read_sections(content, &["extension", "repo"])
111        .map_err(|e| ExtensionError::new(format!("description.yml: {e}")))?;
112    let extension = Fields::new(&sections, "extension");
113    let repo = Fields::new(&sections, "repo");
114    let mut warnings = Vec::new();
115
116    let name = extension.scalar("name")?;
117    if name.is_empty() {
118        return Err(missing("extension.name"));
119    }
120    validate_community_extension_name(&name)
121        .map_err(|e| ExtensionError::new(format!("description.yml: extension.name: {e}")))?;
122
123    let description = extension.required("description")?;
124
125    let version = extension.scalar("version")?;
126    let version = if version.is_empty() {
127        None
128    } else {
129        validate_extension_version(&version)
130            .map_err(|e| ExtensionError::new(format!("description.yml: extension.version: {e}")))?;
131        Some(version)
132    };
133
134    let language = extension.required("language")?;
135    let build = extension.required("build")?;
136
137    let license = match (extension.get("license"), extension.get("licence")) {
138        (Some(first), Some(second)) => {
139            return Err(ExtensionError::new(format!(
140                "description.yml: both 'extension.license' (line {}) and \
141                 'extension.licence' (line {}) are set",
142                first.line, second.line
143            )))
144        }
145        (None, Some(_)) => {
146            warnings.push(
147                "'extension.licence' is the British spelling; DuckDB's documentation and \
148                 tooling use 'license'"
149                    .to_string(),
150            );
151            extension.scalar("licence")?
152        }
153        _ => extension.scalar("license")?,
154    };
155    if license.is_empty() {
156        return Err(missing("extension.license"));
157    }
158    if let Err(e) = validate_spdx_license(&license) {
159        warnings.push(format!("extension.license: {e}"));
160    }
161
162    // `requires_toolchains` is optional. Only 143 of the 346 published
163    // descriptors (community-extensions `5ae7df8`) set it, and the
164    // community-extensions documentation does not list it as required.
165    let requires_toolchains = extension.semicolon_list("requires_toolchains", &mut warnings)?;
166
167    let (excluded_platforms, opt_in_platforms) = extension.platforms(&mut warnings)?;
168
169    let maintainers = extension.maintainers(&mut warnings)?;
170    if maintainers.is_empty() {
171        return Err(ExtensionError::new(
172            "description.yml: 'extension.maintainers' must list at least one maintainer",
173        ));
174    }
175
176    let github = repo.scalar("github")?;
177    if github.is_empty() {
178        return Err(missing("repo.github"));
179    }
180    if !crate::validate::is_github_repo(&github) {
181        return Err(ExtensionError::new(format!(
182            "description.yml: 'repo.github' must be in 'owner/repo' format — the repository, \
183             not its URL or a path inside it, in the characters GitHub allows (the community \
184             build passes it to actions/checkout as `repository`), got '{github}'"
185        )));
186    }
187
188    let git_ref = repo.scalar("ref")?;
189    if git_ref.is_empty() {
190        return Err(missing("repo.ref"));
191    }
192    warn_unless_commit_hash("repo.ref", &git_ref, &mut warnings);
193    let git_ref_next = repo.scalar("ref_next")?;
194    let git_ref_variegata = repo.release_ref("variegata", "v1.5.6", &mut warnings)?;
195    let git_ref_andium = repo.release_ref("andium", "v1.4.5", &mut warnings)?;
196    warn_about_unknown_repo_keys(&repo, &mut warnings);
197
198    warn_about_yaml_1_1_retyping(&[&extension, &repo], &mut warnings);
199
200    Ok(DescriptionYml {
201        name,
202        description,
203        version,
204        language,
205        build,
206        license,
207        requires_toolchains,
208        excluded_platforms,
209        opt_in_platforms,
210        maintainers,
211        github,
212        git_ref,
213        git_ref_next,
214        git_ref_variegata,
215        git_ref_andium,
216        warnings,
217    })
218}
219
220/// The `repo:` keys `scripts/build.py` in `duckdb/community-extensions`
221/// reads.
222const REPO_KEYS: [&str; 6] = [
223    "github",
224    "ref",
225    "ref_next",
226    "variegata",
227    "andium",
228    "canonical_name",
229];
230
231/// Warns about each `repo:` key the community build does not read: a typo
232/// such as `next:` (one published file has it) silently does nothing.
233fn warn_about_unknown_repo_keys(repo: &Fields<'_>, warnings: &mut Vec<String>) {
234    for entry in repo.entries {
235        if !REPO_KEYS.contains(&entry.key.as_str()) {
236            warnings.push(format!(
237                "repo.{}: the community build does not read this key (it reads {}), so it has \
238                 no effect",
239                entry.key,
240                REPO_KEYS.join(", ")
241            ));
242        }
243    }
244}
245
246/// Warns about each platform group name in `list`, the value of `field`.
247///
248/// `DUCKDB_PLATFORM_GROUPS` are the top-level keys of the distribution matrix,
249/// and published files do write them in `excluded_platforms`. The script that
250/// applies the field, `scripts/modify_distribution_matrix.py` in
251/// `extension-ci-tools`, tests `arch in excluded_arch_values` on every branch
252/// the community build uses (`v1.4-andium`, `v1.5-variegata`, `main`,
253/// `v2.0-cyanoptera`), so a group name matches no platform and the build
254/// carries on building all of them.
255fn warn_about_platform_groups(field: &str, list: &str, warnings: &mut Vec<String>) {
256    for group in list
257        .split(';')
258        .filter(|p| DUCKDB_PLATFORM_GROUPS.contains(p))
259    {
260        warnings.push(format!(
261            "extension.{field}: '{group}' is a platform group, not a platform. The community \
262             build compares exact platform names, so it matches nothing here; list the \
263             platforms instead: \"{}\"",
264            platforms_in_group(group).join(";")
265        ));
266    }
267}
268
269/// Warns about each name in `opt_in_platforms` the community build would ignore.
270///
271/// The build only looks the field up for platforms that are opt-in
272/// ([`DUCKDB_OPT_IN_PLATFORMS`]), so anything else in it does nothing, and a
273/// misspelt platform is a silent omission.
274fn warn_about_ignored_opt_ins(list: &str, warnings: &mut Vec<String>) {
275    warn_about_platform_groups("opt_in_platforms", list, warnings);
276    for platform in list.split(';').filter(|p| !p.is_empty()) {
277        if DUCKDB_PLATFORM_GROUPS.contains(&platform) || is_opt_in_platform(platform) {
278            continue;
279        }
280        let why = if validate_platform(platform).is_ok() {
281            "is not an opt-in platform, so the community build ignores it here"
282        } else {
283            "is not a recognized DuckDB build target, so the community build ignores it"
284        };
285        warnings.push(format!(
286            "extension.opt_in_platforms: '{platform}' {why} (the opt-in platforms are {})",
287            DUCKDB_OPT_IN_PLATFORMS.join(", ")
288        ));
289    }
290}
291
292/// Warns about each field this parser reads as text whose unquoted value
293/// `PyYAML` — which the community build reads the file with, following YAML
294/// 1.1 — reads as something else: an unquoted `yes` is a boolean there and
295/// `0.10` the float 0.1. Fields this parser does not read
296/// (`custom_toolchain_script: true`, a real boolean) are left alone.
297fn warn_about_yaml_1_1_retyping(sections: &[&Fields<'_>], warnings: &mut Vec<String>) {
298    const READ_AS_TEXT: [&str; 15] = [
299        "name",
300        "description",
301        "version",
302        "language",
303        "build",
304        "license",
305        "licence",
306        "requires_toolchains",
307        "excluded_platforms",
308        "opt_in_platforms",
309        "github",
310        "ref",
311        "ref_next",
312        "variegata",
313        "andium",
314    ];
315    for fields in sections {
316        for entry in fields.entries {
317            let Value::Scalar(ref text) = entry.value else {
318                continue;
319            };
320            if !entry.plain || !READ_AS_TEXT.contains(&entry.key.as_str()) {
321                continue;
322            }
323            if let Some(kind) = yaml11::implicit_type(text) {
324                warnings.push(format!(
325                    "{}.{}: the unquoted value '{text}' is {kind}, not text, to YAML 1.1 \
326                     readers such as PyYAML, which the community-extensions build uses; quote \
327                     it",
328                    fields.section, entry.key
329                ));
330            }
331        }
332    }
333}
334
335fn missing(field: &str) -> ExtensionError {
336    ExtensionError::new(format!("description.yml: missing required field '{field}'"))
337}
338
339/// The entries of one section, with typed accessors that name the field in
340/// every error.
341struct Fields<'a> {
342    section: &'static str,
343    entries: &'a [Entry],
344}
345
346impl<'a> Fields<'a> {
347    fn new(sections: &'a [Section], section: &'static str) -> Self {
348        let entries = sections
349            .iter()
350            .find(|s| s.name == section)
351            .map_or(&[][..], |s| s.entries.as_slice());
352        Self { section, entries }
353    }
354
355    fn get(&self, key: &str) -> Option<&'a Entry> {
356        self.entries.iter().find(|e| e.key == key)
357    }
358
359    /// A single-valued field; `""` when absent or empty.
360    fn scalar(&self, key: &str) -> Result<String, ExtensionError> {
361        match self.get(key).map(|e| (&e.value, e.line)) {
362            None | Some((Value::Null, _)) => Ok(String::new()),
363            // Not trimmed: the reader already strips a plain scalar, so trimming
364            // here would only alter a quoted one — `name: " my_ext"` is
365            // " my_ext" to PyYAML and the community build.
366            Some((Value::Scalar(s), _)) => Ok(s.clone()),
367            Some((_, line)) => Err(ExtensionError::new(format!(
368                "description.yml: line {line}: '{}.{key}' must be a single value, not a list \
369                 or mapping",
370                self.section
371            ))),
372        }
373    }
374
375    /// `repo.variegata` / `repo.andium`: the revision the community build
376    /// checks out for the `DuckDB` release `release` (`scripts/build.py`: `if
377    /// 'andium' in desc['repo']`) instead of `ref`. Empty when absent; present but empty is
378    /// an error, because the build then gets no revision at all for that
379    /// release. Anything but a commit hash is a warning, as for `ref`.
380    fn release_ref(
381        &self,
382        key: &str,
383        release: &str,
384        warnings: &mut Vec<String>,
385    ) -> Result<String, ExtensionError> {
386        let Some(entry) = self.get(key) else {
387            return Ok(String::new());
388        };
389        let value = self.scalar(key)?;
390        if value.is_empty() {
391            return Err(ExtensionError::new(format!(
392                "description.yml: line {}: 'repo.{key}' is present but empty: the community \
393                 build then checks out no revision when it builds for DuckDB {release}; give \
394                 a commit hash or remove the key",
395                entry.line
396            )));
397        }
398        warn_unless_commit_hash(&format!("repo.{key}"), &value, warnings);
399        Ok(value)
400    }
401
402    /// A single-valued field that must be present and non-empty.
403    fn required(&self, key: &str) -> Result<String, ExtensionError> {
404        let value = self.scalar(key)?;
405        if value.is_empty() {
406            return Err(missing(&format!("{}.{key}", self.section)));
407        }
408        Ok(value)
409    }
410
411    /// A `;`-separated field. A YAML list is accepted and joined with `;`, with
412    /// a warning: `scripts/build.py` in `duckdb/community-extensions` writes the
413    /// value into the build environment as-is, so a list arrives as its Python
414    /// `repr` (`['a', 'b']`), which `extension-ci-tools` then splits on `;` —
415    /// and matches nothing.
416    fn semicolon_list(
417        &self,
418        key: &str,
419        warnings: &mut Vec<String>,
420    ) -> Result<String, ExtensionError> {
421        let Some(Entry {
422            value: Value::Seq(items),
423            line,
424            ..
425        }) = self.get(key)
426        else {
427            return self.scalar(key);
428        };
429        let mut joined = Vec::with_capacity(items.len());
430        for item in items {
431            match item {
432                Value::Scalar(s) => joined.push(s.trim().to_string()),
433                Value::Null => {}
434                _ => {
435                    return Err(ExtensionError::new(format!(
436                        "description.yml: line {line}: '{}.{key}' entries must be plain values",
437                        self.section
438                    )))
439                }
440            }
441        }
442        let joined = joined.join(";");
443        warnings.push(format!(
444            "{}.{key} is a YAML list; the community build copies the value into its \
445             environment as-is, so a list arrives as Python's ['a', 'b'] rather than the \
446             ';'-separated string it splits — write \"{joined}\"",
447            self.section
448        ));
449        Ok(joined)
450    }
451
452    /// `excluded_platforms` and `opt_in_platforms`, in that order: the first
453    /// validated, both warned about where the community build would ignore them.
454    fn platforms(&self, warnings: &mut Vec<String>) -> Result<(String, String), ExtensionError> {
455        let excluded = self.semicolon_list("excluded_platforms", warnings)?;
456        validate_excluded_platforms_str(&excluded).map_err(|e| {
457            ExtensionError::new(format!(
458                "description.yml: extension.excluded_platforms: {e}"
459            ))
460        })?;
461        for retired in excluded
462            .split(';')
463            .filter(|p| DUCKDB_RETIRED_PLATFORMS.contains(p))
464        {
465            warnings.push(format!(
466                "extension.excluded_platforms: '{retired}' is no longer built by DuckDB, so \
467                 excluding it has no effect"
468            ));
469        }
470        warn_about_platform_groups("excluded_platforms", &excluded, warnings);
471
472        let opt_in = self.semicolon_list("opt_in_platforms", warnings)?;
473        warn_about_ignored_opt_ins(&opt_in, warnings);
474        Ok((excluded, opt_in))
475    }
476
477    /// `maintainers`: a list of names. A lone name is accepted with a warning.
478    fn maintainers(&self, warnings: &mut Vec<String>) -> Result<Vec<String>, ExtensionError> {
479        let Some(entry) = self.get("maintainers") else {
480            return Ok(Vec::new());
481        };
482        match &entry.value {
483            Value::Null => Ok(Vec::new()),
484            Value::Scalar(name) => {
485                warnings.push(format!(
486                    "extension.maintainers should be a list; read '{name}' as its only entry"
487                ));
488                Ok(vec![name.trim().to_string()])
489            }
490            Value::Seq(items) => {
491                let mut names = Vec::with_capacity(items.len());
492                for item in items {
493                    match item {
494                        Value::Scalar(name) if !name.trim().is_empty() => {
495                            names.push(name.trim().to_string());
496                        }
497                        Value::Scalar(_) | Value::Null => {}
498                        _ => {
499                            return Err(ExtensionError::new(format!(
500                                "description.yml: line {}: 'extension.maintainers' entries must \
501                                 be names, not lists or mappings",
502                                entry.line
503                            )))
504                        }
505                    }
506                }
507                Ok(names)
508            }
509            Value::Mapping => Err(ExtensionError::new(format!(
510                "description.yml: line {}: 'extension.maintainers' must be a list of names",
511                entry.line
512            ))),
513        }
514    }
515}
516
517/// Warns when `field` (`repo.ref` and its per-release variants) is not a
518/// commit hash: the community build checks out exactly that revision, and a
519/// branch or a tag (11 of the 395 published refs at community-extensions
520/// `845779e`) can be moved to another commit.
521fn warn_unless_commit_hash(field: &str, git_ref: &str, warnings: &mut Vec<String>) {
522    if !is_commit_hash(git_ref) {
523        warnings.push(format!(
524            "{field} '{git_ref}' is not a commit hash: the community build checks out \
525             exactly this revision, and a branch or tag can be moved to another commit, so \
526             the build is not pinned; DuckDB's documentation asks for the hash of the commit \
527             to build"
528        ));
529    }
530}
531
532/// Whether `git_ref` looks like a commit hash: 7 to 64 hexadecimal digits
533/// (an abbreviated or full SHA-1, or a SHA-256 object name).
534fn is_commit_hash(git_ref: &str) -> bool {
535    (7..=64).contains(&git_ref.len()) && git_ref.bytes().all(|b| b.is_ascii_hexdigit())
536}