From 222421e77396629cd86fb0290b5d0b7b8c5d5c9e Mon Sep 17 00:00:00 2001 From: Ben Stull Date: Thu, 4 Jun 2026 06:26:21 -0700 Subject: [PATCH] feat: per-archive license normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per sub-project-2 plan Task 6 / spec §5.3. normalize_license maps CC URLs / identifiers and public-domain markers to the LICENSES vocab, builds non-empty attribution for cc_by/cc_by_nc, and rejects unmappable licenses at ingest. librivox_license() helper returns public_domain. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_licensing.py | 84 +++++++++++++++++++++++++++++++++++++++++ tools/licensing.py | 63 +++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 tests/test_licensing.py create mode 100644 tools/licensing.py diff --git a/tests/test_licensing.py b/tests/test_licensing.py new file mode 100644 index 0000000..aab4fe3 --- /dev/null +++ b/tests/test_licensing.py @@ -0,0 +1,84 @@ +import pytest + +from hef.catalog import LICENSES +from tools.licensing import librivox_license, normalize_license + + +def test_cc_by_url_maps_and_requires_attribution(): + lic, attr = normalize_license( + "https://creativecommons.org/licenses/by/4.0/", creator="Jane Doe" + ) + assert lic == "cc_by" and "Jane Doe" in attr + + +def test_cc_by_identifier_maps(): + lic, attr = normalize_license("CC BY 4.0", creator="Sam") + assert lic == "cc_by" and attr + + +def test_cc_by_nc_maps(): + lic, attr = normalize_license( + "https://creativecommons.org/licenses/by-nc/4.0/", creator="Artist" + ) + assert lic == "cc_by_nc" and "Artist" in attr + + +def test_cc_by_without_creator_still_has_nonempty_attribution(): + # validate() rejects an attribution license with empty attribution, so the + # normalizer must always produce a non-empty string for cc_by/cc_by_nc. + lic, attr = normalize_license("https://creativecommons.org/licenses/by/4.0/") + assert lic == "cc_by" and attr != "" + + +def test_cc0_maps_with_empty_attribution(): + lic, attr = normalize_license("https://creativecommons.org/publicdomain/zero/1.0/") + assert lic == "cc0" and attr == "" + + +def test_cc0_identifier(): + lic, attr = normalize_license("CC0") + assert lic == "cc0" and attr == "" + + +def test_public_domain_mark(): + lic, attr = normalize_license( + "https://creativecommons.org/publicdomain/mark/1.0/" + ) + assert lic == "public_domain" and attr == "" + + +def test_no_known_copyright_is_public_domain(): + lic, attr = normalize_license("No known copyright") + assert lic == "public_domain" and attr == "" + + +def test_public_domain_literal(): + lic, attr = normalize_license("public_domain") + assert lic == "public_domain" and attr == "" + + +def test_unmappable_rejected(): + with pytest.raises(ValueError): + normalize_license("All Rights Reserved") + + +def test_empty_rejected(): + with pytest.raises(ValueError): + normalize_license("") + + +def test_librivox_license_helper(): + lic, attr = librivox_license() + assert lic == "public_domain" and attr == "" + + +def test_all_outputs_in_vocab(): + samples = [ + "https://creativecommons.org/licenses/by/4.0/", + "https://creativecommons.org/licenses/by-nc/4.0/", + "CC0", + "public domain", + ] + for raw in samples: + lic, _ = normalize_license(raw, creator="x") + assert lic in LICENSES diff --git a/tools/licensing.py b/tools/licensing.py new file mode 100644 index 0000000..56684a8 --- /dev/null +++ b/tools/licensing.py @@ -0,0 +1,63 @@ +"""Normalize per-archive origin license metadata to the hef.catalog LICENSES vocab. + +Maps Creative Commons URLs / identifiers and public-domain markers to +{public_domain, cc0, cc_by, cc_by_nc}. For the attribution licenses (cc_by, +cc_by_nc) it builds a non-empty attribution string — hef.catalog.validate() +rejects those licenses with an empty attribution. Anything unmappable is +rejected (a piece whose license can't be established is not ingested). +""" + +from __future__ import annotations + +import re + + +def _attribution(creator: str, license_name: str, raw: str, default_label: str) -> str: + label = license_name or default_label or raw + parts = [] + if creator: + parts.append(creator) + parts.append(label) + return " — ".join(parts) + + +def normalize_license(raw, *, creator="", license_name=""): + """Return (license, attribution) for an origin license string. + + Raises ValueError when `raw` does not map to an allowed license. + """ + text = (raw or "").strip().lower() + if not text: + raise ValueError(f"unmappable license: {raw!r}") + + # CC0 / public-domain dedication (check before the generic publicdomain + # markers, since the CC0 URL also contains "publicdomain"). + if "publicdomain/zero" in text or re.search(r"\bcc[ _-]?0\b", text): + return ("cc0", "") + + # Public Domain Mark / explicit public domain / no-known-copyright. + pd_markers = ( + "publicdomain/mark", + "public_domain", + "public domain", + "publicdomain", + "no known copyright", + "pdm", + ) + if any(m in text for m in pd_markers): + return ("public_domain", "") + + # CC BY-NC must be checked before the bare CC BY. + if re.search(r"by[ _-]?nc", text): + return ("cc_by_nc", _attribution(creator, license_name, raw, "CC BY-NC")) + + # CC BY. + if re.search(r"\bcc[ _-]?by\b", text) or "licenses/by" in text: + return ("cc_by", _attribution(creator, license_name, raw, "CC BY")) + + raise ValueError(f"unmappable license: {raw!r}") + + +def librivox_license(): + """LibriVox recordings are public domain by charter (attribution not required).""" + return ("public_domain", "")