import json import pytest from tools.http import HttpClient from tools.ingest.internet_archive import InternetArchiveFetcher from tools.ingest.librivox import LibriVoxFetcher from tools.ingest.nasa import NasaFetcher class _Resp: def __init__(self, data: bytes): self._data = data def read(self): return self._data def __enter__(self): return self def __exit__(self, *a): return False def fake_opener(mapping): """Dispatch a urllib Request to a canned payload by URL substring.""" def opener(req, timeout=None): url = req.full_url for key, payload in mapping.items(): if key in url: if isinstance(payload, (bytes, bytearray)): return _Resp(bytes(payload)) return _Resp(json.dumps(payload).encode("utf-8")) raise AssertionError(f"unexpected url: {url}") return opener def test_librivox_fetcher(): payload = { "books": [ { "id": "123", "title": "Meditations", "url_librivox": "https://librivox.org/meditations/", "url_zip_file": "https://archive.org/download/meditations/meditations_mp3.zip", "authors": [{"first_name": "Marcus", "last_name": "Aurelius"}], "description": "Stoic philosophy.", } ] } client = HttpClient(opener=fake_opener({"librivox.org/api": payload})) cands = LibriVoxFetcher(client).search("medit", limit=5) c = cands[0] assert c.source_archive == "librivox" assert c.license == "public_domain" and c.attribution == "" assert c.suggested_id == "librivox-meditations" assert c.media_url.endswith(".zip") assert c.media_ext == "zip" assert "Aurelius" in c.description def test_nasa_fetcher(): search_payload = { "collection": { "items": [ { "data": [ { "nasa_id": "as08-14-2383", "title": "Earthrise", "description": "View of Earth from the Moon", "media_type": "video", } ], "href": "https://images-assets.nasa.gov/video/as08-14-2383/collection.json", } ] } } asset_payload = [ "https://images-assets.nasa.gov/video/as08-14-2383/as08-14-2383~orig.mp4", "https://images-assets.nasa.gov/video/as08-14-2383/as08-14-2383~thumb.jpg", ] client = HttpClient( opener=fake_opener( { "images-api.nasa.gov/search": search_payload, "collection.json": asset_payload, } ) ) c = NasaFetcher(client).search("earth", limit=3)[0] assert c.suggested_id == "nasa-as08-14-2383" assert c.license == "public_domain" and c.attribution == "" assert c.media_url.endswith("orig.mp4") assert c.media_ext == "mp4" assert "Earth" in c.description def test_internet_archive_cc_by(): meta_payload = { "metadata": { "identifier": "earthrise", "title": "Earthrise", "licenseurl": "http://creativecommons.org/licenses/by/4.0/", "creator": "NASA", "description": "Apollo 8 footage.", }, "files": [ {"name": "earthrise.mp4", "format": "h.264", "source": "original"}, {"name": "earthrise.png", "format": "PNG", "source": "derivative"}, ], "server": "ia800100.us.archive.org", "dir": "/12/items/earthrise", } client = HttpClient(opener=fake_opener({"archive.org/metadata/earthrise": meta_payload})) c = InternetArchiveFetcher(client).resolve("earthrise") assert c.suggested_id == "ia-earthrise" assert c.license == "cc_by" and "NASA" in c.attribution assert c.media_url == "https://ia800100.us.archive.org/12/items/earthrise/earthrise.mp4" assert c.media_ext == "mp4" assert c.source_url == "https://archive.org/details/earthrise" def test_internet_archive_no_license_assumed_public_domain_and_flagged(): meta_payload = { "metadata": {"identifier": "oldfilm", "title": "Old Film"}, "files": [{"name": "oldfilm.mp4", "source": "original"}], "server": "ia.example.org", "dir": "/x/items/oldfilm", } client = HttpClient(opener=fake_opener({"archive.org/metadata/oldfilm": meta_payload})) c = InternetArchiveFetcher(client).resolve("oldfilm") assert c.license == "public_domain" assert "verify" in c.description.lower() def test_internet_archive_search_resolves_each_hit(): search_payload = {"response": {"docs": [{"identifier": "earthrise"}]}} meta_payload = { "metadata": { "identifier": "earthrise", "title": "Earthrise", "licenseurl": "https://creativecommons.org/publicdomain/mark/1.0/", }, "files": [{"name": "earthrise.mp4", "source": "original"}], "server": "ia.example.org", "dir": "/x", } client = HttpClient( opener=fake_opener( { "advancedsearch.php": search_payload, "archive.org/metadata/earthrise": meta_payload, } ) ) cands = InternetArchiveFetcher(client).search("earthrise", limit=5) assert [c.suggested_id for c in cands] == ["ia-earthrise"] assert cands[0].license == "public_domain" def test_deferred_stubs_raise_not_implemented(): from tools.ingest.fma import FmaFetcher from tools.ingest.freesound import FreesoundFetcher from tools.ingest.musopen import MusopenFetcher with pytest.raises(NotImplementedError): MusopenFetcher(None).search("x", limit=1) with pytest.raises(NotImplementedError): FmaFetcher(None).search("x", limit=1) with pytest.raises(NotImplementedError): FreesoundFetcher(None).search("x", limit=1) def test_fetchers_expose_archive_label(): assert LibriVoxFetcher(None).archive == "librivox" assert NasaFetcher(None).archive == "nasa" assert InternetArchiveFetcher(None).archive == "internet_archive"