Sub-project 2: ingest & tagging / review tools #1

Merged
benstull merged 14 commits from sub-project-2-ingest-tagging-review into main 2026-06-04 14:16:28 +00:00
5 changed files with 44 additions and 1 deletions
Showing only changes of commit db4f036729 - Show all commits
+1
View File
@@ -2,3 +2,4 @@ __pycache__/
*.pyc
.pytest_cache/
.venv/
media/
+5 -1
View File
@@ -12,4 +12,8 @@ requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
[tool.setuptools]
packages = ["hef"]
packages = ["hef", "tools"]
[project.scripts]
hef-ingest = "tools.ingest_cli:main"
hef-review = "tools.review_cli:main"
+5
View File
@@ -0,0 +1,5 @@
def test_tools_package_imports():
import tools
import tools.http
assert tools is not None
View File
+33
View File
@@ -0,0 +1,33 @@
"""Minimal HTTP client over urllib: timeout, small retry, user-agent. Injectable."""
from __future__ import annotations
import json as _json
import time
import urllib.request
USER_AGENT = "human-experience-filter-ingest/0.1 (+local art installation)"
class HttpClient:
def __init__(self, *, timeout=30.0, retries=2, opener=None):
self.timeout, self.retries = timeout, retries
self._opener = opener or urllib.request.urlopen
def get_bytes(self, url, *, headers=None):
last = None
for attempt in range(self.retries + 1):
try:
req = urllib.request.Request(
url, headers={"User-Agent": USER_AGENT, **(headers or {})}
)
with self._opener(req, timeout=self.timeout) as resp:
return resp.read()
except Exception as exc: # noqa: BLE001 - retried below
last = exc
if attempt < self.retries:
time.sleep(0.2 * (attempt + 1))
raise last
def get_json(self, url, *, headers=None):
return _json.loads(self.get_bytes(url, headers=headers).decode("utf-8"))