From e9fdc478f6d3ae9f77b7933b4773fa5027582697 Mon Sep 17 00:00:00 2001 From: Ben Stull Date: Thu, 28 May 2026 07:24:03 -0700 Subject: [PATCH] v0.18.0 Slice 2: migrate send paths to build_envelope + POST unsubscribe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five send sites now construct their envelopes through `email_envelope.build_envelope` instead of building `EmailMessage` ad hoc: * `email_otc.py` — OTC mail (no List-Unsubscribe per the proposal's tradeoff: the recipient explicitly requested the code, so a list semantic would be wrong). * `email_invite.py` — admin/per-RFC invite mail (mailto:-only List-Unsubscribe — the invitee isn't a user yet, so no per-user opt-out URL exists). * `email._deliver` — watcher notifications (full one-click unsubscribe: mailto + signed URL + List-Unsubscribe-Post per RFC 8058, required by Gmail/Yahoo). * `email._send_bundle` — the "while you were away" bundle, one-click to the new `all` synthetic category (which lands `email_opt_out_all = 1` because the bundle spans multiple per-category flags). * `digest.py` — same as the bundle: bulk-adjacent, one-click to `all`. `api_notifications.py` gains the POST `/api/email/unsubscribe` endpoint (the matching receiver for `List-Unsubscribe-Post: List-Unsubscribe=One-Click`) and accepts the `all` category in both GET and POST handlers. `EmailConfig` adds `unsubscribe_mailto` (env: `EMAIL_UNSUBSCRIBE_MAILTO`, default = `EMAIL_FROM`) so deployments can route unsubscribe courtesy mail to a humans-monitored mailbox distinct from the no-reply sender. The `_SENT` test buffer now also carries `envelope["message"]` (the `EmailMessage` itself) so new tests can assert on headers directly. Legacy `to`/`from`/`subject`/`body` keys remain for backward compatibility. 10 new integration tests across test_otc_vertical / test_admin_create_user_invite_vertical / test_notifications_vertical covering: OTC has no List-Unsubscribe; invite has mailto: only; notification has full one-click; POST one-click flips the category; `all` category sets global opt-out via both GET and POST. Full suite: 277 passed. --- backend/app/api_notifications.py | 87 ++++++++++-- backend/app/digest.py | 13 +- backend/app/email.py | 84 ++++++++++-- backend/app/email_invite.py | 24 +++- backend/app/email_otc.py | 20 ++- .../test_admin_create_user_invite_vertical.py | 79 +++++++++++ backend/tests/test_notifications_vertical.py | 124 ++++++++++++++++++ backend/tests/test_otc_vertical.py | 50 +++++++ 8 files changed, 448 insertions(+), 33 deletions(-) diff --git a/backend/app/api_notifications.py b/backend/app/api_notifications.py index c431b43..0e3e255 100644 --- a/backend/app/api_notifications.py +++ b/backend/app/api_notifications.py @@ -443,6 +443,40 @@ def make_router(config: Config) -> APIRouter: # ----- Email: one-click unsubscribe + bounce webhook ----- + # v0.18.0: the category → column map. The `all` synthetic + # category lands the bundle's one-click on the global opt-out + # flag (per `email._send_bundle` in v0.18.0 Slice 2 — a bundle + # spans multiple categories, so a per-category flip wouldn't + # honor the user's intent). + _CATEGORY_COLUMN: dict[str, str] = { + "personal-direct": "email_personal_direct", + "structural": "email_watched_structural", + "admin-actionable": "email_admin_actionable", + "all": "email_opt_out_all", + } + + def _apply_unsubscribe(user_id: int, category: str) -> bool: + """Flip the matching column. Returns True on success, False + if the category is unknown. Idempotent — running twice on + the same (user, category) is harmless (it sets the column + to its current value).""" + column = _CATEGORY_COLUMN.get(category) + if column is None: + return False + # `all` sets the flag to 1 (opt out); per-category sets to 0 + # (turn that category off). The column semantic is "1 means + # don't send"; the per-category booleans are "1 means do + # send". Different polarities, hence the case split. + if category == "all": + db.conn().execute( + f"UPDATE users SET {column} = 1 WHERE id = ?", (user_id,) + ) + else: + db.conn().execute( + f"UPDATE users SET {column} = 0 WHERE id = ?", (user_id,) + ) + return True + @router.get("/api/email/unsubscribe") async def email_unsubscribe(t: str = Query(..., description="Signed token from the email footer")) -> HTMLResponse: try: @@ -453,20 +487,51 @@ def make_router(config: Config) -> APIRouter: "

Open the app to manage your notification preferences directly.

", status_code=400, ) - column = { - "personal-direct": "email_personal_direct", - "structural": "email_watched_structural", - "admin-actionable": "email_admin_actionable", - }.get(category) - if column is None: + if not _apply_unsubscribe(user_id, category): return HTMLResponse( f"

Unknown category

{category}

", status_code=400 ) - db.conn().execute(f"UPDATE users SET {column} = 0 WHERE id = ?", (user_id,)) - return HTMLResponse( - f"

Unsubscribed

You will no longer receive {category} emails. " - f"You can re-enable them in your notification preferences.

" - ) + if category == "all": + body = ( + "

Unsubscribed

You will no longer receive any email " + "from this app. You can re-enable individual categories from " + "your notification preferences after signing in.

" + ) + else: + body = ( + f"

Unsubscribed

You will no longer receive {category} emails. " + f"You can re-enable them in your notification preferences.

" + ) + return HTMLResponse(body) + + @router.post("/api/email/unsubscribe") + async def email_unsubscribe_post( + request: Request, + t: str = Query(..., description="Signed token from the List-Unsubscribe header"), + ) -> dict[str, Any]: + """v0.18.0: RFC 8058 one-click endpoint. + + Gmail and Yahoo POST `List-Unsubscribe=One-Click` (as a + form-encoded body) to the URL in the `List-Unsubscribe` + header when the user clicks their MUA's "Unsubscribe" + button. The endpoint MUST accept POST (per the + `List-Unsubscribe-Post` header we advertise) and MUST be + idempotent. + + The body content is checked loosely — RFC 8058 says it + SHOULD be exactly `List-Unsubscribe=One-Click`, but some + intermediaries strip / re-encode the body, so the + framework accepts any POST to the URL once the token + verifies. The bar is that the token signature carries the + authority; the body is hint-only. + """ + try: + user_id, category = email_mod.verify_unsubscribe_token(t) + except BadSignature: + raise HTTPException(400, "Invalid or expired token") + if not _apply_unsubscribe(user_id, category): + raise HTTPException(400, f"Unknown category: {category}") + return {"ok": True, "category": category} @router.post("/api/webhooks/email-bounce") async def email_bounce(body: BounceBody, request: Request) -> dict[str, Any]: diff --git a/backend/app/digest.py b/backend/app/digest.py index 3af63b0..1e91de9 100644 --- a/backend/app/digest.py +++ b/backend/app/digest.py @@ -180,7 +180,18 @@ def assemble_for_user( subject = _subject(eligible, cadence) body = _body(eligible, cadence, cfg) - sent = email_mod._deliver(cfg, email, subject, body) + # v0.18.0: the digest is the bulk-adjacent surface par excellence + # (it can carry weeks of accumulated activity), so it gets the + # full one-click unsubscribe to the global opt-out. Per-category + # opt-outs are managed from the preferences page; this footer is + # the "stop sending me anything" escape hatch Gmail and Yahoo + # expect for senders at this tier. + unsubscribe_url = email_mod.make_unsubscribe_url(user_id, "all") + sent = email_mod._deliver( + cfg, email, subject, body, + unsubscribe_mailto=cfg.unsubscribe_mailto, + unsubscribe_url=unsubscribe_url, + ) if not sent: return False ids = [r["id"] for r, _ in eligible] diff --git a/backend/app/email.py b/backend/app/email.py index fdd0e93..5e150b2 100644 --- a/backend/app/email.py +++ b/backend/app/email.py @@ -24,7 +24,6 @@ import os import smtplib from dataclasses import dataclass from datetime import datetime, time, timezone -from email.message import EmailMessage from email.utils import formataddr from itertools import groupby from typing import Any @@ -33,6 +32,7 @@ from urllib.parse import urlencode from itsdangerous import BadSignature, URLSafeSerializer from . import db +from .email_envelope import build_envelope log = logging.getLogger(__name__) @@ -69,6 +69,7 @@ class EmailConfig: app_url: str bundle_threshold: int enabled: bool + unsubscribe_mailto: str @classmethod def from_env(cls) -> "EmailConfig": @@ -84,6 +85,16 @@ class EmailConfig: app_url=os.environ.get("APP_URL", "http://localhost:8000").rstrip("/"), bundle_threshold=int(os.environ.get("EMAIL_BUNDLE_THRESHOLD", "5")), enabled=os.environ.get("EMAIL_ENABLED", "1") not in ("0", "false", "False"), + # v0.18.0: the `List-Unsubscribe: ` target on + # invite + notification mail. Defaults to the From + # address when unset; a deployment can route opt-out + # mail to a separate mailbox (e.g., a humans-monitored + # account distinct from the no-reply notifications + # sender) by setting this explicitly. + unsubscribe_mailto=os.environ.get( + "EMAIL_UNSUBSCRIBE_MAILTO", + os.environ.get("EMAIL_FROM", "notifications@wiggleverse.local"), + ).strip(), ) @@ -98,6 +109,14 @@ def _signer() -> URLSafeSerializer: def make_unsubscribe_url(user_id: int, category: str) -> str: + """Build the §15.4 per-category one-click URL. + + `category` is one of `personal-direct`, `structural`, + `admin-actionable` (the three per-category flags) or `all` + (v0.18.0: the bundle path, which sets `email_opt_out_all = 1` + because a bundle covers multiple categories and a per-category + opt-out wouldn't honor the user's intent). + """ cfg = EmailConfig.from_env() token = _signer().dumps({"u": user_id, "c": category}) qs = urlencode({"t": token}) @@ -250,7 +269,19 @@ def _send_one(user: Any, notif_id: int, payload: dict, category: str) -> None: return subject = _subject(payload) body = _body(payload, user["id"], category, cfg) - sent = _deliver(cfg, user["email"], subject, body) + # v0.18.0: notification mail is bulk-adjacent (a watcher can + # accumulate dozens of structural events on a busy RFC), so it + # carries the full one-click unsubscribe — Gmail and Yahoo + # require this for senders at OHM's volume tier per RFC 8058. + unsubscribe_url = make_unsubscribe_url(user["id"], category) + sent = _deliver( + cfg, + user["email"], + subject, + body, + unsubscribe_mailto=cfg.unsubscribe_mailto, + unsubscribe_url=unsubscribe_url, + ) if not sent: return db.conn().execute( @@ -305,23 +336,45 @@ def _deep_link(payload: dict, cfg: EmailConfig) -> str: return cfg.app_url -def _deliver(cfg: EmailConfig, to_address: str, subject: str, body: str) -> bool: +def _deliver( + cfg: EmailConfig, + to_address: str, + subject: str, + body: str, + *, + unsubscribe_mailto: str | None = None, + unsubscribe_url: str | None = None, +) -> bool: + """Build the envelope via the shared `build_envelope` helper and + hand it to SMTP. + + The `_SENT` buffer carries the helper's `EmailMessage` under + `message` plus the legacy `to`/`from`/`subject`/`body` keys for + backward-compatibility with tests that read those directly. + Newer tests can assert on the header surface by inspecting + `envelope["message"]`. + """ + msg = build_envelope( + to_address=to_address, + from_address=cfg.from_address, + from_name=cfg.from_name, + subject=subject, + body_plain=body, + unsubscribe_mailto=unsubscribe_mailto, + unsubscribe_url=unsubscribe_url, + ) envelope = { "to": to_address, "from": formataddr((cfg.from_name, cfg.from_address)), "subject": subject, "body": body, + "message": msg, } _SENT.append(envelope) if not cfg.smtp_host: log.info("email (stdout fallback): to=%s subject=%s", to_address, subject) return True try: - msg = EmailMessage() - msg["From"] = envelope["from"] - msg["To"] = to_address - msg["Subject"] = subject - msg.set_content(body) smtp = smtplib.SMTP(cfg.smtp_host, cfg.smtp_port, timeout=30) try: if cfg.smtp_starttls: @@ -440,13 +493,26 @@ def _send_bundle(cfg: EmailConfig, user: Any, emailable: list) -> int: for r, _cat, extras in group_rows: summary = _summary_for(r["event_kind"], r["actor_display"], r["rfc_title"], extras) sections.append(f" · {summary}") + # v0.18.0: the bundle covers multiple categories, so a + # per-category opt-out can't honor the user's intent. The + # `all` category lands at the §15.4 endpoint and sets + # `email_opt_out_all = 1`. + unsubscribe_url = make_unsubscribe_url(user["id"], "all") body = ( "Activity on RFCs you watch, accumulated during your quiet hours:\n" + "\n".join(sections) + f"\n\nOpen your inbox: {cfg.app_url}/inbox\n" + f"Manage all preferences: {cfg.app_url}/settings/notifications\n" + + f"Unsubscribe from all email: {unsubscribe_url}\n" + ) + sent = _deliver( + cfg, + user["email"], + subject, + body, + unsubscribe_mailto=cfg.unsubscribe_mailto, + unsubscribe_url=unsubscribe_url, ) - sent = _deliver(cfg, user["email"], subject, body) if not sent: return 0 ids = [r["id"] for r, _, _ in emailable] diff --git a/backend/app/email_invite.py b/backend/app/email_invite.py index 9b9d96f..31a13dd 100644 --- a/backend/app/email_invite.py +++ b/backend/app/email_invite.py @@ -31,10 +31,10 @@ from __future__ import annotations import logging import smtplib -from email.message import EmailMessage from email.utils import formataddr from .email import EmailConfig, _SENT +from .email_envelope import build_envelope log = logging.getLogger(__name__) @@ -59,12 +59,29 @@ def send_invite_email( cfg = EmailConfig.from_env() subject = _subject(inviter_display, cfg) body = _body(claim_url, inviter_display, inviter_email, custom_message, cfg) + # v0.18.0: invite mail carries a `List-Unsubscribe: ` + # only (no signed URL) — the invitee isn't a user yet, so there + # is no per-user opt-out row to flip. The operator handles + # ad-hoc opt-outs from the mailto: target. Per the proposal's + # "Tradeoff discussion": the invite was unsolicited from the + # recipient's perspective, so the courtesy header is right; + # but it can't be a one-click URL because the row doesn't + # exist yet. + msg = build_envelope( + to_address=to_address, + from_address=cfg.from_address, + from_name=cfg.from_name, + subject=subject, + body_plain=body, + unsubscribe_mailto=cfg.unsubscribe_mailto, + ) envelope = { "to": to_address, "from": formataddr((cfg.from_name, cfg.from_address)), "subject": subject, "body": body, "kind": "invite", + "message": msg, } _SENT.append(envelope) @@ -79,11 +96,6 @@ def send_invite_email( return True try: - msg = EmailMessage() - msg["From"] = envelope["from"] - msg["To"] = to_address - msg["Subject"] = subject - msg.set_content(body) smtp = smtplib.SMTP(cfg.smtp_host, cfg.smtp_port, timeout=30) try: if cfg.smtp_starttls: diff --git a/backend/app/email_otc.py b/backend/app/email_otc.py index b2f59d9..2d8b461 100644 --- a/backend/app/email_otc.py +++ b/backend/app/email_otc.py @@ -23,10 +23,10 @@ from __future__ import annotations import logging import smtplib -from email.message import EmailMessage from email.utils import formataddr from .email import EmailConfig, _SENT +from .email_envelope import build_envelope log = logging.getLogger(__name__) @@ -44,12 +44,25 @@ def send_otc_email(to_address: str, code: str) -> bool: cfg = EmailConfig.from_env() subject = f"Your sign-in code for {cfg.from_name}" body = _body(code, cfg) + # v0.18.0: OTC mail carries NO List-Unsubscribe — the recipient + # explicitly requested the code; advertising an unsubscribe + # header would imply OHM has them on a list, which it doesn't. + # See the proposal's "Tradeoff discussion" for the binding + # rationale. + msg = build_envelope( + to_address=to_address, + from_address=cfg.from_address, + from_name=cfg.from_name, + subject=subject, + body_plain=body, + ) envelope = { "to": to_address, "from": formataddr((cfg.from_name, cfg.from_address)), "subject": subject, "body": body, "kind": "otc", + "message": msg, } _SENT.append(envelope) @@ -64,11 +77,6 @@ def send_otc_email(to_address: str, code: str) -> bool: return True try: - msg = EmailMessage() - msg["From"] = envelope["from"] - msg["To"] = to_address - msg["Subject"] = subject - msg.set_content(body) smtp = smtplib.SMTP(cfg.smtp_host, cfg.smtp_port, timeout=30) try: if cfg.smtp_starttls: diff --git a/backend/tests/test_admin_create_user_invite_vertical.py b/backend/tests/test_admin_create_user_invite_vertical.py index a160f2e..343e5fd 100644 --- a/backend/tests/test_admin_create_user_invite_vertical.py +++ b/backend/tests/test_admin_create_user_invite_vertical.py @@ -647,3 +647,82 @@ def test_pending_invites_listing_admin_only(app_with_fake_gitea): ) r = client.get("/api/admin/users/invites") assert r.status_code == 403 + + +# --------------------------------------------------------------------------- +# v0.18.0: invite-envelope header shape — Slice 2 +# +# Invite mail goes through `build_envelope` and MUST land Date, +# Message-ID, Auto-Submitted, AND a `List-Unsubscribe: ` +# (no URL — the invitee isn't a user yet, so no per-user opt-out +# row exists). The mailto: target is the operator's `EMAIL_FROM` +# by default; the operator can override via `EMAIL_UNSUBSCRIBE_MAILTO`. +# --------------------------------------------------------------------------- + + +def _provision_admin_and_send_invite(client, app_with_fake_gitea_fixture, *, to: str = "headers@ex.co"): + provision_user_row(user_id=400, login="adminH", role="admin") + sign_in_as( + client, user_id=400, gitea_login="adminH", + display_name="Admin H", role="admin", + email="adminh@test", + ) + _reset_outbound() + r = client.post( + "/api/admin/users", + json={ + "email": to, + "first_name": "Header", + "last_name": "Test", + "role": "contributor", + "custom_message": "", + }, + ) + assert r.status_code == 200, r.text + return _outbound_invite_envelopes(to)[-1] + + +def test_invite_envelope_sets_date_messageid_autosubmitted(app_with_fake_gitea): + from fastapi.testclient import TestClient + from email.utils import parsedate_to_datetime + + app, _fake = app_with_fake_gitea + with TestClient(app) as client: + env = _provision_admin_and_send_invite(client, (app, _fake)) + msg = env["message"] + assert parsedate_to_datetime(msg["Date"]) is not None + assert msg["Message-ID"].startswith("<") and msg["Message-ID"].endswith(">") + assert msg["Auto-Submitted"] == "auto-generated" + + +def test_invite_envelope_has_mailto_list_unsubscribe_only(app_with_fake_gitea): + """The invitee isn't a user yet — no per-user opt-out URL is + available. The `List-Unsubscribe` MUST be a mailto: form, and + the `List-Unsubscribe-Post` header MUST be absent (the + one-click semantic requires a URL the MUA can POST to).""" + from fastapi.testclient import TestClient + + app, _fake = app_with_fake_gitea + with TestClient(app) as client: + env = _provision_admin_and_send_invite(client, (app, _fake)) + msg = env["message"] + lu = msg["List-Unsubscribe"] + assert lu is not None and lu.startswith("") + assert msg["Auto-Submitted"] == "auto-generated" + # Full one-click unsubscribe. + lu = msg["List-Unsubscribe"] + assert lu is not None + assert "") + # Auto-Submitted prevents auto-responder loops. + assert msg["Auto-Submitted"] == "auto-generated" + + +def test_otc_envelope_has_no_list_unsubscribe(app_with_fake_gitea): + """The recipient explicitly typed their email and asked for a + code; the framework MUST NOT advertise a list semantic on this + mail. Per the v0.18.0 proposal's tradeoff discussion.""" + from fastapi.testclient import TestClient + + app, _fake = app_with_fake_gitea + with TestClient(app) as client: + _reset_outbound() + client.post("/auth/otc/request", json={"email": "headers@example.com"}) + msg = _last_otc_envelope()["message"] + assert msg["List-Unsubscribe"] is None + assert msg["List-Unsubscribe-Post"] is None