From 80447092e9cb1e0770e22c0a19d6fbec81940e9a Mon Sep 17 00:00:00 2001 From: domverse Date: Mon, 13 Jul 2026 13:51:41 +0200 Subject: [PATCH] Bootstrap project and safe work-item workflow --- .gitignore | 18 + .work-item.toml | 46 + AGENTS.md | 50 + CLAUDE.md | 3 + DASHBOARD_PLAN.md | 257 ++ INTEGRATED_PIPELINE_CONCEPT.md | 1923 +++++++++++++ README.md | 32 + WEBAPP_CONCEPT.md | 285 ++ album_naming.md | 204 ++ compare_models.py | 179 ++ .../E01-shared-identity-inventory.md | 21 + .../E02-unified-workflow-shell.md | 20 + delivery_backlog/E03-album-proposals.md | 17 + delivery_backlog/E04-guarded-renaming.md | 18 + delivery_backlog/E05-immich-upload.md | 18 + delivery_backlog/E06-archive-lifecycle.md | 18 + delivery_backlog/E07-hardening-release.md | 19 + delivery_backlog/README.md | 49 + .../stories/US01-01-donor-characterization.md | 20 + .../US01-02-app-database-foundation.md | 24 + .../stories/US01-03-inventory-identity.md | 23 + .../stories/US01-04-duplicate-engine.md | 24 + .../stories/US01-05-thumbnail-service.md | 24 + .../stories/US01-06-inventory-review-ui.md | 25 + .../stories/US01-07-phase-a-e2e.md | 24 + .../stories/US02-01-safety-ui-donors.md | 22 + .../stories/US02-02-durable-jobs.md | 22 + .../stories/US02-03-worker-recovery.md | 22 + .../stories/US02-04-job-api-events.md | 23 + .../stories/US02-05-frontend-shell.md | 24 + .../stories/US02-06-workflow-views.md | 25 + .../stories/US02-07-phase-b-e2e.md | 23 + .../stories/US03-01-album-evidence.md | 23 + .../stories/US03-02-naming-policy.md | 22 + .../stories/US03-03-proposal-generation.md | 25 + .../stories/US03-04-proposal-ui.md | 24 + .../stories/US03-05-phase-c-e2e.md | 22 + .../stories/US04-01-rename-plan.md | 23 + .../stories/US04-02-rename-journal.md | 23 + .../stories/US04-03-rename-apply.md | 24 + .../stories/US04-04-rename-recovery.md | 23 + delivery_backlog/stories/US04-05-rename-ui.md | 22 + .../stories/US04-06-phase-d-e2e.md | 22 + .../stories/US05-01-upload-preflight.md | 23 + .../stories/US05-02-upload-batches.md | 23 + .../stories/US05-03-upload-reports.md | 22 + .../stories/US05-04-upload-verification.md | 22 + delivery_backlog/stories/US05-05-upload-ui.md | 23 + .../stories/US05-06-phase-e-e2e.md | 23 + .../stories/US06-01-archive-preflight.md | 23 + .../stories/US06-02-archive-transfer.md | 23 + .../stories/US06-03-offline-assets.md | 22 + delivery_backlog/stories/US06-04-restore.md | 24 + .../stories/US06-05-archive-ui.md | 23 + .../stories/US06-06-phase-f-e2e.md | 23 + .../stories/US07-01-donor-archive.md | 24 + .../stories/US07-02-security-hardening.md | 23 + .../stories/US07-03-media-hardening.md | 24 + .../stories/US07-04-fault-concurrency.md | 25 + .../stories/US07-05-backup-operations.md | 24 + .../stories/US07-06-performance.md | 24 + .../stories/US07-07-release-e2e.md | 29 + nsfw_tag.py | 11 + nsfwtag/README.md | 282 ++ nsfwtag/__init__.py | 25 + nsfwtag/__main__.py | 65 + nsfwtag/bench.py | 426 +++ nsfwtag/exif.py | 123 + nsfwtag/review.html | 426 +++ nsfwtag/scoring.py | 119 + nsfwtag/server.py | 126 + nsfwtag/webapp.py | 91 + photo_analyzer.py | 2454 +++++++++++++++++ pyproject.toml | 7 + scripts/work-item | 11 + test_dedup.py | 122 + test_nsfw_skip.py | 15 + tests/test_cli_e2e.py | 221 ++ tests/test_core.py | 178 ++ webapp/README.md | 62 + webapp/__init__.py | 11 + webapp/__main__.py | 42 + webapp/analyzer.html | 490 ++++ webapp/page.py | 12 + webapp/query.py | 214 ++ webapp/runner.py | 110 + webapp/server.py | 162 ++ work_item/__init__.py | 5 + work_item/__main__.py | 3 + work_item/core.py | 547 ++++ 90 files changed, 10562 insertions(+) create mode 100644 .gitignore create mode 100644 .work-item.toml create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 DASHBOARD_PLAN.md create mode 100644 INTEGRATED_PIPELINE_CONCEPT.md create mode 100644 README.md create mode 100644 WEBAPP_CONCEPT.md create mode 100644 album_naming.md create mode 100644 compare_models.py create mode 100644 delivery_backlog/E01-shared-identity-inventory.md create mode 100644 delivery_backlog/E02-unified-workflow-shell.md create mode 100644 delivery_backlog/E03-album-proposals.md create mode 100644 delivery_backlog/E04-guarded-renaming.md create mode 100644 delivery_backlog/E05-immich-upload.md create mode 100644 delivery_backlog/E06-archive-lifecycle.md create mode 100644 delivery_backlog/E07-hardening-release.md create mode 100644 delivery_backlog/README.md create mode 100644 delivery_backlog/stories/US01-01-donor-characterization.md create mode 100644 delivery_backlog/stories/US01-02-app-database-foundation.md create mode 100644 delivery_backlog/stories/US01-03-inventory-identity.md create mode 100644 delivery_backlog/stories/US01-04-duplicate-engine.md create mode 100644 delivery_backlog/stories/US01-05-thumbnail-service.md create mode 100644 delivery_backlog/stories/US01-06-inventory-review-ui.md create mode 100644 delivery_backlog/stories/US01-07-phase-a-e2e.md create mode 100644 delivery_backlog/stories/US02-01-safety-ui-donors.md create mode 100644 delivery_backlog/stories/US02-02-durable-jobs.md create mode 100644 delivery_backlog/stories/US02-03-worker-recovery.md create mode 100644 delivery_backlog/stories/US02-04-job-api-events.md create mode 100644 delivery_backlog/stories/US02-05-frontend-shell.md create mode 100644 delivery_backlog/stories/US02-06-workflow-views.md create mode 100644 delivery_backlog/stories/US02-07-phase-b-e2e.md create mode 100644 delivery_backlog/stories/US03-01-album-evidence.md create mode 100644 delivery_backlog/stories/US03-02-naming-policy.md create mode 100644 delivery_backlog/stories/US03-03-proposal-generation.md create mode 100644 delivery_backlog/stories/US03-04-proposal-ui.md create mode 100644 delivery_backlog/stories/US03-05-phase-c-e2e.md create mode 100644 delivery_backlog/stories/US04-01-rename-plan.md create mode 100644 delivery_backlog/stories/US04-02-rename-journal.md create mode 100644 delivery_backlog/stories/US04-03-rename-apply.md create mode 100644 delivery_backlog/stories/US04-04-rename-recovery.md create mode 100644 delivery_backlog/stories/US04-05-rename-ui.md create mode 100644 delivery_backlog/stories/US04-06-phase-d-e2e.md create mode 100644 delivery_backlog/stories/US05-01-upload-preflight.md create mode 100644 delivery_backlog/stories/US05-02-upload-batches.md create mode 100644 delivery_backlog/stories/US05-03-upload-reports.md create mode 100644 delivery_backlog/stories/US05-04-upload-verification.md create mode 100644 delivery_backlog/stories/US05-05-upload-ui.md create mode 100644 delivery_backlog/stories/US05-06-phase-e-e2e.md create mode 100644 delivery_backlog/stories/US06-01-archive-preflight.md create mode 100644 delivery_backlog/stories/US06-02-archive-transfer.md create mode 100644 delivery_backlog/stories/US06-03-offline-assets.md create mode 100644 delivery_backlog/stories/US06-04-restore.md create mode 100644 delivery_backlog/stories/US06-05-archive-ui.md create mode 100644 delivery_backlog/stories/US06-06-phase-f-e2e.md create mode 100644 delivery_backlog/stories/US07-01-donor-archive.md create mode 100644 delivery_backlog/stories/US07-02-security-hardening.md create mode 100644 delivery_backlog/stories/US07-03-media-hardening.md create mode 100644 delivery_backlog/stories/US07-04-fault-concurrency.md create mode 100644 delivery_backlog/stories/US07-05-backup-operations.md create mode 100644 delivery_backlog/stories/US07-06-performance.md create mode 100644 delivery_backlog/stories/US07-07-release-e2e.md create mode 100644 nsfw_tag.py create mode 100644 nsfwtag/README.md create mode 100644 nsfwtag/__init__.py create mode 100644 nsfwtag/__main__.py create mode 100644 nsfwtag/bench.py create mode 100644 nsfwtag/exif.py create mode 100644 nsfwtag/review.html create mode 100644 nsfwtag/scoring.py create mode 100644 nsfwtag/server.py create mode 100644 nsfwtag/webapp.py create mode 100644 photo_analyzer.py create mode 100644 pyproject.toml create mode 100755 scripts/work-item create mode 100644 test_dedup.py create mode 100644 test_nsfw_skip.py create mode 100644 tests/test_cli_e2e.py create mode 100644 tests/test_core.py create mode 100644 webapp/README.md create mode 100644 webapp/__init__.py create mode 100644 webapp/__main__.py create mode 100644 webapp/analyzer.html create mode 100644 webapp/page.py create mode 100644 webapp/query.py create mode 100644 webapp/runner.py create mode 100644 webapp/server.py create mode 100644 work_item/__init__.py create mode 100644 work_item/__main__.py create mode 100644 work_item/core.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed83884 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.coverage +htmlcov/ +.DS_Store +.env +*.db +*.db-* +*.sqlite* +*.log +data/ +downloads/ +_archive/ +_todo/ +pictures/ +photos/ +_IGNORE/ diff --git a/.work-item.toml b/.work-item.toml new file mode 100644 index 0000000..3ad1bcf --- /dev/null +++ b/.work-item.toml @@ -0,0 +1,46 @@ +[repository] +slug = "domverse/photoanalyzer" +login = "domverse" +assignee = "domverse" +remote = "origin" +main_branch = "main" + +[workflow] +branch_prefix = "us" +require_ci = false +required_tests = ["python3 -m unittest discover -s tests -v"] + +[safety] +max_file_bytes = 5000000 +allow = ["tests/fixtures/**"] +deny = [ + "_IGNORE/**", + "**/_IGNORE/**", + "pictures/**", + "photos/**", + "_archive/**", + "_todo/**", + "data/**", + "downloads/**", + "*.db", + "*.db-*", + "*.sqlite", + "*.sqlite3", + "*.log", + "*.env", + "*.pem", + "*.key", + "*.jpg", + "*.jpeg", + "*.png", + "*.gif", + "*.webp", + "*.heic", + "*.heif", + "*.tif", + "*.tiff", + "*.mp4", + "*.mov", + "*.avi", + "*.mkv", +] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..01b7353 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# Agent Instructions — Photo Analyzer + +Read `CLAUDE.md` and `INTEGRATED_PIPELINE_CONCEPT.md` before implementation. Preserve +the established project invariants: never inspect or process content beneath any +`_IGNORE/` directory; write verified EXIF before Immich upload; use managed +`immich-go` uploads rather than External Libraries; keep operations resume-safe; and +treat the existing CLI implementations as primary donors rather than rewriting them. + +## Mandatory implementation workflow + +When asked to start, resume, or continue implementation, work from this isolated Git +repository and use `scripts/work-item`. Gitea issues and their dependencies are the +authoritative backlog. + +1. Run `scripts/work-item claim`. +2. Work only on the claimed story and its generated feature branch. +3. Read the entire issue, linked specification, dependencies, and acceptance criteria. +4. Inspect the legacy CLI donors before replacing applicable behavior. Update the donor + ledger and characterization tests required by the story. +5. Implement every acceptance criterion and its automated tests. +6. Run story-specific tests and the accumulated regression suite required by the epic. +7. Run `scripts/work-item submit --test ""` without `--yes` and + review the reported diff and file list. +8. If the changes are correct and safe, rerun with `--yes`. This commits, pushes, opens + a pull request, records test evidence, and moves the issue to review. +9. After review and successful CI, run `scripts/work-item complete --merge`. This + verifies the merge, closes the issue, returns to updated `main`, and removes the + local feature branch. +10. Continue with the next eligible story by returning to step 1. + +If genuinely blocked, run `scripts/work-item block --reason ""` and +stop. Never skip to a later story whose dependencies are open. Never mark work done +when tests are missing, skipped, or failing. + +## Git and privacy safety + +- Never run automated Git operations from the personal photo-library directory. +- Never stage or commit real photos, videos, `_IGNORE/` content, databases, logs, + credentials, environment files, caches, generated thumbnails, or runtime data. +- Never use force-push, destructive reset, untracked-file deletion, or automatic merge + conflict resolution. +- Do not bypass `scripts/work-item` safety checks or manually close implementation + issues to conceal a failed transition. +- Preserve unrelated user changes and stop if the working tree is unexpectedly dirty. + +## Completion standard + +A story is complete only when its pull request is merged, required automated tests and +CI pass, test evidence is recorded on the issue, the issue is closed with +`status/done`, and the local repository is back on a clean, current `main` branch. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6b46d26 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,3 @@ +# CLAUDE.md + +Project guidance lives in [AGENTS.md](AGENTS.md). Read it before making any changes. diff --git a/DASHBOARD_PLAN.md b/DASHBOARD_PLAN.md new file mode 100644 index 0000000..0301b24 --- /dev/null +++ b/DASHBOARD_PLAN.md @@ -0,0 +1,257 @@ +# Rich Live Dashboard — Upgrade Plan + +Replace the scrolling `Progress` context in `photo_analyzer.py` with a persistent +`Live + Layout` that shows **album progress**, a **photo feed**, and the **progress +bar** simultaneously — without touching the DB logic or threading model. + +> **Focus: robustness.** The dashboard is a *view layer only* — it reads state, +> never owns it. SQLite stays the source of truth at all times. + +--- + +## Target layout + +``` +┌─ python photo_analyzer.py --library pictures/ ─────────────────┐ +│ ┌─ Album Progress — leaf folder = album ───────────────────┐ │ +│ │ Camera Roll 238 / 550 ████████░░░░░░░░░░░ 43% │ │ +│ │ Urlaub/Rom 45 / 45 ██████████████████ ✓ │ │ +│ │ Urlaub/Venedig 12 / 60 ███░░░░░░░░░░░░░░░░ 20% │ │ +│ │ WhatsApp Images 0 / 120 ░░░░░░░░░░░░░░░░░░ 0% │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ ┌─ Recent ─────────────────────────────────────────────────┐ │ +│ │ ✓ IMG_4821.jpg │ │ +│ │ A golden retriever playing fetch on a sunny beach │ │ +│ │ ✗ DSC_0042.jpg — Exhausted 4 retries: connection timeout │ │ +│ │ ✓ IMG_4820.jpg │ │ +│ │ Two children building a sandcastle at sunset │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ ┌─ Analyzing — Camera Roll ────────────────────────────────┐ │ +│ │ ████████░░░░░░░░░░ 238/550 43% ok=235 err=3 │ │ +│ │ tokens=284,192 est. remaining ~340,000 ETA 14:23 │ │ +│ └──────────────────────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Core rule: album = leaf folder + +An **album is the leaf folder directly containing the photos**, not a top-level +parent. `Urlaub/Rom` and `Urlaub/Venedig` are two separate albums, never collapsed +into `Urlaub`. Key every stats dict on `album_label(path, library)` (already added +to the script for `--stats`): + +```python +def album_label(photo: Path, library: Path = None) -> str: + """Leaf folder relative to library: Urlaub/Rom/IMG.jpg -> 'Urlaub/Rom'.""" + parent = photo.parent + if library: + try: + rel = parent.relative_to(library) + return "(root)" if str(rel) == "." else str(rel) + except ValueError: + pass + return parent.name or str(parent) +``` + +The analysis loop already groups on `Path(p).parent` (the leaf), so each sub-album +completes fully before the next starts. Only the *display* side needs this fix. + +--- + +## Phases + +### Phase 01 — Scaffold Live + Layout *(structural, ~30 min)* + +Replace `with Progress(...) as progress:` with a `Live` context wrapping a +`Layout`. **Progress becomes a renderable, not a context manager** — it lives +inside the layout rather than owning the terminal. All DB logic, threading, and +retry logic stays unchanged. + +```python +from rich.layout import Layout +from rich.live import Live + +layout = Layout() +layout.split_column( + Layout(name="stats", ratio=2), + Layout(name="feed", ratio=2), + Layout(name="progress", size=5), +) +progress = Progress(...) # same columns as today +task_id = progress.add_task(...) +layout["progress"].update(progress) + +with Live(layout, console=console, refresh_per_second=4, transient=False): + # ... existing folder loop unchanged ... +``` + +- **Test:** script starts, layout renders with placeholder panels, progress bar + advances, Ctrl-C stops cleanly. +- Do not move on until this is stable — later phases build on this scaffold. + +--- + +### Phase 02 — Thread-safe log buffer *(concurrency, ~20 min)* + +Workers must **never write to the terminal directly**. Replace every +`progress.console.print()` call with an append to a shared `deque`. The Live +renderer reads from the deque on each refresh cycle and renders it as the feed +panel. Threads write, the renderer reads — no other direction. + +```python +from collections import deque + +log_buffer = deque(maxlen=10) # newest first +log_lock = threading.Lock() + +# in process() — replace progress.console.print(...) with: +with log_lock: + log_buffer.appendleft(entry) + +# in the Live render function: +def render_feed() -> Table: + with log_lock: + rows = list(log_buffer) # snapshot + # build Rich Table from rows ... +``` + +- **Never** call `live.console.print()` from a worker thread — always via the buffer. +- `deque(maxlen=10)` auto-evicts old entries; no memory growth. +- The lock protects the *snapshot* read, not the render. + +--- + +### Phase 03 — In-memory album stats *(performance, ~20 min)* + +The stats panel refreshes at 4 Hz — **no DB queries in the render loop**. Key +every dict on `album_label(path, library)`. Compute `folder_total` from the sorted +`pending` list, seed `folder_done` from one upfront DB query, and have workers +increment it in memory after each success. + +```python +# album = leaf folder relative to library — reuse the existing helper +folder_total: dict[str, int] = {} +folder_done: dict[str, int] = {} + +for folder, paths in folder_groups: # folder = abs leaf dir + album = album_label(Path(paths[0]), library) + folder_total[album] = len(paths) + folder_done[album] = 0 + +# Seed already-done counts from DB (single query): +for path, status in conn.execute( + "SELECT path, status FROM photos WHERE status IN ('analyzed','exif_written')" +): + album = album_label(Path(path), library) + if album in folder_done: + folder_done[album] += 1 + +# In the worker result handler (album computed once per folder): +with token_lock: # reuse existing lock + folder_done[album] = folder_done.get(album, 0) + 1 +``` + +- **Album = leaf folder, not parent.** Nested trees like `Urlaub/*` become one + row per sub-album. +- Render reads the dicts under the existing `token_lock` — no new lock needed. +- Shows **all** albums (future ones at 0%); long paths left-truncated via `fit_label()`. +- Errors do *not* increment `folder_done` — an album shows ✓ only when every + photo succeeded. + +--- + +### Phase 04 — Non-TTY fallback *(safety, ~10 min)* + +Rich's `Live` behaves unexpectedly when stdout is piped or redirected (log +capture, CI, `nohup`). **Gate the Live path on `console.is_terminal`.** Extract +the current Progress-based loop into `_run_analysis_plain()` unchanged; the new +Live implementation goes in `_run_analysis_live()`. + +```python +def run_analysis(args, conn, client): + ... # shared setup (discover, pending, folder_groups) + if console.is_terminal: + _run_analysis_live(args, conn, client, pending, folder_groups, ...) + else: + _run_analysis_plain(args, conn, client, pending, folder_groups, ...) +``` + +- Plain mode keeps every existing behavior — no regression for scripted use. +- File logs always write regardless of mode. +- No flag needed — detection is automatic. + +--- + +### Phase 05 — Error containment *(robustness, ~15 min)* + +The Live layer must never take down the pipeline. **DB writes happen in worker +threads, outside the render path** — a renderer crash cannot corrupt the +database. Wrap Live construction in try/except: if Rich fails to start, fall back +to plain mode. SIGINT already works (`_stop` set, executor drains, `with` exits). + +```python +try: + with Live(layout, console=console, refresh_per_second=4): + _run_folder_loop(executor, folder_groups, ...) +except Exception as e: + log.warning(f"Live render failed ({e}), falling back to plain mode") + _run_analysis_plain(args, conn, client, pending, folder_groups, ...) +``` + +- **Worker exception path:** `mark_error()` runs before anything is appended to + `log_buffer` — DB is always consistent even if the render crashes mid-frame. +- **Render exceptions:** catch inside the render function, return a plain + `Text("render error")` panel — never let a bad render kill the loop. +- **Resume safety:** unchanged — `--stats` and restart behavior identical to today. + +--- + +## Robustness constraints + +| Constraint | Detail | +|---|---| +| No DB queries in render path | Renderer fires 4×/s. Album counts kept in in-memory dicts, seeded once at startup. SQLite never touched during animation. | +| Workers never touch the terminal | Only interface between workers and display is `log_buffer` (deque + lock). Rich's Live is single-threaded by design. | +| Render errors isolated | Each panel's render function wraps its body in try/except, returns a fallback `Text` on failure. A bad render is cosmetic, not fatal. | +| Ctrl-C still works | `_stop` event and SIGINT handler unchanged. Live context exits when the `with` block ends. In-flight requests complete; queued ones cancel. | +| Non-TTY produces no garbage | Piped output, `nohup`, log capture → plain mode via `console.is_terminal`. No ANSI escapes leak into log files. | +| Resume unchanged | All state lives in SQLite. Kill at any point; restart picks up where it left off. Dashboard reads state, never owns it. | + +--- + +## Status: implemented ✓ (2026-06-22) + +All five phases shipped. Old script archived as `photo_analyzer.py.bak-20260622-091208`. + +Structure: `run_analysis()` does shared setup, then dispatches to +`_run_analysis_live()` (TTY) or `_run_analysis_plain()` (piped/redirected), both +driving the shared `_run_folder_loop()` engine. Live uses a `_Dashboard.__rich__` +that rebuilds stats/feed/progress panels each tick from in-memory dicts + a deque. + +**Extra hardening found during testing (not in original plan):** the shared +`sqlite3.Connection` was committed from multiple worker threads, racing with +"cannot commit - no transaction is active". Real API latency hid it; instant +dry-run exposed it. Fixed with a module-level `_db_lock` serialising all writes +(`mark_analyzed` / `mark_error` / `mark_exif_written` / `upsert_pending`). +Verified: 200 photos × 8 workers, zero spurious errors. + +Tested: plain mode (piped), live mode (PTY, 6 and 200 photos), Live-failure +fallback, render-error isolation, album windowing (>14 albums summarised), +leaf-folder album grouping. + +--- + +## Implementation order + +1. **Phase 1** — scaffold Live+Layout. Test: layout renders, progress bar + advances, Ctrl-C exits. *(must pass before continuing)* +2. **Phase 3** — in-memory album stats panel. Test: album rows appear with correct + counts, numbers increment live as photos complete. +3. **Phase 2** — log buffer + feed panel. Test: photo results appear in the feed, + errors show in red, old entries scroll off. +4. **Phase 4** — non-TTY fallback. Test: `python photo_analyzer.py ... | cat` + produces clean plain output. +5. **Phase 5** — error containment. Test: force a render exception, confirm + pipeline continues; interrupt mid-album, confirm DB is consistent on restart. diff --git a/INTEGRATED_PIPELINE_CONCEPT.md b/INTEGRATED_PIPELINE_CONCEPT.md new file mode 100644 index 0000000..a2b3239 --- /dev/null +++ b/INTEGRATED_PIPELINE_CONCEPT.md @@ -0,0 +1,1923 @@ +# Photo Library Pipeline — Integrated App Concept + +Unify duplicate detection, safety review, content analysis, album naming, guarded +renaming, and Immich upload into one local web application. + +> **North star:** every photo moves through one visible, resume-safe workflow. +> Destructive or irreversible actions always require a preview and explicit approval. +> A file path is mutable metadata, never the identity of a photo. + +--- + +## 1. Product goals + +The app should: + +- guide the user through the stages in the correct order; +- preserve the existing NSFW review and Photo Analyzer UI/UX; +- prevent NSFW photos from reaching the cloud vision provider while still allowing + reviewed, EXIF-tagged NSFW photos to be uploaded to Immich; +- detect duplicates before spending API quota or changing metadata; +- keep database records valid when files or folders are renamed; +- create a verified DB↔EXIF consistency checkpoint after every metadata-producing + stage, always merging with and preserving existing metadata; +- upload each approved album through `immich-go` with a visible audit trail; +- optionally archive successfully uploaded albums out of active storage while retaining + their complete identity, metadata, thumbnails, and deduplication evidence; +- remain safe to stop and resume at every stage; +- exclude every `_IGNORE/` directory from discovery, counts, previews, and actions. + +The application remains local and binds only to `127.0.0.1` by default. It preserves +the existing visual design while replacing the ad-hoc stdlib servers with a typed, +versioned application API. + +--- + +## 2. Canonical workflow + +```text +0. Inventory & duplicates + ↓ approved canonical set +1. Safety score & human review + ├── SFW assets → content analysis + └── NSFW assets → skip analysis, retain safety EXIF +2. Content analysis & analysis EXIF for SFW assets + ↓ reviewed, metadata-ready assets +3. Album suggestions & guarded rename + ↓ approved final folder layout +4. Immich upload + ↓ verified managed assets +5. Archive + ↓ removed from active storage, retained in the permanent ledger +``` + +Stages are sequential **gates**, not merely navigation tabs. Later stages may be +viewed at any time, but actions are disabled until their prerequisites are met. + +### Gate rules + +| Stage | Ready when | Blocks next stage when | +|---|---|---| +| Inventory | discovery and hashes are complete | unclassified duplicate clusters or missing files remain | +| Safety | every canonical asset is `sfw`, `nsfw`, or explicitly deferred | undecided assets remain unless the user accepts a partial run | +| Analysis | every eligible SFW asset has DB data and verified analysis EXIF | pending/error or EXIF-divergent records remain unless explicitly deferred | +| Albums | every selected album proposal is applied or dismissed | rename plan is pending, partially applied, or inconsistent | +| Upload | final file hashes are current and credentials validate | metadata changed after upload preparation | +| Archive | selected album has a verified upload and validated archive destination | upload is uncertain, files changed, archive target unavailable, or transfer verification failed | + +The app can support a **partial pipeline** for selected folders, but it must show +that the library is only partially complete. + +--- + +## 3. Core architecture + +### Real application, not CLI scripts with attached pages + +The target is a cohesive Python application with a documented API, database-backed +services, persistent jobs, and one frontend. The current CLIs are the primary donor +implementations for the new services: reuse their proven code, algorithms, prompts, +format handling, subprocess adapters, and edge-case behavior wherever compatible with +the target architecture. They are not the final application boundary and will be +archived only after their donated behavior has automated parity coverage. + +```text +Browser + │ JSON API + Server-Sent Events + ▼ +FastAPI application + ├── inventory / duplicate service + ├── thumbnail service + ├── safety service + ├── analysis service + ├── album and rename service + ├── upload service + └── job coordinator + │ claims durable jobs + ▼ +Python worker process ── exiftool / local model / vision API / immich-go + │ + ▼ +SQLite database in WAL mode + managed application cache +``` + +Recommended backend foundation: + +- **FastAPI** for typed HTTP APIs, validation, OpenAPI documentation, and streaming; +- **Pydantic** request/response models so browser input never reaches services raw; +- **SQLAlchemy 2.x** for explicit transactions and repository boundaries; +- **Alembic** for versioned, testable database migrations; +- **SQLite WAL mode** initially, with short write transactions and one job writer; +- a separate Python **worker process** that claims durable jobs from SQLite; +- Server-Sent Events for live job/activity updates, with polling as fallback; +- structured JSON logging carrying `job_id`, `asset_id`, and `operation_id`. + +SQLite is appropriate for a single-user local application. Repository and service +interfaces must avoid SQLite-specific business logic so PostgreSQL remains a possible +future deployment option, not an immediate dependency. + +### Domain services + +Current code should be progressively extracted into importable services: + +- `InventoryService` — discovery, path reconciliation, hashes, duplicate candidates; +- `ThumbnailService` — safe, cached previews for every supported image format; +- `DuplicateService` — clusters, recommendations, decisions, canonical membership; +- `SafetyService` — scoring, review decisions, EXIF safety keywords; +- `AnalysisService` — vision requests, result validation, DB and EXIF writes; +- `AlbumService` — folder summaries, AI proposals, naming policy; +- `RenameService` — plans, validation, journaled apply/recovery; +- `UploadService` — preflight, batches, `immich-go`, verification; +- `ArchiveService` — archive plans, transfer verification, offline state, restore; +- `JobService` — durable job lifecycle, locks, progress, cancellation, events. + +The CLI entry points may call these same services for backwards compatibility. The +web API never shells out to `photo_analyzer.py` or `nsfw_tag.py`; only genuine external +tools such as `exiftool` and `immich-go` remain subprocesses. + +### Donor-first CLI migration and archival + +This is an extraction and consolidation project, not a greenfield rewrite. Before +implementing a replacement service, inspect the corresponding CLI implementation and +classify each relevant function or behavior as **reuse unchanged**, **extract with a +thin adapter**, **refactor while preserving behavior**, or **replace with documented +reason**. Default to extraction; replacement requires a concrete incompatibility such +as unsafe concurrency, path-keyed identity, or inability to support durable jobs. + +Create and maintain a donor ledger mapping legacy modules/functions to their target +service location, characterization tests, migration status, intentional behavior +changes, and final parity test. Important donor material includes: + +- file discovery, extension handling, folder grouping, and path exclusion policy; +- exact/perceptual hashing, duplicate clustering, reconciliation, and resume behavior; +- image decode, orientation, resize, HEIC conversion, and thumbnail behavior; +- vision prompts, request construction, JSON validation, retries, and rate limiting; +- NSFW inference, scoring thresholds, review decisions, and model integration; +- EXIF field projection, preservation rules, subprocess invocation, and read-back; +- SQLite schema knowledge, migrations, FTS behavior, status transitions, and stats; +- existing frontend design tokens, layouts, interactions, keyboard behavior, and + useful browser-side components; +- operational logging, cancellation, error reporting, configuration, and CLI flags. + +For each donor area, first add characterization tests against the current CLI code and +record representative outputs using the deterministic fixture corpus. Extract the +smallest coherent implementation into the new service layer, then run the same fixture +expectations against both implementations. Preserve known behavior unless the concept +explicitly changes it; every intentional difference needs a rationale and a new +acceptance test. Avoid copying code into parallel implementations—after extraction, +the legacy entry point should temporarily import the shared service where practical. + +Archive a CLI script only when all of its in-scope donor rows are resolved, its +replacement passes unit, characterization, integration, and end-to-end parity tests, +database migration/reconciliation is verified, and no supported workflow still +depends on the old entry point. Archival means moving the original sources and their +documentation into a read-only `legacy_cli_archive/` area with their final dependency +lock, schema notes, sample configuration with secrets removed, donor ledger, and a +recorded final version/checksum. Archived scripts are reference material and rollback +evidence; production code must not import or execute them. + +### API boundaries + +All mutations are commands against stable IDs, never raw filesystem paths. Example +API groups: + +| Method | Endpoint | Purpose | +|---|---|---| +| GET | `/api/v1/workflow` | stage readiness, blockers, totals, active job | +| POST | `/api/v1/inventory/scan` | enqueue discovery/reconciliation job | +| GET | `/api/v1/duplicates/clusters` | paged clusters by confidence/state | +| GET | `/api/v1/duplicates/clusters/{id}` | members, thumbnails, evidence | +| POST | `/api/v1/duplicates/clusters/{id}/decision` | canonical/variants/not-duplicate/defer | +| GET | `/api/v1/assets/{id}` | stable asset record and stage history | +| GET | `/api/v1/assets/{id}/thumbnail?size=512` | cached, oriented preview | +| POST | `/api/v1/safety/jobs` | enqueue scoring for eligible assets | +| POST | `/api/v1/safety/decisions` | persist reviewed decisions | +| POST | `/api/v1/analysis/jobs` | enqueue content analysis | +| POST | `/api/v1/albums/proposals` | generate album suggestions | +| POST | `/api/v1/rename-plans/{id}/apply` | apply a validated plan | +| POST | `/api/v1/upload-batches/{id}/start` | start approved Immich upload | +| POST | `/api/v1/archive-plans` | create and validate an archive plan | +| POST | `/api/v1/archive-plans/{id}/apply` | transfer/remove active folder after verification | +| POST | `/api/v1/assets/{id}/restore` | restore an archived asset/album to active storage | +| GET | `/api/v1/jobs/{id}` | durable status and progress | +| POST | `/api/v1/jobs/{id}/cancel` | cooperative cancellation request | +| GET | `/api/v1/events` | SSE activity stream | + +API responses use explicit enums and error codes. Conflicts such as stale hashes, +running jobs, or changed rename plans return `409 Conflict`; validation errors return +`422`; unknown IDs return `404`. OpenAPI becomes the contract between backend and UI. + +### Persistent job model + +Long operations are database jobs, not in-memory threads: + +```text +queued → running → cancelling → cancelled + ↘ succeeded + ↘ failed → retry_queued +``` + +The worker claims a queued job atomically, records a heartbeat, persists incremental +progress and events, and checks cancellation between items. On application restart, +stale `running` jobs are detected from their heartbeat and offered for safe resume. +Mutating job types declare locks such as `library_write`, `rename`, or `upload`, which +the coordinator uses to prevent unsafe concurrency. + +### One durable database + +Extend the current SQLite database rather than introduce separate score and state +files. CSV export may remain available, but it is not authoritative. + +Most importantly, replace path identity with a stable internal asset identifier. + +```sql +CREATE TABLE assets ( + id TEXT PRIMARY KEY, -- generated UUID, never changes + current_path TEXT UNIQUE, -- active-library path; NULL when offline + original_path TEXT NOT NULL, + current_sha256 TEXT, -- current file bytes; refreshed after EXIF + pixel_sha256 TEXT, -- normalized decoded pixels; ignores EXIF + phash TEXT, -- resilient to resize/recompression + hash_version INTEGER NOT NULL, -- decoder/normalization algorithm version + byte_size INTEGER, + discovered_at TEXT NOT NULL, + missing_at TEXT, + + canonical_asset_id TEXT REFERENCES assets(id), + safety_score REAL, + availability_state TEXT NOT NULL DEFAULT 'active', + archive_location_id TEXT, + state_version INTEGER NOT NULL DEFAULT 1 +); + +CREATE TABLE asset_paths ( + asset_id TEXT NOT NULL REFERENCES assets(id), + path TEXT NOT NULL, + valid_from TEXT NOT NULL, + valid_until TEXT, + reason TEXT, + PRIMARY KEY (asset_id, path, valid_from) +); + +CREATE TABLE asset_stage_states ( + asset_id TEXT NOT NULL REFERENCES assets(id), + stage TEXT NOT NULL, + state TEXT NOT NULL, + substate TEXT, + input_version TEXT, + attempt_count INTEGER NOT NULL DEFAULT 0, + active_attempt_id TEXT, + blocker_code TEXT, + error_code TEXT, + error_message TEXT, + updated_at TEXT NOT NULL, + completed_at TEXT, + version INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY (asset_id, stage) +); + +CREATE TABLE asset_stage_attempts ( + id TEXT PRIMARY KEY, + asset_id TEXT NOT NULL REFERENCES assets(id), + stage TEXT NOT NULL, + job_id TEXT NOT NULL REFERENCES jobs(id), + attempt_number INTEGER NOT NULL, + input_version TEXT NOT NULL, + started_at TEXT NOT NULL, + heartbeat_at TEXT, + finished_at TEXT, + outcome TEXT, + last_substate TEXT, + error_code TEXT, + error_message TEXT, + worker_id TEXT, + fencing_token INTEGER NOT NULL, + UNIQUE (asset_id, stage, attempt_number) +); +``` + +Analysis fields can remain in `photos` during migration, but `photos` should gain an +`asset_id` foreign key and eventually use it as its stable relationship. All FTS, +EXIF, duplicate, rename, and upload records refer to `asset_id`, not path. + +### Per-asset workflow state + +Every asset has one durable state row per stage. This is the authoritative answer to +“what happened to this picture?” The broad workflow totals are derived from these rows, +not maintained independently. + +Stages include: + +```text +inventory → duplicate_review → safety → analysis → album → upload → archive +``` + +The common state vocabulary is: + +| State | Meaning | +|---|---| +| `blocked` | prerequisite is incomplete; `blocker_code` explains why | +| `ready` | all prerequisites satisfied | +| `queued` | assigned to a durable job but not started | +| `running` | worker owns a valid lease and attempt | +| `complete` | stage output and required verification are durable | +| `deferred` | user intentionally postponed the stage | +| `skipped` | stage does not apply, with a recorded reason | +| `failed` | attempt ended with a retryable or terminal error | +| `cancelled` | user/system cancelled before completion | +| `interrupted` | worker disappeared or application stopped mid-attempt | +| `stale` | previously complete output no longer matches current inputs | + +`substate` identifies the precise point within a running/interrupted stage, for example: + +```text +safety: decoding | scoring | awaiting_review | writing_exif | verifying_exif +analysis: preparing | calling_provider | validating | storing_result | + writing_exif | verifying_exif +album: summarizing | generating_name | awaiting_approval | renaming | verifying_paths +upload: preflight | hashing | uploading | verifying_remote +archive: preflight | transferring | verifying_archive | removing_active | + verifying_absence | complete_offline +``` + +An interruption is not guessed from a browser disconnect. A stage becomes +`interrupted` only when its active attempt was `running`, the worker heartbeat/lease +expired or the process shut down uncleanly, and no valid newer attempt owns it. The +recovery service records the interrupted attempt, inspects its last substate and real +file/external state, then chooses one of: + +- resume safely from a verified checkpoint; +- restart the stage idempotently; +- mark `unknown_requires_verification` and request user action. + +The immutable `asset_stage_attempts` table preserves every start, retry, interruption, +cancellation, error, and successful completion. The current state row is a projection +for fast UI queries; history is never overwritten. + +### Dependency and invalidation rules + +Stage completion is versioned against its inputs. Examples: + +- a changed file invalidates inventory hashes and may make duplicate review stale; +- changing the duplicate canonical can skip later stages for one asset and ready them + for another; +- changing SFW→NSFW makes analysis `skipped` and invalidates any in-flight result, but + does not invalidate upload eligibility after safety EXIF verification; +- changing NSFW→SFW makes analysis `ready` and upload `blocked` until analysis completes; +- external EXIF changes make the relevant EXIF projection `stale` or `divergent`; +- a rename changes paths but does not invalidate content analysis or safety state; +- any byte-changing EXIF checkpoint makes upload `stale` until its exact hashes are + refreshed. +- archiving changes availability and path state but does not invalidate duplicate, + safety, analysis, EXIF, or upload history; +- restoring an archived asset creates a new active path occurrence and requires a + reconciliation scan before mutation. + +State transitions use optimistic `version` checks and are committed with their audit +event. Workers may only complete the attempt they own with the current fencing token. +There is no generic “set status” endpoint; services expose valid domain transitions. + +### Additional tables + +- `duplicate_clusters` / `duplicate_members` — reviewable cluster decisions. +- `safety_reviews` — score, final decision, reviewer, timestamp, prior decision. +- `analysis_runs` — model/config/prompt version, token usage, status, timestamps. +- `album_proposals` — source folder, proposed name, rationale, confidence, status. +- `rename_plans` / `rename_operations` — complete preview and crash-safe journal. +- `upload_batches` / `upload_items` — command, album, verified content checksums, + result, log. +- `archive_locations` / `archive_plans` / `archive_operations` — target configuration, + manifests, journaled transfers, verification, removal, and restore history. +- `jobs` / `job_events` — shared progress and activity history across all stages. +- `thumbnails` — cache key, dimensions, format, path, generation state, error. + +### Database ownership and transactions + +- API routes call application services; they do not contain SQL. +- Services use repositories; repositories are the only database access layer. +- Every state change and corresponding audit event is written in one transaction. +- Filesystem operations use persisted journals because they cannot participate in a + database transaction. +- SQLite foreign keys are enabled, WAL mode is required, and migrations run once at + startup under an application lock. +- Database backups use SQLite's online backup API rather than copying a live DB file. +- Secrets live in environment/keychain-backed configuration, never database events. + +### EXIF ownership and consistency checkpoints + +EXIF is not a final export performed only before upload. It is a durable projection of +completed workflow decisions. Every stage that produces metadata ends with an EXIF +checkpoint for each affected asset: + +```text +persist desired stage result in DB +→ read current EXIF +→ merge only the stage-owned fields into current metadata +→ write through exiftool +→ read EXIF back +→ verify desired values and preserved values +→ refresh current full-file SHA-256 +→ mark checkpoint verified +``` + +The database stores both the desired projection and the last verified projection. +Suggested table: + +```sql +CREATE TABLE exif_projections ( + asset_id TEXT NOT NULL REFERENCES assets(id), + stage TEXT NOT NULL, -- safety | analysis | album + projection_version INTEGER NOT NULL, + desired_json TEXT NOT NULL, + verified_json TEXT, + source_file_sha256 TEXT NOT NULL, + result_file_sha256 TEXT, + state TEXT NOT NULL, -- pending | writing | verified | divergent | error + verified_at TEXT, + PRIMARY KEY (asset_id, stage) +); +``` + +Field ownership is intentionally narrow: + +| Owner | May modify | Must preserve | +|---|---|---| +| User/pre-existing metadata | nothing automatically owns it | all dates, GPS, camera, ratings, people, captions, and keywords | +| Safety stage | only mutually exclusive `nsfw` / `sfw` entries in `Keywords` and `Subject` | all non-safety keywords and all other fields | +| Analysis stage | managed AI caption segment plus additive AI keywords | user caption text, safety keywords, existing keywords, and unrelated metadata | +| Album stage | no EXIF by default; optional managed album field only when enabled | every existing field | + +Non-destructive means **semantic preservation**, not that the file bytes remain +unchanged: `exiftool` may rewrite the container. Before writing, retain the relevant +EXIF snapshot in the operation journal; afterward, compare all fields outside the +stage's ownership. Any unexpected change marks the asset `divergent`, blocks the next +mutating stage, and exposes a recovery action. + +Analysis captions should use a recognizable managed segment so re-analysis can replace +only its own text, for example an application-owned `AI:` portion, without overwriting +a user's description. Keywords are merged as a set. Because standard EXIF keywords do +not record ownership, removing an old AI keyword could also remove a user keyword with +the same value. The conservative default is therefore additive: never automatically +remove non-safety keywords during re-analysis. Potentially stale generated keywords +are reported for optional review rather than silently deleted. + +Stage consistency rules: + +- **Inventory/duplicate stage:** read-only; records an EXIF fingerprint and proves it + did not modify the file. +- **Safety stage:** complete only after the chosen `sfw` or `nsfw` keyword is written, + the opposite safety keyword is absent, all unrelated metadata is preserved, and the + new file SHA-256 is stored. +- **Analysis stage:** applies only to SFW assets and completes per asset only after its + DB result, managed caption segment, additive keywords, preservation checks, and new + file SHA-256 are verified. +- **Album/rename stage:** renaming does not itself require an EXIF rewrite. If optional + album metadata is enabled, it receives its own projection/checkpoint after rename. +- **Upload stage:** writes no metadata. It verifies every required prior projection, + recalculates the current file SHA-256, and uploads exactly those verified bytes. + +If a user edits EXIF externally after a checkpoint, the next scan compares the current +projection/hash with the verified state. User additions are preserved and incorporated; +missing or conflicting managed values create a visible `divergent` checkpoint requiring +repair or explicit acceptance. Upload never silently repairs metadata as a side effect. + +--- + +## 4. Stage 0 — Inventory and duplicate review + +### Discovery + +Discovery registers supported photos without modifying them. It records current +path, size, dates, SHA-256, normalized-pixel hash, and perceptual hash. It always skips `_IGNORE/` and generated +thumbnail/cache directories. + +The inventory screen shows: + +- total files and folders; +- exact duplicates by SHA-256; +- perceptual duplicate clusters by phash distance; +- likely variants such as edited, resized, or motion-photo derivatives; +- unreadable and unsupported files; +- estimated API calls saved by resolving duplicates. + +### Duplicate review + +Do not immediately mark the largest file canonical without review. Exact byte or +normalized-pixel matches can be recommended with very high confidence; every fuzzy +match receives a visual decision surface. + +### Viable thumbnail pipeline + +Duplicate review must never depend on the browser being able to render the original. +The backend generates managed thumbnails for JPEG, PNG, WebP, TIFF, HEIC/HEIF, and +other supported image types. + +Thumbnail generation rules: + +1. read the original through Pillow plus the configured HEIC decoder; +2. apply EXIF orientation before resizing; +3. convert safely to sRGB/RGB without modifying the original; +4. generate at least `256px`, `512px`, and `1280px` long-edge variants; +5. encode browser-compatible WebP, with JPEG fallback; +6. write to a temporary file and atomically rename into the cache; +7. persist success/error state so broken originals do not create retry loops. + +The cache lives in application data outside the photo library, for example: + +```text +data/cache/thumbs/{pixel_hash_prefix}/{pixel_hash}-512-v2.webp +``` + +The key contains the normalized pixel hash, requested size, and thumbnail algorithm +version. Therefore: + +- EXIF-only changes reuse the same thumbnail; +- folder moves and renames reuse the same thumbnail; +- visually different files never collide merely because filenames match; +- changing decoder/color/orientation behavior invalidates cache through the version; +- cache files can be deleted and regenerated without losing application state. + +Thumbnail endpoints accept only an `asset_id` and bounded size enum. They resolve the +current path through the database, set immutable-cache headers for versioned keys, and +never accept an arbitrary path from the browser. + +### Fuzzy duplicate comparison UI + +Show uncertain clusters in a dedicated comparison workspace rather than a small grid: + +- synchronized side-by-side thumbnails, initially fit-to-window; +- click/keyboard toggle between candidates; +- linked zoom and pan using the `1280px` preview; +- optional blink/overlay mode to reveal framing and crop differences; +- pixel dimensions, aspect ratio, file size, format, dates, folder, camera, EXIF + richness, file hash, pixel hash, pHash distance, and confidence explanation; +- badges for exact bytes, metadata-only difference, resized/recompressed copy, + probable crop, edited variant, or uncertain similarity; +- recommended canonical clearly marked, but never silently selected for fuzzy matches; +- prefetch the next cluster's thumbnails for fast keyboard review. + +The decision applies to a complete cluster and is stored as an auditable event. If a +new copy appears later, the cluster is reopened only when it changes the confidence or +conflicts with the previous decision; otherwise the new member inherits the established +canonical relationship. + +The app recommends a canonical item using a visible scoring policy: + +1. original/highest resolution; +2. least recompressed format; +3. richest reliable metadata; +4. preferred folder rules; +5. largest file only as a final tie-breaker. + +Actions: + +- **Keep recommended**; +- **Choose another canonical**; +- **Keep all as variants**; +- **Not duplicates**; +- **Defer**. + +No file is deleted automatically. Non-canonical items are excluded from later stages +and upload, but remain on disk unless a separate future cleanup feature is approved. + +--- + +## 5. Stage 1 — Safety identification and EXIF tagging + +Preserve the current NSFW review experience: folder tree, threshold, score ordering, +bulk selection, per-image decision control, lightbox, EXIF panel, and activity log. + +Changes for integration: + +- run only on canonical, non-ignored assets; +- store score and decisions in SQLite as well as EXIF; +- write mutually exclusive `nsfw` or `sfw` keywords to `Keywords` and `Subject`; +- merge safety keywords with all existing EXIF and verify unrelated metadata remains; +- refresh `current_sha256` after the verified EXIF checkpoint; +- replace `nsfw_scores.csv` as the source of truth with database state; +- make `_IGNORE/` exclusion mandatory in the shared discovery service; +- offer filters for undecided, SFW, NSFW, deferred, and write errors. + +Privacy invariant: + +> Only assets with a confirmed `sfw` decision may enter cloud content analysis. +> Confirmed `nsfw` assets skip that analyzer but remain eligible for Immich upload. + +An optional setting may treat unreviewed low-score images as SFW, but it should be +off by default and clearly marked as reducing safety. + +--- + +## 6. Stage 2 — Content analysis and analysis EXIF + +Preserve the current Library, Analyze, and Stats views, but scope the engine to +confirmed SFW canonical assets. + +For each eligible asset: + +1. prepare a resized in-memory image; +2. call the configured OpenAI-compatible vision model; +3. validate and store the response by `asset_id`; +4. read the latest EXIF and merge the managed AI caption segment and additive tags; +5. write and read back EXIF without removing safety or user metadata; +6. compare non-owned fields with the pre-write snapshot; +7. refresh the full-file SHA-256 after verification; +8. mark the analysis EXIF checkpoint verified and the asset metadata-complete. + +The UI shows model and prompt version, progress, errors, retries, tokens, estimated +remaining cost/quota, and EXIF verification state. + +The stage is complete only when every included database result and on-disk EXIF +projection agree. Verification is per asset, not merely a sample; the UI may additionally +offer a human-readable sample review before proceeding. + +--- + +## 7. Stage 3 — AI album suggestions and guarded renaming + +This stage operates primarily on **folders**, not individual filenames. The default +goal is to create useful Immich albums while minimizing filesystem changes. + +### Inputs for an album proposal + +For each leaf folder, build a compact summary from: + +- current folder name and parent path; +- capture-date range and dominant year; +- number of photos and videos; +- common analysis tags, settings, places, and descriptions; +- GPS/location hints where present; +- detected events or themes; +- neighboring folder names and date ranges; +- existing user-provided naming conventions. + +Prefer using already-stored analysis data. Do not resend every image. If visual +sampling is useful, make it an explicit option using a few representative SFW images. + +### Suggested naming format + +The default recommendation should be predictable and sortable: + +```text +YYYY-MM — Place — Event +YYYY — Event +Place — Event +``` + +Examples: + +```text +2019-07 — Rome — Summer Holiday +2016 — Capgemini Team Event +Family Portraits — Selected +``` + +Names should be concise, human-readable, filesystem-safe, and suitable for Immich's +folder-as-album behavior. The app shows why each name was proposed and flags uncertain +date/place inferences. + +### Proposal review UI + +Use a two-pane Albums view: + +- left: folder tree with proposal state and confidence; +- right: current name, proposed name, representative thumbnails, date range, common + tags, rationale, and editable final name. + +Bulk actions may accept high-confidence proposals, but never apply them immediately. +Accepted edits first become a **rename plan**. + +### Guarded rename plan + +Before any filesystem operation, validate the entire plan: + +- source exists and is still the registered folder; +- destination is inside the configured library; +- neither source nor destination is inside `_IGNORE/`; +- destination names are unique on the target filesystem; +- case-only renames are handled safely on case-insensitive filesystems; +- no asset is currently being analyzed, EXIF-written, or uploaded; +- final paths do not exceed configured portability limits; +- all affected assets are present and hashes match the latest inventory; +- database and filesystem are writable; +- no previously uploaded asset would be silently changed. + +The preview displays every old → new path and supports exporting the plan as JSON. +Applying requires an explicit confirmation such as **Apply 12 folder renames**. + +### Crash-safe rename transaction + +Filesystem and SQLite cannot share a true transaction, so use a journaled state +machine: + +```text +planned → validated → moving → moved → database_updated → verified → complete + ↘ failed / rollback_required +``` + +For each operation: + +1. persist the operation and expected hashes; +2. rename on the same filesystem using an atomic move where possible; +3. update `assets.current_path` by `asset_id` in one SQLite transaction; +4. append old/new entries to `asset_paths`; +5. verify all files at their new paths and refresh folder summaries; +6. mark the operation complete. + +On restart, the app inspects the journal and filesystem to resume or offer rollback. +It never guesses based only on a missing old path. + +Because records use stable asset IDs, renaming does not invalidate analysis, safety, +duplicate, or upload history. + +--- + +## 8. Stage 4 — Immich upload via immich-go + +Upload is an explicit final stage, never an automatic consequence of rename. + +### Safety and upload eligibility + +Safety classification controls **analysis routing**, not whether a reviewed photo may +exist in Immich: + +| Safety state | Cloud content analysis | Immich upload | +|---|---|---| +| `sfw` | eligible/required by normal workflow | eligible after verified safety and analysis EXIF checkpoints | +| `nsfw` | forbidden | eligible after verified `nsfw` EXIF tag | +| `unreviewed` | forbidden | blocked until reviewed | +| `deferred` | forbidden | blocked unless an explicit future policy permits it | + +Immich therefore receives both reviewed SFW and reviewed NSFW canonical assets. The +`nsfw` keyword is written before upload so those assets can be found, filtered, or +managed in Immich without exposing them to the external content-analysis provider. + +### Preflight + +Before enabling upload, verify: + +- asset is canonical and has a confirmed `sfw` or `nsfw` safety decision; +- SFW assets have verified safety and analysis EXIF checkpoints, unless explicitly + waived with a recorded exception; +- NSFW assets carry their verified `nsfw` EXIF keyword and have not been sent to the + cloud content analyzer; +- rename journal is empty and album paths are final; +- current SHA-256 matches the latest verified EXIF checkpoint and post-rename inventory; +- Immich server and API key validate; +- `immich-go` is installed; +- no External Library workflow is configured for the same source; +- the album name preview matches the folder-as-album result. + +### Upload batches + +Create one batch per selected leaf folder. The UI previews the exact command without +printing the API key. A batch stores: + +- album/folder and asset IDs; +- redacted command and immich-go version; +- pre-upload SHA-256 plus the SHA-1 used by Immich/immich-go for every item, both + calculated from the latest verified bytes; +- stdout/stderr activity stream; +- new, upgraded, metadata-updated, duplicate, skipped, and failed counts; +- start/end timestamps and exit status. + +The Upload view provides **Validate**, **Dry run** where supported, **Upload selected**, +**Stop after current album**, and **Retry failed** actions. + +After a successful upload, an asset becomes `uploaded` only when its result is parsed +or otherwise verified. If the file bytes later change, the app marks it +`changed_after_upload` and warns that uploading again may create or upgrade an asset. + +--- + +## 9. Stage 5 — Archive and active-storage reclamation + +Archive is an optional final lifecycle stage. It moves a completed album out of the +active library branch to a configured archive location, then marks its assets archived +and unavailable from active storage while preserving their permanent database records. + +“Archived” means **no longer present in the active photo library**. The archive may be +another local disk, external disk, NAS-backed archive workflow, or removable medium. +When that destination is not mounted, the asset is `archived_offline`, not `missing`. + +Archive is not deduplication and never deletes database history. It preserves: + +- stable asset ID and complete path history; +- exact file SHA-256 and Immich SHA-1 from the archived bytes; +- normalized pixel hash and perceptual hashes; +- duplicate cluster and canonical decision; +- safety decision and verified EXIF projections; +- content analysis, FTS data, album proposal, rename history, and upload history; +- a durable browser-compatible review thumbnail, preferably the `1280px` variant; +- archive location, relative archive path, media/volume identity, and transfer manifest. + +### Availability states + +Availability is independent from workflow completion: + +| State | Meaning | +|---|---| +| `active` | original is present in the active library | +| `archiving` | journaled transfer/removal is in progress | +| `archived_online` | original exists at a currently reachable archive location | +| `archived_offline` | original is archived but its medium/location is unavailable | +| `restoring` | copy back to active storage is in progress | +| `missing_unexpected` | neither active nor recorded archive location explains absence | + +The ordinary inventory scanner must not prune archived assets. It scans active roots +and currently mounted archive roots separately and interprets absence using +`availability_state` and archive-location identity. + +### Archive preflight + +Before allowing an album to be archived: + +- every selected canonical asset has a confirmed safety decision; +- every SFW asset has verified analysis EXIF and every NSFW asset has verified safety + EXIF; +- the selected upload batch is verified, not merely process-successful; +- current file checksums match the exact uploaded/checkpointed bytes; +- no rename, EXIF, analysis, upload, restore, or other archive job holds a conflicting + lease; +- the archive destination is configured, mounted, writable, outside `_IGNORE/`, and + has sufficient free space plus a reserve; +- the destination does not already contain unexpected paths; +- the durable comparison thumbnail exists and is verified before the original leaves + active storage; +- a database backup and archive manifest can be written successfully. + +The UI previews source, destination, number of files, total bytes, expected reclaimed +active space, transfer method, archive medium, and every blocker. Archive requires an +explicit confirmation such as **Archive 1 album · reclaim 84.3 GB**. + +### Transfer and removal semantics + +Archive uses a journaled plan similar to rename, but cross-filesystem movement is +expected: + +```text +planned → validated → transferring → archive_verified → removing_active + → active_absence_verified → database_updated → complete +``` + +For each file: + +1. persist source, archive destination, expected size and hashes; +2. copy to a temporary destination file; +3. flush/close it and calculate its SHA-256; +4. atomically rename the verified temporary file to its archive destination; +5. record the verified archive occurrence and manifest entry; +6. only then remove the active source; +7. verify the active path is absent and archive path still matches; +8. set `current_path=NULL`, close the active path-history entry, and set availability + to `archived_online` or `archived_offline` according to location reachability. + +If source and archive destination are demonstrably on the same filesystem, an atomic +move may replace copy-verify-delete, but the operation still verifies the resulting +file and persists the journal before updating database state. + +The source is never removed merely because Immich reports success. Immich managed +storage is an ingest target, not automatically the only backup. A future explicit +“uploaded-only retention” policy may exist, but it must be a separate high-risk option +and is not the default Archive behavior. + +### Future deduplication against archived assets + +Archived assets remain in every hash index. When a new active photo is discovered: + +- exact SHA-256 or normalized-pixel matches can link directly to the archived canonical; +- perceptual matches create/reopen a fuzzy duplicate cluster; +- the retained thumbnail, metadata, dimensions, and hash evidence appear in duplicate + review even when the archive disk is offline; +- if full-resolution comparison is required, the UI requests that the named archive + location/medium be mounted rather than guessing; +- choosing the archived asset as canonical keeps the new active copy excluded from + analysis/upload unless the user deliberately restores or changes the canonical. + +This is why durable comparison thumbnails are part of archival preflight rather than +an evictable cache entry. They may use a separate protected archive-preview store with +its own backup policy. + +### Restore + +Restore is also planned and journaled. It requires the archive location online, copies +and verifies the archived bytes into a collision-free active destination, registers a +new active path occurrence, changes availability to `active`, and runs reconciliation. +Existing safety, analysis, EXIF, duplicate, and upload states remain valid when hashes +match. Any mismatch produces `stale`/`divergent` state instead of silently accepting a +different file. + +--- + +## 10. Information architecture and UX + +Retain the dark OLED design, cards, folder sidebar, lightbox, segmented controls, +progress bars, status colors, activity drawer, and keyboard navigation. The visual +language remains familiar even though the frontend is now an API client rather than +HTML generated by Python. + +### Frontend architecture + +Keep the first implementation deliberately lightweight and dependency-free: semantic +HTML, CSS, and modular browser JavaScript served as separate static assets by FastAPI. +`index.html` contains only the persistent application shell and references the +stylesheets and JavaScript entry point; it does not embed the application's CSS, +JavaScript, cards, or operational data. Organize JavaScript by API client, state/store, +views, and reusable components rather than putting everything in one HTML file or one +large script. If the duplicate comparison and workflow state outgrow this approach, +the same versioned API can support a later TypeScript framework without changing the +backend. + +Frontend rules: + +- no Python string interpolation of cards or operational state; +- `index.html`, CSS, and JavaScript are separate source files; +- the initial HTML response is only the application shell; JavaScript fetches all + operational data from `/api/v1` after the shell loads; +- `/api/v1` endpoints return JSON (and SSE only for the documented event stream), + never HTML fragments or the application shell; API errors use a consistent JSON + error envelope and appropriate HTTP status codes; +- HTML navigation routes may fall back to `index.html`, but requests under `/api/v1` + must never use that fallback; +- one shared API client handles errors, cancellation, and optimistic concurrency; +- SSE updates the job/activity store; failed streams fall back to polling; +- URL routes encode the current view, cluster, asset, album, and filters; +- destructive confirmations include the server-issued plan/version token so stale + browser state cannot apply changed operations. + +### Global header + +```text +[Photo Pipeline] Library: pictures/ [Workflow] [Review] [Library] [Albums] [Upload] [Archive] + 25,318 files · 21 warnings · no job running [Activity] +``` + +### Workflow home + +The default page is a five-step vertical or horizontal stepper: + +```text +✓ 0 Inventory 24,901 canonical · 417 duplicates +! 1 Safety 24,620 SFW · 83 NSFW · 198 undecided +● 2 Analysis 20,110 complete · 4,510 pending +○ 3 Albums blocked by analysis · 43 suggestions ready +○ 4 Upload not ready +○ 5 Archive blocked by upload · 0 B reclaimable +``` + +Each stage card contains status, prerequisite warnings, last-run time, primary action, +and a link to details. Status is communicated with text/icons as well as color. + +### Shared interaction rules + +- one long-running mutating job at a time; +- read-only browsing remains available during jobs; +- every bulk mutation has preview → validate → confirm → execute → verify; +- stop requests drain safely and leave a resumable job record; +- every error links to the affected asset, album, or operation; +- all actions are scoped to the visible selection and show exact counts; +- credentials are never returned to the browser or written to logs; +- all paths served by image/EXIF endpoints are validated by asset ID and current path. + +--- + +## 11. Consistency rules + +The integrated app enforces these invariants centrally: + +1. `_IGNORE/` is excluded by one shared path-policy function used everywhere. +2. Duplicate resolution precedes safety scoring and paid analysis. +3. NSFW assets never reach the external vision API, but remain eligible for Immich + upload after their safety EXIF keyword is verified. +4. Safety keywords survive content EXIF writes. +5. Every metadata stage ends with a non-destructive, read-back-verified EXIF checkpoint; + upload uses the SHA-256 of the latest verified bytes. +6. Rename plans cannot run concurrently with analysis, EXIF, or upload jobs. +7. Asset identity is stable across moves and renames. +8. Upload history is tied to asset ID plus the exact uploaded SHA-256 and Immich SHA-1. +9. Changing bytes after upload produces a visible stale-upload warning. +10. No automatic deletion, rename, or upload occurs without explicit approval. +11. Archiving never removes an active source until the archive bytes and durable + manifest are verified. +12. Archived assets remain in deduplication indexes even when their original storage + is offline. + +--- + +## 12. Proposed project layout + +```text +pipeline_app/ +├── pyproject.toml +├── alembic.ini +├── migrations/ +├── src/photo_pipeline/ +│ ├── __main__.py # API/worker management CLI +│ ├── config.py # typed configuration and secret references +│ ├── db.py # engine, sessions, WAL/foreign-key setup +│ ├── models/ # SQLAlchemy persistence models +│ ├── schemas/ # Pydantic API contracts +│ ├── repositories/ # database access only +│ ├── services/ +│ │ ├── inventory.py +│ │ ├── thumbnails.py +│ │ ├── duplicates.py +│ │ ├── safety.py +│ │ ├── analysis.py +│ │ ├── albums.py +│ │ ├── renames.py +│ │ ├── uploads.py +│ │ ├── archives.py +│ │ └── workflow.py +│ ├── jobs/ +│ │ ├── coordinator.py # enqueue, locks, cancellation, recovery +│ │ ├── worker.py # durable worker loop +│ │ └── handlers.py # job-type dispatch +│ ├── api/ +│ │ ├── app.py # FastAPI factory and lifecycle +│ │ ├── dependencies.py +│ │ ├── errors.py +│ │ └── routes/ # versioned /api/v1 route modules +│ ├── integrations/ +│ │ ├── exiftool.py +│ │ ├── vision.py +│ │ ├── nsfw_model.py +│ │ └── immich_go.py +│ └── path_policy.py # library boundary and _IGNORE rules +├── frontend/ +│ ├── index.html # markup-only application shell +│ ├── css/ +│ │ └── app.css # shared design tokens, layout, components +│ └── js/ +│ ├── app.js # module entry point and application bootstrap +│ ├── api.js +│ ├── store.js +│ ├── router.js +│ ├── components/ +│ └── views/ +├── legacy_cli_archive/ # frozen donor sources after verified parity +│ ├── README.md # provenance, final versions, and archive rationale +│ └── donor-ledger.yaml # source → service/test mapping and intentional deltas +├── tests/ +│ ├── unit/ +│ ├── integration/ +│ └── e2e/ +└── data/ # runtime DB/cache; ignored by source control + └── cache/thumbs/ +``` + +Existing CLIs remain available during migration. Their proven internals should be +extracted into the new service layer, after which the CLI entry points should call +those shared services until the archival gates above pass. Business logic must have +one implementation shared by API, worker, and transitional CLI. This avoids both a +needless rewrite and two subtly different discovery, path-exclusion, EXIF, or state +models surviving in parallel. + +--- + +## 13. Recommended implementation phases + +### Automated acceptance policy for every phase + +Automated tests are part of implementation, not deferred hardening. Every user story +must have at least one automated acceptance test that exercises its observable +behavior and failure conditions. Maintain a traceable mapping from story identifier +to test name/path; a story is incomplete while its mapped tests are missing, skipped, +or failing. + +Every phase below must add and pass an end-to-end suite covering the user journeys +introduced or changed by that phase through the real browser, HTTP/SSE API, worker, +isolated database, and temporary photo library. External services and executables may +use controlled fakes at their integration boundaries, but browser-visible behavior +and backend orchestration must not be mocked. The full end-to-end regression suite for +all completed phases must also remain green. A phase cannot be declared complete or +merged merely because its unit or API tests pass. + +### Phase A — Shared identity and inventory + +- inventory both CLI codebases into the donor ledger and add characterization tests + before replacing or relocating any in-scope behavior; +- extract reusable discovery, hashing, reconciliation, image/EXIF, model, database, + configuration, and UI donor components into stable modules with provenance noted; +- scaffold FastAPI, SQLAlchemy, Alembic, configuration, and application lifecycle; +- add stable asset IDs and path history; +- centralize discovery and `_IGNORE/` policy; +- migrate current photo and NSFW state; +- implement normalized pixel hashes plus exact and perceptual indexes; +- implement managed thumbnail generation and its asset-ID API; +- build inventory and visual duplicate-review UI. + +**End-to-end gate:** automated journeys cover scan/reconciliation, `_IGNORE/` +exclusion, stable identity across a move, thumbnail loading/orientation, exact and +fuzzy duplicate clusters, canonical selection, and persistence after reload. Donor +characterization and parity suites pass for every CLI behavior migrated in this phase. + +This is the foundation. Do not implement renaming before it is complete. + +### Phase B — Unified workflow shell + +- implement durable jobs, worker heartbeats, recovery, locks, and SSE events; +- create Workflow home and shared job/activity model; +- port the existing safety review to the shared APIs; +- port the existing Library/Analyze/Stats views to the shared APIs; +- enforce gates and one-mutating-job policy. + +**End-to-end gate:** automated journeys cover the workflow shell loading data from +JSON APIs, job start/progress/reconnect/cancel/resume, activity SSE with polling +fallback, safety decisions, Library/Analyze/Stats behavior, prerequisite blockers, +and rejection of a second concurrent mutating job. + +### Phase C — Album proposals + +- aggregate folder metadata from existing DB results; +- define configurable naming templates and forbidden characters; +- add AI proposal generation, rationale, confidence, editing, and approval. + +**End-to-end gate:** automated journeys cover proposal generation, editing, approval, +invalid names, collisions, provider failures, stale proposal versions, and persistence +after reload. + +### Phase D — Guarded renaming + +- implement plan validation and JSON export; +- implement journaled rename/apply/verify; +- test interruption at every state transition; +- add recovery and rollback UI. + +**End-to-end gate:** automated journeys cover preview/validate/confirm/apply/verify, +stale confirmation rejection, collisions and case-only renames, interruption at each +journal state, restart recovery, rollback, and refusal to overwrite unexpected files. + +### Phase E — Immich upload + +- add credential preflight and redacted command preview; +- run one album at a time through `immich-go`; +- parse and persist reports; +- add retry, verification, and changed-after-upload warnings. + +**End-to-end gate:** automated journeys cover credential preflight without secret +leakage, album scope and command preview, successful upload, exact duplicate, upgrade, +retryable failure, uncertain outcome verification, cancellation/resume, and stale-byte +blocking after EXIF changes. + +### Phase F — Archive lifecycle + +- configure archive locations and stable volume/media identities; +- implement archive preview, capacity checks, and protected thumbnails; +- implement journaled copy-verify-remove and same-filesystem move paths; +- retain offline assets in hash indexes and duplicate review; +- implement mount detection, restore plans, and recovery UI; +- test interruption before and after source removal. + +**End-to-end gate:** automated journeys cover archive preview and capacity blockers, +copy/hash verification, same- and cross-filesystem paths, offline browsing through +protected thumbnails, interruption before and after source removal, mount changes, +restart recovery, restore, and collision-safe restoration. + +### Phase G — Hardening + +- resolve every donor-ledger row, document intentional deltas, and archive the CLI + sources only after their replacement parity and end-to-end gates pass; +- migrate remaining CSV state into SQLite; +- add end-to-end tests with a temporary library and fake uploader; +- test API authorization assumptions, path validation, stale versions, and malformed + requests; +- test thumbnail orientation, HEIC decoding, corruption, regeneration, and cache + invalidation; +- test case-only renames, collisions, interruptions, EXIF failures, and stale hashes; +- document backup and recovery procedures. + +**End-to-end gate:** the complete automated story matrix passes as one reproducible +command against a fresh temporary environment, including fault injection, restart +recovery, authorization/path attacks, malformed inputs, supported image formats, and +the full discovery-to-upload-and-archive smoke journey. + +--- + +## 14. Definition of done + +The integrated application is successful when a new folder can be taken from discovery +to verified Immich upload and optional archive without using separate commands, while +still allowing every stage to be stopped, reviewed, resumed, restored, or rerun safely. +At every point the UI can +answer: + +- What stage is this asset or album in? +- Why is it blocked? +- What will the next action change? +- Can that change be reversed? +- Which exact bytes were uploaded to Immich? +- Where is the original now, and how can it be restored? + +In addition, every user story is linked to a passing automated acceptance test, every +phase's end-to-end gate passes, and the accumulated end-to-end regression suite runs +successfully from a clean environment with one documented command. Manual testing is +exploratory evidence only and never substitutes for these automated checks. Every +legacy CLI donor-ledger row is resolved, all intentional behavior changes are tested +and documented, and the frozen CLI archive is sufficient to trace the origin of reused +behavior without being a runtime dependency. + +--- + +## 15. Architecture challenge and risk register + +The design is viable, but it can fail in several predictable ways. These risks should +be treated as design inputs, not deferred hardening. + +### Critical risks + +| Risk | Failure mode | Required mitigation | +|---|---|---| +| False duplicate decision | distinct burst/cropped photos are grouped and excluded | auto-resolve only exact byte/pixel matches; fuzzy matches require review; decisions reversible | +| Identity ambiguity | an old path disappears and several matching new paths appear | preserve the old asset; create occurrences; ask which path is canonical; never infer deletion | +| Files change during work | user/editor modifies or moves a file after validation | stat/hash before and after; optimistic asset version; fail stale operations instead of continuing | +| SQLite writer contention | many worker threads produce `database is locked` or long stalls | one serialized DB-write channel, short transactions, bounded queue, busy timeout, backpressure | +| Crash between filesystem and DB | rename succeeds but DB update does not, or EXIF partly completes | durable operation journal, idempotent recovery, pre/postconditions, never hold DB transaction during tool call | +| Duplicate job execution | stale heartbeat causes the same item to run twice | leases, idempotency keys, compare-and-set transitions, idempotent handlers; assume at-least-once delivery | +| Wrong upload state | `immich-go` output changes or process dies after server accepted files | store exact pre-upload hash, raw report, versioned parser; use `unknown_requires_verification`, not success | +| Archive data loss | active source is removed before archive bytes are durable | copy, close, hash-verify, manifest, then remove; journal every item; never overwrite unexpected destinations | +| Offline canonical uncertainty | fuzzy match targets an archived original that cannot be opened | retain protected preview/evidence; require archive mount for full-resolution decisions | +| Resource exhaustion | large images/high concurrency exhaust RAM, file descriptors, GPU, or disk | bounded pools and queues, pixel limits, cache quota, admission control, metrics | +| External path access | symlink or race escapes the configured library | resolve and validate at operation time, reject symlink escapes, use asset IDs, repeat validation before writes | +| Local web attack | another page triggers mutations through localhost | random session token, strict Origin/Host checks, SameSite cookie, CSRF protection, no permissive CORS | + +### Deduplication risks + +- Perceptual hashes have false positives, particularly for screenshots, blank images, + graphics, bursts, and similarly framed portraits. +- They have false negatives for crops, rotations, overlays, strong color edits, or + decoder differences. +- Normalized pixel hashes can change when color-management or decoder versions change. +- Automatically attaching a newly discovered file to an already-reviewed cluster can + propagate an old mistake. +- Selecting one canonical can discard useful metadata present only on another copy. + +Mitigations: + +- store hash algorithm and decoder versions; +- keep exact, normalized-pixel, and perceptual evidence separately; +- define confidence bands rather than a single pHash threshold; +- require visual review for fuzzy matches and for cluster merges; +- preserve all files and make canonical decisions reversible; +- show metadata differences and optionally merge approved metadata before exclusion; +- reopen a cluster when a new member contradicts its existing dimensions/date/evidence; +- keep `not_duplicate` negative links so rescans do not repeatedly suggest rejected pairs. + +### Filesystem and EXIF risks + +- A file can be moved, replaced, truncated, or edited between discovery and action. +- `exiftool` can rewrite the whole container, fail after creating a temporary file, or + behave differently by format. +- Filesystems may be case-insensitive, normalization-sensitive, remote, or lack atomic + directory rename guarantees. +- Symlinks can redirect a previously validated path. +- File timestamps can change due to metadata writes and cannot identify content. +- Rename rollback may be impossible if another process creates the old destination. + +Required behavior: + +- store a lightweight snapshot `(device, inode where useful, size, mtime_ns, hash, + asset_version)` when planning; +- revalidate immediately before mutation and again afterward; +- perform EXIF writes one asset at a time with a durable item state; +- verify safety keywords and analysis fields by reading EXIF back; +- treat cross-filesystem moves as copy-verify-delete and require stronger confirmation; +- refuse rename rollback when either path no longer matches recorded preconditions; +- never overwrite an unexpected destination; +- mark ambiguous outcomes for manual recovery instead of guessing. + +### External integration risks + +- Vision providers rate-limit, return malformed JSON, change model behavior, or block + content despite an SFW review. +- A model or prompt upgrade makes old and new descriptions inconsistent. +- NSFW model downloads or GPU libraries may be unavailable. +- `immich-go` flags and report text can change between versions. +- Immich may accept an upload even if the local process loses its response. + +Mitigations: + +- persist provider/model/prompt/schema versions with every result; +- validate structured responses and quarantine invalid results; +- use bounded retries with jitter and a circuit breaker, never infinite retry; +- pin and record supported `immich-go` versions; +- keep raw subprocess output and parse through version-specific adapters; +- represent uncertain external outcomes explicitly and provide verification/retry tools; +- make every external step restartable at item level. + +### Migration and operational risks + +- Importing the current path-keyed DB can accidentally merge records or lose FTS data. +- Running old CLIs while the new app is active can bypass locks and mutate files. +- Database corruption or an incorrect migration can strand the whole library state. +- Logs, raw model responses, and thumbnails can expose private information. + +Mitigations: + +- migration begins with a verified online backup and produces a reconciliation report; +- run migrations transactionally where SQLite permits and test downgrade/recovery; +- add a library-level application lock understood by both new and migrated CLI paths; +- warn or refuse when an incompatible legacy process is detected; +- keep application data owner-readable only and support log/thumbnail retention limits; +- provide database integrity checks and backup restore drills. + +--- + +## 16. Concurrency, locking, and race-condition model + +High internal concurrency is allowed for image decoding, hashing, inference, and API +calls. It must not translate into uncontrolled database or filesystem concurrency. + +### Concurrency lanes + +```text +Scanner/read pool bounded threads, read-only filesystem work +CPU image pool bounded processes for decode/hash/thumbnail work +GPU safety lane normally one process/bounded batch queue +Vision API pool bounded async requests with rate limiter +Database writer one serialized writer queue +Filesystem mutator one library-wide lane for EXIF/rename operations +Uploader one active album batch by default +Archiver one active archive plan by default +``` + +Defaults must derive from resources, not a single global `MAX_WORKERS`. For example, +thumbnail decoding may allow four workers while GPU inference allows one and network +analysis allows twenty. Every lane has a bounded queue; when full, producers block or +pause rather than allocating indefinitely. + +### Database rules + +1. SQLAlchemy sessions are never shared across threads or tasks. +2. Worker threads produce result messages; a dedicated writer persists them in small + batches or individual short transactions. +3. No transaction remains open while hashing, decoding, calling a model, running + `exiftool`, moving files, or uploading. +4. Reads use separate short-lived sessions. Long UI queries are paged and cannot hold + a WAL read transaction indefinitely. +5. Job claiming is an atomic compare-and-set update from `queued` to `running` with a + lease owner and expiry. +6. Asset updates include `WHERE id=? AND version=?`; zero updated rows means stale data + and produces a conflict/retry decision. +7. The write queue has a maximum depth and exposes pressure metrics. +8. WAL checkpoints are monitored; repeated checkpoint starvation is an operational + warning, not silently ignored. + +SQLite still has one writer. If sustained write pressure or UI latency exceeds agreed +limits after batching and backpressure, PostgreSQL becomes a planned scale transition. +Adding more SQLite writer threads is not the solution. + +### Lock hierarchy + +Locks have a fixed acquisition order to prevent deadlocks: + +```text +library lease → stage/job lease → album/folder lease → asset lease +``` + +- Never acquire a broader lock while holding a narrower one. +- Database row/lease ownership is authoritative; in-process mutexes are optimizations. +- Leases have owner IDs, heartbeats, expiry, and fencing tokens. +- A recovered worker with an old fencing token cannot commit after a new worker has + taken ownership. +- Read-only thumbnail generation may run with analysis but not for an asset currently + being renamed unless it already has a cache hit. +- Rename takes an exclusive library mutation lease. +- EXIF writes take an asset mutation lease and are blocked during rename. +- Upload takes album leases and validates that no EXIF/rename lease is active. +- Archive takes an exclusive album mutation lease plus archive-location lease and is + blocked by analysis, EXIF, rename, upload, restore, or another archive operation. + +### TOCTOU protection + +Every mutating operation follows: + +```text +load asset + version +→ acquire lease/fencing token +→ resolve and validate current path +→ verify expected snapshot/hash +→ perform one external/filesystem action +→ verify result +→ commit state with version + fencing-token check +→ release lease +``` + +If the commit loses its version race, the result is not discarded. It is recorded as +an ambiguous operation event and reconciled from the actual file state. + +### Idempotency + +Exactly-once execution is not achievable across SQLite, filesystem, subprocesses, and +Immich. The design promises **at-least-once execution with idempotent recovery**. + +- API mutation requests accept or generate idempotency keys. +- Job items have unique `(job_type, asset_id, input_version)` constraints. +- EXIF writes merge desired fields and verify the final value. +- Thumbnail writes target content/version keys and use atomic replacement. +- Rename operations persist source, destination, and expected hashes before moving. +- Upload retries consult both local upload history and Immich/immich-go results. + +--- + +## 17. Resource allocation and backpressure + +### Memory + +Decoded images, not file sizes, determine memory pressure. A compressed 100 MB image +can require far more RAM when decoded. + +- reject or specially queue images exceeding configurable pixel/dimension limits; +- use Pillow decompression-bomb protection and treat warnings as reviewable errors; +- decode thumbnails directly to bounded target sizes where decoder support permits; +- avoid retaining full pixel arrays after hashing/thumbnail generation; +- limit concurrent decoded-pixel budget, not only worker count; +- isolate crash-prone/large decoders in worker processes so memory is reclaimed. + +### Disk + +- thumbnail cache has a configurable quota and LRU eviction; +- active thumbnails are pinned while a review page references them; +- temporary files live beside their destination when atomic rename is required; +- startup cleans only recognized stale temporary files, never arbitrary files; +- database, WAL, logs, raw responses, backups, and thumbnails expose separate size + metrics and low-disk warnings; +- mutating stages stop before the disk reaches a safety reserve. + +### CPU, GPU, and network + +- CPU pools default to a conservative fraction of available cores; +- GPU inference has an explicit single-owner semaphore unless proven safe otherwise; +- network concurrency and requests-per-minute are separate controls; +- retry traffic consumes the same rate-limit budget as first attempts; +- file descriptor use is bounded by closing images/subprocess pipes promptly; +- subprocess stdout is streamed with a size cap while the complete raw report may be + stored in a rotated file when required. + +--- + +## 18. Testing strategy + +Testing must prove crash safety and concurrency behavior, not only successful API +responses. Tests use synthetic images and temporary directories; they never access the +real photo library or `_IGNORE/` contents. + +### Deterministic fixture photo library + +Maintain a small, generated, non-private, source-controlled fixture specification from +which the test harness creates a fresh photo library for each test. Do not depend on +the developer's photo library, network downloads, current time, random model output, +filesystem enumeration order, or mutable shared fixture directories. If binary +fixtures are committed, they must be redistributable and accompanied by their source +and license; prefer reproducible generation where format support permits it. + +The fixture library must include enough deliberately distinct files and folders to +exercise every workflow state: + +- root-level photos, nested albums, empty folders, similarly named folders, Unicode + and whitespace in names, long names, mixed-case extensions, and names that collide + on case-insensitive or Unicode-normalizing filesystems; +- an exclusion-policy sentinel whose identifier is asserted to be absent from scan + results, counts, logs, provider calls, thumbnails, and upload requests; tests must + never open or inspect excluded content to prove exclusion; +- unique synthetic scenes with known visible properties, including zero, one, and + multiple people; indoor/outdoor; day/night; readable synthetic signs; portrait and + landscape orientation; transparency; and simple SFW/NSFW-classification fixtures; +- exact byte copies, renamed copies, same pixels with different metadata, EXIF-only + changes, resized/recompressed variants, orientation variants, crops, bursts, + watermarks, color edits, blank images, and intentionally similar non-duplicates; +- JPEG, PNG, WebP, TIFF, and HEIC/HEIF where the pinned CI decoder supports them, plus + uppercase extensions, unsupported extensions, corrupt/truncated images, zero-byte + files, and decompression-bomb/huge-dimension headers that remain safe to test; +- files with no EXIF, representative existing user EXIF, previous safety keywords, + previous analysis metadata, conflicting managed fields, malformed metadata, known + timestamps, and timezone/offset edge cases; +- read-only, missing, externally moved, externally copied, replaced-after-scan, and + symlink-escape cases created by the harness only on platforms that support them; +- album/upload/archive cases for valid names, invalid names, collisions, offline + archive media, insufficient capacity, already-uploaded bytes, changed-after-upload + bytes, and restore destination collisions. + +A versioned machine-readable manifest is the authority for the corpus. For every +fixture it records a stable logical ID (never an absolute path), relative path, role, +generation recipe or source/license, byte SHA-256, decoded dimensions/orientation, +relevant metadata, duplicate-family membership and confidence class, expected stage +eligibility, and expected fake-provider outcomes. Expected results are keyed by logical +ID so tests remain valid after copying the corpus into a random temporary directory. +Hash and thumbnail goldens also record algorithm, decoder, encoder, and fixture-schema +versions; intentional changes require an explicit golden update and review. + +Fixture generation must be deterministic: pin image-library/codec versions in CI; +use fixed seeds for pixels and generated names; fix clock, timezone, and locale; write +explicit timestamps instead of using creation time; sort all discovered paths; avoid +lossy re-encoding unless its exact output is committed and checksummed; and skip a +format with an explicit platform marker when byte-identical generation cannot be +guaranteed. Tests compare semantic pixels/metadata where cross-platform encoders are +not byte-stable. + +Deterministic fakes map stable fixture IDs to fixed vision JSON, safety scores, +`exiftool` behavior, and `immich-go` reports. The manifest includes success, delay, +rate-limit, malformed-response, retry, cancellation, and uncertain-outcome scripts. +Timing-sensitive tests synchronize on observable events or fault-controller barriers, +never sleeps. Each test clones or regenerates the minimal named fixture subset, and +the harness verifies its checksums before startup, resets the database/cache/archive, +and fails on undeclared files or unexpected provider/uploader calls. + +Keep the default corpus small enough for every change while retaining a separate +extended corpus for format/platform and soak coverage. CI must provide one fixture +validation command that regenerates the corpus twice, compares manifests/checksums, +validates all expected relationships, and proves that a complete test run leaves the +source corpus unchanged. + +#### Concrete fixture-corpus TODO checklist + +Build the default corpus below as the first implementation target. Counts describe +logical source images; explicitly listed copies/variants are additional files. All +people and sensitive-content fixtures must be synthetic, illustrated, generated, or +properly licensed adult-only material—never personal photos and never minors in any +sexualized or ambiguous context. + +**Folder topology** + +- [ ] Create `00_root/` behavior with 3 supported images placed directly at the + library root: 1 JPEG, 1 PNG with transparency, and 1 uppercase `.JPG`. +- [ ] Create 3 ordinary album folders with distinct conditions: + `01_family_day/` (outdoor/day/people), `02_city_night/` + (outdoor/night/readable sign), and `03_home_indoor/` + (indoor/mixed lighting/no location hint), with 5 unique source images each. +- [ ] Create 3 nested leaf albums under one parent, with 2 unique images per leaf, to + prove leaf-album counts are not collapsed into their parent. +- [ ] Create 3 naming-edge folders containing 1 image each: one with spaces, one with + Unicode characters, and one mixed-case name. Add a pair whose names differ only by + case and a pair using composed/decomposed Unicode for collision simulation. +- [ ] Create 1 empty album folder and verify it is handled consistently and never + invents an asset. +- [ ] Create the exclusion-policy sentinel directory and assert only from outside the + boundary that it is never traversed, counted, logged, thumbnailed, sent to a + provider, or uploaded. + +**Known visual content** + +- [ ] Create 5 deterministic NSFW-class fixtures spanning the fake classifier's score + bands: 2 clearly NSFW, 1 boundary/review-required, and 2 clearly SFW controls that + are visually similar enough to catch simplistic color/skin heuristics. +- [ ] Create 5 additional SFW fixtures: 1 landscape with no people, 1 single adult, + 1 group of 3 adults, 1 indoor pet, and 1 document/screenshot with readable synthetic + text. +- [ ] Across the ordinary albums, include at least 3 portrait, 3 landscape, and 2 + square images; 3 indoor and 3 outdoor scenes; and morning, afternoon, evening, + night, and unknown time-of-day expectations. +- [ ] Include exactly 3 synthetic location-hint cases: 1 readable street sign, 1 + unmistakable generated landmark, and 1 ambiguous sign that must yield no confident + location. + +**Duplicate and similarity families** + +- [ ] Family A: 1 canonical JPEG plus 2 byte-identical copies with different paths. +- [ ] Family B: 1 canonical image plus 2 files with identical decoded pixels but + different EXIF/timestamps. +- [ ] Family C: 1 canonical plus 3 near-duplicates: resized, JPEG-recompressed, and + WebP-converted. +- [ ] Family D: 1 canonical plus 2 orientation variants: EXIF rotation and physically + rotated pixels. +- [ ] Family E: 1 canonical plus 3 review-required variants: crop, watermark, and + color edit. +- [ ] Create 3 intentionally similar non-duplicate pairs: blank/near-blank images, + burst-like frames with a meaningful subject change, and two screenshots sharing the + same layout. Expected decisions must be `not_duplicate` or `review_required`, never + automatic merge. + +**Formats and damaged inputs** + +- [ ] Include at least 3 JPEG, 3 PNG, 2 WebP, 2 TIFF, and 2 HEIC/HEIF files in the + extended corpus; keep one supported example of each CI-stable format in the default + corpus. +- [ ] Add 1 unsupported-extension file, 1 zero-byte file, 1 truncated JPEG, 1 corrupt + PNG, and 1 safe synthetic huge-dimension/decompression-bomb header. +- [ ] Add 1 grayscale, 1 transparent RGBA, 1 wide-gamut/profile-bearing, and 1 very + small image to exercise decoding and thumbnail normalization. + +**Metadata and time cases** + +- [ ] Create 2 files without EXIF, 2 with ordinary user EXIF that must survive, 2 with + existing safety keywords, 2 with existing analysis fields, 1 with conflicting + managed fields, and 1 with malformed metadata. +- [ ] Create 5 fixed capture-time cases: UTC, positive offset, negative offset, + daylight-saving boundary, and missing timezone. Freeze filesystem mtimes separately + so capture-time fallback behavior is explicit. +- [ ] Create 1 file whose EXIF write changes its byte hash without changing decoded + pixels, then assert upload uses the post-checkpoint hash. + +**Runtime mutations produced by the harness** + +- [ ] From clean source fixtures, create one moved file, one copied file, one renamed + folder, one file replaced after scan, one file modified after upload, and one file + removed after scan. +- [ ] Where supported, create one read-only file, one read-only destination folder, + one safe in-library symlink, and one symlink escape attempt. +- [ ] Create 3 rename-plan cases: valid multi-file rename, destination collision, and + case-only rename; add one stale plan by modifying a source after validation. +- [ ] Create 4 fake-upload results: new upload, exact duplicate, server upgrade, and + accepted-but-response-lost uncertainty. +- [ ] Create 3 archive cases: successful copy/verify, insufficient capacity, and + interruption after verification but before source removal; add one offline restore + and one restore-destination collision. + +**Manifest and acceptance totals** + +- [ ] Give every file a stable logical ID and manifest entry; declare its expected + discovery status, safety class, analysis response, duplicate family/decision, EXIF + projection, album membership, and upload/archive eligibility. +- [ ] Record the exact expected counts after each workflow stage, including canonical, + duplicate, review-required, excluded, error, analyzed, EXIF-verified, uploaded, and + archived totals. Tests must compare the complete count object, not selected fields. +- [ ] Add a corpus-lint test that fails on missing IDs, duplicate IDs, undeclared + files, checksum drift, inconsistent family membership, missing expected outcomes, + or a fixture not exercised by at least one automated test. +- [ ] Add a reproducibility test that generates the corpus twice in separate temporary + roots and compares manifests, relative paths, semantic metadata, and checksums where + byte stability is promised. + +### Unit tests + +- path-policy normalization, symlink escape rejection, and `_IGNORE/` exclusion; +- normalized pixel hashing across EXIF-only changes and orientation variants; +- pHash distance/confidence classification and negative-link behavior; +- canonical recommendation scoring; +- stage-gate evaluation and blocker explanations; +- state-machine transitions for jobs, renames, uploads, and duplicate decisions; +- idempotency-key and optimistic-version handling; +- naming sanitization across macOS/Linux and case-insensitive collision simulation; +- `immich-go` parser fixtures for every supported version and unknown output. + +Use property-based tests for path transformations, rename plans, hash grouping, state +machines, and arbitrary API payloads. Important properties include: paths never escape +the library, canonical clusters never contain cycles, retries do not duplicate events, +and applying a valid rename plan preserves the asset set. + +### Golden-image tests + +Maintain a small generated/non-private corpus containing: + +- exact copies with different names; +- identical pixels with different EXIF; +- resized and recompressed copies; +- rotations represented by EXIF and by transformed pixels; +- crops, watermarks, color edits, screenshots, blank images, and burst-like images; +- JPEG, PNG, WebP, TIFF, HEIC fixtures where licensing permits; +- corrupt, truncated, huge-dimension, and unsupported files. + +Assert hashes, confidence bands, thumbnail orientation/dimensions, and which cases must +require human review. Do not assert that fuzzy hashing is universally correct. + +### Repository and API integration tests + +- run every migration from an empty DB and from anonymized snapshots of each supported + historical schema; +- verify foreign keys, FTS rebuilds, WAL settings, online backup, and integrity checks; +- test API status codes, schemas, pagination, idempotency, stale versions, CSRF/Origin, + path validation, and event-stream reconnect behavior; +- replace vision, NSFW, `exiftool`, and `immich-go` integrations with deterministic + fakes that can succeed, delay, emit malformed output, or fail at chosen points. +- snapshot representative EXIF before and after every stage; assert only owned fields + change, previous-stage values survive, read-back matches the desired projection, and + the stored post-write hashes match the actual file bytes. + +### Black-box API end-to-end tests + +API end-to-end tests must not import service or repository internals. They launch the +real FastAPI server and worker as child processes, create a temporary application-data +directory and photo library, and communicate only over HTTP/SSE. This validates the +same process boundaries used in production. + +Recommended harness: + +- `pytest` owns a session-scoped temporary directory and allocates free ports; +- Alembic creates a fresh SQLite database through the production startup path; +- the API and worker receive an isolated test configuration through environment vars; +- deterministic fake executables/providers replace `exiftool`, the vision API, + NSFW inference, and `immich-go`, but are invoked through the real integration layer; +- readiness waits on `/api/v1/health/ready`, never an arbitrary sleep; +- tests use an HTTP client and consume SSE events with reconnect support; +- processes are terminated and checked for leaked children after each test session. + +Do not add a powerful test-only mutation API to production. Fixtures are prepared +before process startup through files, migrations, and fake integrations. When runtime +fault control is required, use a separate fake-provider control server bound to its +own random localhost port and enabled only by test configuration. + +Core API scenarios: + +1. **Fresh workflow** — scan, wait for job completion, review duplicate cluster, mark + safety decisions, verify the safety EXIF checkpoint, analyze, verify the analysis + EXIF checkpoint and preservation snapshot, generate/apply rename plan, upload, + archive, restart, and verify the offline asset remains searchable/deduplicable. +2. **Move and copy outside the app** — stop jobs, manually move A→B and copy B→C, + rescan, verify the original asset ID survives and C becomes a duplicate candidate. +3. **Idempotent commands** — submit the same idempotency key concurrently and assert + one job/decision plus identical responses. +4. **Optimistic conflict** — submit a decision with an old asset/cluster version and + require `409 Conflict` without mutation. +5. **Cancellation and resume** — cancel each job type mid-item, restart processes, + and verify a valid resumable/terminal state. +6. **Lease recovery** — kill the worker, let its lease expire, start another worker, + then prove the old worker cannot commit with its stale fencing token. +7. **Safety race** — change an asset to NSFW while an analysis fake is delayed; the + eventual analysis result must be discarded and must not write analysis EXIF. The + asset remains upload-eligible after its NSFW EXIF keyword is verified. +8. **Rename uncertainty** — change a source after plan validation and assert apply is + rejected without moving any unaffected paths. +9. **Unknown upload result** — fake server acceptance followed by process failure; + require `unknown_requires_verification`, never automatic retry/success. +10. **Privacy boundary** — verify `_IGNORE/` identifiers never appear in provider or + uploader logs; NSFW identifiers never appear in analysis-provider logs but do + appear in uploader logs when their album is explicitly uploaded. +11. **External EXIF edit** — add user metadata after a verified checkpoint, rescan, and + verify the next stage preserves the addition; remove or conflict with a managed + value and require `divergent` instead of silent repair/upload. +12. **Archive interruption** — terminate during transfer, after archive verification, + and after active-source removal; restart and assert no verified source is lost or + removed twice. +13. **Offline deduplication** — unmount the fake archive, discover an exact and a fuzzy + active copy, and verify hash matching plus protected-thumbnail review without + treating the archived canonical as unexpectedly missing. + +For every scenario, assert both API state and durable state after a full server/worker +restart. An HTTP `200` before restart is not sufficient evidence of durability. + +API contract tests also validate the generated OpenAPI schema. Breaking response or +enum changes require an explicit API version/migration decision rather than silently +breaking the frontend. + +### Playwright browser end-to-end tests + +Playwright tests run against the same real API/worker test stack. They validate user +journeys and visible safety controls, while API tests remain responsible for exhaustive +state combinations. Browser tests should not duplicate every backend assertion. + +Use stable accessible locators first (`getByRole`, labels, names) and add +`data-testid` only for dynamic structures without reliable semantics. Never locate by +CSS styling classes or incidental text generated by AI fixtures. + +Critical browser journeys: + +1. **Workflow overview** — stage counts render, blockers are understandable, and + disabled actions explain their prerequisite. +2. **Duplicate comparison** — thumbnails load in the correct orientation; keyboard + navigation, synchronized zoom, blink/overlay, metadata evidence, and canonical + selection work; fuzzy matches require explicit confirmation. +3. **Safety review** — individual and bulk SFW/NSFW decisions persist after reload; + filters/counts update; an NSFW asset disappears from analysis-eligible totals while + remaining visible in upload-eligible totals. +4. **Analysis job** — start, live progress via SSE, reconnect after page reload, cancel, + resume, and inspect an error without losing completed work. +5. **Album proposal** — edit a suggestion, detect collision/invalid name, preview the + complete path plan, and verify stale confirmation is rejected visibly. +6. **Rename recovery** — simulate a controlled interrupted rename, reopen the UI, and + confirm the recovery screen prevents unrelated mutations. +7. **Upload preflight** — blocked reasons render, secrets never appear in DOM/logs, + exact album scope is shown, and uncertain results require verification. +8. **Move/copy reconciliation** — after the harness changes files externally, rescan + from the UI and resolve the newly formed duplicate cluster. +9. **Archive and restore** — preview reclaimable bytes, inspect blockers, confirm an + archive plan, observe progress/recovery, browse the offline asset from its retained + thumbnail, and restore it when the fake archive location is mounted again. +10. **Accessibility and keyboard** — complete duplicate and safety decisions without a + mouse; focus remains visible; dialogs trap/restore focus; status is not color-only. +11. **Responsive layout** — desktop is primary, but tablet/narrow widths retain access + to blockers, decisions, logs, and confirmations without horizontal action loss. + +Playwright execution policy: + +- each test gets a fresh browser context and deterministic seeded library state; +- tests wait for semantic UI state or API responses, never fixed timeouts; +- external provider calls are observed at the fake integration boundary, not mocked + inside the browser, so frontend-to-backend behavior remains real; +- use Playwright's API request context only for setup when the action is also available + through the public API and does not bypass the behavior under test; +- run Chromium on every change; run Firefox/WebKit on scheduled or release builds; +- collect trace, console, network log, screenshot, and video on first retry/failure; +- fail on uncaught page errors, unexpected console errors, failed API responses, and + unhandled dialogs; +- visual snapshots cover only stable layouts/components, with deterministic fonts, + viewport, timezone, locale, animation disabling, and generated thumbnails; +- accessibility scans complement—not replace—keyboard journey assertions. + +Avoid large all-in-one browser tests. Each journey should establish state through the +API, perform the behavior under test through the UI, and assert the visible result plus +one authoritative API outcome. Keep one longer smoke journey for the complete pipeline. + +### Test-stack topology + +```text +pytest / Playwright + │ + ├── browser ───────────────► real FastAPI + static frontend + │ │ + ├── HTTP/SSE client ───────────────┤ + │ ▼ + │ isolated SQLite DB + │ ▲ + └── fixture/fault controller │ + │ real worker process + └── fake vision / NSFW / exiftool / immich-go +``` + +The harness must expose a single command for local and CI use, for example: + +```text +pytest tests/e2e/api +playwright test tests/e2e/web +pytest tests/e2e/full_pipeline +``` + +Exact commands depend on the final frontend tooling, but both suites must provision +the same stack through one reusable test-harness module. + +### Concurrency and race tests + +Run repeatedly with randomized timing: + +- many hash/thumbnail workers completing simultaneously into the DB writer queue; +- two workers attempting to claim the same job; +- lease expiry followed by a late commit from the old worker; +- user moves/replaces a file between validation and EXIF write; +- thumbnail request races with rename and cache eviction; +- analysis result races with a safety decision changing to NSFW; +- upload preparation races with an EXIF update; +- two rename plans target the same destination; +- long UI reads create WAL checkpoint pressure; +- cancellation arrives during decoding, API retry, EXIF write, rename, and upload. + +Assertions include no deadlocks, bounded queue depth, no duplicate terminal state, no +lost audit events, no writes from stale fencing tokens, and eventual database integrity. + +### Crash/fault-injection tests + +Use real child processes and terminate them at every persisted transition: + +- after journal write but before rename; +- after filesystem rename but before DB update; +- during EXIF subprocess execution; +- after upload acceptance but before report persistence; +- while a DB writer batch is committing; +- while thumbnail temporary files exist. + +Restart the application and assert it either resumes safely or presents a precise +manual-recovery state. Never accept silent guessing as a passing result. + +Fault injection should also cover disk full, read-only directories, permission changes, +database busy timeouts, corrupt DB copies, network timeouts, malformed model JSON, GPU +out-of-memory, subprocess hangs, and unavailable binaries. + +### End-to-end workflow tests + +Build a temporary library, including an `_IGNORE/` sentinel, then run: + +```text +scan → duplicate decisions → safety review → analysis → EXIF verification +→ album proposal → rename → rescan/reconciliation → fake Immich upload +→ archive → offline deduplication → restore +``` + +Repeat after manually moving and copying files between stages. Assert stable asset IDs, +preserved decisions, duplicate exclusion, verified EXIF checkpoint order, upload +hashes, archive manifests, offline hash-index membership, successful restore, and that the +sentinel is never discovered, counted, opened, or modified. + +### Load and soak tests + +- synthetic databases at 25k, 100k, and 500k asset rows; +- duplicate clusters with thousands of members; +- sustained thumbnail browsing while analysis writes complete; +- multi-hour worker soak with cancellation/retry and periodic backups; +- cache eviction under quota pressure; +- memory/RSS, open-file, DB latency, WAL size, queue depth, and event throughput limits. + +Define acceptance budgets before implementation, for example: + +- API list/search p95 under 250 ms on 100k rows; +- no unbounded queue or cache growth; +- stable RSS during a multi-hour run; +- clean restart with no lost completed item after forced termination; +- zero automatic fuzzy-duplicate exclusions without recorded user approval. + +### Release gates + +1. Unit/property/golden suites pass. +2. Migration test succeeds from a copied current database. +3. Concurrency suite passes repeatedly with randomized scheduling. +4. Every rename crash point passes recovery tests. +5. Black-box API workflow passes through real server and worker processes, including a + restart after every completed stage. +6. Critical Playwright journeys pass in Chromium with no unexpected browser console, + network, accessibility, or page errors. +7. Scheduled cross-browser Playwright suite passes before a release. +8. Full end-to-end pipeline passes with fake external services and retained artifacts. +9. Archive crash/recovery suite proves that active sources are removed only after + verified durable copies and remain deduplicable while offline. +10. A read-only dry run against the real library produces an approved reconciliation + report before any mutation is enabled. +11. The donor ledger has no unresolved in-scope rows; characterization/parity tests + pass, intentional deltas are approved, and archived CLI sources are not imported + or executed by the production application. diff --git a/README.md b/README.md new file mode 100644 index 0000000..12213b2 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Photo Analyzer + +Integrated, restart-safe photo analysis, duplicate review, metadata, upload, and +archive workflow. Planning lives in `INTEGRATED_PIPELINE_CONCEPT.md` and +`delivery_backlog/`. + +## Agent workflow helper + +Requirements: Python 3.11+, Git, and an authenticated `tea` login named `domverse`. + +```bash +scripts/work-item next +scripts/work-item claim +scripts/work-item status +scripts/work-item submit --test "python3 -m unittest discover -s tests -v" +scripts/work-item submit --test "python3 -m unittest discover -s tests -v" --yes +scripts/work-item complete --merge +scripts/work-item block --reason "precise blocking condition" +``` + +The helper selects the lowest numbered open story whose dependencies are closed, +creates a story branch, runs required tests, rejects private or unsafe files, creates a +pull request, and closes the issue only after the merge is verified. + +Workflow state is stored inside `.git/` and is never committed. Status labels are the +machine-readable source of truth; the Gitea Project board is the visual view. + +## Tests + +```bash +python3 -m unittest discover -s tests -v +``` diff --git a/WEBAPP_CONCEPT.md b/WEBAPP_CONCEPT.md new file mode 100644 index 0000000..6813795 --- /dev/null +++ b/WEBAPP_CONCEPT.md @@ -0,0 +1,285 @@ +# Photo Analyzer — Web App UX Concept + +Transform `photo_analyzer.py` from a terminal pipeline into a **local web app** that +reuses the `nsfwtag` review-app design language and architecture. Concept first — +implementation follows after review. + +> **North star:** the analyzed data (descriptions, tags, mood, people, year, …) is +> currently invisible — it lives in a SQLite DB nobody can see. The web app's *primary* +> job is to surface it (browse + FTS5 search); the analysis pipeline becomes a +> monitored background job instead of a foreground terminal takeover. + +Design validated with the `ui-ux-pro-max` skill: **Data-Dense Dashboard** pattern +(KPI cards, status colors green/amber/red, filtering-first, hover tooltips, row +highlight), debounced search with a real "no results" state, list virtualization for +25k+ photos, and — to stay dependency-free — **CSS progress bars with values-always- +visible** for stats rather than a chart library (the skill's AAA-accessible option). + +--- + +## 1. Architecture (reuse nsfwtag's, unchanged in spirit) + +- One **stdlib `ThreadingHTTPServer`** on `127.0.0.1:`, opened in the + browser. No framework, no build step, no external assets (matches nsfwtag). +- One **token-injected HTML page** (`analyzer.html`, `%%TOKEN%%` replacement) + JSON + endpoints. Same "real HTML file, not a Python string" rule. +- **SQLite stays the single source of truth.** The server never owns pipeline state; + it reads the DB and an in-memory run-state object (exactly like the terminal + `_Dashboard` reads dicts + a deque). A render/HTTP crash can never corrupt the DB. +- **Path args validated against the DB** on every request (nsfwtag validates against + the scanned candidate set) — the server can't be pointed at arbitrary files. +- The existing analysis engine (`_run_folder_loop`, workers, retry/backoff, `_db_lock`, + variant grouping, nsfw-skip, `_IGNORE` exclusion) is **reused verbatim**, driven from + a background thread. The web layer is a new view + control surface, not a rewrite. + +### Endpoints + +| Method | Route | Purpose | +|--------|-------|---------| +| GET | `/` | The app page | +| GET | `/img?path=` | Thumbnail bytes *(reuse nsfwtag)* | +| GET | `/exif?path=` | Raw EXIF + decimal GPS + Maps URL *(reuse nsfwtag)* | +| GET | `/photo?path=` | Full analyzed record for one photo (all DB fields) | +| GET | `/search?q=&setting=&tod=&season=&people=&year_min=&year_max=&status=&album=&sort=&offset=` | Paged result set (FTS5 when `q` present) → `{rows:[…], total, offset}` | +| GET | `/facets` | Distinct filter values + counts (for the toolbar) | +| GET | `/stats` | Aggregates for the Stats view | +| GET | `/progress` | Live run state: `{running, totals, tokens, eta, albums:[…], feed:[…]}` | +| POST | `/run` | Start analysis with a config body (the CLI flags) | +| POST | `/stop` | Signal the current run to stop (reuses the existing `_stop` event) | +| GET | `/balance` · `/quota-check` | Proxy the existing functions → JSON | +| GET | `/log` | Activity log *(reuse nsfwtag)* | + +Live updates use **polling** (`/progress` ~1.5 s, `/log` ~2 s) — same pattern as +nsfwtag's log panel. No SSE/WebSocket dependency. + +--- + +## 2. Information architecture + +Flat, three top-level views behind a header **segmented control** (deep-linked via +`#library` / `#analyze` / `#stats`; active state highlighted per the skill's +`nav-state-active`). An **Activity-log** slide-in panel is available from any view. + +``` +Header (sticky): [◧ logo] Photo Analyzer ⟨ Library · Analyze · Stats ⟩ [ total 25,318 · analyzed 25,316 · errors 8 ] [☰ log] + ^ segmented, active = --text on --elev +``` + +- **Library** (default) — browse + search the analyzed photos. The payoff. +- **Analyze** — run controls + the live dashboard (album progress, feed, tokens/ETA). +- **Stats** — aggregates, breakdowns, error list. + +--- + +## 3. View: Library *(browse + FTS5 search)* + +``` +┌ Header ─ view: Library ──────────────────────────────────────────────────────────┐ +├ Toolbar (sticky) ─────────────────────────────────────────────────────────────────┤ +│ 🔍 Search description, tags, mood, place… Setting▾ Time▾ Season▾ People▾ │ +│ Year 1998 ●──────● 2019 ☐ has location Status▾ Sort: Relevance▾ │ +├──────────────┬────────────────────────────────────────────────────────────────────┤ +│ Folders │ ┌───────┐ ┌───────┐ ┌───────┐ ┌───────┐ ← virtualized / infinite │ +│ ▸ Urlaub │ │ [img] │ │ [img] │ │ [img] │ │ [img] │ scroll, server-paged │ +│ 2015 USA │ │ '15 👤2│ │ '04 │ │ '11 👤5│ │ ⚠err │ │ +│ ▸ job │ └───────┘ └───────┘ └───────┘ └───────┘ │ +│ ▸ selfies │ beach sunset… cathedral… wedding… (error: timeout) │ +│ … │ … 25,316 results … │ +└──────────────┴────────────────────────────────────────────────────────────────────┘ +``` + +- **Search** (`.field`): debounced 250 ms → `/search` FTS5 over `description, tags, + mood, location_hint`. Autocomplete-style live results (skill `Search/Autocomplete`). + Empty query = browse-all (paged). **No-results state** shows a message + suggestions + ("try a broader term, or clear filters"), never a blank grid. +- **Facet filters** map every analyzed field: + - Setting (indoor/outdoor) · Time of day · Season → `select` or `.chip` toggles. + - People → buckets `0 / 1 / 2 / 3+`. + - Year → dual-handle range slider (`approx_year` min–max), tabular numbers. + - Has-location → toggle (`location_hint` not null). + - Status → analyzed / exif_written / error / pending. + - Filters AND with the search; each updates the live result count. +- **Folder sidebar** (reuse nsfwtag `.sidebar`/`.fitem`): album = **leaf folder** + (reuse `album_label`), with counts; filter to a folder + its subfolders. +- **Card** (reuse `.card`/`.thumb`): lazy `/img` thumbnail; overlays — **year badge** + + **people badge** (reuse `.score` badge styling), a **status dot** (error=red, + pending=faint), and the first line of the description under the thumbnail. Tag chips + on hover. +- **Grid scale:** 25k+ photos → **server-paged infinite scroll** (append on scroll), + not load-all (skill `virtualize-lists`, `content-jumping`). Reserve card height to + avoid layout shift. +- **Lightbox** (reuse nsfwtag lightbox + keyboard + prev/next within the current result + set): left = full image; right = a two-section panel: + 1. **Analysis** — description, tags, mood, setting, time-of-day, season, people, + year, location. (The EXIF caption Immich will index, made human-readable.) + 2. **EXIF** — raw fields + GPS **Google Maps** link (reuse `/exif`). + Per-photo action: **Re-analyze this one** (POST `/run` scoped to one path). + +--- + +## 4. View: Analyze *(run controls + live dashboard)* + +Mirrors the terminal `_Dashboard` (see `DASHBOARD_PLAN.md`) as a web surface. + +``` +┌ Header ─ view: Analyze ───────────────────────────────────────────────────────────┐ +├ Run bar ──────────────────────────────────────────────────────────────────────────┤ +│ Library [ /…/pictures ▾ ] ☐ Dry-run ☐ Re-analyze ☐ No-EXIF ☐ EXIF-only ☐ Variants │ +│ [ Balance ] [ Quota check ] [▶ Start] │ +├ Overall ──────────────────────────────────────────────────────────────────────────┤ +│ ████████████████░░░░░░░░ 238 / 550 43 % ok 235 · err 3 │ +│ tokens 284,192 · est. remaining ~340,000 · ETA 14:23 [■ Stop] (danger) │ +├ Albums (data-dense) ──────────────────┬ Recent feed ───────────────────────────────┤ +│ Camera Roll 238/550 ███████░░ 43% │ ✓ IMG_4821.jpg │ +│ Urlaub/Rom 45/ 45 ████████ ✓ │ A golden retriever on a sunny beach │ +│ Urlaub/Venedig 12/ 60 ██░░░░░░ 20% │ ✗ DSC_0042.jpg — 4 retries: timeout │ +│ WhatsApp Images 0/120 ░░░░░░░░ 0% │ ✓ IMG_4820.jpg │ +│ … │ Two children building a sandcastle │ +└───────────────────────────────────────┴─────────────────────────────────────────────┘ +``` + +- **Run bar** = the CLI flags as controls: + Library path (default from `LIBRARY` env) · **Dry-run · Re-analyze · No-EXIF · + EXIF-only · Group-variants** toggles · **Start** (primary) / **Stop** (danger). + One run at a time (guarded); Start on an interrupted library simply **resumes** + (DB is truth) — resume-safety is automatic, no separate control. +- **Overall panel**: big progress bar, ok/err counts, tokens used + estimated + remaining + ETA (all tabular). Status colors: green done, amber running, red error. +- **Albums panel**: one row per **leaf-folder album** with a mini progress bar + `✓` + when fully done (the DASHBOARD_PLAN target, now clickable → jumps to that album in + Library). Future albums shown at 0 %. +- **Recent feed**: newest-first list — thumbnail + filename + one-line description; + errors in red with the message. Reuses the activity-log line styling. +- **Balance / Quota-check**: small buttons → `/balance`, `/quota-check` → toast/inline + result. **Debug log**: available via the Activity-log panel. +- Everything here is driven by polling `/progress`; the panel degrades to "Idle — no + run in progress" with a Start hint when nothing is running (empty state with action). + +--- + +## 5. View: Stats *(aggregates + errors, no chart lib)* + +``` +┌ KPI cards ────────────────────────────────────────────────────────────────────────┐ +│ 25,318 total 25,316 analyzed 25,316 EXIF-written 8 errors 2 pending │ +├ Breakdowns (CSS bars, counts always visible) ───────────────────────────────────────┤ +│ Setting outdoor ██████████████ 14,902 indoor ████████ 9,120 │ +│ Season summer ████████ · autumn ██████ · spring ████ · winter ███ · unknown ██ │ +│ People 0 ████████ · 1 ██████ · 2 ████ · 3+ ███ │ +│ By year 1998▁ 2003▃ 2004▅ 2010▇ 2011█ 2015█ … (histogram of bars) │ +│ Top tags beach 1,204 · wedding 980 · city 900 · food 610 … │ +├ Errors ─────────────────────────────────────────────────────────────────────────────┤ +│ DSC_0042.jpg — connection timeout (4 retries) [Retry] │ +│ … │ +└──────────────────────────────────────────────────────────────────────────────────────┘ +``` + +- KPI cards reuse `.metric`. Breakdowns are **horizontal CSS bars with the count + printed** (skill: values-always-visible, AAA; no dependency, no CSP issue). +- Error table lists `status='error'` rows with `error_message` + a **Retry** action + (re-queues that path for the next run — matches "re-running retries error records"). + +--- + +## 6. Component inventory (reuse existing tokens) + +Everything below already exists in `nsfwtag/review.html`'s `:root` token set and class +library — the web app inherits it wholesale (same dark OLED palette, radii, focus +rings, reduced-motion guard). + +| Component | Source | New/changed | +|---|---|---| +| Header `.app`/`.brand`/`.logo`/`.titles` | reuse | title copy | +| KPI `.metrics`/`.metric` | reuse | used for header counts + Stats cards | +| View switcher | reuse `.seg` | active-state = current view | +| Toolbar `.toolbar` | reuse | — | +| Search `.field` | reuse | — | +| Facet `.chip` / `select` / `.seg` | reuse | People/Setting/Season/Status | +| Range slider `input[type=range]` | reuse | **dual-handle** year range (small JS) | +| Sidebar `.sidebar`/`.fitem` | reuse | album tree | +| Grid `.grid`/`.card`/`.thumb` | reuse | + year/people badges, status dot | +| Tag chips | reuse `.chip` (mini) | — | +| Lightbox + EXIF panel | reuse | + Analysis section | +| Buttons `.btn`/`.primary`/`.danger`/`.ghost` | reuse | Start/Stop/Balance | +| Toast `.toast` | reuse | — | +| Activity log panel | reuse | run events too | +| **Progress bar** | **new** `.bar` (fill = `--primary`) | overall + per-album + stat bars | +| **Run toggles** | **new** switch styled from tokens | the 5 flags | + +**One new token:** `--warn:#f59e0b` (amber) for "running / partial" states — the only +gap in the current green/red/faint status set. + +--- + +## 7. Interaction model + +- **View switching**: header segmented control; state deep-linked in the URL hash and + preserved on back/forward (skill `deep-linking`, `state-preservation`). +- **Search + filters**: debounced; combine as AND; the URL hash encodes `q` + active + filters so a search is shareable and survives reload. Result count updates live. +- **Browse at scale**: infinite scroll appends server-paged results; card heights + reserved (no CLS); thumbnails lazy-load. +- **Lightbox**: `←/→` navigate the current result set, `Esc`/backdrop close; EXIF and + full record fetched lazily on open. +- **Analyze**: Start → POST `/run` (config) → server spawns the run on a background + thread → dashboard polls `/progress`. Stop → POST `/stop` → sets the existing + `_stop` event; in-flight requests drain, DB stays consistent, restart resumes. +- **Accessibility/motion**: keyboard nav + visible focus rings (inherited), status + never by color alone (badges carry text/icons), `prefers-reduced-motion` respected + (token media query already present), all touch targets ≥ 36–44 px. + +--- + +## 8. CLI → Web feature parity + +| CLI feature | Web surface | Preserved how | +|---|---|---| +| `--library` | Analyze › Run bar path (default `LIBRARY` env) | same discovery | +| `--dry-run` | Run toggle | passed to engine | +| `--reanalyze` | Run toggle | passed to engine | +| `--no-exif` | Run toggle | passed to engine | +| `--exif-only` | Run toggle | passed to engine | +| `--group-variants` | Run toggle | reuses variant grouping | +| `--stats` | **Stats view** | `/stats` aggregates | +| `--balance` | Balance button | `/balance` | +| `--quota-check` | Quota-check button | `/quota-check` | +| `--debug` | Activity-log panel | server log | +| resume-safety | automatic (Start resumes) | DB source of truth | +| `_IGNORE/` exclusion | server-side discovery | `discover_photos` unchanged | +| nsfw-skip | automatic in every run | `filter_nsfw_tagged` unchanged | +| FTS5 search *(CLI had none)* | **Library search** | new surface for existing FTS5 table | +| DB (all fields) | Library cards + lightbox + facets | new surface for existing schema | + +Nothing is dropped. The only genuinely new capability is **surfacing** what the DB +already stores (search/browse/stats) — everything else is the same engine behind a +web control panel instead of a terminal. + +--- + +## 9. Proposed file layout (mirrors `nsfwtag/`) + +``` +webapp/ (or analyzer_web/) +├── __main__.py # CLI: python -m webapp → starts server, opens browser +├── server.py # ThreadingHTTPServer + routes (reuse nsfwtag/server.py shape) +├── page.py # render analyzer.html with %%TOKENS%% (reuse webapp.py shape) +├── analyzer.html # the single-page frontend (inherits nsfwtag tokens/classes) +├── query.py # DB read helpers: search (FTS5), facets, stats, one-photo +└── runner.py # drive photo_analyzer's engine on a background thread + run-state +``` + +`photo_analyzer.py` stays the pipeline library; `runner.py` imports and drives it. No +duplication of analysis logic. + +--- + +## 10. Build order (for after approval) + +1. **Read-only Library first** — server + page + `/search` + `/facets` + grid + + lightbox over the *existing* DB. Immediately useful, zero risk to the pipeline. +2. **Stats view** — `/stats` + CSS breakdown bars. +3. **Analyze view** — `/run` + `/stop` + `/progress` polling + run-state object, + driving the existing engine on a thread. +4. **Balance/quota/log** wiring + polish (empty states, deep-link hash, reduced-motion). + +Each step ships independently; the pipeline keeps working from the CLI throughout. diff --git a/album_naming.md b/album_naming.md new file mode 100644 index 0000000..c33af9d --- /dev/null +++ b/album_naming.md @@ -0,0 +1,204 @@ +# Immich album naming scheme + +Goal: turn the messy on-disk album folders into clean, consistent Immich albums. + +## The scheme + +**Pattern:** `YYYY[-MM[-DD]] Name` for anything datable; `Category – Detail` for +undated evergreen sets. + +Rules: + +1. **Year leads, ISO format.** `2015 USA`, not `USA 2015`. Add month/day only when + you actually know it: weddings get `YYYY-MM-DD`, month-tagged trips get `YYYY-MM`. + A leading 4-digit year sorts every dated album chronologically in Immich's + (alphabetical) album list; lettered evergreen albums fall below them. +2. **One name, one album.** Keep German, keep umlauts (Immich is UTF-8). Title Case. +3. **Separator:** a single space after the date; ` – ` (en dash) or `( )` for a + qualifier. No underscores, no double spaces, no cryptic tokens. +4. **Decode export artifacts.** Embedded month numbers (`Brindisi 9 2010`, + `Kuba 1 2011`) become `YYYY-MM`. Drop `alle` / `_MobileMe` / `MobileMe` — those + are old Apple/full-export copies, not real albums (see *Duplicates* below). +5. **Flatten the fake `Urlaub/` tree.** `Urlaub` (holiday) currently holds weddings, + job photos, and applications too. Drop it as a parent; the year prefix does the + grouping. + +## How to apply it (no disk renames) + +immich-go derives the album from the folder, but you can override per upload: + +```bash +immich-go upload \ + --server https://immich.domverse-berlin.eu --api-key --no-ui \ + from-folder \ + --album "2015 USA" \ + "/Users/domverse/Documents/Bilder/pictures/Urlaub/2015 USA/" +``` + +One `--album` per folder, using the *Proposed* name from the table below. This +keeps folder paths (and content hashes) untouched — no re-upload risk. Multiple +folders can map to the *same* `--album` name to merge them (that's how the +duplicates collapse). + +> Verify the exact flag against your immich-go version — recent builds use +> `--album ""` on `from-folder`; older ones used `--create-album-name`. + +--- + +## Mapping + +`[m?]` = month inferred from an embedded number, confirm before merging. +`[y?]` = year is a guess, confirm. + +### Travel (chronological) + +| Current folder | Proposed album | +|---|---| +| Urlaub/Indien 2003 | `2003 Indien` | +| Urlaub/Irina München 2003 | `2003 München (Irina)` | +| Urlaub/Mallorca 2003 | `2003 Mallorca` | +| Urlaub/Italien 2004 | `2004 Italien` | +| Urlaub/Irina 2004 | `2004 Irina` | +| Urlaub/Melania 30.Geb 2004 | `2004 Melania 30. Geburtstag` | +| Urlaub/Spanien 2007 | `2007 Spanien` | +| Urlaub/Tauchsafari 2008 | `2008 Tauchsafari` | +| Urlaub/Marokko 2009 + Urlaub/Marokko | `2009 Marokko` *(merge)* | +| Urlaub/Tauchen 2009 | `2009 Tauchen` | +| Urlaub/Ägypten 2010 | `2010 Ägypten` | +| Urlaub/Karwendel 2010 | `2010 Karwendel` | +| Urlaub/Maui 2010 | `2010 Maui` | +| Urlaub/Bootsausflug 2010 | `2010 Bootsausflug` | +| Urlaub/Kuba 2010 | `2010 Kuba` | +| Urlaub/Brindisi 9 2010 | `2010-09 Brindisi` `[m?]` | +| Urlaub/Wanderurlaub 6 2010 (+ …alle) | `2010-06 Wanderurlaub` `[m?]` *(merge)* | +| Urlaub/Italien 2011 | `2011 Italien` | +| Urlaub/Italien 7 2011 (+ …alle) | `2011-07 Italien` `[m?]` *(merge)* | +| Urlaub/Kuba 1 2011 (+ …alle, …_MobileMe, Kuba alle) | `2011-01 Kuba` `[m?]` *(merge)* | +| 2012 Dienstreise Zürich | `2012 Dienstreise Zürich` | +| 2012 Safari Hochzeitsreise | `2012 Hochzeitsreise (Safari)` | +| Urlaub/2013 Paris | `2013 Paris` | +| Urlaub/Schwedenurlaub (Edit) 07-2013 | `2013-07 Schweden (Edit)` | +| Urlaub/Schwedenurlaub (Tanja) 07-2013 | `2013-07 Schweden (Tanja)` | +| Urlaub/2014 Graz Nürnberg | `2014 Graz & Nürnberg` | +| Urlaub/2014 London | `2014 London` | +| Urlaub/2015 USA + Urlaub/USA 2015 | `2015 USA` *(merge)* | +| Urlaub/2015 XMas Paris | `2015-12 Paris (Weihnachten)` | +| Urlaub/2016 Tauchen | `2016 Tauchen` | +| Urlaub/2017 Paris | `2017 Paris` | +| Urlaub/2017 Ägypten | `2017 Ägypten` | +| Urlaub/2017 Mama in HH | `2017 Mama in Hamburg` | +| Urlaub/2017 Mammutmarsch + Mammutmarsch | `2017 Mammutmarsch` *(merge)* | +| Urlaub/2018 Italien | `2018 Italien` | +| Urlaub/2018 Ausflug Sylvester | `2018 Ausflug Silvester` | +| Urlaub/2019 Tauchen | `2019 Tauchen` | +| Urlaub/Berlin | `Berlin` `[y?]` | +| Urlaub/Uckermark | `Uckermark` `[y?]` | +| Urlaub/Malerweg | `Malerweg` `[y?]` | +| Urlaub/Dresden_im_Winter | `Dresden im Winter` `[y?]` | +| Sächsische Schweiz | `Sächsische Schweiz` `[y?]` | +| Urlaub/Kurzurlaube | `Kurzurlaube` | +| Urlaub/foodlog | `Foodlog` | + +### Weddings + +| Current folder | Proposed album | +|---|---| +| Urlaub/Hochzeit Alex _ Tanja | `2012-04-05 Hochzeit Alex & Tanja` *(your wedding)* | +| Urlaub/Hochzeit 05.04.2012 - Dia | `2012-04-05 Hochzeit Alex & Tanja (Dia-Scan)` | +| Urlaub/Hochzeit 05.04.2012 - Farbe | `2012-04-05 Hochzeit Alex & Tanja (Farbscan)` | +| Urlaub/Hochzeit 05.04.2012 - SW | `2012-04-05 Hochzeit Alex & Tanja (SW-Scan)` | +| 2012 Hochzeit | → fold into `2012-04-05 Hochzeit Alex & Tanja` `[?]` | +| Urlaub/Hochzeit (934) + Urlaub/Hochzeit_Gesamt (577) | `2012-04-05 Hochzeit Alex & Tanja (Gesamt)` `[?]` *(confirm same wedding)* | +| Urlaub/Hochzeit_Astrid_Paul | `Hochzeit Astrid & Paul` `[y?]` | +| Urlaub/Hochzeit_Carlos | `Hochzeit Carlos` `[y?]` | +| Urlaub/Hochzeit_Carlos Freunde | `Hochzeit Carlos (Freunde)` `[y?]` | +| Urlaub/Hochzeit_Cosimo | `Hochzeit Cosimo` `[y?]` | +| Urlaub/Hochezeit_Bjoern | `Hochzeit Björn` `[y?]` *(typo fixed)* | +| Urlaub/Sardinien & Hochzeit Alberto Kerstin | `Sardinien – Hochzeit Alberto & Kerstin` `[y?]` | +| Urlaub/Hochzeitessen Italien | `Hochzeitsessen Italien` `[y?]` | + +### Applications / Job + +| Current folder | Proposed album | +|---|---| +| Urlaub/Bewerbung 2006 | `2006 Bewerbung` | +| 2008_erste Bewerbungsbilder | `2008 Bewerbungsbilder (erste)` | +| Urlaub/Bewerbungsfotos 2011 | `2011 Bewerbungsfotos` | +| Urlaub/Bewerbungsfotos (Auswahl) | `Bewerbungsfotos (Auswahl)` | +| bewerbungsbilder | `Bewerbungsbilder` | +| bewerbungsbilder/foto_karoline_wolf | `Bewerbungsbilder (Karoline Wolf)` | +| Urlaub/FOKUS SE 2007 + Urlaub/Fokus 2007 | `2007 Fokus SE` *(merge)* | +| job/Capgemini | `Job – Capgemini` | +| job/Capgemini/jugendhackt | `Job – Capgemini (Jugend Hackt)` | +| job/adesso | `Job – adesso` | +| job/lucanet | `Job – LucaNet` | +| job/TS MMS | `Job – TS MMS` | +| Urlaub/2014 Jugend Hackt | `2014 Jugend Hackt` | +| Urlaub/2016_GTA | `2016 GTA` `[?]` | + +### People / Family / Personal + +| Current folder | Proposed album | +|---|---| +| Urlaub/Fotos Tanja _ Alex 3 2008 | `2008-03 Tanja & Alex` `[m?]` | +| Teenie Jahre | `Teenie Jahre` | +| Teenie Jahre/[Originals] | `Teenie Jahre (Originals)` | +| eltern | `Familie – Eltern` | +| eltern_tanja | `Familie – Eltern (Tanja)` | +| tanja | `Tanja` | +| Camera_Tanja + 0_Open/Camera_Tanja | `Kamera – Tanja` *(merge, same 243)* | +| 0_Open/Tanja Best Of Egypt | `Ägypten – Best Of (Tanja)` `[y?]` | +| selfies | `Selfies` | +| selfies/alex | `Selfies – Alex` | +| selfies/tanja | `Selfies – Tanja` | +| selfies/wir | `Selfies – Wir` | +| profile | `Profilbilder` | +| workout | `Workout` | + +### Evergreen / camera roll / misc + +| Current folder | Proposed album | +|---|---| +| Camera Roll | `Camera Roll` | +| Urlaub/iphone | `iPhone` | +| Urlaub/Handyvideos | `Handyvideos` | +| Wohnung | `Wohnung` | +| 2012_Wohnung DD | `2012 Wohnung Dresden` | +| Urlaub/Zu Hause | `Zu Hause` | +| fun_pics | `Fun Pics` | +| Urlaub/Auswahl Abzug Bild | `Abzüge (Auswahl)` | +| Urlaub/Unterschrift | `Unterschrift` `[?]` | +| Wallpaper/UW_Bilder | `Wallpaper – Unterwasser` | +| Wallpaper/wallpaper_sfw | `Wallpaper` | +| Wallpaper/wallpaper_ww | `Wallpaper – WW` `[?]` | + +--- + +## Duplicates to merge (point several folders at one `--album`) + +- **Marokko** — `Marokko 2009` + `Marokko` → `2009 Marokko` +- **Kuba** — `Kuba 1 2011` + `…alle` + `…_MobileMe` + `Kuba alle` → `2011-01 Kuba`; keep `Kuba 2010` separate as `2010 Kuba` +- **Italien 7 2011** + `…alle` → `2011-07 Italien` +- **Wanderurlaub 6 2010** + `…alle` → `2010-06 Wanderurlaub` +- **USA 2015** + `2015 USA` → `2015 USA` +- **Mammutmarsch** (top-level) + `Urlaub/2017 Mammutmarsch` → `2017 Mammutmarsch` +- **Camera_Tanja** (top-level) + `0_Open/Camera_Tanja` → `Kamera – Tanja` +- **Fokus** — `FOKUS SE 2007` + `Fokus 2007` → `2007 Fokus SE` + +immich-go dedupes on content hash, so uploading two folders into the same album +never creates duplicate assets — identical files land once, the album just gains +both source sets. + +## Needs your call (I guessed or couldn't tell) + +- **Years** for: Berlin, Uckermark, Malerweg, Dresden im Winter, Sächsische Schweiz, + and all the friends' weddings (`Astrid & Paul`, `Carlos`, `Cosimo`, `Björn`, + `Sardinien`). Add `YYYY ` if you know them. +- **`2016_GTA`** (620) — game screenshots? Greater Toronto? a person? → real name. +- **`Unterschrift`** (378, "signature"), **`SiS`** (24), **`ZGIDC`** (18), + **`Horka`** (1), **`Neues Album`** (4), **`Tanja`** (4) — cryptic/tiny; rename or + skip. +- **Big `Hochzeit` (934) / `Hochzeit_Gesamt` (577)** — confirm these are your own + 2012 wedding before merging into that album. +- **Month inferences** `[m?]` — verify `9 2010`, `1 2011`, `7 2011`, `6 2010`, + `3 2008` are months, not sequence numbers, before locking the `-MM`. diff --git a/compare_models.py b/compare_models.py new file mode 100644 index 0000000..28d7929 --- /dev/null +++ b/compare_models.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +""" +Side-by-side model comparison for photo_analyzer. + +Reads already-analyzed photos from the DB (the stored result — no re-run, no +extra cost on the model that produced them) and runs a second model (default: +Gemini 2.5 Flash via its OpenAI-compatible endpoint) live on the same images, +then prints the two results next to each other plus measured token cost. + +Reuses photo_analyzer's prepare_image, ANALYSIS_PROMPT, get_db and env loader — +no logic is duplicated. + +Usage: + GEMINI_API_KEY=... python compare_models.py -n 10 + python compare_models.py --selftest # offline check, no API call + +Put GEMINI_API_KEY in photo_analyzer.env (the same file the main pipeline reads). +""" +import argparse +import json +import os +import random +import sys +from collections import defaultdict +from pathlib import Path + +from openai import OpenAI + +import photo_analyzer as pa + +# Gemini's OpenAI-compatible endpoint — the existing OpenAI client just works. +GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/" +DEFAULT_MODEL = "gemini-2.5-flash" + +# $/million tokens, for the live model's measured cost. Update if you change model. +PRICES = { # (input, output) + "gemini-2.5-flash": (0.30, 2.50), + "gemini-2.5-flash-lite": (0.10, 0.40), + "gemini-3.5-flash": (1.50, 9.00), +} + + +def _strip_json_fences(raw: str) -> str: + """Same fence-stripping photo_analyzer uses — models often wrap JSON in ```.""" + raw = raw.strip() + if raw.startswith("```"): + raw = raw.split("```")[1] + if raw.startswith("json"): + raw = raw[4:] + raw = raw.strip() + return raw + + +def analyze_with(client: OpenAI, model: str, path: Path) -> tuple[dict, dict]: + """Run `model` on one image. Returns (parsed_result, usage_dict).""" + b64, mime = pa.prepare_image(path) + resp = client.chat.completions.create( + model=model, + max_tokens=4096, + messages=[{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}}, + {"type": "text", "text": pa.ANALYSIS_PROMPT}, + ], + }], + ) + result = json.loads(_strip_json_fences(resp.choices[0].message.content)) + u = resp.usage + usage = {"prompt": u.prompt_tokens if u else 0, + "completion": u.completion_tokens if u else 0} + return result, usage + + +def fetch_analyzed(conn, limit: int) -> list: + """ + Randomly sample already-analyzed photos, spread across folders. Groups rows by + parent directory, shuffles, then round-robins one per folder so the sample + covers as many different folders as possible. Only picked files are checked on + disk (cheap), not the whole library. + """ + # "analyzed already" = has a description, regardless of status — once EXIF is + # written the status flips from 'analyzed' to 'exif_written', but the result stays. + rows = conn.execute( + """SELECT path, description, tags, people_count, mood, setting, approx_year + FROM photos WHERE description IS NOT NULL AND description != ''""").fetchall() + buckets = defaultdict(list) + for r in rows: + buckets[str(Path(r["path"]).parent)].append(r) + folders = list(buckets.keys()) + random.shuffle(folders) + for f in folders: + random.shuffle(buckets[f]) + + out, exhausted = [], False + while len(out) < limit and not exhausted: + exhausted = True + for f in folders: # one per folder per pass → spread + if not buckets[f]: + continue + exhausted = False + r = buckets[f].pop() + if Path(r["path"]).exists(): + out.append(r) + if len(out) >= limit: + break + return out + + +def _fmt(label: str, desc, tags, people, mood) -> str: + tags_s = ", ".join(tags) if isinstance(tags, list) else (tags or "") + return (f" {label}\n" + f" desc: {desc}\n" + f" tags: {tags_s}\n" + f" people: {people} mood: {mood}") + + +def selftest(): + assert _strip_json_fences('```json\n{"a":1}\n```') == '{"a":1}' + assert _strip_json_fences('{"a":1}') == '{"a":1}' + assert _strip_json_fences('```\n{"b":2}\n```') == '{"b":2}' + print("selftest OK") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("-n", "--num", type=int, default=10, help="photos to compare") + ap.add_argument("--model", default=DEFAULT_MODEL, help="live model to compare against the stored results") + ap.add_argument("--db", default="photo_analysis.db") + ap.add_argument("--selftest", action="store_true", help="run offline check and exit") + args = ap.parse_args() + + if args.selftest: + selftest() + return + + pa.load_env_file() # picks up GEMINI_API_KEY from photo_analyzer.env + key = os.environ.get("GEMINI_API_KEY") + if not key: + sys.exit("GEMINI_API_KEY not set (add it to photo_analyzer.env or export it).") + + conn = pa.get_db(args.db) + rows = fetch_analyzed(conn, args.num) + if not rows: + sys.exit("No analyzed photos found in the DB to compare against.") + + client = OpenAI(api_key=key, base_url=GEMINI_BASE_URL) + p_in, p_out = PRICES.get(args.model, (0.0, 0.0)) + tot_in = tot_out = 0 + ok = 0 + + for r in rows: + path = Path(r["path"]) + print(f"\n=== {path.parent.name}/{path.name} ===") + # stored result (already in the DB — no re-run) + try: + stored_tags = json.loads(r["tags"]) if r["tags"] else [] + except Exception: + stored_tags = r["tags"] + print(_fmt("stored (DB)", r["description"], stored_tags, + r["people_count"], r["mood"])) + # live call to the comparison model + try: + res, usage = analyze_with(client, args.model, path) + tot_in += usage["prompt"]; tot_out += usage["completion"]; ok += 1 + print(_fmt(f"{args.model} (live)", res.get("description"), + res.get("tags", []), res.get("people_count"), res.get("mood"))) + except Exception as e: + print(f" {args.model} (live)\n ERROR: {e}") + + if ok: + cost = tot_in / 1e6 * p_in + tot_out / 1e6 * p_out + print(f"\n— {args.model}: {ok} photos, {tot_in:,} in + {tot_out:,} out tokens, " + f"measured cost ${cost:.4f} (${cost / ok * 1000:.2f}/1k photos)") + + +if __name__ == "__main__": + main() diff --git a/delivery_backlog/E01-shared-identity-inventory.md b/delivery_backlog/E01-shared-identity-inventory.md new file mode 100644 index 0000000..8b9bd25 --- /dev/null +++ b/delivery_backlog/E01-shared-identity-inventory.md @@ -0,0 +1,21 @@ +# E01 — Shared Identity and Inventory + +Concept phase: A. Establish the application foundation, stable asset identity, +inventory, duplicate review, and managed thumbnails by extracting proven CLI donor +code wherever possible. + +## Stories + +1. [US01-01 — Inventory and characterize donor code](stories/US01-01-donor-characterization.md) +2. [US01-02 — Establish application and database foundation](stories/US01-02-app-database-foundation.md) +3. [US01-03 — Discover and reconcile stable assets](stories/US01-03-inventory-identity.md) +4. [US01-04 — Detect and decide duplicates](stories/US01-04-duplicate-engine.md) +5. [US01-05 — Generate and serve managed thumbnails](stories/US01-05-thumbnail-service.md) +6. [US01-06 — Review inventory and duplicates in the browser](stories/US01-06-inventory-review-ui.md) +7. [US01-07 — Automate Phase A end-to-end acceptance](stories/US01-07-phase-a-e2e.md) + +## Epic outcome + +The system discovers the fixture library without violating path policy, preserves +asset identity across moves, creates explainable duplicate clusters, serves oriented +thumbnails, and persists review decisions through restart. diff --git a/delivery_backlog/E02-unified-workflow-shell.md b/delivery_backlog/E02-unified-workflow-shell.md new file mode 100644 index 0000000..10c2e60 --- /dev/null +++ b/delivery_backlog/E02-unified-workflow-shell.md @@ -0,0 +1,20 @@ +# E02 — Unified Workflow Shell + +Concept phase: B. Add durable jobs and the shared browser application, then migrate +safety review and the existing Library, Analyze, and Stats experiences onto the common +API and service layer. + +## Stories + +1. [US02-01 — Extract safety and existing web UI donors](stories/US02-01-safety-ui-donors.md) +2. [US02-02 — Persist and coordinate durable jobs](stories/US02-02-durable-jobs.md) +3. [US02-03 — Execute jobs with leases, locks, and recovery](stories/US02-03-worker-recovery.md) +4. [US02-04 — Expose job APIs and activity events](stories/US02-04-job-api-events.md) +5. [US02-05 — Build the static application shell](stories/US02-05-frontend-shell.md) +6. [US02-06 — Deliver workflow, safety, library, analysis, and stats views](stories/US02-06-workflow-views.md) +7. [US02-07 — Automate Phase B end-to-end acceptance](stories/US02-07-phase-b-e2e.md) + +## Epic outcome + +Users can operate and observe the shared workflow through one browser application; +mutating jobs are durable, cancellable, recoverable, and limited to safe concurrency. diff --git a/delivery_backlog/E03-album-proposals.md b/delivery_backlog/E03-album-proposals.md new file mode 100644 index 0000000..dcd850a --- /dev/null +++ b/delivery_backlog/E03-album-proposals.md @@ -0,0 +1,17 @@ +# E03 — Album Proposals + +Concept phase: C. Turn analyzed folder metadata into editable, explainable, versioned +album naming proposals without mutating files. + +## Stories + +1. [US03-01 — Aggregate album evidence](stories/US03-01-album-evidence.md) +2. [US03-02 — Define album naming policy](stories/US03-02-naming-policy.md) +3. [US03-03 — Generate and persist versioned proposals](stories/US03-03-proposal-generation.md) +4. [US03-04 — Review and approve proposals in the browser](stories/US03-04-proposal-ui.md) +5. [US03-05 — Automate Phase C end-to-end acceptance](stories/US03-05-phase-c-e2e.md) + +## Epic outcome + +Users receive deterministic, evidence-backed album suggestions, can edit them safely, +and can approve a validated proposal without any rename occurring yet. diff --git a/delivery_backlog/E04-guarded-renaming.md b/delivery_backlog/E04-guarded-renaming.md new file mode 100644 index 0000000..4cf4880 --- /dev/null +++ b/delivery_backlog/E04-guarded-renaming.md @@ -0,0 +1,18 @@ +# E04 — Guarded Renaming + +Concept phase: D. Convert approved proposals into validated, journaled, recoverable +filesystem rename operations. + +## Stories + +1. [US04-01 — Build and export rename plans](stories/US04-01-rename-plan.md) +2. [US04-02 — Journal rename state and preconditions](stories/US04-02-rename-journal.md) +3. [US04-03 — Apply and verify guarded renames](stories/US04-03-rename-apply.md) +4. [US04-04 — Recover or roll back interrupted renames](stories/US04-04-rename-recovery.md) +5. [US04-05 — Operate rename plans in the browser](stories/US04-05-rename-ui.md) +6. [US04-06 — Automate Phase D end-to-end acceptance](stories/US04-06-phase-d-e2e.md) + +## Epic outcome + +No rename occurs without a current validated plan and explicit confirmation; crashes +produce deterministic recovery or a precise manual-recovery state. diff --git a/delivery_backlog/E05-immich-upload.md b/delivery_backlog/E05-immich-upload.md new file mode 100644 index 0000000..2cc6110 --- /dev/null +++ b/delivery_backlog/E05-immich-upload.md @@ -0,0 +1,18 @@ +# E05 — Immich Upload + +Concept phase: E. Upload verified album bytes to Immich through controlled, +observable, restart-safe `immich-go` batches. + +## Stories + +1. [US05-01 — Validate credentials and upload readiness](stories/US05-01-upload-preflight.md) +2. [US05-02 — Orchestrate album upload batches](stories/US05-02-upload-batches.md) +3. [US05-03 — Parse and persist uploader outcomes](stories/US05-03-upload-reports.md) +4. [US05-04 — Verify, retry, and resolve uncertain uploads](stories/US05-04-upload-verification.md) +5. [US05-05 — Operate uploads in the browser](stories/US05-05-upload-ui.md) +6. [US05-06 — Automate Phase E end-to-end acceptance](stories/US05-06-phase-e-e2e.md) + +## Epic outcome + +Users can upload an explicitly scoped album only after metadata checkpoints pass and +can distinguish new, duplicate, upgraded, failed, and uncertain outcomes. diff --git a/delivery_backlog/E06-archive-lifecycle.md b/delivery_backlog/E06-archive-lifecycle.md new file mode 100644 index 0000000..f644c0e --- /dev/null +++ b/delivery_backlog/E06-archive-lifecycle.md @@ -0,0 +1,18 @@ +# E06 — Archive Lifecycle + +Concept phase: F. Reclaim active storage only after verified archival, while retaining +identity, duplicate evidence, browsing, recovery, and restoration. + +## Stories + +1. [US06-01 — Configure and preflight archive destinations](stories/US06-01-archive-preflight.md) +2. [US06-02 — Transfer, verify, and remove active sources](stories/US06-02-archive-transfer.md) +3. [US06-03 — Preserve offline identity and review evidence](stories/US06-03-offline-assets.md) +4. [US06-04 — Plan and execute safe restores](stories/US06-04-restore.md) +5. [US06-05 — Operate archive and restore in the browser](stories/US06-05-archive-ui.md) +6. [US06-06 — Automate Phase F end-to-end acceptance](stories/US06-06-phase-f-e2e.md) + +## Epic outcome + +Uploaded albums can be archived without data loss and later browsed, deduplicated, +recovered, and restored even when archive media is temporarily offline. diff --git a/delivery_backlog/E07-hardening-release.md b/delivery_backlog/E07-hardening-release.md new file mode 100644 index 0000000..fd22eef --- /dev/null +++ b/delivery_backlog/E07-hardening-release.md @@ -0,0 +1,19 @@ +# E07 — Hardening and Release + +Concept phase: G. Resolve migration debt, harden hostile and failure conditions, +validate performance, document operations, and prove release readiness. + +## Stories + +1. [US07-01 — Complete donor migration and freeze the CLI archive](stories/US07-01-donor-archive.md) +2. [US07-02 — Harden API authorization and path boundaries](stories/US07-02-security-hardening.md) +3. [US07-03 — Harden media and metadata edge cases](stories/US07-03-media-hardening.md) +4. [US07-04 — Prove concurrency and crash recovery](stories/US07-04-fault-concurrency.md) +5. [US07-05 — Deliver backup and operational recovery](stories/US07-05-backup-operations.md) +6. [US07-06 — Validate performance and resource bounds](stories/US07-06-performance.md) +7. [US07-07 — Automate full release acceptance](stories/US07-07-release-e2e.md) + +## Epic outcome + +The integrated application meets security, durability, recovery, performance, donor +parity, and full workflow release gates with reproducible automated evidence. diff --git a/delivery_backlog/README.md b/delivery_backlog/README.md new file mode 100644 index 0000000..aee2881 --- /dev/null +++ b/delivery_backlog/README.md @@ -0,0 +1,49 @@ +# Integrated Photo Pipeline Delivery Backlog + +This backlog decomposes the phases in +[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md) into seven +epics and small, independently verifiable user stories. + +## Numbering and file naming + +- Epics: `E01` through `E07`, matching concept Phases A through G. +- Stories: `US-`, for example `US03-02`. +- Epic files: `E01-.md`. +- Story files: `stories/US01-01-.md`. +- IDs are permanent. Renaming or reordering a story must not reuse its ID. + +## Delivery rules + +- Implement epics in numeric order unless an epic explicitly permits overlap. +- A story should produce one coherent, reviewable outcome and be small enough for one + focused implementation change. +- Donor-first migration is mandatory: characterize and extract useful CLI behavior + before replacing it. Record provenance in the donor ledger. +- Every implementation story includes automated unit, property, golden, contract, or + integration tests appropriate to its risk. +- Every epic ends with a dedicated automated end-to-end story that exercises the real + browser, API/SSE, worker, isolated database, and temporary fixture library. +- A story is complete only when all acceptance criteria and automated tests pass with + no unexplained skips. An epic is complete only when its cumulative regression suite + and end-to-end gate pass. + +## Epics + +1. [E01 — Shared identity and inventory](E01-shared-identity-inventory.md) +2. [E02 — Unified workflow shell](E02-unified-workflow-shell.md) +3. [E03 — Album proposals](E03-album-proposals.md) +4. [E04 — Guarded renaming](E04-guarded-renaming.md) +5. [E05 — Immich upload](E05-immich-upload.md) +6. [E06 — Archive lifecycle](E06-archive-lifecycle.md) +7. [E07 — Hardening and release](E07-hardening-release.md) + +## Shared definition of done + +- Acceptance criteria are demonstrably satisfied. +- Public API/schema changes are documented and contract-tested. +- Database and filesystem mutations are restart-safe and idempotent where required. +- Paths are constrained by the shared library policy. +- No credentials or private image data appear in logs or browser responses. +- Relevant donor-ledger rows and story-to-test traceability entries are current. +- Automated tests pass locally and in CI; changed user behavior has an automated + acceptance test. diff --git a/delivery_backlog/stories/US01-01-donor-characterization.md b/delivery_backlog/stories/US01-01-donor-characterization.md new file mode 100644 index 0000000..6008db5 --- /dev/null +++ b/delivery_backlog/stories/US01-01-donor-characterization.md @@ -0,0 +1,20 @@ +# US01-01 — Inventory and Characterize Donor Code + +Epic: [E01](../E01-shared-identity-inventory.md) + +As a maintainer, I want a donor ledger and characterization suite for the current CLI +implementations so proven behavior is reused deliberately instead of rewritten. + +## Acceptance criteria + +- Relevant functions in both CLI codebases are classified as reuse, extract, refactor, + or replace, with rationale and target module. +- The ledger covers discovery, hashing, imaging, NSFW, vision, EXIF, database, UI, + configuration, logging, cancellation, and error behavior. +- Representative current outputs are captured against stable fixture IDs. +- No legacy file is moved or archived in this story. + +## Automated tests + +- Characterization tests run against current donor entry points and core functions. +- A ledger-lint test rejects missing source references, target locations, or test IDs. diff --git a/delivery_backlog/stories/US01-02-app-database-foundation.md b/delivery_backlog/stories/US01-02-app-database-foundation.md new file mode 100644 index 0000000..ae51f6d --- /dev/null +++ b/delivery_backlog/stories/US01-02-app-database-foundation.md @@ -0,0 +1,24 @@ +# US01-02 — Establish Application and Database Foundation + +Epic: [E01](../E01-shared-identity-inventory.md) + +As an operator, I want the API, worker configuration, and database lifecycle to start +consistently so later services share one durable foundation. + +## Acceptance criteria + +- FastAPI factory, typed configuration, SQLAlchemy sessions, Alembic, WAL, foreign + keys, structured logging, and readiness are configured. +- Stable asset IDs, path occurrences, versions, and audit timestamps have migrations. +- Secrets are referenced through configuration and never returned or logged. +- Startup and shutdown leave no leaked worker or database resources. + +## Automated tests + +- Migrations pass from an empty database and supported legacy schema snapshots. +- Process-level integration tests verify readiness, restart, WAL, foreign keys, and + clean shutdown. + +## Dependencies + +- US01-01 diff --git a/delivery_backlog/stories/US01-03-inventory-identity.md b/delivery_backlog/stories/US01-03-inventory-identity.md new file mode 100644 index 0000000..9882f27 --- /dev/null +++ b/delivery_backlog/stories/US01-03-inventory-identity.md @@ -0,0 +1,23 @@ +# US01-03 — Discover and Reconcile Stable Assets + +Epic: [E01](../E01-shared-identity-inventory.md) + +As a user, I want scans to discover eligible photos and retain identity across moves +so reorganizing folders does not lose prior work. + +## Acceptance criteria + +- Donor discovery and reconciliation logic is extracted behind `InventoryService`. +- All supported roots use one boundary and exclusion policy; unsafe symlink paths fail. +- New, moved, copied, replaced, missing, active, and archived occurrences are distinct. +- Enumeration is deterministic and rescanning unchanged input is idempotent. + +## Automated tests + +- Unit/property tests cover path normalization, boundaries, and reconciliation. +- Integration tests move, copy, replace, remove, and rescan fixture assets, asserting + stable IDs and exact durable states after restart. + +## Dependencies + +- US01-02 diff --git a/delivery_backlog/stories/US01-04-duplicate-engine.md b/delivery_backlog/stories/US01-04-duplicate-engine.md new file mode 100644 index 0000000..ec3b952 --- /dev/null +++ b/delivery_backlog/stories/US01-04-duplicate-engine.md @@ -0,0 +1,24 @@ +# US01-04 — Detect and Decide Duplicates + +Epic: [E01](../E01-shared-identity-inventory.md) + +As a user, I want explainable duplicate clusters and reversible decisions so redundant +copies are excluded without collapsing distinct photos automatically. + +## Acceptance criteria + +- Donor exact, normalized-pixel, and perceptual hash behavior is extracted and + versioned. +- Confidence bands distinguish automatic exact matches from fuzzy review candidates. +- Canonical, variant, not-duplicate, and deferred decisions persist with evidence. +- New contradictory members reopen reviewed clusters; canonical links cannot cycle. + +## Automated tests + +- Golden corpus tests assert hash relationships and confidence classes. +- Property/integration tests cover clustering, negative links, reversals, new members, + and persistence after restart. + +## Dependencies + +- US01-03 diff --git a/delivery_backlog/stories/US01-05-thumbnail-service.md b/delivery_backlog/stories/US01-05-thumbnail-service.md new file mode 100644 index 0000000..3ae8cb8 --- /dev/null +++ b/delivery_backlog/stories/US01-05-thumbnail-service.md @@ -0,0 +1,24 @@ +# US01-05 — Generate and Serve Managed Thumbnails + +Epic: [E01](../E01-shared-identity-inventory.md) + +As a reviewer, I want safe, correctly oriented previews so I can compare photos without +exposing arbitrary filesystem paths. + +## Acceptance criteria + +- Donor decode, orientation, resize, and HEIC behavior is reused where compatible. +- Thumbnails are requested by asset ID, cached atomically, quota-managed, and + invalidated when source identity changes. +- Corrupt, oversized, missing, and unsupported inputs produce bounded typed errors. +- Requests cannot escape configured roots or access excluded paths. + +## Automated tests + +- Golden tests assert orientation, dimensions, transparency handling, and cache keys. +- Integration tests cover concurrent generation, corruption, invalidation, eviction, + and path attacks. + +## Dependencies + +- US01-03 diff --git a/delivery_backlog/stories/US01-06-inventory-review-ui.md b/delivery_backlog/stories/US01-06-inventory-review-ui.md new file mode 100644 index 0000000..d0e6596 --- /dev/null +++ b/delivery_backlog/stories/US01-06-inventory-review-ui.md @@ -0,0 +1,25 @@ +# US01-06 — Review Inventory and Duplicates in the Browser + +Epic: [E01](../E01-shared-identity-inventory.md) + +As a user, I want to browse inventory and compare duplicate candidates visually so I +can make informed canonical decisions. + +## Acceptance criteria + +- Inventory lists are paged, filterable, keyboard accessible, and use stable URLs. +- Duplicate comparison shows oriented previews, metadata evidence, confidence, and + synchronized comparison controls. +- Fuzzy decisions require explicit confirmation; stale versions receive a visible + conflict and never mutate state. +- Reloading restores the selected view and persisted decisions. + +## Automated tests + +- Component tests cover state, routing, and API error presentation. +- Playwright tests cover keyboard review, fuzzy confirmation, stale conflicts, and + persistence after reload. + +## Dependencies + +- US01-04, US01-05 diff --git a/delivery_backlog/stories/US01-07-phase-a-e2e.md b/delivery_backlog/stories/US01-07-phase-a-e2e.md new file mode 100644 index 0000000..1e4633e --- /dev/null +++ b/delivery_backlog/stories/US01-07-phase-a-e2e.md @@ -0,0 +1,24 @@ +# US01-07 — Automate Phase A End-to-End Acceptance + +Epic: [E01](../E01-shared-identity-inventory.md) + +As a delivery owner, I want reproducible Phase A end-to-end tests so identity, +inventory, duplicates, and thumbnails cannot regress unnoticed. + +## Acceptance criteria + +- The harness launches the real API and worker with a fresh database and deterministic + temporary fixture library. +- Journeys cover scan, exclusion, move reconciliation, exact/fuzzy duplicate review, + thumbnail orientation, canonical selection, reload, and full process restart. +- API and durable database state are asserted after restart. +- Story IDs US01-01 through US01-06 map to passing automated tests. + +## Automated tests + +- One documented command runs Phase A API and Playwright suites without network access. +- CI retains diagnostics on failure and rejects skipped, flaky, or unexercised fixtures. + +## Dependencies + +- US01-01 through US01-06 diff --git a/delivery_backlog/stories/US02-01-safety-ui-donors.md b/delivery_backlog/stories/US02-01-safety-ui-donors.md new file mode 100644 index 0000000..2d86c3a --- /dev/null +++ b/delivery_backlog/stories/US02-01-safety-ui-donors.md @@ -0,0 +1,22 @@ +# US02-01 — Extract Safety and Existing Web UI Donors + +Epic: [E02](../E02-unified-workflow-shell.md) + +As a maintainer, I want safety logic and proven web interactions extracted from the +existing tools so the unified workflow retains working behavior and visual language. + +## Acceptance criteria + +- The donor ledger maps scoring, thresholds, decisions, EXIF keywords, filters, + layouts, tokens, keyboard controls, and error states to new modules. +- Behavior changes required by stable identity or shared jobs are documented. +- Shared modules have no dependency on archived entry points. + +## Automated tests + +- Characterization tests compare donor and extracted behavior on the same fixture IDs. +- Snapshot/interaction tests cover reused design tokens and critical controls. + +## Dependencies + +- E01 complete diff --git a/delivery_backlog/stories/US02-02-durable-jobs.md b/delivery_backlog/stories/US02-02-durable-jobs.md new file mode 100644 index 0000000..2f83bb6 --- /dev/null +++ b/delivery_backlog/stories/US02-02-durable-jobs.md @@ -0,0 +1,22 @@ +# US02-02 — Persist and Coordinate Durable Jobs + +Epic: [E02](../E02-unified-workflow-shell.md) + +As an operator, I want jobs and item progress stored durably so work can resume after +browser, API, or worker interruption. + +## Acceptance criteria + +- Job, item, event, attempt, progress, cancellation, and terminal states are persisted. +- Commands support idempotency keys and atomic state transitions. +- Prerequisites and one-mutating-job policy produce explicit blockers. +- Progress derives from durable item state rather than browser memory. + +## Automated tests + +- State-machine/property tests reject invalid transitions and duplicate terminal state. +- Repository tests cover concurrent idempotent submission and restart persistence. + +## Dependencies + +- US01-02 diff --git a/delivery_backlog/stories/US02-03-worker-recovery.md b/delivery_backlog/stories/US02-03-worker-recovery.md new file mode 100644 index 0000000..1961bd2 --- /dev/null +++ b/delivery_backlog/stories/US02-03-worker-recovery.md @@ -0,0 +1,22 @@ +# US02-03 — Execute Jobs with Leases, Locks, and Recovery + +Epic: [E02](../E02-unified-workflow-shell.md) + +As an operator, I want workers to claim, cancel, and recover jobs safely so crashes or +duplicate workers cannot corrupt state. + +## Acceptance criteria + +- Workers use leases, heartbeats, fencing tokens, bounded queues, and typed lanes. +- Cancellation is cooperative and leaves resumable or terminal item states. +- Lock ordering prevents conflicting analysis, EXIF, rename, upload, and archive work. +- A recovered job rejects late commits from its former worker. + +## Automated tests + +- Process tests kill workers, expire leases, recover jobs, and attempt stale commits. +- Randomized concurrency tests assert no deadlocks, duplicate completion, or lost events. + +## Dependencies + +- US02-02 diff --git a/delivery_backlog/stories/US02-04-job-api-events.md b/delivery_backlog/stories/US02-04-job-api-events.md new file mode 100644 index 0000000..e0ffea5 --- /dev/null +++ b/delivery_backlog/stories/US02-04-job-api-events.md @@ -0,0 +1,23 @@ +# US02-04 — Expose Job APIs and Activity Events + +Epic: [E02](../E02-unified-workflow-shell.md) + +As a frontend client, I want versioned job commands, status APIs, and activity events +so the browser can operate workflows without HTML responses or internal imports. + +## Acceptance criteria + +- `/api/v1` exposes typed job start, status, cancel, blocker, and event contracts. +- Operational endpoints return JSON; the event endpoint returns SSE with resumable IDs. +- SSE reconnects without duplicate/lost durable events and polling is sufficient as a + fallback. +- Errors use the shared JSON envelope and correct status codes, including `409`. + +## Automated tests + +- OpenAPI/contract tests validate schemas, status codes, errors, and content types. +- Black-box tests cover SSE disconnect/reconnect, polling, idempotency, and cancellation. + +## Dependencies + +- US02-02, US02-03 diff --git a/delivery_backlog/stories/US02-05-frontend-shell.md b/delivery_backlog/stories/US02-05-frontend-shell.md new file mode 100644 index 0000000..3b7ab95 --- /dev/null +++ b/delivery_backlog/stories/US02-05-frontend-shell.md @@ -0,0 +1,24 @@ +# US02-05 — Build the Static Application Shell + +Epic: [E02](../E02-unified-workflow-shell.md) + +As a user, I want a fast, navigable application shell so all workflow views share one +consistent interface without server-generated operational HTML. + +## Acceptance criteria + +- `index.html`, CSS, and modular JavaScript are separate static files. +- Shared API client, store, router, error handling, cancellation, and SSE/polling + adapters are implemented. +- Navigation routes restore current view and filters after reload. +- HTML fallback never intercepts `/api/v1` requests. + +## Automated tests + +- JavaScript tests cover API errors, store transitions, routing, and event fallback. +- Browser tests assert asset loading, deep links, reload, JSON-only APIs, and no + unexpected console/network errors. + +## Dependencies + +- US02-04 diff --git a/delivery_backlog/stories/US02-06-workflow-views.md b/delivery_backlog/stories/US02-06-workflow-views.md new file mode 100644 index 0000000..ce0c2eb --- /dev/null +++ b/delivery_backlog/stories/US02-06-workflow-views.md @@ -0,0 +1,25 @@ +# US02-06 — Deliver Workflow, Safety, Library, Analysis, and Stats Views + +Epic: [E02](../E02-unified-workflow-shell.md) + +As a user, I want one workflow home and migrated operational views so I can understand +readiness, review safety, browse assets, run analysis, and inspect results. + +## Acceptance criteria + +- Workflow cards show counts, blockers, last run, action, and details without relying + on color alone. +- Safety decisions persist and prevent NSFW assets from reaching vision analysis while + retaining verified upload eligibility. +- Library, Analyze, and Stats preserve useful donor features on shared APIs. +- Read-only browsing remains available during jobs; conflicting actions explain why + they are disabled. + +## Automated tests + +- View tests cover counts, filters, blockers, decisions, progress, errors, and reload. +- Integration tests assert provider-call privacy and verified safety EXIF behavior. + +## Dependencies + +- US02-01, US02-05 diff --git a/delivery_backlog/stories/US02-07-phase-b-e2e.md b/delivery_backlog/stories/US02-07-phase-b-e2e.md new file mode 100644 index 0000000..5c1b1d1 --- /dev/null +++ b/delivery_backlog/stories/US02-07-phase-b-e2e.md @@ -0,0 +1,23 @@ +# US02-07 — Automate Phase B End-to-End Acceptance + +Epic: [E02](../E02-unified-workflow-shell.md) + +As a delivery owner, I want Phase B workflow automation so durable jobs and migrated +views are proven through real process and browser boundaries. + +## Acceptance criteria + +- Journeys cover shell loading, workflow blockers, safety review, analysis start, + progress, SSE reconnect, polling fallback, cancellation, resume, and error inspection. +- A second conflicting mutation is rejected while read-only browsing continues. +- NSFW fixture IDs never reach the fake vision provider and decisions survive restart. +- Story IDs US02-01 through US02-06 map to passing automated tests. + +## Automated tests + +- One documented command runs Phase B API, worker-recovery, and Playwright suites. +- The Phase A suite runs unchanged as a required regression gate. + +## Dependencies + +- US02-01 through US02-06 diff --git a/delivery_backlog/stories/US03-01-album-evidence.md b/delivery_backlog/stories/US03-01-album-evidence.md new file mode 100644 index 0000000..3a07630 --- /dev/null +++ b/delivery_backlog/stories/US03-01-album-evidence.md @@ -0,0 +1,23 @@ +# US03-01 — Aggregate Album Evidence + +Epic: [E03](../E03-album-proposals.md) + +As a user, I want album suggestions based on summarized photo evidence so names reflect +the folder without exposing unnecessary raw responses. + +## Acceptance criteria + +- The service aggregates canonical, eligible asset descriptions, tags, dates, + locations, people counts, and current folder identity. +- Missing, conflicting, deferred, duplicate, and NSFW evidence follows documented rules. +- Aggregation is versioned, deterministic, paged where needed, and invalidated by + relevant asset changes. + +## Automated tests + +- Unit/property tests cover aggregation, exclusions, conflicts, ordering, and versions. +- Repository tests verify results before and after relevant asset changes. + +## Dependencies + +- E02 complete diff --git a/delivery_backlog/stories/US03-02-naming-policy.md b/delivery_backlog/stories/US03-02-naming-policy.md new file mode 100644 index 0000000..460f106 --- /dev/null +++ b/delivery_backlog/stories/US03-02-naming-policy.md @@ -0,0 +1,22 @@ +# US03-02 — Define Album Naming Policy + +Epic: [E03](../E03-album-proposals.md) + +As a user, I want configurable naming templates and validation so proposed album names +are readable and safe on supported filesystems. + +## Acceptance criteria + +- Templates define allowed evidence, formatting, length, separators, and fallbacks. +- Validation handles forbidden characters, reserved names, whitespace, Unicode, + case-insensitive collisions, existing destinations, and empty results. +- Validation returns structured issues and suggestions without mutating files. + +## Automated tests + +- Property tests generate names across platforms and prove paths stay inside the root. +- Golden cases cover valid names, collisions, normalization, and template fallbacks. + +## Dependencies + +- US03-01 diff --git a/delivery_backlog/stories/US03-03-proposal-generation.md b/delivery_backlog/stories/US03-03-proposal-generation.md new file mode 100644 index 0000000..1a43865 --- /dev/null +++ b/delivery_backlog/stories/US03-03-proposal-generation.md @@ -0,0 +1,25 @@ +# US03-03 — Generate and Persist Versioned Proposals + +Epic: [E03](../E03-album-proposals.md) + +As a user, I want AI-assisted album proposals with rationale and confidence so I can +evaluate suggestions before approving them. + +## Acceptance criteria + +- A provider adapter receives only the minimum aggregated evidence and returns a + validated structured proposal. +- Proposal, evidence version, model/prompt version, rationale, confidence, edits, and + approval state are durable. +- Malformed, rate-limited, failed, duplicate, and stale responses are explicit and + safely retryable. +- Approval validates the latest proposal version and performs no rename. + +## Automated tests + +- Deterministic fake-provider tests cover success, malformed output, retry, and failure. +- API tests cover creation, edits, optimistic conflicts, approval, and restart. + +## Dependencies + +- US03-01, US03-02 diff --git a/delivery_backlog/stories/US03-04-proposal-ui.md b/delivery_backlog/stories/US03-04-proposal-ui.md new file mode 100644 index 0000000..f9e81bd --- /dev/null +++ b/delivery_backlog/stories/US03-04-proposal-ui.md @@ -0,0 +1,24 @@ +# US03-04 — Review and Approve Proposals in the Browser + +Epic: [E03](../E03-album-proposals.md) + +As a user, I want to inspect evidence, edit a proposal, and approve it so I retain +control over album naming. + +## Acceptance criteria + +- The view shows source album, suggested name, evidence summary, rationale, confidence, + validation issues, and affected count. +- Edits validate promptly; collisions and invalid names are actionable. +- Approval requires explicit action and rejects stale state visibly. +- Reload preserves edits and approval state through the API. + +## Automated tests + +- UI tests cover editing, errors, collision guidance, approval, stale conflict, and + keyboard operation. +- Playwright asserts no file path changes occur during proposal approval. + +## Dependencies + +- US03-03 diff --git a/delivery_backlog/stories/US03-05-phase-c-e2e.md b/delivery_backlog/stories/US03-05-phase-c-e2e.md new file mode 100644 index 0000000..c1945d6 --- /dev/null +++ b/delivery_backlog/stories/US03-05-phase-c-e2e.md @@ -0,0 +1,22 @@ +# US03-05 — Automate Phase C End-to-End Acceptance + +Epic: [E03](../E03-album-proposals.md) + +As a delivery owner, I want automated proposal journeys so generation, editing, and +approval remain deterministic and non-mutating. + +## Acceptance criteria + +- Journeys cover evidence aggregation, success, provider failure, invalid name, + collision, editing, stale approval, valid approval, reload, and restart. +- Tests assert provider inputs, persisted versions, and unchanged fixture paths. +- Story IDs US03-01 through US03-04 map to passing automated tests. + +## Automated tests + +- One command runs Phase C API and Playwright suites with the deterministic provider. +- Phase A–B end-to-end suites remain green. + +## Dependencies + +- US03-01 through US03-04 diff --git a/delivery_backlog/stories/US04-01-rename-plan.md b/delivery_backlog/stories/US04-01-rename-plan.md new file mode 100644 index 0000000..77adaa3 --- /dev/null +++ b/delivery_backlog/stories/US04-01-rename-plan.md @@ -0,0 +1,23 @@ +# US04-01 — Build and Export Rename Plans + +Epic: [E04](../E04-guarded-renaming.md) + +As a user, I want a complete validated rename plan before mutation so I can understand +every source, destination, collision, and blocker. + +## Acceptance criteria + +- Approved album proposals produce itemized plans with asset IDs, source/destination, + expected hashes, versions, and required operations. +- Validation detects missing/changed sources, duplicate targets, case-only and Unicode + collisions, root escapes, symlinks, and cross-filesystem behavior. +- A portable JSON export contains no credentials and has a schema version/checksum. + +## Automated tests + +- Property tests validate collision-free plans and asset-set preservation. +- API/golden tests cover valid, invalid, stale, case-only, Unicode, and exported plans. + +## Dependencies + +- E03 complete diff --git a/delivery_backlog/stories/US04-02-rename-journal.md b/delivery_backlog/stories/US04-02-rename-journal.md new file mode 100644 index 0000000..d58f2b5 --- /dev/null +++ b/delivery_backlog/stories/US04-02-rename-journal.md @@ -0,0 +1,23 @@ +# US04-02 — Journal Rename State and Preconditions + +Epic: [E04](../E04-guarded-renaming.md) + +As an operator, I want every rename transition journaled durably so interrupted work +can be diagnosed and recovered without guessing. + +## Acceptance criteria + +- Plan and item state machines record preconditions, intent, attempts, outcomes, + timestamps, fencing tokens, and verification evidence. +- Journal writes occur before filesystem mutations and terminal transitions are + idempotent. +- Startup classifies incomplete transitions as resumable, rollback-safe, or manual. + +## Automated tests + +- State-machine tests cover all valid/invalid transitions and repeated recovery. +- Migration/repository tests prove durable state after process restart. + +## Dependencies + +- US04-01, US02-03 diff --git a/delivery_backlog/stories/US04-03-rename-apply.md b/delivery_backlog/stories/US04-03-rename-apply.md new file mode 100644 index 0000000..e82bb57 --- /dev/null +++ b/delivery_backlog/stories/US04-03-rename-apply.md @@ -0,0 +1,24 @@ +# US04-03 — Apply and Verify Guarded Renames + +Epic: [E04](../E04-guarded-renaming.md) + +As a user, I want confirmed plans applied without overwrites so reorganizing the +library preserves identity and bytes. + +## Acceptance criteria + +- Apply requires current plan/version confirmation and an exclusive mutation lease. +- Preconditions are rechecked immediately before each mutation; unexpected targets + are never overwritten. +- Case-only and cross-filesystem operations use safe staged procedures. +- Postconditions verify bytes and paths, update occurrences, and reconcile stable IDs. + +## Automated tests + +- Filesystem integration tests cover normal, case-only, collision, cross-filesystem, + stale source, and repeated apply. +- Tests assert byte preservation and database/path consistency after restart. + +## Dependencies + +- US04-02 diff --git a/delivery_backlog/stories/US04-04-rename-recovery.md b/delivery_backlog/stories/US04-04-rename-recovery.md new file mode 100644 index 0000000..e615f5d --- /dev/null +++ b/delivery_backlog/stories/US04-04-rename-recovery.md @@ -0,0 +1,23 @@ +# US04-04 — Recover or Roll Back Interrupted Renames + +Epic: [E04](../E04-guarded-renaming.md) + +As an operator, I want safe recovery choices after interruption so partially renamed +albums never trigger blind retries or data loss. + +## Acceptance criteria + +- Recovery derives action from journal plus current source/destination evidence. +- Resume and rollback revalidate recorded hashes and refuse changed paths. +- Ambiguous states block unrelated mutations and explain manual recovery precisely. +- Recovery and rollback are idempotent across repeated restarts. + +## Automated tests + +- Fault injection terminates the process at every persisted rename transition. +- Tests cover resume, rollback, changed paths, unexpected destinations, and repeated + restart with no overwrite or asset loss. + +## Dependencies + +- US04-03 diff --git a/delivery_backlog/stories/US04-05-rename-ui.md b/delivery_backlog/stories/US04-05-rename-ui.md new file mode 100644 index 0000000..644b020 --- /dev/null +++ b/delivery_backlog/stories/US04-05-rename-ui.md @@ -0,0 +1,22 @@ +# US04-05 — Operate Rename Plans in the Browser + +Epic: [E04](../E04-guarded-renaming.md) + +As a user, I want to preview, confirm, monitor, and recover renames in the browser so +filesystem changes remain understandable and deliberate. + +## Acceptance criteria + +- Preview shows every affected path, validation issue, operation type, and count. +- Confirmation includes the current server-issued plan/version token. +- Progress, cancellation limits, terminal verification, and errors are visible. +- Recovery blocks unrelated mutations and offers only evidence-safe actions. + +## Automated tests + +- UI/Playwright tests cover preview, stale confirmation, apply, progress, collision, + interruption, recovery, rollback, focus, and reload. + +## Dependencies + +- US04-03, US04-04 diff --git a/delivery_backlog/stories/US04-06-phase-d-e2e.md b/delivery_backlog/stories/US04-06-phase-d-e2e.md new file mode 100644 index 0000000..1edc950 --- /dev/null +++ b/delivery_backlog/stories/US04-06-phase-d-e2e.md @@ -0,0 +1,22 @@ +# US04-06 — Automate Phase D End-to-End Acceptance + +Epic: [E04](../E04-guarded-renaming.md) + +As a delivery owner, I want exhaustive automated rename journeys so every crash point +and stale condition is proven safe. + +## Acceptance criteria + +- Journeys cover plan/export, confirmation, valid apply, case-only rename, collision, + stale source, cancellation boundary, every journal crash point, recovery, and rollback. +- Stable IDs, hashes, asset set, journal, and UI state are asserted after restart. +- Story IDs US04-01 through US04-05 map to passing automated tests. + +## Automated tests + +- One command runs API, filesystem fault-injection, and Playwright rename suites. +- Phases A–C remain green and the source fixture corpus remains unchanged. + +## Dependencies + +- US04-01 through US04-05 diff --git a/delivery_backlog/stories/US05-01-upload-preflight.md b/delivery_backlog/stories/US05-01-upload-preflight.md new file mode 100644 index 0000000..3d6d9b0 --- /dev/null +++ b/delivery_backlog/stories/US05-01-upload-preflight.md @@ -0,0 +1,23 @@ +# US05-01 — Validate Credentials and Upload Readiness + +Epic: [E05](../E05-immich-upload.md) + +As a user, I want upload blockers and scope validated before execution so unverified or +changed files never reach Immich accidentally. + +## Acceptance criteria + +- Preflight checks credentials without exposing them, server reachability, selected + album scope, canonical/safety/analysis/EXIF readiness, and current byte hashes. +- The redacted command preview shows exact folders, album mapping, and asset counts. +- Partial scope and every blocker require explicit resolution or approved policy. +- Preflight produces a versioned token invalidated by relevant changes. + +## Automated tests + +- Integration tests cover valid, missing/invalid credential, blocked stage, changed + bytes, partial album, stale token, and secret redaction. + +## Dependencies + +- E04 complete diff --git a/delivery_backlog/stories/US05-02-upload-batches.md b/delivery_backlog/stories/US05-02-upload-batches.md new file mode 100644 index 0000000..6455a56 --- /dev/null +++ b/delivery_backlog/stories/US05-02-upload-batches.md @@ -0,0 +1,23 @@ +# US05-02 — Orchestrate Album Upload Batches + +Epic: [E05](../E05-immich-upload.md) + +As a user, I want one approved album uploaded at a time so scope, progress, and retries +remain controlled. + +## Acceptance criteria + +- A durable upload batch records album, asset IDs, pre-upload hashes, command version, + attempts, progress, cancellation, and raw report location. +- `immich-go` is invoked through the integration adapter with bounded output and no + shell interpolation or secret logging. +- Only one configured uploader lane runs; restart resumes at safe item/batch boundaries. + +## Automated tests + +- Fake-executable integration tests verify arguments, album isolation, concurrency, + cancellation, restart, output limits, and credential privacy. + +## Dependencies + +- US05-01, US02-03 diff --git a/delivery_backlog/stories/US05-03-upload-reports.md b/delivery_backlog/stories/US05-03-upload-reports.md new file mode 100644 index 0000000..ea07287 --- /dev/null +++ b/delivery_backlog/stories/US05-03-upload-reports.md @@ -0,0 +1,22 @@ +# US05-03 — Parse and Persist Uploader Outcomes + +Epic: [E05](../E05-immich-upload.md) + +As an operator, I want uploader reports converted into durable typed outcomes so I can +distinguish new, upgraded, duplicate, failed, and unknown assets. + +## Acceptance criteria + +- Version-specific parsers preserve bounded raw evidence and classify every item. +- Unknown versions or lines never become success; they require verification. +- Uploaded SHA-1/content evidence, timestamps, counts, and parser version are stored. +- Reprocessing the same report is idempotent. + +## Automated tests + +- Golden parser fixtures cover every supported outcome/version and malformed output. +- Repository tests cover idempotent import, count reconciliation, and restart. + +## Dependencies + +- US05-02 diff --git a/delivery_backlog/stories/US05-04-upload-verification.md b/delivery_backlog/stories/US05-04-upload-verification.md new file mode 100644 index 0000000..5ccad2f --- /dev/null +++ b/delivery_backlog/stories/US05-04-upload-verification.md @@ -0,0 +1,22 @@ +# US05-04 — Verify, Retry, and Resolve Uncertain Uploads + +Epic: [E05](../E05-immich-upload.md) + +As a user, I want uncertain uploads verified before retry so a lost response cannot +create duplicate server assets. + +## Acceptance criteria + +- Retry policy distinguishes safe failures from accepted-but-unknown outcomes. +- Verification compares recorded local bytes with authoritative available evidence. +- Changed-after-upload bytes create a visible stale warning and block unsafe actions. +- Manual resolution records evidence and audit history; it never silently assumes success. + +## Automated tests + +- Fault tests cover timeout before acceptance, acceptance then lost response, parser + uncertainty, safe retry, changed bytes, and repeated verification. + +## Dependencies + +- US05-03 diff --git a/delivery_backlog/stories/US05-05-upload-ui.md b/delivery_backlog/stories/US05-05-upload-ui.md new file mode 100644 index 0000000..9ddbc96 --- /dev/null +++ b/delivery_backlog/stories/US05-05-upload-ui.md @@ -0,0 +1,23 @@ +# US05-05 — Operate Uploads in the Browser + +Epic: [E05](../E05-immich-upload.md) + +As a user, I want to preflight, confirm, monitor, and verify uploads so I know exactly +what reached Immich and what remains uncertain. + +## Acceptance criteria + +- The view shows album scope, blockers, redacted configuration, counts, and exact + confirmation before start. +- Progress distinguishes new, upgraded, duplicate, failed, cancelled, and uncertain. +- Uncertain outcomes expose verification actions, not an automatic retry button. +- Secrets never appear in DOM, logs, URLs, traces, or browser storage. + +## Automated tests + +- Playwright covers blocked/valid preflight, start, progress, cancellation, report, + uncertainty, verification, stale bytes, reload, and secret scanning. + +## Dependencies + +- US05-04 diff --git a/delivery_backlog/stories/US05-06-phase-e-e2e.md b/delivery_backlog/stories/US05-06-phase-e-e2e.md new file mode 100644 index 0000000..b548b23 --- /dev/null +++ b/delivery_backlog/stories/US05-06-phase-e-e2e.md @@ -0,0 +1,23 @@ +# US05-06 — Automate Phase E End-to-End Acceptance + +Epic: [E05](../E05-immich-upload.md) + +As a delivery owner, I want deterministic upload journeys so metadata ordering, +deduplication outcomes, and uncertainty handling cannot regress. + +## Acceptance criteria + +- Journeys cover credential failure, preflight blockers, new, exact duplicate, upgrade, + retryable failure, acceptance-response loss, verification, cancellation, and resume. +- Tests prove EXIF precedes upload and persisted hashes match submitted bytes. +- No secret appears in retained artifacts; state is asserted after restart. +- Story IDs US05-01 through US05-05 map to passing automated tests. + +## Automated tests + +- One command runs fake uploader, black-box API, and Playwright upload suites. +- Phases A–D remain green. + +## Dependencies + +- US05-01 through US05-05 diff --git a/delivery_backlog/stories/US06-01-archive-preflight.md b/delivery_backlog/stories/US06-01-archive-preflight.md new file mode 100644 index 0000000..085f463 --- /dev/null +++ b/delivery_backlog/stories/US06-01-archive-preflight.md @@ -0,0 +1,23 @@ +# US06-01 — Configure and Preflight Archive Destinations + +Epic: [E06](../E06-archive-lifecycle.md) + +As a user, I want archive destinations identified and validated so an album is never +moved to the wrong, unavailable, or undersized storage. + +## Acceptance criteria + +- Archive locations record stable media/volume identity, root, capabilities, and state. +- Preflight validates verified upload, current bytes, destination identity, capacity, + writability, boundaries, collisions, locks, backup, and manifest creation. +- Preview shows transfer method, exact scope, reclaimable bytes, and every blocker. +- Confirmation uses a versioned token invalidated by source/destination changes. + +## Automated tests + +- Integration tests cover valid, offline, wrong-volume, low-space, read-only, collision, + stale source, lock conflict, and unsafe path cases. + +## Dependencies + +- E05 complete diff --git a/delivery_backlog/stories/US06-02-archive-transfer.md b/delivery_backlog/stories/US06-02-archive-transfer.md new file mode 100644 index 0000000..fd64803 --- /dev/null +++ b/delivery_backlog/stories/US06-02-archive-transfer.md @@ -0,0 +1,23 @@ +# US06-02 — Transfer, Verify, and Remove Active Sources + +Epic: [E06](../E06-archive-lifecycle.md) + +As a user, I want source files removed only after durable archive verification so +reclaiming space cannot cause data loss. + +## Acceptance criteria + +- A journal records planned, transferring, verified, removing, and terminal item state. +- Cross-filesystem flow copies to a temporary destination, closes, hashes, atomically + publishes, records manifest, then removes the matching source. +- Same-filesystem optimization is used only when proven safe and still verified. +- Unexpected source/destination changes stop the item without overwrite or deletion. + +## Automated tests + +- Filesystem/process tests interrupt every transition before and after source removal. +- Tests assert manifest/hash correctness, idempotent recovery, and zero verified loss. + +## Dependencies + +- US06-01, US02-03 diff --git a/delivery_backlog/stories/US06-03-offline-assets.md b/delivery_backlog/stories/US06-03-offline-assets.md new file mode 100644 index 0000000..1618d03 --- /dev/null +++ b/delivery_backlog/stories/US06-03-offline-assets.md @@ -0,0 +1,22 @@ +# US06-03 — Preserve Offline Identity and Review Evidence + +Epic: [E06](../E06-archive-lifecycle.md) + +As a user, I want archived assets searchable and usable for deduplication while their +media is offline so new copies are not mistaken for unrelated photos. + +## Acceptance criteria + +- Availability distinguishes active, archived online/offline, and unexpectedly missing. +- Archived hashes remain in indexes and inventory scans do not prune offline assets. +- Protected comparison previews and evidence remain available under a durable policy. +- Exact and fuzzy active matches link to archived canonicals with appropriate review. + +## Automated tests + +- Integration tests toggle fake mounts, rescan, discover exact/fuzzy copies, review + previews, and assert stable state across restart. + +## Dependencies + +- US06-02 diff --git a/delivery_backlog/stories/US06-04-restore.md b/delivery_backlog/stories/US06-04-restore.md new file mode 100644 index 0000000..96f2c42 --- /dev/null +++ b/delivery_backlog/stories/US06-04-restore.md @@ -0,0 +1,24 @@ +# US06-04 — Plan and Execute Safe Restores + +Epic: [E06](../E06-archive-lifecycle.md) + +As a user, I want archived assets restored to collision-free active paths so I can +recover originals without losing identity or previous decisions. + +## Acceptance criteria + +- Restore preflight requires the correct archive identity, matching bytes, capacity, + safe destination, and no conflicting lease. +- Restore copies, closes, verifies, atomically publishes, and registers an active + occurrence while preserving asset identity and stage history. +- Collisions never overwrite; mismatches become stale/divergent states. +- Restore is journaled, resumable, and idempotent. + +## Automated tests + +- Tests cover offline media, wrong volume, normal restore, collision, changed archive + bytes, interruption, repeated restart, and preserved decisions. + +## Dependencies + +- US06-03 diff --git a/delivery_backlog/stories/US06-05-archive-ui.md b/delivery_backlog/stories/US06-05-archive-ui.md new file mode 100644 index 0000000..48d013c --- /dev/null +++ b/delivery_backlog/stories/US06-05-archive-ui.md @@ -0,0 +1,23 @@ +# US06-05 — Operate Archive and Restore in the Browser + +Epic: [E06](../E06-archive-lifecycle.md) + +As a user, I want to preview, confirm, monitor, recover, and restore archives so storage +reclamation remains understandable and reversible. + +## Acceptance criteria + +- Archive preview shows exact scope, destination identity, transfer method, capacity, + blockers, and reclaimable bytes. +- Progress separates transfer, verification, and source-removal states. +- Offline assets remain browsable with clear availability and mount instructions. +- Recovery and restore views expose only safe evidence-backed actions. + +## Automated tests + +- Playwright covers preflight blockers, confirmation, progress, interruption/recovery, + offline browsing, restore, collision, keyboard operation, and reload. + +## Dependencies + +- US06-02, US06-03, US06-04 diff --git a/delivery_backlog/stories/US06-06-phase-f-e2e.md b/delivery_backlog/stories/US06-06-phase-f-e2e.md new file mode 100644 index 0000000..3582c7f --- /dev/null +++ b/delivery_backlog/stories/US06-06-phase-f-e2e.md @@ -0,0 +1,23 @@ +# US06-06 — Automate Phase F End-to-End Acceptance + +Epic: [E06](../E06-archive-lifecycle.md) + +As a delivery owner, I want archive lifecycle automation so removal, offline behavior, +recovery, and restore are proven without risking real files. + +## Acceptance criteria + +- Journeys cover preflight blockers, copy/verify/remove, same-filesystem behavior, + every interruption point, offline deduplication, mount return, restore, and collision. +- API, journal, manifest, paths, hashes, previews, and durable state are asserted after + restart; no source is removed before archive verification. +- Story IDs US06-01 through US06-05 map to passing automated tests. + +## Automated tests + +- One command runs archive fault-injection, black-box API, and Playwright suites. +- Phases A–E remain green. + +## Dependencies + +- US06-01 through US06-05 diff --git a/delivery_backlog/stories/US07-01-donor-archive.md b/delivery_backlog/stories/US07-01-donor-archive.md new file mode 100644 index 0000000..7af4a90 --- /dev/null +++ b/delivery_backlog/stories/US07-01-donor-archive.md @@ -0,0 +1,24 @@ +# US07-01 — Complete Donor Migration and Freeze the CLI Archive + +Epic: [E07](../E07-hardening-release.md) + +As a maintainer, I want all useful legacy behavior resolved and original CLIs frozen +with provenance so production has one implementation without losing historical evidence. + +## Acceptance criteria + +- Every in-scope donor-ledger row has a target, characterization test, parity result, + and documented intentional delta where applicable. +- Remaining CSV state is migrated and reconciled into SQLite with a report. +- Original sources, docs, dependency lock, schema/config notes, version, and checksums + are placed in the read-only legacy archive with secrets removed. +- Production imports and runtime paths cannot load or execute archived scripts. + +## Automated tests + +- Ledger/archive lint verifies completeness, checksums, redaction, and non-importability. +- Full parity and database migration/reconciliation suites pass. + +## Dependencies + +- E01–E06 complete diff --git a/delivery_backlog/stories/US07-02-security-hardening.md b/delivery_backlog/stories/US07-02-security-hardening.md new file mode 100644 index 0000000..143c9af --- /dev/null +++ b/delivery_backlog/stories/US07-02-security-hardening.md @@ -0,0 +1,23 @@ +# US07-02 — Harden API Authorization and Path Boundaries + +Epic: [E07](../E07-hardening-release.md) + +As an operator, I want the local application resistant to cross-origin and path attacks +so another process or webpage cannot access photos or trigger mutations. + +## Acceptance criteria + +- Session authentication, Origin/Host checks, SameSite cookies, CSRF protection, and + restrictive CORS/default headers cover all mutation and media endpoints. +- Asset IDs, repeated root validation, symlink defense, upload limits, and schema + validation prevent raw-path and race escapes. +- Errors reveal no secrets, filesystem internals, or private metadata unnecessarily. + +## Automated tests + +- Black-box security tests cover missing/invalid auth, CSRF, hostile origins/hosts, + traversal, symlink races, malformed/oversized requests, and information leakage. + +## Dependencies + +- US07-01 diff --git a/delivery_backlog/stories/US07-03-media-hardening.md b/delivery_backlog/stories/US07-03-media-hardening.md new file mode 100644 index 0000000..95e404b --- /dev/null +++ b/delivery_backlog/stories/US07-03-media-hardening.md @@ -0,0 +1,24 @@ +# US07-03 — Harden Media and Metadata Edge Cases + +Epic: [E07](../E07-hardening-release.md) + +As a user, I want unusual or damaged media handled safely so one file cannot corrupt +metadata, exhaust resources, or stop the library workflow. + +## Acceptance criteria + +- Supported format/orientation/profile combinations have bounded decode behavior. +- Corrupt, truncated, huge-dimension, unsupported, and malformed-metadata files become + precise item errors without worker failure. +- EXIF checkpoints preserve user fields and previous-stage fields, read back owned + fields, and mark conflicts divergent rather than silently repairing them. +- Cache regeneration/invalidation and temporary-file cleanup are safe and scoped. + +## Automated tests + +- Extended golden corpus covers all declared format, orientation, corruption, and EXIF + cases with memory/time bounds and before/after metadata snapshots. + +## Dependencies + +- US07-01 diff --git a/delivery_backlog/stories/US07-04-fault-concurrency.md b/delivery_backlog/stories/US07-04-fault-concurrency.md new file mode 100644 index 0000000..a2ca2ae --- /dev/null +++ b/delivery_backlog/stories/US07-04-fault-concurrency.md @@ -0,0 +1,25 @@ +# US07-04 — Prove Concurrency and Crash Recovery + +Epic: [E07](../E07-hardening-release.md) + +As an operator, I want randomized race and fault tests so concurrency cannot produce +deadlocks, stale commits, duplicate terminal states, or data loss. + +## Acceptance criteria + +- Test control points exist at persisted transitions without exposing production + mutation APIs. +- Tests cover database pressure, worker claim races, file changes, thumbnail races, + safety/analysis races, EXIF/upload races, rename/archive conflicts, and cancellation. +- Faults cover process death, disk full, read-only paths, DB busy/corruption, network + failures, malformed providers, GPU exhaustion, subprocess hangs, and missing tools. +- Recovery is deterministic or enters an explicit manual-recovery state. + +## Automated tests + +- Randomized suites repeat with recorded seeds and assert invariants after restart. +- CI retains journals, logs, DB, seed, and filesystem manifest for any failure. + +## Dependencies + +- US07-02, US07-03 diff --git a/delivery_backlog/stories/US07-05-backup-operations.md b/delivery_backlog/stories/US07-05-backup-operations.md new file mode 100644 index 0000000..1c113fe --- /dev/null +++ b/delivery_backlog/stories/US07-05-backup-operations.md @@ -0,0 +1,24 @@ +# US07-05 — Deliver Backup and Operational Recovery + +Epic: [E07](../E07-hardening-release.md) + +As an operator, I want documented and tested backup/recovery procedures so application +state can be restored after migration failure, database damage, or storage loss. + +## Acceptance criteria + +- Online backup includes database consistency, required manifests, configuration + references, and retention guidance without copying secrets into logs. +- Integrity checks, migration rollback/recovery, archive media handling, and restore + drills have step-by-step documentation. +- Operational diagnostics report DB/WAL/cache/log/thumbnail sizes and low-disk risks. +- Legacy-process locking prevents incompatible simultaneous mutations. + +## Automated tests + +- Recovery drills restore backups into fresh roots and run integrity/reconciliation. +- Tests cover failed migration, corrupt copy detection, retention, and process locking. + +## Dependencies + +- US07-01, US07-04 diff --git a/delivery_backlog/stories/US07-06-performance.md b/delivery_backlog/stories/US07-06-performance.md new file mode 100644 index 0000000..598b35b --- /dev/null +++ b/delivery_backlog/stories/US07-06-performance.md @@ -0,0 +1,24 @@ +# US07-06 — Validate Performance and Resource Bounds + +Epic: [E07](../E07-hardening-release.md) + +As an operator, I want measured performance and bounded resource use so large libraries +remain responsive and do not exhaust memory, disk, handles, or queues. + +## Acceptance criteria + +- Agreed budgets cover API latency, throughput, RSS, open files, WAL, queues, cache, + disk reserve, and event delivery at 25k, 100k, and 500k synthetic assets. +- Duplicate clusters with thousands of members remain usable and paged. +- Multi-hour work plus browsing, cancellation, retry, backup, and cache eviction shows + no unbounded growth or starvation. +- Budget exceptions are measured, documented, and approved before release. + +## Automated tests + +- Repeatable load/soak commands export machine-readable metrics and fail budget breaches. +- CI runs a short profile; scheduled infrastructure runs the full soak matrix. + +## Dependencies + +- US07-04, US07-05 diff --git a/delivery_backlog/stories/US07-07-release-e2e.md b/delivery_backlog/stories/US07-07-release-e2e.md new file mode 100644 index 0000000..f024e0c --- /dev/null +++ b/delivery_backlog/stories/US07-07-release-e2e.md @@ -0,0 +1,29 @@ +# US07-07 — Automate Full Release Acceptance + +Epic: [E07](../E07-hardening-release.md) + +As a release owner, I want one reproducible automated release gate so every user story +and complete workflow is proven before real-library mutation is enabled. + +## Acceptance criteria + +- A story-to-test matrix contains every backlog story with no missing, skipped, or + failing required test. +- One fresh-environment journey runs discovery, duplicate review, safety, analysis, + EXIF verification, album proposal, rename, rescan, upload, archive, offline + deduplication, restore, and full process restarts. +- Security, migration, donor parity, crash recovery, accessibility, responsive UI, + browser console/network, fixture reproducibility, and archive safety gates pass. +- A read-only real-library dry run produces an explicitly approved reconciliation + report before mutation can be configured. + +## Automated tests + +- One documented release command provisions and destroys the full isolated stack and + retains signed/versioned evidence. +- Chromium passes every change; supported cross-browser and full soak suites pass for + release; all earlier epic suites run unchanged. + +## Dependencies + +- US07-01 through US07-06 diff --git a/nsfw_tag.py b/nsfw_tag.py new file mode 100644 index 0000000..178bea6 --- /dev/null +++ b/nsfw_tag.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +"""Backwards-compatible shim — the code now lives in the nsfwtag/ package. + +Run either of these (identical behaviour): + python nsfw_tag.py "" -r + python -m nsfwtag "" -r +""" +from nsfwtag.__main__ import main + +if __name__ == "__main__": + main() diff --git a/nsfwtag/README.md b/nsfwtag/README.md new file mode 100644 index 0000000..c5ac0e8 --- /dev/null +++ b/nsfwtag/README.md @@ -0,0 +1,282 @@ +# nsfwtag + +Local, on-device NSFW image review and tagging for large photo libraries — built to feed [Immich](https://immich.app) search and to complement the `photo_analyzer.py` captioning pipeline. + +It scores a photo library with a **local** vision model (Apple GPU via MPS), opens a fast **browser review app**, and writes an `nsfw` (or `sfw`) keyword into each photo's EXIF for the ones you confirm — **preserving every other tag**. Nothing ever leaves the machine: no API calls, no uploads, no telemetry. + +``` +python -m nsfwtag "pictures/" -r +``` + +--- + +## Table of contents + +- [The idea](#the-idea) +- [Features](#features) +- [How it works](#how-it-works) +- [Model choice](#model-choice) +- [Requirements & install](#requirements--install) +- [CLI usage](#cli-usage) +- [The review app](#the-review-app) +- [EXIF / review-state model](#exif--review-state-model) +- [HTTP API](#http-api) +- [Data & caching](#data--caching) +- [Project layout](#project-layout) +- [Design decisions](#design-decisions) +- [Immich integration](#immich-integration) + +--- + +## The idea + +A large personal photo library has a handful of sensitive images scattered through +thousands of holiday snaps. To filter them in [Immich](https://immich.app) you need a +searchable marker on each one — but going through every photo by hand is infeasible, and +sending a private library to a cloud moderation API is a non-starter. + +**The model does recall; you do precision.** A local classifier is good at narrowing +thousands of photos down to a few dozen candidates, but it is *not* trustworthy as the +final word — it over-flags swimwear, art, medical shots. So this is a **human-in-the-loop** +tool: the model proposes, you confirm in a fast visual review app, and only your confirmed +decisions are written to disk. Nothing ever leaves the machine — no API, no upload, no +telemetry. + +The decision lives **in the file's own EXIF** as an `nsfw`/`sfw` keyword, not in a +side database, so it survives across tools, backups and restarts, and Immich indexes it +for free at ingest. + +--- + +## Features + +- **On-device scoring** with `AdamCodd/vit-base-nsfw-detector` (ViT, ~350 MB, downloaded once then fully offline). Chosen over Falconsai after benchmarking against the library — better recall, fewer false positives (see `bench.py`). +- **Resume-safe**: scores are cached to a CSV; a rerun reuses them and only scores newly-added files — skipping the model load entirely when nothing is new. +- **Browser review app** (dark OLED UI, zero web dependencies — served by a stdlib HTTP server): + - Per-image **checkmark** to mark NSFW / SFW; frame colors show state at a glance. + - **Folder browser** sidebar, live search, a single **Show** filter (All / To-handle / NSFW / SFW), sort, NSFW-score threshold, and a contextual action group that appears only when images are selected. + - **Lightbox** with prev/next, keyboard control, and an **EXIF panel** (incl. a clickable **Google Maps** link for geotagged photos). + - **Activity log** tab with a per-file audit trail of every EXIF write. + - Lazy, per-folder tag scan with a **progress ring**. +- **Reversible & non-destructive**: tag, untag, or toggle at any time; only `Keywords`/`Subject` are modified, so Caption, dates, GPS, camera info, etc. are always preserved. +- **Persistent decisions**: marking an image safe writes an `sfw` keyword, so your review survives restarts. + +--- + +## How it works + +``` + discover images score (cached) review in browser write EXIF + folder ─────────────────▶ [*.jpg …] ───────────────▶ nsfw_scores.csv ─────────────────▶ exiftool: Keywords/Subject += nsfw|sfw + (recursive, de-duped) supported local ViT on MPS (path,score) local HTTP server + review.html (all other tags preserved) +``` + +1. **Discover** — walk the folder (optionally recursive), keep supported image types, de-duplicate by resolved path. +2. **Score** — load cached scores; run the model only on files not in the cache; each image is resized to 224×224 and scored 0..1 for the `nsfw` class. Merge and persist. +3. **Review** — candidates (score ≥ `--review-min`) are served as a single-page app. A lazy per-folder scan reads existing `nsfw`/`sfw` keywords and colors each card. +4. **Tag** — your actions POST to the local server, which calls `exiftool` to add/remove the keyword. EXIF is written **before** the file is later uploaded to Immich. + +--- + +## Model choice + +The tool ships with **`AdamCodd/vit-base-nsfw-detector`** — a ~350 MB ViT binary +classifier that runs offline on Apple MPS. It was picked by benchmarking candidates +against **your own library**, using the photos you'd already hand-tagged `nsfw`/`sfw` +as ground truth (`bench.py` reads those EXIF marks). A generic public benchmark wouldn't +predict how a model behaves on *these* photos, so the comparison is run on the real data. + +Candidates compared (`nsfwtag/bench.py`): + +| Model | Notes | +|-------|-------| +| **AdamCodd/vit-base-nsfw-detector** ✅ | Chosen — best recall with the fewest false positives on this library | +| Falconsai/nsfw_image_detection | Previous default; missed more true positives and needed a high `0.85` cutoff | +| Marqo/nsfw-image-detection-384 | Tiny/fast, but noisier on borderline shots | +| OwenElliott/image-safety-classifier-xs | 3-class (SFW/NSFW/NSFL); useful signal, weaker overall here | + +**Selection criterion — recall first.** Because a human confirms every candidate, a false +*positive* costs one extra glance, but a false *negative* means a sensitive photo silently +slips through untagged. So the model is judged on catching everything at a threshold low +enough to be safe without burying you in noise. AdamCodd's sweet spot is **`0.6`** (vs +Falconsai's `0.85`) — that is the shipped default. + +**Reproduce it** on any sample folder: + +```bash +python -m nsfwtag.bench "pictures/SomeFolder" -r +# writes bench_scores.csv + bench_report.html (thumbnails, per-model scores, +# disagreements highlighted) and prints precision/recall vs your manual tags. +``` + +To switch models, change `MODEL_ID` in `nsfwtag/__init__.py` (any HF image-classification +id) and re-tune `DEFAULT_THRESHOLD`. + +--- + +## Requirements & install + +- **Python 3.10+** +- **exiftool** on `PATH` — `brew install exiftool` (macOS) / `apt install libimage-exiftool-perl` (Debian/Ubuntu) +- Python packages: `pip install torch transformers pillow numpy` +- A GPU is optional — uses Apple **MPS** if available, else CPU. + +The model downloads automatically on first run (~350 MB) and is cached in `~/.cache/huggingface`; every run after that is offline (`local_files_only=True`, which also skips the slow HF Hub update check). + +--- + +## CLI usage + +Run from the directory that contains the `nsfwtag/` package: + +```bash +# Score + open the review app +python -m nsfwtag "pictures/" # this folder only +python -m nsfwtag "pictures/" -r # recurse into subfolders +python nsfw_tag.py "pictures/" -r # identical (backwards-compat shim) + +# Tag directly by score, no review UI +python -m nsfwtag "pictures/" -r --apply + +# Tag every path listed in a text file (one path per line) +python -m nsfwtag confirmed.txt [more.txt ...] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `-r, --recursive` | off | Descend into every subfolder | +| `--threshold` | `0.6` | Score ≥ this is treated as "NSFW to review" (and tagged directly in `--apply`) | +| `--review-min` | `0.20` | Lowest score shown in the review app | +| `--limit N` | `0` | Only the first N images (calibration) | +| `--apply` | off | Tag score ≥ threshold directly, skip the review UI | +| `--rescore` | off | Ignore the score cache and re-run the model on everything | + +The argument is auto-detected: a **folder** opens the review app; one or more **`.txt`** files tag the listed paths directly. + +--- + +## The review app + +Served on `http://127.0.0.1:/` and opened in your browser automatically. Press `Ctrl-C` in the terminal when done. + +There are **two independent channels** per card: the **decision** (the EXIF tag, +shown as the frame + tick color) and the **selection** (a blue ring, for bulk +actions). A card can be both selected *and* decided. + +### Frame colors (the decision) + +| Color | Meaning | EXIF | +|-------|---------|------| +| 🩷 **pink** | tagged NSFW | `nsfw` keyword | +| 🟢 **green** | tagged safe | `sfw` keyword | +| 🔴 **red** | undecided **and** selected — a bulk candidate, nothing written yet | none | +| ⚪ neutral | undecided, not selected | none | +| 🔵 blue ring | selected for a bulk action (independent of the frame color) | — | + +Each card has a **tick** (bottom-right circle) that makes a per-image decision and +**writes EXIF on every click**, cycling: undecided → **NSFW** (pink) → **SFW** +(green) → undecided. One click is enough — it no longer depends on the selection +state. Clicking elsewhere on a card does nothing (no accidental toggles). + +### Controls + +The toolbar separates **viewing** (always visible) from **acting** (contextual). It reads left-to-right: + +``` +🔍 Search… Show ▾ Sort ▾ NSFW ≥ 0.6 ●───── Select flagged Select all ⟨ N selected · Deselect · Remove · Mark safe · Mark NSFW ⟩ +``` + +- **Folder browser** (left sidebar, toggleable): filter to a folder + its subfolders; counts per folder. +- **Search**: filter by filename or folder. +- **Show** (single filter): **All** / **To-handle** (no keyword yet — what's left to decide) / **NSFW** (tagged nsfw) / **SFW** (tagged sfw). +- **Sort**: score / name / folder. +- **NSFW ≥ slider**: score threshold used by *Select flagged* and the red "identified" coloring. +- **Select flagged**: selects visible, untagged images at/above the threshold. +- **Select all**: selects every currently-visible image (respects the folder/filter). Pair with **Deselect** to clear. Use this to build a selection after a filter where *Select flagged* matches nothing. +- **Per-image tick**: one click writes EXIF and cycles undecided → NSFW → SFW → undecided; the tick and frame colors show the current decision. +- **Action group** (appears only when ≥1 image is selected): **Mark NSFW** / **Mark safe** / **Remove** (clears either keyword) / **Deselect**. Counts and actions are scoped to the current view (folder/filter), so the count matches what you see. +- **Lightbox** (⤢ on a thumbnail): full-res image, EXIF side panel (incl. Google Maps link), `←`/`→` navigate, `Space` toggle selection, `Esc`/backdrop close. +- **Activity log** (header button): timestamped, per-file record of every EXIF write. + +--- + +## EXIF / review-state model + +The review decision is stored **in the file's EXIF** as one of two mutually-exclusive keywords, written to both `Keywords` (IPTC) and `Subject` (XMP): + +| Keyword | Meaning | +|---------|---------| +| `nsfw` | confirmed NSFW | +| `sfw` | confirmed safe | +| *(neither)* | not yet reviewed | + +Rules: + +- Writing `nsfw` removes any `sfw`, and vice-versa — a file can hold **at most one**. +- On read, **`nsfw` takes precedence**: a stray file with both is reported as NSFW, never as safe. +- **Only `Keywords` and `Subject` are ever modified.** All other metadata — `ImageDescription`/Caption, GPS, capture dates, camera make/model, ratings — is left byte-for-byte intact in both directions (add and remove). +- `exiftool` runs with `-overwrite_original` (no sidecar `_original` files) and `-m` (ignore minor warnings so writes still apply to slightly-nonconforming files). + +Immich indexes `Keywords`, so tagged photos become searchable there (search `nsfw`). + +--- + +## HTTP API + +Stdlib `ThreadingHTTPServer` bound to `127.0.0.1` on an OS-assigned free port. **Every path argument is validated against the scanned candidate set** — the server can't be pointed at arbitrary files. + +| Method | Route | Purpose | +|--------|-------|---------| +| GET | `/` | The review page | +| GET | `/img?path=` | Thumbnail bytes for a candidate image | +| GET | `/exif?path=` | Filtered EXIF for the lightbox panel (decimal GPS + Google Maps URL) | +| GET | `/log` | Activity-log entries `[{t, m}, …]` | +| POST | `/tagged` | Body: `[path,…]` → `{nsfw:[…], sfw:[…]}` (batched keyword read) | +| POST | `/apply` | Body: `[path,…]` → add `nsfw` (strips `sfw`) → `{tagged:n}` — "Mark NSFW" | +| POST | `/mark?state=sfw` | Body: `[path,…]` → add `sfw` (strips `nsfw`) — "Mark safe" / checkmark→green | +| POST | `/untag` | Body: `[path,…]` → remove both keywords (un-review) → `{untagged:n}` — "Remove" | +| POST | `/mark?state=clear` | Body: `[path,…]` → remove `sfw` only (checkmark→red) | +| POST | `/toggle` | Body: `[path,…]` → flip `nsfw` per file → `{tagged:[…]}` (endpoint retained; not used by the current UI) | + +--- + +## Data & caching + +- **`nsfw_scores.csv`** (written in the scanned folder): `path,nsfw_score`. This is the resume cache — reruns reuse it and only score new files. Delete it or pass `--rescore` to force a full re-score. +- **Review state** is **not** in the CSV; it lives in each file's EXIF (see above), so it persists independently of the score cache and survives restarts. +- The activity log is in-memory (last 2000 events) and resets when the server stops; messages are also echoed to stderr. + +--- + +## Project layout + +``` +nsfwtag/ +├── __init__.py # constants (model id, extensions, defaults), warning suppression +├── exif.py # read/write/remove nsfw|sfw via exiftool; read_marks, read_exif (+GPS) +├── scoring.py # discover images, cached model scoring +├── webapp.py # render the page: fill review.html tokens (cards, folder tree, totals) +├── review.html # the frontend — HTML/CSS/JS (no build step, no dependencies) +├── server.py # local HTTP server + routes + activity log +├── __main__.py # CLI entry point (python -m nsfwtag) +└── README.md +nsfw_tag.py # thin shim: `python nsfw_tag.py` == `python -m nsfwtag` +``` + +The frontend lives in a real `review.html` (dynamic parts injected via `%%TOKEN%%` replacement) rather than a Python string, so it edits with normal HTML/CSS/JS tooling. + +--- + +## Design decisions + +- **No pre-selection at render.** Cards render unselected; the **lazy per-folder scan** decides each card's state from its EXIF keywords (pink/green) or score (red). This is deliberate: it guarantees a photo you marked safe never reappears pre-selected-red after a restart, and keeps startup instant (the browser opens before any `exiftool` scan runs). +- **Lazy, chunked, per-folder scanning** with a progress ring, instead of one blocking scan at startup — so switching into a folder shows real progress and huge libraries stay responsive. +- **Score cache first, model second** — the model (and its load) is skipped entirely on reruns with no new files. +- **View-scoped counters/actions** — Apply/Toggle/Remove and their counts act on the folder/filter you're viewing, so the number always matches what's on screen. +- **Stdlib-only server + dependency-free frontend** — nothing to build or install for the UI; the whole app is one HTML file plus a ~120-line Python server. + +--- + +## Immich integration + +Order matters: **write EXIF first, then upload.** immich-go deduplicates on the file's SHA-1 content hash, so writing EXIF after upload would create duplicates. This tool only writes metadata in place; ingest into Immich is a separate, deliberate `immich-go upload` step. Once uploaded, `Keywords` (including `nsfw`) are searchable in Immich alongside its own ML pipeline. **Do not** also point Immich at the folder as an External Library. diff --git a/nsfwtag/__init__.py b/nsfwtag/__init__.py new file mode 100644 index 0000000..f7bb9a4 --- /dev/null +++ b/nsfwtag/__init__.py @@ -0,0 +1,25 @@ +"""nsfwtag — local, on-device NSFW image tagging for Immich. + +Score a photo library with a local ViT model (Apple GPU via MPS), review the +flagged images in a browser, and write an 'nsfw' EXIF keyword into the ones you +confirm — preserving every other tag. Nothing leaves the machine. + +Entry point: `python -m nsfwtag [-r]` (or the nsfw_tag.py shim). +""" +import os +import warnings + +os.environ.setdefault("PYTHONWARNINGS", "ignore") # quiet numpy-2/-1 ABI noise from optional libs +warnings.filterwarnings("ignore") + +# AdamCodd/vit-base-nsfw-detector — chosen after benchmarking against the user's own +# library (much better recall + fewer false positives than Falconsai). See bench.py. +MODEL_ID = "AdamCodd/vit-base-nsfw-detector" +EXTS = {".jpg", ".jpeg", ".png", ".heic", ".heif", ".webp"} +CSV_NAME = "nsfw_scores.csv" +DEFAULT_THRESHOLD = 0.6 # this model's sweet spot; Falconsai used 0.85 +DEFAULT_REVIEW_MIN = 0.2 +BATCH = 16 +KEYWORD = "nsfw" + +__version__ = "1.1.0" diff --git a/nsfwtag/__main__.py b/nsfwtag/__main__.py new file mode 100644 index 0000000..58b7f5b --- /dev/null +++ b/nsfwtag/__main__.py @@ -0,0 +1,65 @@ +"""CLI entry point. + + python -m nsfwtag "" # score + open the review app + python -m nsfwtag "" -r # recurse into subfolders + python -m nsfwtag list.txt [list2.txt] # tag every path in the .txt list(s) + +The argument is auto-detected: a folder opens the review app; .txt file(s) tag +directly. A tiny stdlib HTTP server serves the page and tags on Apply — no data +leaves the machine. +""" +import argparse +import sys +from pathlib import Path + +from . import CSV_NAME, DEFAULT_REVIEW_MIN, DEFAULT_THRESHOLD +from .exif import apply_list, write_keyword +from .scoring import discover_images, score_images +from .server import serve_review + + +def main(): + ap = argparse.ArgumentParser(prog="nsfwtag", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("target", nargs="+", help="a FOLDER to scan (opens the review app) OR one/more .txt lists to tag") + ap.add_argument("-r", "--recursive", action="store_true", help="descend into every subfolder") + ap.add_argument("--apply", action="store_true", help="folder mode: tag score>=threshold directly, skip review") + ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD, help=f"score >= this is pre-selected / tagged (default {DEFAULT_THRESHOLD})") + ap.add_argument("--review-min", type=float, default=DEFAULT_REVIEW_MIN, help=f"lowest score shown in the review app (default {DEFAULT_REVIEW_MIN})") + ap.add_argument("--limit", type=int, default=0, help="only the first N images (calibration)") + ap.add_argument("--rescore", action="store_true", help="ignore cached scores and re-run the model on everything") + args = ap.parse_args() + + # auto-detected: .txt list(s) = apply mode; a single folder = review mode. + targets = [Path(t).expanduser() for t in args.target] + if targets and all(t.is_file() for t in targets): + for t in targets: + apply_list(str(t)) + return + if len(targets) != 1 or not targets[0].is_dir(): + sys.exit("Pass ONE folder (review), or one or more .txt lists (apply).") + target = targets[0] + + imgs = discover_images(target, recursive=args.recursive, limit=args.limit) + if not imgs: + sys.exit("No images found.") + + csv_path = target / CSV_NAME + scored, errors = score_images(imgs, csv_path, rescore=args.rescore) + print(f"\n{len(scored)} images ready, {errors} errors (cache -> {csv_path})") + + if args.apply: + tagged = sum(1 for p, s in scored if s >= args.threshold and write_keyword(str(p))) + print(f" tagged 'nsfw' in EXIF: {tagged} (score >= {args.threshold})") + return + + # live review app: one page for ALL candidates across the tree, tag on click + cands = sorted([(p, s) for p, s in scored if s >= args.review_min], key=lambda x: -x[1]) + if not cands: + print(f" no candidates >= {args.review_min} — nothing to review.") + return + serve_review(cands, args.threshold) # tag-state is checked lazily in-browser, per folder + + +if __name__ == "__main__": + main() diff --git a/nsfwtag/bench.py b/nsfwtag/bench.py new file mode 100644 index 0000000..d1e4e9b --- /dev/null +++ b/nsfwtag/bench.py @@ -0,0 +1,426 @@ +"""Benchmark / compare NSFW models on YOUR OWN photos. + + python -m nsfwtag.bench "" [-r] [--models a,b,c] [--limit N] [--threshold 0.85] + +Scores every image with each selected model and writes, into the sample folder: + * bench_scores.csv — path + one nsfw-score column per model + * bench_report.html — thumbnails with per-model scores, sortable, disagreements + highlighted (open it to eyeball which model over-flags) +…plus a console summary (flag counts, agreement, biggest disagreements). + +Every score is normalized to nsfw = 1 - P(safe-class) so binary and multi-class +models are directly comparable. Nothing leaves the machine. + +Models (need `pip install timm` for the timm ones; each downloads once): + falconsai Falconsai/nsfw_image_detection (current, transformers, binary) + adamcodd AdamCodd/vit-base-nsfw-detector (transformers, binary) + marqo Marqo/nsfw-image-detection-384 (timm, binary, tiny) + owen-xs OwenElliott/image-safety-classifier-xs (timm, 3-class SFW/NSFW/NSFL) +""" +import argparse +import csv +import gc +import html +import json +import os +import sys +import webbrowser +from pathlib import Path +from urllib.parse import quote + +from .exif import read_marks +from .scoring import discover_images + +BATCH = 16 +# label (lowercased) treated as "safe"; nsfw score = 1 - sum(P over these) +SAFE_LABELS = {"sfw", "safe", "normal", "neutral"} + +MODELS = { # name -> (kind, hf_id) + "falconsai": ("hf", "Falconsai/nsfw_image_detection"), + "adamcodd": ("hf", "AdamCodd/vit-base-nsfw-detector"), + "marqo": ("timm", "Marqo/nsfw-image-detection-384"), + "owen-xs": ("timm", "OwenElliott/image-safety-classifier-xs"), +} + + +def _device(): + import torch + return "mps" if torch.backends.mps.is_available() else "cpu" + + +def _unsafe_indices(labels): + """Given lowercased class labels, return (safe_idx, use_safe). If any safe + label is present we score 1 - P(safe); otherwise P(labels that look unsafe).""" + safe = [i for i, l in enumerate(labels) if l in SAFE_LABELS] + if safe: + return safe, True + unsafe = [i for i, l in enumerate(labels) + if any(k in l for k in ("nsfw", "nsfl", "porn", "sexy", "hentai", "explicit"))] + return unsafe, False + + +def load_hf(model_id): + import torch + from transformers import AutoImageProcessor, AutoModelForImageClassification + dev = _device() + proc = AutoImageProcessor.from_pretrained(model_id) + model = AutoModelForImageClassification.from_pretrained(model_id).to(dev).eval() + labels = [model.config.id2label[i].lower() for i in range(len(model.config.id2label))] + idx, use_safe = _unsafe_indices(labels) + + def score(imgs): + inputs = proc(images=imgs, return_tensors="pt").to(dev) + with torch.no_grad(): + p = model(**inputs).logits.softmax(-1) + part = p[:, idx].sum(-1) if idx else p[:, :1].sum(-1) * 0 + return (1 - part if use_safe else part).cpu().tolist() + + return score, model + + +def load_timm(model_id): + import timm + import torch + dev = _device() + model = timm.create_model(f"hf_hub:{model_id}", pretrained=True).to(dev).eval() + cfg = timm.data.resolve_model_data_config(model) + tf = timm.data.create_transform(**cfg, is_training=False) + labels = [l.lower() for l in model.pretrained_cfg["label_names"]] + idx, use_safe = _unsafe_indices(labels) + + def score(imgs): + batch = torch.stack([tf(im) for im in imgs]).to(dev) + with torch.no_grad(): + p = model(batch).softmax(-1) + part = p[:, idx].sum(-1) if idx else p[:, :1].sum(-1) * 0 + return (1 - part if use_safe else part).cpu().tolist() + + return score, model + + +def score_all(names, imgs): + """Return {model_name: {path: nsfw_score}}; loads one model at a time.""" + from PIL import Image, ImageFile + ImageFile.LOAD_TRUNCATED_IMAGES = True + dev = _device() + out = {} + for name in names: + kind, mid = MODELS[name] + print(f"\n[{name}] loading {mid} on {dev} …", file=sys.stderr) + try: + score, model = (load_hf if kind == "hf" else load_timm)(mid) + except Exception as e: + print(f" SKIP {name}: {e}", file=sys.stderr) + continue + res = {} + for i in range(0, len(imgs), BATCH): + pil, paths = [], [] + for p in imgs[i:i + BATCH]: + try: + pil.append(Image.open(p).convert("RGB")); paths.append(p) + except Exception as e: + print(f"\n error {p}: {e}", file=sys.stderr) + if not pil: + continue + for p, s in zip(paths, score(pil)): + res[str(p)] = float(s) + print(f"\r {min(i + BATCH, len(imgs))}/{len(imgs)}", end="", file=sys.stderr, flush=True) + print(file=sys.stderr) + out[name] = res + del score, model + gc.collect() + return out + + +def _chip_color(s): + hue = round((1 - s) * 130) # green (safe) -> red (unsafe) + return f"hsl({hue} 70% 40%)" + + +def write_report(rows, names, target, threshold, gt): + """rows: list of (path, {name: score}). gt: {path: 1(nsfw)/0(sfw)}. Writes + bench_report.html in target. Chips that disagree with your manual tag get a + red ring; sortable by error count.""" + target = Path(target) + cards = [] + for p, scores in rows: + rel = quote(os.path.relpath(p, target)) + vals = [scores.get(n) for n in names if scores.get(n) is not None] + spread = (max(vals) - min(vals)) if len(vals) > 1 else 0.0 + gtv = gt.get(p) # 1 nsfw / 0 sfw / None + errors, chips = 0, [] + for n in names: + s = scores.get(n) + if s is None: + chips.append(f'{n} —'); continue + hot = s >= threshold + err = gtv is not None and (hot != (gtv == 1)) + errors += err + cls = "chip" + (" hot" if hot else "") + (" err" if err else "") + chips.append(f'{n} {s:.2f}') + gtbadge = (f'you: {"nsfw" if gtv==1 else "sfw"}' + if gtv is not None else "") + data = html.escape(json.dumps({n: scores.get(n) for n in names}), quote=True) + cards.append( + f'
' + f'{gtbadge}' + f'
{"".join(chips)}
' + f'
{html.escape(Path(p).name)}
') + opts = [] + if gt: + opts.append('') + opts.append('') + opts += [f'' for n in names] + opts.append('') + doc = _REPORT.replace("%%CARDS%%", "".join(cards)).replace("%%SORTOPTS%%", "".join(opts)) \ + .replace("%%N%%", str(len(rows))).replace("%%THR%%", f"{threshold:.2f}") + out = target / "bench_report.html" + out.write_text(doc, encoding="utf-8") + return out + + +_REPORT = """ +NSFW model bench (%%N%%) + +
NSFW model bench%%N%% images · white ring = ≥ %%THR%% · red ring = disagrees with your tag +
+
%%CARDS%%
+""" + + +def summarize(rows, names, threshold): + print(f"\n=== {len(rows)} images · flagged = score ≥ {threshold} ===") + print(f"{'model':<12}{'flagged':>10}{'mean':>8}{'median':>8}") + for n in names: + vals = sorted(s[n] for _, s in rows if s.get(n) is not None) + if not vals: + print(f"{n:<12}{'(skipped)':>10}"); continue + flagged = sum(v >= threshold for v in vals) + mean = sum(vals) / len(vals) + med = vals[len(vals) // 2] + print(f"{n:<12}{f'{flagged}/{len(vals)}':>10}{mean:>8.3f}{med:>8.3f}") + + ok = [n for n in names if any(s.get(n) is not None for _, s in rows)] + if len(ok) > 1: + print("\npairwise agreement (same side of threshold):") + for i in range(len(ok)): + for j in range(i + 1, len(ok)): + a, b = ok[i], ok[j] + both = [(s[a], s[b]) for _, s in rows if s.get(a) is not None and s.get(b) is not None] + agree = sum((x >= threshold) == (y >= threshold) for x, y in both) + print(f" {a:<10} vs {b:<10} {agree}/{len(both)} ({100*agree/max(1,len(both)):.0f}%)") + + if len(ok) > 1: + print("\nbiggest disagreements (open the HTML to judge):") + spread = sorted(rows, key=lambda r: -(max(v for v in (r[1].get(n) for n in ok) if v is not None) + - min(v for v in (r[1].get(n) for n in ok) if v is not None))) + for p, s in spread[:8]: + print(" " + Path(p).name + " " + " ".join(f"{n}={s[n]:.2f}" for n in ok if s.get(n) is not None)) + + +def metrics(rows, names, gt, threshold): + """Per-model precision/recall/F1 vs. your manual tags, at the given threshold, + plus each model's F1-optimal threshold.""" + labs = [gt[p] for p, _ in rows if p in gt] + npos, nneg = sum(labs), len(labs) - sum(labs) + single = npos == 0 or nneg == 0 # a known single-class folder (--assume) + print(f"\n=== vs. ground truth · {npos} nsfw + {nneg} sfw · flag = score ≥ {threshold} ===") + print(f"{'model':<11}{'acc':>7}{'prec':>7}{'recall':>8}{'F1':>6}{'FP':>5}{'FN':>5}{' best-thr':>11}{'F1@best':>8}") + for n in names: + pairs = [(gt[p], s[n]) for p, s in rows if p in gt and s.get(n) is not None] + if not pairs: + print(f"{n:<11}{'(skipped)':>7}"); continue + + def conf(t): + tp = fp = fn = tn = 0 + for y, sc in pairs: + pred = sc >= t + if y == 1 and pred: tp += 1 + elif y == 0 and pred: fp += 1 + elif y == 1: fn += 1 + else: tn += 1 + return tp, fp, fn, tn + + def f1(t): + tp, fp, fn, _ = conf(t) + pr = tp / (tp + fp) if tp + fp else 0 + rc = tp / (tp + fn) if tp + fn else 0 + return 2 * pr * rc / (pr + rc) if pr + rc else 0 + + tp, fp, fn, tn = conf(threshold) + pr = tp / (tp + fp) if tp + fp else 0 + rc = tp / (tp + fn) if tp + fn else 0 + f = 2 * pr * rc / (pr + rc) if pr + rc else 0 + acc = (tp + tn) / len(pairs) + if single: + bt_col = f"{'—':>11}{'—':>8}" # threshold tuning needs both classes + else: + cand = sorted({round(sc, 3) for _, sc in pairs} | {threshold}) + bt = max(cand, key=f1) + bt_col = f"{bt:>11.2f}{f1(bt):>8.2f}" + print(f"{n:<11}{acc:>7.2f}{pr:>7.2f}{rc:>8.2f}{f:>6.2f}{fp:>5}{fn:>5}{bt_col}") + if single and npos: + print("single-class NSFW folder: recall = catch rate, FN = missed. (precision/FP need safe images too — " + "run the bench on a mixed/SFW folder for those.)") + elif single: + print("single-class SFW folder: FP = false alarms, acc = correct-reject rate. (recall needs NSFW images.)") + else: + print("prec = of what it flagged, how much you'd tagged nsfw (fewer false alarms) · " + "recall = of your nsfw, how much it caught · FP/FN at the current threshold") + + +def combine(csv_paths, threshold): + """Pool the labeled rows from several bench_scores.csv files and print a + precision/recall/F1 threshold sweep per model — the real cross-dataset verdict.""" + per_model, seen = {}, set() + pos = neg = 0 + for cp in csv_paths: + with open(cp, newline="", encoding="utf-8") as fh: + r = csv.DictReader(fh) + models = [c for c in r.fieldnames if c not in ("path", "manual")] + for row in r: + man = row.get("manual", "") + if man not in ("nsfw", "sfw") or row["path"] in seen: + continue + seen.add(row["path"]) + y = 1 if man == "nsfw" else 0 + pos += y; neg += (1 - y) + for m in models: + v = row.get(m, "") + if v != "": + per_model.setdefault(m, []).append((y, float(v))) + if not pos or not neg: + sys.exit("Need both nsfw and sfw labels pooled — include a mixed/SFW run and an NSFW run.") + print(f"pooled {pos} nsfw + {neg} sfw from {len(csv_paths)} file(s)") + + def prf(pairs, t): + tp = sum(y == 1 and s >= t for y, s in pairs) + fp = sum(y == 0 and s >= t for y, s in pairs) + fn = sum(y == 1 and s < t for y, s in pairs) + pr = tp / (tp + fp) if tp + fp else 0 + rc = tp / (tp + fn) if tp + fn else 0 + return pr, rc, (2 * pr * rc / (pr + rc) if pr + rc else 0), fp, fn + + ranking = [] + for m, pairs in per_model.items(): + cand = sorted({round(s, 3) for _, s in pairs}) + best = max(cand, key=lambda t: prf(pairs, t)[2]) + ranking.append((prf(pairs, best)[2], best, m)) + print(f"\n[{m}] best-F1 threshold = {best:.2f}") + print(f" {'thr':>5}{'prec':>7}{'recall':>8}{'F1':>6}{'FP':>6}{'FN':>6}") + for t in sorted(set([0.5, 0.7, 0.85, 0.9, 0.95, 0.99, round(best, 2)])): + pr, rc, f, fp, fn = prf(pairs, t) + print(f" {t:>5.2f}{pr:>7.2f}{rc:>8.2f}{f:>6.2f}{fp:>6}{fn:>6}{' <- best F1' if abs(t - best) < 5e-3 else ''}") + ranking.sort(reverse=True) + print("\nranking by best F1: " + " ".join(f"{m}={f:.2f}@{t:.2f}" for f, t, m in ranking)) + + +def main(): + ap = argparse.ArgumentParser(prog="nsfwtag.bench", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("target", nargs="?", help="folder of photos to score with each model") + ap.add_argument("--combine", nargs="+", metavar="CSV", help="pool these bench_scores.csv files (using their manual labels) and print a precision/recall/threshold sweep per model — no scoring") + ap.add_argument("-r", "--recursive", action="store_true", help="descend into subfolders") + ap.add_argument("--models", default=",".join(MODELS), help=f"comma list (default: all) — {', '.join(MODELS)}") + ap.add_argument("--limit", type=int, default=0, help="cap number of images scored (quick pass)") + ap.add_argument("--threshold", type=float, default=0.85, help="flag cutoff for the report (default 0.85)") + ap.add_argument("--all", action="store_true", help="score ALL images, not just your manually-tagged ones") + ap.add_argument("--assume", choices=["nsfw", "sfw"], help="treat EVERY image in the folder as this label (a known folder) instead of reading EXIF tags") + args = ap.parse_args() + + if args.combine: + combine(args.combine, args.threshold) + return + if not args.target: + ap.error("a target folder is required (or use --combine)") + target = Path(args.target).expanduser() + if not target.is_dir(): + sys.exit(f"Not a folder: {target}\n Check the path/spelling (that folder does not exist).") + names = [n.strip() for n in args.models.split(",") if n.strip()] + bad = [n for n in names if n not in MODELS] + if bad: + sys.exit(f"Unknown model(s): {bad}. Available: {list(MODELS)}") + + imgs = discover_images(target, recursive=args.recursive) + if not imgs: + has_subdirs = any(p.is_dir() for p in target.iterdir()) + hint = " Images look to be in subfolders — add -r to scan recursively." if (has_subdirs and not args.recursive) else "" + sys.exit(f"No images found directly in {target}.\n{hint}".rstrip()) + + if args.assume: # whole folder is a known class + if args.limit: + imgs = imgs[:args.limit] + gt = {str(p): (1 if args.assume == "nsfw" else 0) for p in imgs} + print(f"assuming all {len(imgs)} images are {args.assume.upper()} (folder is ground truth).", file=sys.stderr) + else: + # ground truth from your manual nsfw/sfw EXIF tags (one batched exiftool read) + print(f"found {len(imgs)} images — reading your manual tags …", file=sys.stderr) + marks = read_marks([str(p) for p in imgs]) + gt = {p: 1 for p in marks["nsfw"]} + gt.update({p: 0 for p in marks["sfw"] if p not in marks["nsfw"]}) + if gt and not args.all: + imgs = [p for p in imgs if str(p) in gt] # evaluate only what you've tagged + print(f"evaluating on {len(imgs)} tagged images " + f"({sum(gt.values())} nsfw / {len(gt) - sum(gt.values())} sfw). Use --all to score everything.", file=sys.stderr) + elif not gt: + print("no manual nsfw/sfw tags found — falling back to a relative model comparison.", file=sys.stderr) + if args.limit: + imgs = imgs[:args.limit] + print(f"scoring {len(imgs)} images with {len(names)} model(s): {names}", file=sys.stderr) + + scores = score_all(names, imgs) + names = [n for n in names if n in scores] # keep only models that loaded + if not names: + sys.exit("No models loaded (need: pip install torch transformers timm).") + + rows = [(str(p), {n: scores[n].get(str(p)) for n in names}) for p in imgs] + + csv_path = target / "bench_scores.csv" + with open(csv_path, "w", newline="", encoding="utf-8") as fh: + w = csv.writer(fh); w.writerow(["path", "manual", *names]) + for p, s in rows: + man = {1: "nsfw", 0: "sfw"}.get(gt.get(p), "") + w.writerow([p, man, *[("" if s[n] is None else f"{s[n]:.4f}") for n in names]]) + + if any(p in gt for p, _ in rows): + metrics(rows, names, gt, args.threshold) + else: + summarize(rows, names, args.threshold) + report = write_report(rows, names, target, args.threshold, gt) + print(f"\nscores -> {csv_path}\nreport -> {report}", file=sys.stderr) + webbrowser.open(report.resolve().as_uri()) + + +if __name__ == "__main__": + main() diff --git a/nsfwtag/exif.py b/nsfwtag/exif.py new file mode 100644 index 0000000..f5981e9 --- /dev/null +++ b/nsfwtag/exif.py @@ -0,0 +1,123 @@ +"""EXIF keyword read/write via exiftool. + +Only the Keywords + Subject fields are ever touched, so every other tag +(Caption, ImageDescription, dates, GPS, Artist, …) is preserved on tag/untag. +""" +import json +import os +import subprocess +import sys +from pathlib import Path + +from . import KEYWORD + + +def write_keyword(path: str, kw: str = KEYWORD) -> bool: + """Idempotently add a keyword to EXIF Keywords + Subject (no duplicates on re-run).""" + return subprocess.run( + ["exiftool", "-m", "-overwrite_original", + f"-Keywords-={kw}", f"-Keywords+={kw}", + f"-Subject-={kw}", f"-Subject+={kw}", path], + capture_output=True, text=True, + ).returncode == 0 + + +def remove_keyword(path: str, kw: str = KEYWORD) -> bool: + """Reverse of write_keyword: strip the keyword from EXIF Keywords + Subject.""" + return subprocess.run( + ["exiftool", "-m", "-overwrite_original", + f"-Keywords-={kw}", f"-Subject-={kw}", path], + capture_output=True, text=True, + ).returncode == 0 + + +def _read_keywords(paths) -> dict: + """{path: set(lowercased Keywords+Subject values)} via one batched exiftool + call (paths fed via stdin). Empty dict if exiftool isn't available.""" + paths = list(paths) + if not paths: + return {} + want = {os.path.normpath(p): p for p in paths} + try: + r = subprocess.run( + ["exiftool", "-m", "-j", "-Keywords", "-Subject", "-@", "-"], + input="\n".join(paths), capture_output=True, text=True, + ) + except FileNotFoundError: + return {} + out = {} + try: + for obj in json.loads(r.stdout or "[]"): + vals = [] + for f in ("Keywords", "Subject"): + v = obj.get(f) + if isinstance(v, list): + vals += v + elif v is not None: + vals.append(v) + key = want.get(os.path.normpath(obj.get("SourceFile", ""))) + if key is not None: + out[key] = {str(x).strip().lower() for x in vals} + except (ValueError, TypeError): + pass + return out + + +def read_tagged(paths, kw: str = KEYWORD) -> set: + """Subset of `paths` whose EXIF already contains keyword `kw`.""" + kw = kw.lower() + return {p for p, kws in _read_keywords(paths).items() if kw in kws} + + +def read_marks(paths) -> dict: + """Which paths carry the 'nsfw' vs the 'sfw' review keyword, in one read: + {'nsfw': set, 'sfw': set}. They should be mutually exclusive; if a stray file + somehow has both, nsfw wins so it never reads as safe.""" + got = _read_keywords(paths) + nsfw = {p for p, k in got.items() if "nsfw" in k} + sfw = {p for p, k in got.items() if "sfw" in k and p not in nsfw} + return {"nsfw": nsfw, "sfw": sfw} + + +_EXIF_NOISE = { + "SourceFile", "ExifToolVersion", "Directory", "FilePermissions", + "FileModifyDate", "FileAccessDate", "FileInodeChangeDate", "FileTypeExtension", +} + + +def read_exif(path: str) -> dict: + """Human-relevant EXIF tags for one file (for the lightbox overlay), in + exiftool's order. Skips file-system noise and binary/oversized values. When + the photo is geotagged, a "Map" entry with a Google Maps URL is prepended. + `-c %+.6f` prints GPS coordinates as signed decimals (other tags unaffected).""" + r = subprocess.run(["exiftool", "-m", "-j", "-c", "%+.6f", path], capture_output=True, text=True) + try: + obj = json.loads(r.stdout or "[]")[0] + except (ValueError, IndexError): + return {} + out = {} + for k, v in obj.items(): + if k in _EXIF_NOISE: + continue + s = ", ".join(str(x) for x in v) if isinstance(v, list) else str(v) + if not s or s.startswith("(Binary data") or len(s) > 300: + continue + out[k] = s + lat, lon = out.get("GPSLatitude"), out.get("GPSLongitude") + if lat and lon: + try: + out = {"Map": f"https://www.google.com/maps?q={float(lat):.6f},{float(lon):.6f}", **out} + except ValueError: + pass + return out + + +def apply_list(list_file: str): + """Write the nsfw keyword to every path in a newline-delimited file.""" + paths = [l.strip() for l in Path(list_file).read_text(encoding="utf-8").splitlines() if l.strip()] + ok = miss = 0 + for p in paths: + if not Path(p).exists(): + miss += 1; print(f" missing: {p}", file=sys.stderr); continue + ok += 1 if write_keyword(p) else 0 + print(f"tagged 'nsfw' in EXIF: {ok}/{len(paths)}" + (f" ({miss} missing)" if miss else "")) diff --git a/nsfwtag/review.html b/nsfwtag/review.html new file mode 100644 index 0000000..d8193a8 --- /dev/null +++ b/nsfwtag/review.html @@ -0,0 +1,426 @@ + + +NSFW Review · %%TOTAL%% images + +
+ +
+ +

NSFW Review

On-device · nothing leaves this Mac

+
+
+
+
%%TOTAL%%images
+
+ +
+
+ +
+
+
+
+ + + + + +
+ +
+
%%CARDS%%
+ +
+ + + + + \ No newline at end of file diff --git a/nsfwtag/scoring.py b/nsfwtag/scoring.py new file mode 100644 index 0000000..07f4ec2 --- /dev/null +++ b/nsfwtag/scoring.py @@ -0,0 +1,119 @@ +"""Discover images and score them with the local NSFW model. + +Scores are cached to a CSV next to the library, so a rerun reuses them and only +scores newly-added files — skipping the model (and its load) entirely when +nothing is new. +""" +import csv +import sys +from pathlib import Path + +from . import BATCH, CSV_NAME, EXTS, MODEL_ID + + +def discover_images(target: Path, recursive: bool = False, limit: int = 0): + """All supported image files under `target`, de-duplicated by resolved path + (guards against symlinks / duplicate paths). Sorted for stable ordering.""" + walk = target.rglob("*") if recursive else target.iterdir() + seen, imgs = set(), [] + for p in sorted(walk): + if not (p.is_file() and p.suffix.lower() in EXTS): + continue + rp = p.resolve() + if rp in seen: + continue + seen.add(rp) + imgs.append(p) + return imgs[:limit] if limit else imgs + + +def load_cache(csv_path: Path) -> dict: + """Read a scores CSV into {path: score}. Missing file -> empty dict.""" + cache = {} + if csv_path.exists(): + with open(csv_path, newline="", encoding="utf-8") as fh: + for row in csv.DictReader(fh): + try: + cache[row["path"]] = float(row["nsfw_score"]) + except (KeyError, ValueError): + pass + return cache + + +def _save_cache(csv_path: Path, cache: dict): + with open(csv_path, "w", newline="", encoding="utf-8") as fh: + w = csv.writer(fh) + w.writerow(["path", "nsfw_score"]) + w.writerows([(pth, f"{sc:.4f}") for pth, sc in cache.items()]) + + +def score_images(imgs, csv_path: Path, rescore: bool = False, batch: int = BATCH): + """Return (scored, errors) where scored is [(Path, score)] for every image in + `imgs` that has a score. Reuses the CSV cache; only `imgs` missing from it are + run through the model, then the merged cache is persisted.""" + cache = {} if rescore else load_cache(csv_path) + todo = [p for p in imgs if str(p) not in cache] + if cache: + print(f"cache: {len(cache)} scores from {csv_path} — {len(todo)} new to score", file=sys.stderr) + + errors = 0 + if todo: + import numpy as np + import torch + from PIL import Image, ImageFile + from transformers import AutoModelForImageClassification + ImageFile.LOAD_TRUNCATED_IMAGES = True + + device = "mps" if torch.backends.mps.is_available() else "cpu" + # Load from local cache without hitting the HF Hub (skips the slow per-run + # update check). First run falls through to a one-time download, then cached. + try: + model = AutoModelForImageClassification.from_pretrained(MODEL_ID, local_files_only=True) + print(f"Loaded {MODEL_ID} from cache on {device}", file=sys.stderr) + except Exception: + print(f"Downloading {MODEL_ID} (one-time, ~350 MB) …", file=sys.stderr) + model = AutoModelForImageClassification.from_pretrained(MODEL_ID) + model = model.to(device).eval() + nsfw_idx = next(i for i, l in model.config.id2label.items() if l.lower() == "nsfw") + + # Input size comes from the model, not a constant — AdamCodd wants 384, + # Falconsai wanted 224. Hardcoding it silently breaks on a model swap. + size = getattr(model.config, "image_size", 224) + + def preprocess(img): + img = img.convert("RGB").resize((size, size), Image.BILINEAR) + a = (np.asarray(img, dtype="float32") / 255.0 - 0.5) / 0.5 + return torch.from_numpy(a).permute(2, 0, 1) + + print(f"Scoring {len(todo)} images …", file=sys.stderr) + done_since_save = 0 + for i in range(0, len(todo), batch): + tens, paths = [], [] + for p in todo[i:i + batch]: + try: + tens.append(preprocess(Image.open(p))); paths.append(p) + except Exception as e: + errors += 1; print(f"\n error {p}: {e}", file=sys.stderr) + if not tens: + continue + try: + with torch.no_grad(): + probs = model(pixel_values=torch.stack(tens).to(device)).logits.softmax(-1) + for p, pr in zip(paths, probs): + cache[str(p)] = pr[nsfw_idx].item() + except Exception as e: + # One bad batch (e.g. MPS OOM / Metal error, common when two GPU + # jobs run at once) must not discard everything scored so far. + errors += len(paths) + print(f"\n batch {i}-{i+len(paths)} failed: {e}", file=sys.stderr) + done_since_save += len(paths) + if done_since_save >= 500: # periodic checkpoint → crash-resumable + _save_cache(csv_path, cache); done_since_save = 0 + print(f"\r {min(i + batch, len(todo))}/{len(todo)}", end="", file=sys.stderr, flush=True) + print(file=sys.stderr) + _save_cache(csv_path, cache) + else: + print("all images already scored — skipping model load", file=sys.stderr) + + scored = [(p, cache[str(p)]) for p in imgs if str(p) in cache] + return scored, errors diff --git a/nsfwtag/server.py b/nsfwtag/server.py new file mode 100644 index 0000000..ffb9da2 --- /dev/null +++ b/nsfwtag/server.py @@ -0,0 +1,126 @@ +"""Local HTTP server for the review app (stdlib only). + +Routes: + GET / the review page + GET /img?path= a thumbnail (restricted to the scanned candidate set) + POST /tagged -> which of the posted paths already carry the nsfw keyword + POST /apply -> add the nsfw keyword to the posted paths + POST /untag -> remove the nsfw keyword from the posted paths +All POST routes are restricted to the allowed candidate set. +""" +import sys +from pathlib import Path + +from .exif import read_exif, read_marks, read_tagged, remove_keyword, write_keyword +from .webapp import render_page + + +def serve_review(cands, threshold, host="127.0.0.1"): + """Serve the review page and tag/untag confirmed images directly on request.""" + import http.server + import json + import mimetypes + import time + import webbrowser + from collections import deque + from urllib.parse import parse_qs, unquote, urlparse + + allowed = {str(p) for p, _ in cands} # only these paths may be served or tagged + page = render_page(cands, threshold).encode("utf-8") + + log_entries = deque(maxlen=2000) # shown in the browser's Activity log tab + + def log(msg): + log_entries.append({"t": time.strftime("%H:%M:%S"), "m": msg}) + print(msg, file=sys.stderr) + + def label(p): # folder + picture name for the log + pp = Path(p) + return f"{pp.parent.name}/{pp.name}" if pp.parent.name else pp.name + + log(f"review ready — {len(cands)} candidate images") + + class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): # quiet + pass + + def _send(self, code, ctype, body): + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + u = urlparse(self.path) + if u.path == "/": + return self._send(200, "text/html; charset=utf-8", page) + if u.path == "/img": + fp = unquote(parse_qs(u.query).get("path", [""])[0]) + # ponytail: browsers won't render HEIC bytes; those thumbs show broken — score/tag still work. + if fp in allowed and Path(fp).exists(): + ctype = mimetypes.guess_type(fp)[0] or "application/octet-stream" + return self._send(200, ctype, Path(fp).read_bytes()) + if u.path == "/exif": # metadata panel for the lightbox + fp = unquote(parse_qs(u.query).get("path", [""])[0]) + if fp in allowed and Path(fp).exists(): + return self._send(200, "application/json", json.dumps(read_exif(fp)).encode()) + if u.path == "/log": # activity log tab + return self._send(200, "application/json", json.dumps(list(log_entries)).encode()) + return self._send(404, "text/plain", b"not found") + + def do_POST(self): + u = urlparse(self.path) + route = u.path + n = int(self.headers.get("Content-Length", 0)) + paths = [p for p in json.loads(self.rfile.read(n) or b"[]") if p in allowed] + if route == "/tagged": # review state of these paths (lazy, chunked): nsfw + sfw + m = read_marks(paths) + return self._send(200, "application/json", + json.dumps({"nsfw": list(m["nsfw"]), "sfw": list(m["sfw"])}).encode()) + if route == "/mark": # persist the manual SFW decision (opposite of nsfw) + state = parse_qs(u.query).get("state", [""])[0] + if state == "sfw": # mark safe: write 'sfw', drop 'nsfw' + for p in paths: + write_keyword(p, "sfw"); remove_keyword(p, "nsfw") + log(f"marked sfw — {label(p)}") + elif state == "clear": # un-mark safe: drop 'sfw' + for p in paths: + remove_keyword(p, "sfw") + log(f"cleared sfw — {label(p)}") + return self._send(200, "application/json", json.dumps({"ok": len(paths)}).encode()) + if route == "/apply": + ok = 0 + for p in paths: + if write_keyword(p): + remove_keyword(p, "sfw") # nsfw and sfw are mutually exclusive + ok += 1; log(f"tagged nsfw — {label(p)}") + return self._send(200, "application/json", json.dumps({"tagged": ok}).encode()) + if route == "/untag": # clear the review keyword (nsfw or sfw) — back to un-decided + ok = 0 + for p in paths: + remove_keyword(p, "nsfw"); remove_keyword(p, "sfw") + ok += 1; log(f"removed tag — {label(p)}") + return self._send(200, "application/json", json.dumps({"untagged": ok}).encode()) + if route == "/toggle": # flip each path's tag both ways; other EXIF preserved + cur = read_tagged(paths) # one batched read, then add/remove per file + now_tagged = [] + for p in paths: + if p in cur: + remove_keyword(p); log(f"removed nsfw — {label(p)}") + elif write_keyword(p): + remove_keyword(p, "sfw") # mutually exclusive + now_tagged.append(p); log(f"tagged nsfw — {label(p)}") + return self._send(200, "application/json", json.dumps({"tagged": now_tagged}).encode()) + return self._send(404, "text/plain", b"not found") + + srv = http.server.ThreadingHTTPServer((host, 0), H) # port 0 = OS picks a free one + url = f"http://{host}:{srv.server_address[1]}/" + print(f"\nReview app: {url}\n Tick images, click 'Apply' — writes the 'nsfw' EXIF tag directly (idempotent, safe to re-apply).\n Press Ctrl-C here when done.", file=sys.stderr) + webbrowser.open(url) + try: + srv.serve_forever() + except KeyboardInterrupt: + print("\nstopped.", file=sys.stderr) + finally: + srv.server_close() diff --git a/nsfwtag/webapp.py b/nsfwtag/webapp.py new file mode 100644 index 0000000..3c46953 --- /dev/null +++ b/nsfwtag/webapp.py @@ -0,0 +1,91 @@ +"""Render the review page. + +The HTML/CSS/JS lives in review.html (loaded once at import). This module fills +in the %%TOKENS%%: the folder-tree sidebar, the image cards, totals. The server +contract is preserved: each .nsfw checkbox's value is the image's absolute path, +thumbnails load via GET /img?path=, actions POST to /apply /untag /tagged. +""" +import html +import os +from pathlib import Path +from urllib.parse import quote + +# Inline Lucide-style SVG icons (no emoji, theme-able via currentColor). +_SVG_EXPAND = '' +_SVG_CHECK = '' + +_PAGE = (Path(__file__).parent / "review.html").read_text(encoding="utf-8") + + +def folder_tree_html(cands): + """Indented folder list for the sidebar file browser, built from the candidate + paths. Each row filters the grid to that folder + its subfolders (prefix match); + 'All folders' resets. Counts are images directly in that folder.""" + folders = {} + for p, _ in cands: + folders[str(p.parent)] = folders.get(str(p.parent), 0) + 1 + if not folders: + return "" + paths = sorted(folders) + root = os.path.commonpath(paths) if len(paths) > 1 else paths[0] + rows = [ + f'' + ] + for folder in paths: + rel = os.path.relpath(folder, root) + depth = 0 if rel == "." else rel.count(os.sep) + 1 + name = os.path.basename(folder) or folder + rows.append( + f'' + ) + return "".join(rows) + + +def render_page(cands, threshold): + """Build the review page. cands: list of (Path, score) sorted desc.""" + cards, seen = [], set() + for p, score in cands: + sp = str(p) + if sp in seen: # never list the same path twice + continue + seen.add(sp) + # No pre-selection at render. The per-folder scan decides state so persisted + # marks win: nsfw -> pink, sfw -> green, undecided high-score -> red. + # (A dismissed/sfw image must never come back pre-selected red on reload.) + checked = "" + src = f"/img?path={quote(sp)}" + esc = html.escape(sp, quote=True) + name = html.escape(p.name) + folder = html.escape(str(p.parent)) + folder_attr = html.escape(str(p.parent), quote=True) + name_low = html.escape(p.name.lower(), quote=True) + folder_low = html.escape(str(p.parent).lower(), quote=True) + cards.append( + f'
' + f'
' + f'' + f'{score:.2f}' + f'' + f'' + f'' + f'' + f'' + f'
' + f'
{name}
' + f'
{folder}
' + f'
' + ) + return (_PAGE + .replace("%%TREE%%", folder_tree_html(cands)) + .replace("%%CARDS%%", "".join(cards)) + .replace("%%TOTAL%%", str(len(cards))) + .replace("%%THRESHOLD%%", f"{threshold:.2f}")) diff --git a/photo_analyzer.py b/photo_analyzer.py new file mode 100644 index 0000000..173f7e1 --- /dev/null +++ b/photo_analyzer.py @@ -0,0 +1,2454 @@ +#!/usr/bin/env python3 +""" +photo_analyzer.py — AI photo analysis pipeline for Immich + +Crawls a photo library, analyzes each image once via an OpenAI-compatible +vision model (any OpenAI-compatible provider — Google Gemini, OpenAI, …), stores results in +SQLite, and writes captions back into EXIF so Immich can index them. +Safe to interrupt and resume at any point. + +Requirements: + pip install openai pillow rich + +External tools: + exiftool (https://exiftool.org — must be on PATH) + +Usage: + export LLM_API_KEY=... # or set it in photo_analyzer.env + python photo_analyzer.py --library /path/to/photos + + # Dry run (no API calls, no EXIF writes): + python photo_analyzer.py --library /path/to/photos --dry-run + + # Re-analyze already-processed files (e.g. after prompt change): + python photo_analyzer.py --library /path/to/photos --reanalyze + + # Skip EXIF writing (index only): + python photo_analyzer.py --library /path/to/photos --no-exif +""" + +import argparse +import base64 +import hashlib +import json +import logging +import os +import re +import select +import shutil +import signal +import sqlite3 +import subprocess +import sys +import termios +import threading +import time +import tty +from collections import deque +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from itertools import groupby +from zoneinfo import ZoneInfo +from pathlib import Path + +from openai import OpenAI, RateLimitError, APIError +from PIL import Image, ImageFile + +# Decode truncated/partial JPEGs instead of erroring ("image file is truncated", +# "broken data stream") — renders the available portion, fine for captioning a +# damaged file. Without this, corrupt source images fail before the API call. +ImageFile.LOAD_TRUNCATED_IMAGES = True +from rich.console import Console, Group +from rich.layout import Layout +from rich.live import Live +from rich.logging import RichHandler +from rich.panel import Panel +from rich.progress import ( + BarColumn, MofNCompleteColumn, Progress, SpinnerColumn, + TaskProgressColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn, +) +from rich.table import Table +from rich.text import Text + +# ────────────────────────────────────────────── +# Configuration +# ────────────────────────────────────────────── + +SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"} + +# Config file for secrets (chiefly the API key). A real shell env var always +# wins; this is just a convenience so the key survives across sessions without +# re-exporting it. Searched in the current dir first, then next to this script. +# NB: deliberately NOT ".env" — that name is taken by Immich's docker-compose. +ENV_FILE = "photo_analyzer.env" +API_KEY_PLACEHOLDER = "sk-REPLACE_WITH_YOUR_KEY" # ignored until replaced + +# OpenAI-compatible LLM endpoint. Defaults to Google Gemini Flash via the +# Generative Language OpenAI-compatibility endpoint, but ANY OpenAI-compatible +# provider works — override in photo_analyzer.env (or the shell) with +# LLM_BASE_URL / LLM_MODEL / LLM_API_KEY. e.g. OpenAI: +# LLM_BASE_URL=https://api.openai.com/v1 +# LLM_MODEL=gpt-4o-mini +# LLM_API_KEY=sk-... +# (GEMINI_API_KEY / GOOGLE_API_KEY are honoured as fallbacks for the key.) +LLM_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/" +LLM_MODEL = "gemini-2.5-flash" + +# Resize before sending — most vision APIs handle ≤ 4K well; 2048px long-edge is a good +# balance between quality and token cost (~30–60% fewer tokens than full-res) +MAX_LONG_EDGE = 2048 + +# Duplicate detection. Each photo gets a 64-bit DCT perceptual hash (phash); two +# photos are "the same picture" when their hashes differ by ≤ PHASH_THRESHOLD +# bits (Hamming distance). 0 = pixel-identical structure; ~5 tolerates +# recompression/resize/format-change across phone backups. Raise for looser +# matching (catches more, risks grouping similar-but-distinct shots), lower for +# stricter. Tunable via PHASH_THRESHOLD in photo_analyzer.env. +PHASH_THRESHOLD = 5 + +# Concurrency: tune to your provider's rate limits (free tiers are often low, +# paid tiers much higher). Keep conservative to avoid 429s; the throttle warning +# box and throttle_events.jsonl tell you when you've pushed too hard. +MAX_WORKERS = 3 +RETRY_ATTEMPTS = 4 +RETRY_BASE_DELAY = 10 # seconds; doubles on each retry + +# Your provider's requests-per-day cap, shown as a live "RPD today: N / limit" +# gauge in the dashboard (Gemini Tier 1 flash = 10000). 0 = unknown → just count. +RPD_LIMIT = 10000 + +# Analysis prompt — structured JSON output for reliable parsing +ANALYSIS_PROMPT = """Analyze this photo and respond ONLY with a valid JSON object. +No markdown, no explanation, just the JSON. + +{ + "description": "One clear sentence describing what is happening in this photo.", + "tags": ["list", "of", "8-12", "descriptive", "keywords"], + "people_count": 0, + "setting": "indoor or outdoor", + "time_of_day": "morning | afternoon | evening | night | unknown", + "season": "spring | summer | autumn | winter | unknown", + "mood": "the specific emotional tone of THIS scene, 1-2 words", + "landmarks": ["recognizable landmarks, monuments, named buildings or place signs visible — [] if none"], + "location_hint": "best guess of WHERE this was taken, e.g. 'Cairo, Egypt' / 'Sardinia, Italy' / 'the Alps' — null if no basis", + "approx_year": null +} + +Rules: +- people_count: count only clearly visible people (partial counts as 1) +- approx_year: integer only if strongly inferable from clothing/tech/decor or the album hint, otherwise null +- tags: specific and useful (prefer 'golden retriever' over 'dog', 'birthday cake' over 'food'); + include any identified landmark or place name as a tag too +- mood: name the specific feeling of this particular scene — e.g. professional, theatrical, + serene, tense, celebratory, nostalgic, melancholic, candid, attentive, romantic. Do NOT + default to 'happy'/'joyful' just because people are smiling: a formal headshot is + 'professional', a quiet landscape 'serene', a certificate 'accomplished'. +- landmarks: identify famous or named places from architecture, monuments, signage or + distinctive landscape (e.g. 'Eiffel Tower', 'Cologne Cathedral', 'Pyramids of Giza'). [] if none. +- location_hint: infer the place from landmarks, readable signs, architecture, vegetation/ + landscape AND the album hint. Be as specific as the evidence allows (city > region > country). + Do NOT invent a precise place from weak evidence — when unsure, give the broader region or null. +- All values must be valid JSON types (string, integer, null, array) +""" + +# ────────────────────────────────────────────── +# Console + Logging +# ────────────────────────────────────────────── + +console = Console(highlight=False) + +# ── Handlers ──────────────────────────────────────────────────────────────── + +_rich_handler = RichHandler(console=console, rich_tracebacks=True, show_path=False, markup=True) +_rich_handler.setLevel(logging.INFO) # bumped to DEBUG by --debug flag at runtime + +_info_file_handler = logging.FileHandler("photo_analyzer.log", encoding="utf-8") +_info_file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-7s %(message)s", datefmt="%H:%M:%S")) +_info_file_handler.setLevel(logging.INFO) + +_debug_file_handler = logging.FileHandler("photo_analyzer_debug.log", encoding="utf-8") +_debug_file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-7s %(name)s %(message)s", datefmt="%H:%M:%S")) +_debug_file_handler.setLevel(logging.DEBUG) + +logging.basicConfig( + level=logging.DEBUG, # root at DEBUG so debug handler receives everything + format="%(message)s", + datefmt="[%H:%M:%S]", + handlers=[_rich_handler, _info_file_handler, _debug_file_handler], +) +log = logging.getLogger(__name__) + +# Silence the per-request HTTP chatter from the OpenAI SDK / httpx — one +# "HTTP Request: POST … 200 OK" line per photo floods the console. Their +# warnings and errors still come through. +for _noisy in ("httpx", "httpcore", "openai"): + logging.getLogger(_noisy).setLevel(logging.WARNING) + +# ── History logger (JSONL — one line per photo) ────────────────────────────── + +_history_handler = logging.FileHandler("photo_analyzer_history.jsonl", encoding="utf-8") +_history_handler.setFormatter(logging.Formatter("%(message)s")) + +history_log = logging.getLogger("history") +history_log.addHandler(_history_handler) +history_log.setLevel(logging.DEBUG) +history_log.propagate = False # never send to root / console + + +# ── Graceful shutdown ──────────────────────────────────────────────────────── + +_stop = threading.Event() + +# Live-mode popup menu (hotkey 'm'). _menu = open/closed; _menu_state holds the +# highlighted row and which view is showing. Both threads touch it, so guard with +# the lock. Each item is (label, action) — action handled in _key_listener. +_menu = threading.Event() +_menu_lock = threading.Lock() +_menu_state = {"sel": 0, "view": "menu"} +_MENU_ITEMS = [ + ("Token usage", "token"), # switch overlay to session token stats + ("Stop after current", "stop"), # graceful: finish in-flight, then quit + ("Force quit now", "force"), # immediate exit + ("Resume", "close"), # close menu, keep analyzing +] + +# A single sqlite3.Connection is shared across worker threads (check_same_thread= +# False). SQLite serialises writes internally, but Python's connection object is +# not thread-safe — concurrent commits race ("cannot commit - no transaction is +# active"). This lock serialises every write; API latency dominates, so the cost +# is negligible. +_db_lock = threading.Lock() + +def _handle_sigint(sig, frame): + # Audible bell on every press — instant confirmation the signal was received, + # even in the full-screen dashboard. The visual banner appears on the next + # render tick (see _Dashboard); plain mode shows the log line below. + try: + sys.__stderr__.write("\a") + sys.__stderr__.flush() + except Exception: + pass + if _stop.is_set(): + log.warning("Force quit.") + sys.exit(1) + _stop.set() + log.warning("[yellow]Stopping after current batch… press Ctrl+C again to force quit.[/]") + +signal.signal(signal.SIGINT, _handle_sigint) + + +def log_history(path: str, status: str, result: dict = None, + tokens: dict = None, error: str = None, + copied_from: str = None) -> None: + # status is typically "analyzed" | "error" | "copied" (variant inherited + # from its primary with no API call — copied_from names that primary). + entry = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "path": path, "status": status} + if copied_from: + entry["copied_from"] = copied_from + if tokens: + entry["tokens_total"] = tokens.get("total", 0) + entry["tokens_prompt"] = tokens.get("prompt", 0) + entry["tokens_completion"] = tokens.get("completion", 0) + if result: + entry["description"] = result.get("description") + entry["tags"] = result.get("tags", []) + entry["mood"] = result.get("mood") + entry["setting"] = result.get("setting") + entry["people_count"] = result.get("people_count", 0) + entry["location_hint"]= result.get("location_hint") + entry["approx_year"] = result.get("approx_year") + if error: + entry["error"] = error + history_log.info(json.dumps(entry, ensure_ascii=False)) + + +# ────────────────────────────────────────────── +# Database +# ────────────────────────────────────────────── + +DB_FILE = "photo_analysis.db" + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS photos ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT UNIQUE NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + -- pending | analyzed | exif_written | error | duplicate + phash TEXT, -- 64-bit perceptual hash (16 hex chars); content-based, survives resize/recompress/EXIF + file_sha1 TEXT, -- SHA-1 of the file bytes; exact identity, for rename/move reconciliation + dup_of TEXT, -- if status='duplicate', the canonical photo's path this duplicates + description TEXT, + tags TEXT, -- JSON array stored as string + people_count INTEGER, + setting TEXT, + time_of_day TEXT, + season TEXT, + mood TEXT, + location_hint TEXT, + approx_year INTEGER, + raw_response TEXT, -- full JSON from the model, for debugging + error_message TEXT, + analyzed_at TEXT, + exif_written_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_status ON photos(status); +CREATE INDEX IF NOT EXISTS idx_path ON photos(path); +-- idx_sha1 / idx_phash are created in _migrate_schema(), AFTER the columns are +-- guaranteed to exist (a pre-existing table won't have them until the ALTER). + +CREATE VIRTUAL TABLE IF NOT EXISTS photos_fts USING fts5( + path, description, tags, mood, location_hint, + content=photos, content_rowid=id +); + +CREATE TRIGGER IF NOT EXISTS photos_ai AFTER INSERT ON photos BEGIN + INSERT INTO photos_fts(rowid, path, description, tags, mood, location_hint) + VALUES (new.id, new.path, new.description, new.tags, new.mood, new.location_hint); +END; + +CREATE TRIGGER IF NOT EXISTS photos_au AFTER UPDATE ON photos BEGIN + INSERT INTO photos_fts(photos_fts, rowid, path, description, tags, mood, location_hint) + VALUES ('delete', old.id, old.path, old.description, old.tags, old.mood, old.location_hint); + INSERT INTO photos_fts(rowid, path, description, tags, mood, location_hint) + VALUES (new.id, new.path, new.description, new.tags, new.mood, new.location_hint); +END; + +CREATE TRIGGER IF NOT EXISTS photos_ad AFTER DELETE ON photos BEGIN + INSERT INTO photos_fts(photos_fts, rowid, path, description, tags, mood, location_hint) + VALUES ('delete', old.id, old.path, old.description, old.tags, old.mood, old.location_hint); +END; +""" + + +def get_db(db_path: str) -> sqlite3.Connection: + conn = sqlite3.connect(db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA) + _migrate_schema(conn) + conn.commit() + return conn + + +def _migrate_schema(conn: sqlite3.Connection) -> None: + """Add columns introduced after a DB was first created. CREATE TABLE IF NOT + EXISTS never alters an existing table, so pre-existing databases (e.g. the + original 25k-row one) need these added explicitly. Idempotent.""" + have = {r["name"] for r in conn.execute("PRAGMA table_info(photos)")} + for col in ("phash", "file_sha1", "dup_of"): + if col not in have: + conn.execute(f"ALTER TABLE photos ADD COLUMN {col} TEXT") + conn.executescript( + "CREATE INDEX IF NOT EXISTS idx_sha1 ON photos(file_sha1);" + "CREATE INDEX IF NOT EXISTS idx_phash ON photos(phash);" + ) + + +def purge_excluded(conn: sqlite3.Connection) -> int: + """Remove DB entries for paths that should be excluded (thumbs, _IGNORE).""" + cur = conn.execute( + "DELETE FROM photos WHERE path LIKE '%/.@__thumb/%' OR path LIKE '%/_IGNORE/%'" + ) + conn.commit() + return cur.rowcount + + +def prune_missing(conn: sqlite3.Connection, library: Path) -> int: + """ + Delete DB rows whose image file no longer exists on disk (moved or deleted), + so the stats/album breakdown don't keep listing folders that are gone. + + Guarded by the library root existing: if the whole library is unreachable + (unmounted drive, wrong path) we skip pruning rather than wipe the database. + """ + if not library.exists(): + log.warning(f"Library root not found ({library}); skipping stale-entry " + "prune so an unavailable filesystem can't wipe the DB.") + return 0 + rows = conn.execute("SELECT path FROM photos").fetchall() + missing = [r["path"] for r in rows if not os.path.exists(r["path"])] + if missing: + with _db_lock: + conn.executemany("DELETE FROM photos WHERE path = ?", + [(p,) for p in missing]) + conn.commit() + return len(missing) + + +def upsert_pending(conn: sqlite3.Connection, path: str): + with _db_lock: + conn.execute( + "INSERT OR IGNORE INTO photos (path, status) VALUES (?, 'pending')", + (path,) + ) + conn.commit() + + +def mark_analyzed(conn: sqlite3.Connection, path: str, result: dict, raw: str): + with _db_lock: + conn.execute( + """UPDATE photos SET + status = 'analyzed', + description = ?, + tags = ?, + people_count = ?, + setting = ?, + time_of_day = ?, + season = ?, + mood = ?, + location_hint = ?, + approx_year = ?, + raw_response = ?, + error_message = NULL, + analyzed_at = datetime('now') + WHERE path = ?""", + ( + result.get("description"), + json.dumps(result.get("tags", []), ensure_ascii=False), + result.get("people_count"), + result.get("setting"), + result.get("time_of_day"), + result.get("season"), + result.get("mood"), + result.get("location_hint"), + result.get("approx_year"), + raw, + path, + ), + ) + conn.commit() + + +def mark_error(conn: sqlite3.Connection, path: str, message: str): + with _db_lock: + conn.execute( + """UPDATE photos SET + status = 'error', + error_message = ?, + analyzed_at = datetime('now') + WHERE path = ?""", + (message, path), + ) + conn.commit() + + +def mark_exif_written(conn: sqlite3.Connection, path: str): + with _db_lock: + conn.execute( + """UPDATE photos SET + status = 'exif_written', + exif_written_at = datetime('now') + WHERE path = ?""", + (path,), + ) + conn.commit() + + +def get_pending(conn: sqlite3.Connection, reanalyze: bool) -> list[str]: + # 'duplicate' rows never hit the API — they inherit from their canonical twin + # (or are simply skipped) so we don't spend a call, or an upload, on a copy. + if reanalyze: + rows = conn.execute( + "SELECT path FROM photos WHERE status != 'duplicate'" + ).fetchall() + else: + rows = conn.execute( + "SELECT path FROM photos WHERE status IN ('pending', 'error')" + ).fetchall() + return [r["path"] for r in rows] + + +def get_analyzed_no_exif(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return conn.execute( + "SELECT * FROM photos WHERE status = 'analyzed'" + ).fetchall() + + +# ────────────────────────────────────────────── +# Image preparation +# ────────────────────────────────────────────── + +def prepare_image(path: Path) -> tuple[str, str]: + """ + Resize to MAX_LONG_EDGE if needed, return (base64_string, mime_type). + Handles HEIC by converting to JPEG in memory. + """ + suffix = path.suffix.lower() + + with Image.open(path) as img: + img = img.convert("RGB") # normalise; drops alpha, converts HEIC + + w, h = img.size + long_edge = max(w, h) + if long_edge > MAX_LONG_EDGE: + scale = MAX_LONG_EDGE / long_edge + img = img.resize((int(w * scale), int(h * scale)), Image.LANCZOS) + + import io + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85) + buf.seek(0) + b64 = base64.b64encode(buf.read()).decode("utf-8") + + return b64, "image/jpeg" + + +# ────────────────────────────────────────────── +# API call with retry +# ────────────────────────────────────────────── + +# Rolling record of recent rate-limit (429) / overload (503) hits so the dashboard +# can warn when the provider starts throttling — the cue to lower MAX_WORKERS. +# Workers append timestamps; the render thread reads a count over a short window. +# Every hit is ALSO appended to throttle_events.jsonl (separate, persistent backend) +# so the full throttling history survives across runs and the live UI. +_THROTTLE_LOG = Path(__file__).with_name("throttle_events.jsonl") +_ratelimit_hits: deque = deque(maxlen=500) +_ratelimit_total = 0 +_ratelimit_lock = threading.Lock() +RATELIMIT_WINDOW = 30 # seconds to look back +RATELIMIT_WARN_AT = 5 # hits within the window → show the warning box + + +def _record_ratelimit(status: int, path, attempt: int, detail: str = ""): + """Record one 429/503 hit: in-memory (for the warning box) + persistent JSONL. + `detail` is the provider's error body — for Gemini it names the exact quota + (PerMinute vs PerDay, TPM vs RPM) or a billing/balance message.""" + global _ratelimit_total + with _ratelimit_lock: + _ratelimit_hits.append(time.monotonic()) + _ratelimit_total += 1 + try: + with _THROTTLE_LOG.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({ + "time": time.strftime("%Y-%m-%d %H:%M:%S"), + "status": status, "model": LLM_MODEL, + "file": Path(path).name, "attempt": attempt, + "detail": (detail or "")[:500], + }, ensure_ascii=False) + "\n") + except Exception as e: # logging must never break the run + log.debug(f"throttle log write failed: {e}") + + +def _ratelimit_recent() -> int: + """How many 429/503 hits in the last RATELIMIT_WINDOW seconds.""" + cutoff = time.monotonic() - RATELIMIT_WINDOW + with _ratelimit_lock: + return sum(1 for t in _ratelimit_hits if t >= cutoff) + + +# Requests-per-day counter. Providers don't return remaining RPD, so count locally: +# every request fired = 1 RPD unit. Persisted per Pacific calendar day (RPD resets +# midnight PT) so the count is cumulative across multiple runs in the same day. +# ponytail: single-process counter — two concurrent runs would each undercount. +_RPD_FILE = Path(__file__).with_name("rpd_count.json") +_rpd_lock = threading.Lock() +_rpd = {"date": None, "count": 0} + + +def _pt_today() -> str: + return datetime.now(ZoneInfo("America/Los_Angeles")).strftime("%Y-%m-%d") + + +def _rpd_load(): + """Seed the counter from disk if it's still the same PT day, else start at 0.""" + today = _pt_today() + try: + d = json.loads(_RPD_FILE.read_text(encoding="utf-8")) + if d.get("date") == today: + _rpd.update(date=today, count=int(d.get("count", 0))) + return + except Exception: + pass + _rpd.update(date=today, count=0) + + +def _rpd_increment(): + with _rpd_lock: + today = _pt_today() + if _rpd["date"] != today: # rolled past midnight PT mid-run + _rpd["date"], _rpd["count"] = today, 0 + _rpd["count"] += 1 + + +def _rpd_persist(): + with _rpd_lock: + snap = dict(_rpd) + try: + _RPD_FILE.write_text(json.dumps(snap), encoding="utf-8") + except Exception as e: + log.debug(f"rpd persist failed: {e}") + + +def _rpd_count() -> int: + with _rpd_lock: + return _rpd["count"] + + +def write_throttle_summary(): + """Append a run-summary line to the throttle log after the run completes.""" + if _ratelimit_total == 0: + return + try: + with _THROTTLE_LOG.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({ + "time": time.strftime("%Y-%m-%d %H:%M:%S"), + "type": "run_summary", "model": LLM_MODEL, + "workers": MAX_WORKERS, "total_throttle_hits": _ratelimit_total, + }, ensure_ascii=False) + "\n") + except Exception as e: + log.debug(f"throttle summary write failed: {e}") + log.warning(f"⚠ {_ratelimit_total} rate-limit/overload hits this run " + f"(logged to {_THROTTLE_LOG.name}). Consider lowering MAX_WORKERS.") + + +def analyze_image(client: OpenAI, path: Path, dry_run: bool) -> tuple[dict, str]: + """ + Call the configured OpenAI-compatible vision model (LLM_MODEL). Returns + (parsed_result, raw_json_string, tokens). Raises on unrecoverable error. + """ + if dry_run: + fake = { + "description": f"[DRY RUN] Photo at {path.name}", + "tags": ["dry-run", "test"], + "people_count": 0, + "setting": "unknown", + "time_of_day": "unknown", + "season": "unknown", + "mood": "neutral", + "location_hint": None, + "approx_year": None, + } + return fake, json.dumps(fake), {"prompt": 0, "completion": 0, "total": 0} + + b64, mime = prepare_image(path) + image_url = f"data:{mime};base64,{b64}" + log.debug(f"Prepared image {path.name} as {mime} ({len(b64)} base64 chars)") + + # Feed the album/folder name as context — folder names like "Ägypten 2010" + # carry place + year the model can't see from pixels alone. The image still + # wins on conflict; non-place folders (selfies, fun_pics) are ignored. + album = path.parent.name + prompt_text = ANALYSIS_PROMPT + ( + f'\n\nAlbum hint: this photo is filed in a folder named "{album}". ' + f'Folder names often contain the place and/or year — use it to inform ' + f'"location_hint" and "approx_year", but trust the image if they conflict. ' + f'Ignore the hint if it is clearly not a place or date (e.g. "selfies", "fun_pics").' + ) + + for attempt in range(1, RETRY_ATTEMPTS + 1): + log.debug(f"API attempt {attempt}/{RETRY_ATTEMPTS} for {path.name}") + try: + _rpd_increment() # every request fired counts toward the daily cap + response = client.chat.completions.create( + model=LLM_MODEL, + max_tokens=4096, + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": image_url}, + }, + { + "type": "text", + "text": prompt_text, + }, + ], + } + ], + ) + choice = response.choices[0] if response.choices else None + msg = choice.message if choice else None + raw = (msg.content if msg else None) or "" + + # Safety/content-filter block: the model returns no message at all. + # Retrying the same model won't help — fail fast with a clear reason + # instead of crashing on None.content. Caption these on another model. + finish = getattr(choice, "finish_reason", "") or "" + if not raw and "content_filter" in finish: + raise RuntimeError(f"blocked by {LLM_MODEL} content filter ({finish})") + + raw = raw.strip() + # Strip accidental markdown fences + if raw.startswith("```"): + raw = raw.split("```")[1] + if raw.startswith("json"): + raw = raw[4:] + raw = raw.strip() + + # Empty response — retryable (model occasionally returns blank) + if not raw: + delay = RETRY_BASE_DELAY * attempt + log.warning(f"Empty response from {LLM_MODEL} for {path} (attempt {attempt}/{RETRY_ATTEMPTS}). Retry in {delay}s") + time.sleep(delay) + continue + + result = json.loads(raw) + usage = response.usage + tokens = { + "prompt": usage.prompt_tokens if usage else 0, + "completion": usage.completion_tokens if usage else 0, + "total": usage.total_tokens if usage else 0, + } + log.debug(f"OK {path.name} — tokens: {tokens['total']} — raw: {raw[:200]}") + return result, raw, tokens + + except RateLimitError as e: + detail = str(getattr(e, "message", "") or e) + _record_ratelimit(429, path, attempt, detail) + delay = RETRY_BASE_DELAY * (2 ** (attempt - 1)) + log.warning(f"429 rate limited on {path.name}. Waiting {delay}s (attempt {attempt}/{RETRY_ATTEMPTS}) — {detail[:160]}") + time.sleep(delay) + + except json.JSONDecodeError as e: + delay = RETRY_BASE_DELAY * attempt + log.warning(f"Bad JSON from {LLM_MODEL} for {path}: {e}. Raw: {raw[:300]!r} — retry in {delay}s (attempt {attempt}/{RETRY_ATTEMPTS})") + time.sleep(delay) + + except APIError as e: + # 503 overload / 429 / any 5xx is a throttle — record + log EVERY one, + # even on the final attempt (so none go unlogged before the raise). + status = getattr(e, "status_code", None) + is_throttle = status in (429, 503) or (status or 0) >= 500 + if is_throttle: + detail = str(getattr(e, "message", "") or e) + _record_ratelimit(status, path, attempt, detail) + log.warning(f"{status} throttle on {path.name} (attempt {attempt}/{RETRY_ATTEMPTS}) — {detail[:160]}") + if attempt < RETRY_ATTEMPTS: + delay = RETRY_BASE_DELAY * attempt + if not is_throttle: + log.warning(f"API error on {path.name}: {e}. Retry in {delay}s") + time.sleep(delay) + else: + raise + + raise RuntimeError(f"Exhausted {RETRY_ATTEMPTS} retries for {path.name}") + + +# ────────────────────────────────────────────── +# EXIF writing via exiftool +# ────────────────────────────────────────────── + +def build_exif_caption(row: sqlite3.Row) -> str: + """Compose the string that will be written to EXIF ImageDescription.""" + parts = [] + if row["description"]: + parts.append(row["description"]) + if row["tags"]: + try: + tags = json.loads(row["tags"]) + parts.append("Tags: " + ", ".join(tags)) + except Exception: + pass + if row["mood"]: + parts.append(f"Mood: {row['mood']}") + if row["location_hint"]: + parts.append(f"Location: {row['location_hint']}") + if row["approx_year"]: + parts.append(f"~{row['approx_year']}") + return " | ".join(parts) + + +def build_exif_caption_from_result(result: dict) -> str: + """Same as build_exif_caption but from a raw API result dict.""" + parts = [] + if result.get("description"): + parts.append(result["description"]) + tags = result.get("tags", []) + if tags: + parts.append("Tags: " + ", ".join(tags)) + if result.get("mood"): + parts.append(f"Mood: {result['mood']}") + if result.get("location_hint"): + parts.append(f"Location: {result['location_hint']}") + if result.get("approx_year"): + parts.append(f"~{result['approx_year']}") + return " | ".join(parts) + + +def read_existing_exif(path: str) -> dict: + """Read current ImageDescription, XPComment, Subject, Keywords from a file.""" + try: + r = subprocess.run( + ["exiftool", "-json", "-ImageDescription", "-XPComment", "-Subject", "-Keywords", path], + capture_output=True, text=True, timeout=30, + ) + if r.returncode == 0 and r.stdout.strip(): + return json.loads(r.stdout)[0] + except Exception: + pass + return {} + + +def _rec_has_nsfw(rec: dict) -> bool: + """True if an exiftool JSON record's Keywords/Subject carries the 'nsfw' tag + written by the nsfwtag tool. Handles both list- and scalar-valued fields.""" + vals = [] + for field in ("Keywords", "Subject"): + v = rec.get(field) + if isinstance(v, list): + vals += v + elif v is not None: + vals.append(v) + return any(str(x).strip().lower() == "nsfw" for x in vals) + + +def filter_nsfw_tagged(paths: list[str]) -> tuple[list[str], list[str]]: + """Split paths into (to_analyze, skipped), skipping any file the nsfwtag tool + marked with an 'nsfw' EXIF keyword. One batched exiftool call reads Keywords + + Subject for the whole list. Fails open (skips nothing) if exiftool can't run, so + a metadata hiccup never aborts an analysis run.""" + if not paths: + return paths, [] + skipped = set() + try: + r = subprocess.run( + ["exiftool", "-j", "-Keywords", "-Subject", "-@", "-"], + input="\n".join(paths), capture_output=True, text=True, + timeout=max(120, len(paths) // 20), # metadata-only read is fast + ) + if r.returncode == 0 and r.stdout.strip(): + for rec in json.loads(r.stdout): + if _rec_has_nsfw(rec): + skipped.add(os.path.normpath(rec.get("SourceFile", ""))) + except Exception as e: + log.warning(f"nsfw tag check failed ({e}); analyzing all files") + return paths, [] + kept = [p for p in paths if os.path.normpath(p) not in skipped] + return kept, [p for p in paths if os.path.normpath(p) in skipped] + + +_REPAIR_LOG = Path(__file__).with_name("repairs.jsonl") + + +def _log_repair(record: dict) -> None: + """Append one repair attempt to repairs.jsonl for later analysis.""" + record["time"] = time.strftime("%Y-%m-%d %H:%M:%S") + try: + with _REPAIR_LOG.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=False) + "\n") + except Exception as e: # logging must never break the run + log.debug(f"could not write repair log: {e}") + + +# exiftool errors that a clean re-encode fixes: truncated/no-EOI JPEGs, corrupt +# EXIF directories (Bad format … IFD0), and extension/content mismatches (a JPEG +# named .png). Re-encoding rebuilds a valid file with sane metadata structure. +_REPAIRABLE_ERRORS = ( + "EOI marker not found", # truncated JPEG + "Bad format", # corrupt IFD0 / EXIF directory + "looks more like", # wrong extension (e.g. "Not a valid PNG (looks more like a JPEG)") + "Bad IFD", +) + +# Map file extension → the sips output format, so a re-encode produces a file that +# actually matches its extension (the fix for a JPEG mistakenly named .png). +_SIPS_FORMAT = {".png": "png", ".jpg": "jpeg", ".jpeg": "jpeg", + ".tif": "tiff", ".tiff": "tiff", ".heic": "jpeg", ".heif": "jpeg"} + + +def repair_image(path: str) -> bool: + """ + Re-encode a structurally broken image into a clean one so exiftool can write + to it. Handles truncated JPEGs (no EOI), corrupt EXIF directories, and + extension/content mismatches. Backs up the original as .orig and + re-encodes via sips to the format matching the file's extension. + Logs the outcome to repairs.jsonl. Returns True on success. + """ + p = Path(path) + fmt = _SIPS_FORMAT.get(p.suffix.lower(), "jpeg") + backup = p.with_suffix(p.suffix + ".orig") + tmp = p.with_suffix(p.suffix + ".repair_tmp") + rec = {"path": path, "action": "reencode", "format": fmt, "backup": str(backup)} + try: + size_before = p.stat().st_size + if not backup.exists(): + shutil.copy2(p, backup) + r = subprocess.run( + ["sips", "-s", "format", fmt, "-s", "formatOptions", "best", + str(backup if backup.exists() else p), "--out", str(tmp)], + capture_output=True, text=True, timeout=120, + ) + if r.returncode != 0 or not tmp.exists(): + rec.update(status="failed", error=(r.stderr or "sips produced no output").strip()) + _log_repair(rec) + log.error(f"repair failed (sips) for {path}: {rec['error']}") + tmp.unlink(missing_ok=True) + return False + os.replace(tmp, p) # same-dir, atomic; avoids cross-volume mv issues + rec.update(status="ok", size_before=size_before, size_after=p.stat().st_size) + _log_repair(rec) + log.warning(f"repaired image (re-encoded to {fmt}, backup at {backup.name}): {path}") + return True + except Exception as e: + rec.update(status="failed", error=str(e)) + _log_repair(rec) + log.error(f"repair failed for {path}: {e}") + tmp.unlink(missing_ok=True) + return False + + +def write_exif(path: str, caption: str, keywords: list[str]) -> bool: + """ + Add AI caption and keywords to EXIF without overwriting existing values. + - ImageDescription / XPComment: appended with ' | AI: ...' if already populated + - Subject / Keywords: merged with existing, deduplicated + exiftool must be on PATH. + """ + existing = read_existing_exif(path) + + # ── Text fields: append if already set, write if empty ────────────────── + def merge_text(field: str) -> str: + cur = (existing.get(field) or "").strip() + if cur: + if caption in cur: # idempotency: don't append twice + return cur + return f"{caption} | {cur}" + return caption + + final_desc = merge_text("ImageDescription") + final_comment = merge_text("XPComment") + + # ── Keyword fields: merge + deduplicate ────────────────────────────────── + def to_list(val) -> list[str]: + if not val: + return [] + return [val] if isinstance(val, str) else list(val) + + existing_kws = to_list(existing.get("Keywords")) + to_list(existing.get("Subject")) + merged_kws = list(dict.fromkeys(existing_kws + keywords)) # preserves order, deduplicates + + # ── Build exiftool command ─────────────────────────────────────────────── + cmd = [ + "exiftool", + "-m", # ignore minor errors/warnings (e.g. bad MakerNotes offsets) so writes still apply + "-overwrite_original", + f"-ImageDescription={final_desc}", + f"-XPComment={final_comment}", + ] + for kw in merged_kws: + cmd.append(f"-Subject={kw}") + cmd.append(f"-Keywords={kw}") + + cmd.append(path) + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if result.returncode != 0: + err = result.stderr.strip() + # Structurally broken image exiftool can't write to — re-encode and retry once. + if any(sig in err for sig in _REPAIRABLE_ERRORS) and repair_image(path): + retry = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if retry.returncode == 0: + return True + log.error(f"exiftool error after repair for {path}: {retry.stderr.strip()}") + return False + log.error(f"exiftool error for {path}: {err}") + return False + return True + except FileNotFoundError: + log.error("exiftool not found on PATH. Install from https://exiftool.org") + sys.exit(1) + except subprocess.TimeoutExpired: + log.error(f"exiftool timed out for {path}") + return False + + +# ────────────────────────────────────────────── +# Discovery +# ────────────────────────────────────────────── + +def discover_photos(library: Path) -> list[Path]: + """Recursively find all supported image files, sorted for deterministic order.""" + photos = [] + for ext in SUPPORTED_EXTENSIONS: + photos.extend(library.rglob(f"*{ext}")) + photos.extend(library.rglob(f"*{ext.upper()}")) + # Never process files inside _IGNORE/ folders (anywhere in the tree) + photos = [p for p in photos if "_IGNORE" not in p.parts and ".@__thumb" not in p.parts] + return sorted(set(photos)) + + +# ────────────────────────────────────────────── +# Variant grouping +# +# Many photos exist as several near-identical files that differ only by an edit +# marker or a downscaled copy, e.g. +# _MG_1432.JPG +# _MG_1432-bearbeitet.jpg (edited) +# _MG_1432 (640x427).jpg (resized) +# _MG_1432 (640x427)-bearbeitet.jpg +# With --group-variants we analyze only ONE per group (the largest file with no +# markers) via the API, then copy that result into the siblings' DB rows and +# EXIF — no extra API calls. +# ────────────────────────────────────────────── + +# " (640x427)", " (1920 x 1080)" etc. — a trailing pixel-dimensions suffix. +_PIXEL_RE = re.compile(r"\s*\(\s*\d+\s*[x×]\s*\d+\s*\)", re.IGNORECASE) +# Edit markers to strip. Extend this tuple if your library uses others. +_EDIT_RE = re.compile(r"[-_ ]?bearbeitet", re.IGNORECASE) + + +def _strip_variant_markers(stem: str) -> str: + """Reduce a filename stem to its base identity (drop pixel size + edit tag).""" + s = _PIXEL_RE.sub("", stem) + s = _EDIT_RE.sub("", s) + return s.strip() + + +def _is_variant(path: Path) -> bool: + """True if the name carries a pixel-size suffix or an edit marker.""" + stem = path.stem + return bool(_PIXEL_RE.search(stem)) or bool(_EDIT_RE.search(stem)) + + +def _variant_key(path: Path) -> tuple[str, str]: + """Group key: same folder + same base name = same picture.""" + return (str(path.parent), _strip_variant_markers(path.stem).lower()) + + +def _safe_size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 + + +def build_variant_groups(paths: list[Path]) -> dict[str, list[str]]: + """ + Group files that are variants of the same picture. Returns a mapping + {primary_path: [secondary_path, ...]} for groups of 2+. The primary is the + largest file with no markers (falling back to the largest file overall). + Standalone pictures are not included. + """ + from collections import defaultdict + buckets: dict[tuple, list[Path]] = defaultdict(list) + for p in paths: + buckets[_variant_key(p)].append(p) + + members_of: dict[str, list[str]] = {} + for members in buckets.values(): + if len(members) < 2: + continue + clean = [p for p in members if not _is_variant(p)] + primary = max(clean or members, key=_safe_size) + secs = sorted(str(p) for p in members if p != primary) + members_of[str(primary)] = secs + return members_of + + +def _row_to_result(row: sqlite3.Row) -> dict: + """Reconstruct an analysis result dict from a stored primary row.""" + try: + tags = json.loads(row["tags"] or "[]") + except Exception: + tags = [] + return { + "description": row["description"], + "tags": tags, + "people_count": row["people_count"], + "setting": row["setting"], + "time_of_day": row["time_of_day"], + "season": row["season"], + "mood": row["mood"], + "location_hint": row["location_hint"], + "approx_year": row["approx_year"], + } + + +def propagate_variants(conn: sqlite3.Connection, members_of: dict[str, list[str]], + args) -> None: + """ + Copy each analyzed primary's data into its sibling variants — DB row first + (durable), then EXIF — without calling the API. Skips siblings already + finished, and groups whose primary isn't analyzed yet. + """ + # Find work: primaries that are analyzed, with siblings not yet done. + todo = [] # (secondary_path, result_dict, primary_path) + for primary, secs in members_of.items(): + prow = conn.execute( + "SELECT * FROM photos WHERE path = ?", (primary,) + ).fetchone() + if not prow or prow["status"] not in ("analyzed", "exif_written"): + continue # primary not ready — its siblings wait for a later run + result = _row_to_result(prow) + for sec in secs: + srow = conn.execute( + "SELECT status FROM photos WHERE path = ?", (sec,) + ).fetchone() + if srow and srow["status"] == "exif_written": + continue # already fully done + if args.no_exif and srow and srow["status"] == "analyzed": + continue # DB already carries the data; nothing more to do + todo.append((sec, result, primary)) + + if not todo: + return + + log.info(f"Propagating analysis to {len(todo):,} variant copies (no API calls)") + + ok = err = 0 + with Progress( + SpinnerColumn(spinner_name="dots", style="magenta"), + TextColumn("[bold magenta] Variants[/]"), + BarColumn(bar_width=None, style="dim magenta", complete_style="magenta", finished_style="green"), + MofNCompleteColumn(), + TaskProgressColumn(style="bold"), + TimeElapsedColumn(), + console=console, + refresh_per_second=4, + ) as progress: + task = progress.add_task("", total=len(todo)) + for sec, result, primary in todo: + # 1) DB first — durable. Record that this row was copied, not analyzed, + # and from which primary (so it's traceable in raw_response). + raw = json.dumps({"copied_variant": True, "copied_from": primary}, + ensure_ascii=False) + mark_analyzed(conn, sec, result, raw) + # History line is marked "copied" (not "analyzed") with its source. + log_history(sec, "copied", result=result, copied_from=primary) + progress.console.print( + f"[magenta][/] [bold]{Path(sec).name}[/] " + f"[dim]copied from[/] {Path(primary).name}" + ) + # 2) EXIF into the sibling image (unless disabled). + if not args.no_exif and not args.dry_run: + caption = build_exif_caption_from_result(result) + keywords = result.get("tags", []) or [] + if write_exif(sec, caption, keywords): + mark_exif_written(conn, sec) + ok += 1 + else: + err += 1 + else: + ok += 1 + progress.update(task, advance=1) + if _stop.is_set(): + progress.console.print("[yellow]Stopped. Progress saved.[/]") + break + + log.info(f"Variant propagation complete: {ok:,} OK, {err:,} errors") + + +# ────────────────────────────────────────────── +# Content hashing, duplicate detection, move reconciliation +# +# Two hashes per photo, both stored in the DB (the permanent, resume-safe +# ledger): +# • file_sha1 — SHA-1 of the raw bytes. Exact identity. Changes if EXIF is +# rewritten, so it tracks a moved/renamed file only while its bytes are +# untouched. Used by reconcile_moved to follow files across a reorg. +# • phash — 64-bit DCT perceptual hash. Content identity that SURVIVES resize, +# recompression, format conversion and EXIF edits. Used to spot the same +# picture arriving from different phone backups. Compared by Hamming +# distance (see PHASH_THRESHOLD). +# ────────────────────────────────────────────── + +def _sha1_file(path: Path) -> str | None: + """SHA-1 of a file's bytes, streamed. None if unreadable.""" + try: + h = hashlib.sha1() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + except OSError as e: + log.debug(f"sha1 failed for {path}: {e}") + return None + + +def _phash_image(path: Path) -> str | None: + """DCT perceptual hash → 16 hex chars (64 bits). Matches the standard + imagehash.phash recipe: grayscale → 32×32 → 2D DCT → keep the top-left 8×8 + low-frequency block → bit = coefficient > median. None if unreadable.""" + try: + from scipy.fftpack import dct + import numpy as np + with Image.open(path) as img: + small = img.convert("L").resize((32, 32), Image.LANCZOS) + a = np.asarray(small, dtype=np.float64) + d = dct(dct(a, axis=0), axis=1) # 2D DCT-II + low = d[:8, :8] # 64 lowest frequencies + bits = (low > np.median(low)).flatten() + val = 0 + for b in bits: + val = (val << 1) | int(b) + return f"{val:016x}" + except Exception as e: + log.debug(f"phash failed for {path}: {e}") + return None + + +def ensure_hashes(conn: sqlite3.Connection, paths, force: bool = False) -> int: + """Compute + store phash and file_sha1 for photos that lack them (or all, if + force). One-time cost per file; resume-safe (stored, committed periodically, + interruptible). Rows must already exist (upsert_pending).""" + if force: + todo = [str(p) for p in paths] + else: + have = {r["path"] for r in conn.execute( + "SELECT path FROM photos WHERE phash IS NOT NULL AND file_sha1 IS NOT NULL" + )} + todo = [str(p) for p in paths if str(p) not in have] + if not todo: + return 0 + + log.info(f"Hashing {len(todo):,} photo(s) (perceptual + SHA-1)…") + done = 0 + for i, p in enumerate(todo, 1): + if _stop.is_set(): + break + ph = _phash_image(Path(p)) + s1 = _sha1_file(Path(p)) + with _db_lock: + # COALESCE(new, old): a failed hash (None) leaves any existing value be. + conn.execute( + "UPDATE photos SET phash = COALESCE(?, phash), " + "file_sha1 = COALESCE(?, file_sha1) WHERE path = ?", + (ph, s1, p), + ) + if i % 200 == 0: + conn.commit() + done += 1 + if i % 1000 == 0: + log.info(f" hashed {i:,}/{len(todo):,}") + with _db_lock: + conn.commit() + log.info(f"Hashing complete: {done:,} updated") + return done + + +def cluster_duplicates(conn: sqlite3.Connection, threshold: int) -> list[list[dict]]: + """Group photos whose perceptual hashes are within `threshold` bits into + duplicate clusters (union-find over the Hamming graph). Each returned cluster + is a list of {path, size, status} dicts sorted largest-file-first — so + cluster[0] is the canonical (highest-quality) copy. Only clusters of ≥2. + + ponytail: O(n²) vectorised Hamming scan — a few seconds at ~25k photos. If the + library grows past ~100k, swap the inner scan for a BK-tree. + """ + import numpy as np + rows = conn.execute( + "SELECT path, phash, status FROM photos WHERE phash IS NOT NULL" + ).fetchall() + n = len(rows) + if n < 2: + return [] + + arr = np.array([int(r["phash"], 16) for r in rows], dtype=np.uint64) + popcount8 = np.array([bin(i).count("1") for i in range(256)], dtype=np.uint16) + + parent = list(range(n)) + def find(i): + while parent[i] != i: + parent[i] = parent[parent[i]] + i = parent[i] + return i + + for i in range(n): + tail = arr[i + 1:] + if not len(tail): + break + x = tail ^ arr[i] # uint64 XOR + dist = popcount8[x.view(np.uint8).reshape(-1, 8)].sum(axis=1) + for off in np.nonzero(dist <= threshold)[0]: + ri, rj = find(i), find(i + 1 + int(off)) + if ri != rj: + parent[rj] = ri + + groups: dict[int, list[int]] = {} + for i in range(n): + groups.setdefault(find(i), []).append(i) + + clusters = [] + for members in groups.values(): + if len(members) < 2: + continue + items = [{"path": rows[m]["path"], + "size": _safe_size(Path(rows[m]["path"])), + "status": rows[m]["status"]} for m in members] + items.sort(key=lambda it: it["size"], reverse=True) + clusters.append(items) + return clusters + + +def list_duplicates(conn: sqlite3.Connection, threshold: int) -> None: + """Report perceptual-duplicate clusters. Read-only — changes nothing.""" + clusters = cluster_duplicates(conn, threshold) + if not clusters: + log.info(f"No perceptual duplicates found (threshold {threshold}).") + return + total_dupes = sum(len(c) - 1 for c in clusters) + console.print(f"\n[bold]{len(clusters):,} duplicate cluster(s), " + f"{total_dupes:,} redundant copy/copies[/] " + f"[dim](Hamming ≤ {threshold})[/]\n") + for c in clusters: + canonical = c[0] + console.print(f"[green] KEEP[/] [bold]{canonical['path']}[/] " + f"[dim]({canonical['size']:,} B)[/]") + for dup in c[1:]: + console.print(f" [yellow] dup[/] {dup['path']} " + f"[dim]({dup['size']:,} B, {dup['status']})[/]") + console.print() + console.print("[dim]Run with --dedupe to mark the copies as 'duplicate' " + "(skipped by analysis + excluded from upload).[/]\n") + + +def mark_duplicates(conn: sqlite3.Connection, threshold: int) -> tuple[int, int]: + """Mark every non-canonical member of each cluster status='duplicate' with + dup_of pointing at the canonical (largest) copy. Returns (marked, clusters).""" + clusters = cluster_duplicates(conn, threshold) + marked = 0 + with _db_lock: + for c in clusters: + canonical = c[0]["path"] + for dup in c[1:]: + conn.execute( + "UPDATE photos SET status = 'duplicate', dup_of = ? WHERE path = ?", + (canonical, dup["path"]), + ) + marked += 1 + conn.commit() + return marked, len(clusters) + + +def reconcile_moved(conn: sqlite3.Connection, library: Path) -> int: + """Follow files that moved/renamed since the last run: match on-disk files + that aren't in the DB against DB rows whose path is gone, keyed by file_sha1, + and update the row's path in place — preserving its analysis, EXIF state and + hashes. Must run BEFORE prune_missing so moved files aren't deleted and + re-analyzed. Cheap when nothing moved (no missing rows → early return).""" + if not library.exists(): + return 0 + db_rows = conn.execute("SELECT path, file_sha1 FROM photos").fetchall() + db_paths = {r["path"] for r in db_rows} + # sha1 → old path, only for rows whose file is gone and that carry a sha1. + missing = {r["file_sha1"]: r["path"] for r in db_rows + if r["file_sha1"] and not os.path.exists(r["path"])} + if not missing: + return 0 + new_files = [p for p in discover_photos(library) if str(p) not in db_paths] + if not new_files: + return 0 + + moved = 0 + for p in new_files: + s = _sha1_file(p) + old = missing.pop(s, None) if s else None + if old: + with _db_lock: + conn.execute("UPDATE photos SET path = ? WHERE path = ?", (str(p), old)) + conn.commit() + moved += 1 + if not missing: + break + if moved: + log.info(f"Reconciled {moved:,} moved/renamed file(s) to their new paths " + f"(analysis + EXIF preserved, no re-upload).") + return moved + + +# ────────────────────────────────────────────── +# Main pipeline +# ────────────────────────────────────────────── + +def run_analysis(args, conn: sqlite3.Connection, client: OpenAI): + """ + Analyze images via the configured vision model. Each worker writes the DB record first, then + immediately writes EXIF into that image (unless --no-exif/--dry-run). The + trailing run_exif_write() pass then only has to mop up any inline failures. + """ + + library = Path(args.library).expanduser().resolve() + if not library.exists(): + log.error(f"Library path does not exist: {library}") + sys.exit(1) + + log.info(f"Scanning {library} for photos...") + all_photos = discover_photos(library) + log.info(f"Found {len(all_photos):,} photos") + + # Remove any previously-registered excluded paths + purged = purge_excluded(conn) + if purged: + log.info(f"Purged {purged:,} excluded entries from DB (.@__thumb, _IGNORE)") + + # Register all files as pending (idempotent) + for p in all_photos: + upsert_pending(conn, str(p)) + + # Content hashes (phash + sha1) for any file that lacks them — the ledger + # duplicate detection and move-reconciliation both read from. Only-missing, + # so it's a one-time cost per file and skipped on resume. + if not args.dry_run: + ensure_hashes(conn, all_photos) + + # Variant grouping: analyze only the primary of each group via the API; its + # siblings inherit the result afterwards (propagate_variants), no API call. + members_of: dict[str, list[str]] = {} + secondaries: set[str] = set() + if getattr(args, "group_variants", False): + members_of = build_variant_groups(all_photos) + secondaries = {s for secs in members_of.values() for s in secs} + if members_of: + log.info(f"Variant grouping: {len(secondaries):,} copies will inherit from " + f"{len(members_of):,} primaries (no API call for the copies)") + + pending = get_pending(conn, args.reanalyze) + if secondaries: + pending = [p for p in pending if p not in secondaries] + # Skip anything the nsfwtag tool flagged 'nsfw' — those stay local, never hit the API. + pending, nsfw_skipped = filter_nsfw_tagged(pending) + if nsfw_skipped: + log.info(f"Skipping {len(nsfw_skipped):,} photo(s) tagged 'nsfw' by nsfwtag") + log.info(f"{len(pending):,} photos to analyze (skipping already processed)") + + # Flush variant copies whose primary is ALREADY analyzed (e.g. from earlier + # runs) up front — before the dashboard seeds its album counts — so those + # copies are counted. Copies whose primary is still pending are skipped here + # and handled inline when that primary is analyzed below. + if members_of: + propagate_variants(conn, members_of, args) + + if not pending: + log.info("No primaries left to analyze.") + else: + # Overall progress across all runs. Count truly-done rows directly — + # not total minus pending — because variant copies are excluded from + # `pending` yet aren't done, which would otherwise inflate the figure. + total_in_db = conn.execute("SELECT COUNT(*) FROM photos").fetchone()[0] + already_done = conn.execute( + "SELECT COUNT(*) FROM photos WHERE status IN ('analyzed', 'exif_written')" + ).fetchone()[0] + log.info(f"Overall progress: {already_done:,} / {total_in_db:,} done " + f"({already_done / total_in_db * 100:.1f}%)" if total_in_db else "") + + # Sort by album (leaf folder) — finish one album fully before the next. + pending_sorted = sorted(pending, key=lambda p: (str(Path(p).parent), p)) + folder_groups = [ + (folder, list(group)) + for folder, group in groupby(pending_sorted, key=lambda p: str(Path(p).parent)) + ] + + # Live dashboard only when attached to a real terminal; piped/redirected + # output (logs, CI, nohup) falls back to plain line-by-line printing. + if console.is_terminal: + try: + ok, err, copied = _run_analysis_live( + args, conn, client, library, pending, folder_groups, + total_in_db, already_done, members_of=members_of, + ) + except Exception as e: + log.warning(f"Live dashboard failed ({e}); falling back to plain mode") + log.debug("Live dashboard traceback", exc_info=True) + ok, err, copied = _run_analysis_plain( + args, conn, client, library, pending, folder_groups, + total_in_db, already_done, members_of=members_of, + ) + else: + ok, err, copied = _run_analysis_plain( + args, conn, client, library, pending, folder_groups, + total_in_db, already_done, members_of=members_of, + ) + + log.info(f"Analysis complete: {ok:,} OK, {err:,} errors, {copied:,} copied to variants") + if total_in_db: + done_total = already_done + ok + copied + log.info(f"Overall: {done_total:,} / {total_in_db:,} done " + f"({done_total / total_in_db * 100:.1f}%)") + + +# ────────────────────────────────────────────── +# Analysis engine + display (shared by live + plain modes) +# ────────────────────────────────────────────── + +def _make_progress() -> Progress: + """The progress bar, identical in both display modes.""" + return Progress( + SpinnerColumn(spinner_name="dots", style="cyan"), + TextColumn("[bold cyan]\uf03e Analyzing[/]"), + BarColumn(bar_width=None, style="dim blue", complete_style="bright_blue", finished_style="green"), + MofNCompleteColumn(), + TaskProgressColumn(style="bold"), + TimeElapsedColumn(), + TextColumn("[dim]·[/]"), + TimeRemainingColumn(), + TextColumn(" [dim]ok=[/][green]{task.fields[ok]}[/] [dim]err=[/][red]{task.fields[err]}[/] [dim]overall=[/][yellow]{task.fields[overall]}[/]"), + console=console, + refresh_per_second=4, + ) + + +def _run_folder_loop(folder_groups, library, pending, conn, client, args, + progress, task_id, total_in_db, already_done, + renderer, stats_lock, folder_done=None, members_of=None, + token_box=None): + """ + Shared engine: walk albums in order, run workers, drive the progress bar, + and hand each result to the renderer. DB writes happen inside worker threads + here — entirely outside any render path — so a render crash can never corrupt + the database. + + When members_of maps a primary to its variant copies, each copy is propagated + (DB + EXIF) right after its primary is analyzed — so copies complete, and show + up in the feed, incrementally instead of in a separate pass at the end. + Returns (ok, err, copied). + """ + ok = err = copied = 0 + total_tokens = 0 + members_of = members_of or {} + + write_exif_inline = not args.no_exif and not args.dry_run + + def process(path_str: str): + # Bail out before any network call if a stop was requested. This makes a + # draining thread-pool queue (e.g. after Ctrl+C → shutdown) collapse into + # instant no-ops instead of firing off the rest of the folder's API calls. + if _stop.is_set(): + return "skip", path_str, None, None, [] + path = Path(path_str) + try: + result, raw, tokens = analyze_image(client, path, args.dry_run) + # 1) DB first — the durable record. If anything below fails, the + # photo is already safely 'analyzed' and can be retried. + mark_analyzed(conn, path_str, result, raw) + log_history(path_str, "analyzed", result=result, tokens=tokens) + # 2) Write EXIF into the image right away, in this same worker. On + # failure we leave status 'analyzed' — the trailing EXIF pass (or + # --exif-only) retries it, so nothing is lost. + caption = build_exif_caption_from_result(result) + keywords = result.get("tags", []) or [] + if write_exif_inline: + if write_exif(path_str, caption, keywords): + mark_exif_written(conn, path_str) + # 3) Propagate to this primary's variant copies — no API call. Same + # DB-first ordering, marked "copied" with the source primary. + copied_secs = [] + for sec in members_of.get(path_str, ()): + raw_c = json.dumps({"copied_variant": True, "copied_from": path_str}, + ensure_ascii=False) + mark_analyzed(conn, sec, result, raw_c) + log_history(sec, "copied", result=result, copied_from=path_str) + if write_exif_inline and write_exif(sec, caption, keywords): + mark_exif_written(conn, sec) + copied_secs.append(sec) + return "ok", path_str, result, tokens, copied_secs + except Exception as e: + mark_error(conn, path_str, str(e)) + log_history(path_str, "error", error=str(e)) + return "err", path_str, str(e), {}, [] + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + for folder, folder_paths in folder_groups: + if _stop.is_set(): + break + + album = album_label(Path(folder_paths[0]), library) + renderer.folder_start(album, len(folder_paths)) + + futures = {executor.submit(process, p): p for p in folder_paths} + for future in as_completed(futures): + outcome = future.result() + if outcome[0] == "skip": # worker no-op'd because we're stopping + continue + est = 0 + copied_secs = [] + if outcome[0] == "ok": + ok += 1 + tokens = outcome[3] + copied_secs = outcome[4] + with stats_lock: + total_tokens += tokens["total"] + est = int((total_tokens / ok) * (len(pending) - (ok + err))) + if folder_done is not None: + folder_done[album] = folder_done.get(album, 0) + 1 + len(copied_secs) + if token_box is not None: + t = tokens["total"] + token_box["n"] += 1 + token_box["total"] += t + token_box["prompt"] += tokens.get("prompt", 0) + token_box["completion"] += tokens.get("completion", 0) + token_box["min"] = t if token_box["n"] == 1 else min(token_box["min"], t) + token_box["max"] = max(token_box["max"], t) + copied += len(copied_secs) + else: + err += 1 + + ctx = {"album": album, "ok": ok, "err": err, + "total_tokens": total_tokens, "est": est} + renderer.photo(outcome, ctx) + # Show each inherited copy in the feed, marked as copied. + for sec in copied_secs: + renderer.copied(sec, outcome[1], outcome[2]) + + done_now = already_done + ok + err + copied + overall = f"{done_now / total_in_db * 100:.1f}%" if total_in_db else "?" + progress.update(task_id, advance=1, ok=ok, err=err, overall=overall) + + if _stop.is_set(): + for f in futures: + f.cancel() + renderer.stopped() + break + + return ok, err, copied + + +# ── Plain renderer (non-TTY / fallback) ─────────────────────────────────────── + +class _PlainRenderer: + """Prints one block per photo — the original behaviour, unchanged.""" + + def __init__(self, progress: Progress): + self.progress = progress + + def folder_start(self, album: str, n: int): + self.progress.console.print( + f"\n[bold blue]\uf07b {album}[/] [dim]({n} photos)[/]" + ) + + def photo(self, outcome, ctx): + if outcome[0] == "ok": + path_str, result, tokens = outcome[1], outcome[2], outcome[3] + tags = result.get("tags", []) + desc = result.get("description", "") + caption = build_exif_caption_from_result(result) + mood = result.get("mood", "") + setting = result.get("setting", "") + people = result.get("people_count", 0) + self.progress.console.print( + f"\n[green]\uf058[/] [bold]{path_str}[/]\n" + f" [cyan]\uf03e[/] [dim]Desc [/] {desc}\n" + f" [yellow]\uf02b[/] [dim]Tags [/] [dim]{', '.join(tags)}[/]\n" + f" [magenta]\uf005[/] [dim]Mood [/] {mood} [dim]·[/] {setting} [dim]·[/] {people} [dim]people[/]\n" + f" [blue]\uf044[/] [dim]EXIF [/] [dim]{caption[:120]}{'…' if len(caption) > 120 else ''}[/]\n" + f" [yellow]⚡[/] [dim]Tokens [/] [bold]{tokens['total']:,}[/] " + f"[dim](prompt {tokens['prompt']:,} + compl {tokens['completion']:,}) · " + f"session [/][bold]{ctx['total_tokens']:,}[/][dim] · est. remaining [/][bold]~{ctx['est']:,}[/]" + ) + else: + self.progress.console.print( + f"\n[red]\uf057[/] [bold red]{outcome[1]}[/]\n" + f" [dim]{outcome[2]}[/]" + ) + + def copied(self, sec, primary, result): + self.progress.console.print( + f"\uf0c5 [bold]{Path(sec).name}[/] " + f"[dim]\u2190 copied from {Path(primary).name} (no API call)[/]" + ) + + def stopped(self): + self.progress.console.print( + "[yellow]Stopped. In-flight requests completed, rest skipped.[/]" + ) + + +def _run_analysis_plain(args, conn, client, library, pending, folder_groups, + total_in_db, already_done, members_of=None): + progress = _make_progress() + stats_lock = threading.Lock() + with progress: + task_id = progress.add_task("", total=len(pending), ok=0, err=0, overall="0.0%") + renderer = _PlainRenderer(progress) + return _run_folder_loop( + folder_groups, library, pending, conn, client, args, + progress, task_id, total_in_db, already_done, + renderer, stats_lock, folder_done=None, members_of=members_of, + ) + + +# ── Live dashboard ──────────────────────────────────────────────────────────── + +_STATS_ROWS = 14 # max album rows shown; the rest are summarised + + +def _text_bar(done: int, total: int, width: int = 18) -> Text: + pct = (done / total) if total else 0.0 + filled = max(0, min(width, int(round(pct * width)))) + complete = total > 0 and done >= total + bar = Text("█" * filled, style="green" if complete else "bright_blue") + bar.append("░" * (width - filled), style="grey30") + return bar + + +def _stats_panel(folder_total, folder_done, stats_lock) -> Panel: + """Album progress — leaf folder = album. Reads in-memory dicts only.""" + try: + with stats_lock: + items = [(a, folder_done.get(a, 0), t) for a, t in folder_total.items()] + + def rank(it): + a, d, t = it + if t and 0 < d < t: # active first + return (0, a) + if d == 0: # not started + return (1, a) + return (2, a) # complete last + items.sort(key=rank) + + done_albums = sum(1 for _, d, t in items if t and d >= t) + title = (f"[b]Album Progress[/] [dim]\u2014 leaf folder = album " + f"({done_albums:,}/{len(items):,} albums)[/]") + + table = Table.grid(padding=(0, 2)) + table.add_column(justify="left", no_wrap=True) + table.add_column(justify="right", no_wrap=True) + table.add_column(no_wrap=True) + table.add_column(justify="right", no_wrap=True) + + for album, done, total in items[:_STATS_ROWS]: + pct = (done / total * 100) if total else 0 + if total and done >= total: + flag = Text("✓", style="green") + elif done == 0: + flag = Text("0%", style="grey50") + else: + flag = Text(f"{pct:3.0f}%", style="yellow") + table.add_row( + Text(fit_label(album, 30), style="white"), + Text(f"{done:,}/{total:,}", style="grey62"), + _text_bar(done, total), + flag, + ) + + hidden = items[_STATS_ROWS:] + if hidden: + hdone = sum(1 for _, d, t in hidden if t and d >= t) + table.add_row( + Text(f"\u2026 +{len(hidden):,} more albums", style="grey50"), + Text(f"{hdone:,} done", style="grey50"), Text(""), Text(""), + ) + + return Panel(table, title=title, title_align="left", + border_style="grey37", padding=(0, 1)) + except Exception as e: + return Panel(Text(f"stats render error: {e}", style="red"), border_style="red") + + +def _feed_panel(log_buffer, log_lock) -> Panel: + """Recent per-photo results. Reads the shared deque under its lock.""" + try: + with log_lock: + rows = list(log_buffer) + + grid = Table.grid(padding=(0, 1)) + grid.add_column(overflow="ellipsis", no_wrap=True) + + if not rows: + grid.add_row(Text("Waiting for first result\u2026", style="grey50")) + + for e in rows: + kind = e.get("kind") + if kind == "folder": + grid.add_row(Text(f"\uf07b {e['album']} ({e['n']})", style="bold blue")) + elif kind == "stop": + grid.add_row(Text("Stopped \u2014 in-flight done, rest skipped.", style="yellow")) + elif kind == "copied": + line = Text("\uf0c5 ", style="magenta") + line.append(e["name"], style="bold magenta") + line.append(f" \u2190 {e.get('from', '')} (copied)", style="grey50") + grid.add_row(line) + if e.get("desc"): + grid.add_row(Text(f" {e['desc']}", style="grey62")) + elif e.get("ok"): + line = Text("✓ ", style="green") + line.append(e["name"], style="bold white") + grid.add_row(line) + if e.get("desc"): + grid.add_row(Text(f" {e['desc']}", style="grey62")) + else: + line = Text("✗ ", style="red") + line.append(e["name"], style="bold red") + line.append(f" \u2014 {e.get('err', '')}", style="grey50") + grid.add_row(line) + + return Panel(grid, title="[b]Recent[/]", title_align="left", + border_style="grey37", padding=(0, 1)) + except Exception as e: + return Panel(Text(f"feed render error: {e}", style="red"), border_style="red") + + +def _balance_text(bal: dict | None) -> Text: + """One-line API balance, coloured by how much is left. Reads cached data.""" + if not bal or bal.get("available_balance") is None: + return Text(" ⚡ balance: unavailable", style="grey50") + avail = bal.get("available_balance") + try: + amount = float(avail) + colour = "red" if amount < 1 else "yellow" if amount < 5 else "green" + shown = f"${amount:,.2f}" + except (TypeError, ValueError): + colour, shown = "grey62", str(avail) + line = Text(" ⚡ balance: ", style="grey62") + line.append(shown, style=f"bold {colour}") + return line + + +def _rpd_text() -> Text: + """One-line requests-per-day gauge (local count vs RPD_LIMIT), coloured by + how close to the daily cap. Resets at midnight Pacific.""" + n = _rpd_count() + line = Text(" 📅 RPD today: ", style="grey62") + if RPD_LIMIT > 0: + frac = n / RPD_LIMIT + colour = "red" if frac >= 0.95 else "yellow" if frac >= 0.8 else "green" + line.append(f"{n:,} / {RPD_LIMIT:,}", style=f"bold {colour}") + else: + line.append(f"{n:,}", style="bold grey70") + return line + + +def _tokens_panel(box, lock) -> Panel: + """Session token usage — overlay toggled with 't'. Reads the shared box.""" + with lock: + n, tot = box["n"], box["total"] + pr, co = box["prompt"], box["completion"] + mn, mx = box["min"], box["max"] + if not n: + body = Text("No analyses yet this session.", style="grey50") + else: + t = Table.grid(padding=(0, 3)) + t.add_column(style="grey62", no_wrap=True) + t.add_column(justify="right", style="bold white", no_wrap=True) + t.add_row("Photos this session", f"{n:,}") + t.add_row("Avg total / photo", f"{tot / n:,.0f}") + t.add_row("Avg prompt (image)", f"{pr / n:,.0f}") + t.add_row("Avg completion", f"{co / n:,.0f}") + t.add_row("Min / max total", f"{mn:,} / {mx:,}") + t.add_row("Session total", f"{tot:,}") + body = t + return Panel(body, title="[b]Token usage[/] [dim]— m or Esc to go back[/]", + title_align="left", border_style="cyan", padding=(0, 1)) + + +def _menu_panel(box, lock) -> Panel: + """The popup overlay: either the navigable menu list or the token view.""" + with _menu_lock: + sel, view = _menu_state["sel"], _menu_state["view"] + if view == "token": + return _tokens_panel(box, lock) + grid = Table.grid(padding=(0, 1)) + grid.add_column(no_wrap=True) + for i, (label, _) in enumerate(_MENU_ITEMS): + if i == sel: + grid.add_row(Text(f" ▸ {label} ", style="bold black on cyan")) + else: + grid.add_row(Text(f" {label} ", style="white")) + hint = Text("↑/↓ or j/k · Enter select · m/Esc close", style="grey42") + return Panel(Group(grid, Text(""), hint), title="[b]Menu[/]", + title_align="left", border_style="cyan", padding=(0, 1)) + + +def _read_key(timeout): + """One keypress from cbreak stdin, normalised. Arrow keys arrive as ESC [ A/B; + a lone ESC (no follow-up byte) is reported as 'esc'.""" + r, _, _ = select.select([sys.stdin], [], [], timeout) + if not r: + return None + ch = sys.stdin.read(1) + if ch == "\x1b": + r2, _, _ = select.select([sys.stdin], [], [], 0.001) + if not r2: + return "esc" + return {"[A": "up", "[B": "down"}.get(sys.stdin.read(2)) + if ch in ("\r", "\n"): + return "enter" + return ch + + +def _key_listener(stop_evt): + """Drive the 'm' popup menu. cbreak (not raw) keeps ISIG on, so Ctrl+C still + raises SIGINT exactly as before. No-op without a real TTY.""" + if not sys.stdin.isatty(): + return + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + try: + tty.setcbreak(fd) + while not stop_evt.is_set(): + k = _read_key(0.2) + if k is None: + continue + if not _menu.is_set(): + if k in ("m", "M"): + with _menu_lock: + _menu_state["sel"], _menu_state["view"] = 0, "menu" + _menu.set() + continue + with _menu_lock: + view = _menu_state["view"] + if view == "token": # token panel: only way out is back + if k in ("m", "M", "esc", "enter"): + with _menu_lock: + _menu_state["view"] = "menu" + continue + if k in ("up", "k"): + with _menu_lock: + _menu_state["sel"] = (_menu_state["sel"] - 1) % len(_MENU_ITEMS) + elif k in ("down", "j"): + with _menu_lock: + _menu_state["sel"] = (_menu_state["sel"] + 1) % len(_MENU_ITEMS) + elif k in ("m", "M", "esc"): + _menu.clear() + elif k == "enter": + with _menu_lock: + action = _MENU_ITEMS[_menu_state["sel"]][1] + if action == "token": + with _menu_lock: + _menu_state["view"] = "token" + elif action == "close": + _menu.clear() + elif action == "stop": + _stop.set() + _menu.clear() + elif action == "force": + # Reuse the tested SIGINT path: _stop already set means the + # handler force-quits on the main thread, which unwinds Live + # (restores the screen) and our finally restores termios. + _stop.set() + os.kill(os.getpid(), signal.SIGINT) + except Exception: + pass + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + + +class _Dashboard: + """Re-rendered by Live every tick; rebuilds the layout from live state.""" + + def __init__(self, progress, folder_total, folder_done, stats_lock, + log_buffer, log_lock, bal_holder=None, token_box=None): + self.progress = progress + self.folder_total = folder_total + self.folder_done = folder_done + self.stats_lock = stats_lock + self.log_buffer = log_buffer + self.log_lock = log_lock + self.bal_holder = bal_holder or {} + self.token_box = token_box + + def __rich__(self): + try: + # Cached balance (refreshed off-thread); never a network call here. + prog_block = Group( + self.progress, + _balance_text(self.bal_holder.get("value")), + _rpd_text(), + Text(" press m for menu", style="grey42"), + ) + sections = [] + # Popup menu overlay, toggled with 'm' — same pattern as the stop + # banner: state is set off-thread, drawn on the next render tick. + if _menu.is_set() and self.token_box is not None: + sections.append(Layout(_menu_panel(self.token_box, self.stats_lock), + name="menu", size=10)) + # Immediate, unmissable feedback that Ctrl+C was received. Shown on + # the next refresh tick (~250 ms) — the log line would be hidden by + # the full-screen Live, so we render it inside the dashboard instead. + if _stop.is_set(): + banner = Text( + " STOPPING — finishing current work. " + "Ctrl+C again to force-quit. ", + style="bold black on yellow", justify="center", + ) + sections.append(Layout(Panel(banner, border_style="bold yellow", + padding=(0, 1)), name="banner", size=3)) + # Throttling warning — provider returning 429/503 in clusters. Auto- + # clears once hits age out of the window. Cue to lower MAX_WORKERS. + rl = _ratelimit_recent() + if rl >= RATELIMIT_WARN_AT: + warn = Text( + f" ⚠ {rl} rate-limit/overload hits in {RATELIMIT_WINDOW}s — " + f"provider is throttling. Lower MAX_WORKERS if this persists. ", + style="bold white on red", justify="center", + ) + sections.append(Layout(Panel(warn, border_style="bold red", + padding=(0, 1)), name="throttle", size=3)) + sections.append(Layout(_stats_panel(self.folder_total, self.folder_done, self.stats_lock), + name="stats", ratio=2)) + sections.append(Layout(_feed_panel(self.log_buffer, self.log_lock), + name="feed", ratio=2)) + sections.append(Layout(Panel(prog_block, title="[b]Analyzing[/]", title_align="left", + border_style="grey37", padding=(0, 1)), + name="progress", size=5)) + layout = Layout() + layout.split_column(*sections) + return layout + except Exception as e: + return Text(f"dashboard render error: {e}", style="red") + + +class _LiveRenderer: + """Workers never touch the terminal — they only append to the shared deque.""" + + def __init__(self, log_buffer, log_lock): + self.log_buffer = log_buffer + self.log_lock = log_lock + + def folder_start(self, album: str, n: int): + with self.log_lock: + self.log_buffer.appendleft({"kind": "folder", "album": album, "n": n}) + + def photo(self, outcome, ctx): + if outcome[0] == "ok": + path_str, result = outcome[1], outcome[2] + entry = {"kind": "photo", "ok": True, "name": Path(path_str).name, + "desc": (result.get("description") or "")} + else: + entry = {"kind": "photo", "ok": False, "name": Path(outcome[1]).name, + "err": outcome[2]} + with self.log_lock: + self.log_buffer.appendleft(entry) + + def copied(self, sec, primary, result): + entry = {"kind": "copied", "name": Path(sec).name, + "from": Path(primary).name, + "desc": (result.get("description") or "")} + with self.log_lock: + self.log_buffer.appendleft(entry) + + def stopped(self): + with self.log_lock: + self.log_buffer.appendleft({"kind": "stop"}) + + +def _run_analysis_live(args, conn, client, library, pending, folder_groups, + total_in_db, already_done, members_of=None): + progress = _make_progress() + task_id = progress.add_task("", total=len(pending), ok=0, err=0, overall="0.0%") + + stats_lock = threading.Lock() + log_lock = threading.Lock() + log_buffer = deque(maxlen=10) + + # Seed album totals + done counts from the DB once, up front. No queries + # run inside the 4 Hz render loop — workers mutate folder_done in memory. + folder_total: dict = {} + folder_done: dict = {} + for row in conn.execute("SELECT path, status FROM photos"): + album = album_label(Path(row["path"]), library) + folder_total[album] = folder_total.get(album, 0) + 1 + if row["status"] in ("analyzed", "exif_written"): + folder_done[album] = folder_done.get(album, 0) + 1 + + # API balance shown in the Analyzing panel. Fetched once now, then refreshed + # on a slow daemon thread — never on the render path, never per photo (would + # waste the 3 RPM budget). The render only reads the cached value. + bal_holder = {"value": fetch_balance(client.api_key) if not args.dry_run else None} + bal_stop = threading.Event() + + _rpd_load() # seed today's request count (cumulative across runs this PT day) + + def _refresh_balance(): + while not bal_stop.wait(90): # every 90s until told to stop + v = fetch_balance(client.api_key) + if v is not None: + bal_holder["value"] = v + _rpd_persist() # checkpoint the RPD count alongside the balance refresh + + bal_thread = None + if not args.dry_run: + bal_thread = threading.Thread(target=_refresh_balance, daemon=True) + bal_thread.start() + + token_box = {"n": 0, "total": 0, "prompt": 0, "completion": 0, "min": 0, "max": 0} + + dashboard = _Dashboard(progress, folder_total, folder_done, stats_lock, + log_buffer, log_lock, bal_holder=bal_holder, + token_box=token_box) + renderer = _LiveRenderer(log_buffer, log_lock) + + key_stop = threading.Event() + key_thread = threading.Thread(target=_key_listener, args=(key_stop,), daemon=True) + key_thread.start() + + try: + with Live(dashboard, console=console, refresh_per_second=4, screen=True): + return _run_folder_loop( + folder_groups, library, pending, conn, client, args, + progress, task_id, total_in_db, already_done, + renderer, stats_lock, folder_done=folder_done, members_of=members_of, + token_box=token_box, + ) + finally: + key_stop.set() # restore terminal mode promptly + bal_stop.set() # stop the refresher promptly + _rpd_persist() # final RPD checkpoint so the count survives this run + + +def run_exif_write(args, conn: sqlite3.Connection): + """Phase 2: write analysis results into EXIF of analyzed files.""" + + if args.no_exif: + log.info("--no-exif set, skipping EXIF write phase.") + return + + rows = get_analyzed_no_exif(conn) + log.info(f"{len(rows):,} photos need EXIF written") + + if not rows: + return + + ok = err = 0 + with Progress( + SpinnerColumn(spinner_name="dots", style="yellow"), + TextColumn("[bold yellow]\uf044 Writing EXIF[/]"), + BarColumn(bar_width=None, style="dim yellow", complete_style="yellow", finished_style="green"), + MofNCompleteColumn(), + TaskProgressColumn(style="bold"), + TimeElapsedColumn(), + TextColumn("[dim]·[/]"), + TimeRemainingColumn(), + TextColumn(" [dim]ok=[/][green]{task.fields[ok]}[/] [dim]err=[/][red]{task.fields[err]}[/]"), + console=console, + refresh_per_second=4, + ) as progress: + task_id = progress.add_task("", total=len(rows), ok=0, err=0) + + for row in rows: + if args.dry_run: + mark_exif_written(conn, row["path"]) + ok += 1 + progress.update(task_id, advance=1, ok=ok, err=err) + continue + + caption = build_exif_caption(row) + try: + keywords = json.loads(row["tags"] or "[]") + except Exception: + keywords = [] + + if write_exif(row["path"], caption, keywords): + mark_exif_written(conn, row["path"]) + ok += 1 + else: + err += 1 + + progress.update(task_id, advance=1, ok=ok, err=err) + + if _stop.is_set(): + progress.console.print("[yellow]Stopped. Progress saved.[/]") + break + + log.info(f"EXIF write complete: {ok:,} OK, {err:,} errors") + + +def album_label(photo: Path, library: Path = None) -> str: + """ + Album = the leaf folder directly containing the photo, shown as a path + relative to the library root so nested albums stay distinct + (Urlaub/Rom vs Urlaub/Venedig). Falls back to the absolute parent if the + photo lies outside the library. + """ + parent = photo.parent + if library: + try: + rel = parent.relative_to(library) + return "(root)" if str(rel) == "." else str(rel) + except ValueError: + pass + return parent.name or str(parent) + + +def fit_label(label: str, width: int) -> str: + """Left-truncate so the album name (path tail) stays visible: Urlaub/…/Rom.""" + if len(label) <= width: + return label + return "…" + label[-(width - 1):] + + +def print_stats(conn: sqlite3.Connection, library: Path = None): + DONE = {"analyzed", "exif_written"} + + rows = conn.execute( + "SELECT status, COUNT(*) as n FROM photos GROUP BY status" + ).fetchall() + log.info("── Database summary ──────────────────────") + for r in rows: + log.info(f" {r['status']:20s} {r['n']:>6,}") + + # Album breakdown — an album is the leaf folder directly containing the + # photos (e.g. Urlaub/Rom and Urlaub/Venedig are two albums, not one). + all_rows = conn.execute("SELECT path, status FROM photos").fetchall() + if not all_rows: + log.info("──────────────────────────────────────────") + return + + folder_total: dict[str, int] = {} + folder_done: dict[str, int] = {} + + for row in all_rows: + folder = album_label(Path(row["path"]), library) + folder_total[folder] = folder_total.get(folder, 0) + 1 + if row["status"] in DONE: + folder_done[folder] = folder_done.get(folder, 0) + 1 + + log.info("── Album breakdown ───────────────────────") + for folder in sorted(folder_total): + done = folder_done.get(folder, 0) + total = folder_total[folder] + pct = done / total * 100 + flag = "[green]✓[/]" if done == total else f"[yellow]{pct:3.0f}%[/]" + log.info(f" {fit_label(folder, 40):<40s} {done:>5,}/{total:<5,} {flag}") + + log.info("──────────────────────────────────────────") + + +# ────────────────────────────────────────────── +# CLI +# ────────────────────────────────────────────── + +def load_env_file() -> Path | None: + """ + Load KEY=VALUE pairs from a .env file into os.environ. Existing shell env + vars are never overwritten, so `export LLM_API_KEY=...` still wins. + Looks in the current directory first, then beside this script. Tiny manual + parser — no python-dotenv dependency. Returns the file used, or None. + """ + candidates = [Path.cwd() / ENV_FILE, Path(__file__).resolve().parent / ENV_FILE] + for path in candidates: + if not path.is_file(): + continue + try: + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export "):] + if "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + value = value.strip() + # Strip an inline comment: a '#' at the start of the value or + # preceded by whitespace. Skip if the value is quoted, and keep + # '#' that's part of a token (e.g. a URL fragment, pass#word). + if value[:1] not in ("'", '"'): + for i, ch in enumerate(value): + if ch == "#" and (i == 0 or value[i - 1].isspace()): + value = value[:i] + break + value = value.strip().strip('"').strip("'") + if key and key not in os.environ: + os.environ[key] = value + except Exception as e: + log.warning(f"Could not read {path}: {e}") + return path + return None + + +# Read config from the env file (or real shell env). These let photo_analyzer.env +# drive every setting, so the script can run with no flags at all. +def _env_str(name: str, fallback=None): + v = os.environ.get(name) + return v if v not in (None, "") else fallback + + +def _env_int(name: str, fallback: int) -> int: + v = os.environ.get(name) + if v in (None, ""): + return fallback + try: + return int(v) + except ValueError: + log.warning(f"{name}={v!r} is not an integer; using {fallback}") + return fallback + + +def _env_bool(name: str) -> bool: + return os.environ.get(name, "").strip().lower() in ("1", "true", "yes", "on") + + +def fetch_balance(api_key: str) -> dict | None: + """GET /v1/users/me/balance. Returns the data dict, or None on any failure.""" + import urllib.request + import urllib.error + + url = LLM_BASE_URL.rstrip("/") + "/users/me/balance" + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {api_key}"}) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + data = json.loads(resp.read().decode("utf-8")) + return data.get("data", data) or {} + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", "replace")[:300] + log.debug(f"Balance query failed: HTTP {e.code} {e.reason} {body}") + except Exception as e: + log.debug(f"Balance query failed: {e}") + return None + + +def query_balance(api_key: str) -> None: + """Print remaining API balance (works on providers that expose a balance endpoint).""" + d = fetch_balance(api_key) + if d is None: + log.error("Balance query failed or unsupported by this provider (see debug log).") + return + log.info("── API balance ───────────────────────────") + log.info(f" available {d.get('available_balance')}") + log.info(f" cash {d.get('cash_balance')}") + log.info(f" voucher {d.get('voucher_balance')}") + log.info("──────────────────────────────────────────") + + +def check_quota(client: OpenAI) -> bool: + """Fire ONE minimal request to test whether today's per-model quota is + available, instead of launching the whole pipeline to find out. Returns + True if the model accepted the request. A 429 here isn't billed and (for a + daily cap) doesn't consume quota.""" + log.info(f"Probing {LLM_MODEL} with one minimal request…") + try: + client.chat.completions.create( + model=LLM_MODEL, max_tokens=1, + messages=[{"role": "user", "content": "ping"}], + ) + log.info(f"✓ {LLM_MODEL} is accepting requests — safe to run.") + return True + except RateLimitError as e: + detail = str(getattr(e, "message", "") or e) + low = detail.lower() + if "per_day" in low or "perday" in low: + log.error(f"✗ Daily quota still exhausted for {LLM_MODEL}. " + f"Wait for reset (~midnight Pacific) or switch model.") + elif "per_minute" in low or "perminute" in low: + log.warning(f"⚠ Per-minute limit only (transient) — daily quota looks OK, " + f"just lower MAX_WORKERS. {detail[:140]}") + else: + log.error(f"✗ Rate limited: {detail[:200]}") + return False + except APIError as e: + log.error(f"Quota check failed: {e}") + return False + + +def main(): + # Load photo_analyzer.env first so every setting below can default from it. + # Real shell env vars still win (load_env_file never overwrites them), and a + # CLI flag still overrides the env value for one-off runs. + env_file = load_env_file() + + # Tuning knobs — env-driven, no flags. See photo_analyzer.env for docs. + global MAX_WORKERS, MAX_LONG_EDGE, RETRY_ATTEMPTS, RETRY_BASE_DELAY + global LLM_BASE_URL, LLM_MODEL + MAX_WORKERS = _env_int("MAX_WORKERS", MAX_WORKERS) + MAX_LONG_EDGE = _env_int("MAX_LONG_EDGE", MAX_LONG_EDGE) + RETRY_ATTEMPTS = _env_int("RETRY_ATTEMPTS", RETRY_ATTEMPTS) + RETRY_BASE_DELAY = _env_int("RETRY_BASE_DELAY", RETRY_BASE_DELAY) + LLM_BASE_URL = _env_str("LLM_BASE_URL", LLM_BASE_URL) + LLM_MODEL = _env_str("LLM_MODEL", LLM_MODEL) + global RPD_LIMIT, PHASH_THRESHOLD + RPD_LIMIT = _env_int("RPD_LIMIT", RPD_LIMIT) + PHASH_THRESHOLD = _env_int("PHASH_THRESHOLD", PHASH_THRESHOLD) + + parser = argparse.ArgumentParser( + description="Analyze photos with an OpenAI-compatible vision model and write results to EXIF for Immich. " + "Every option can be set in photo_analyzer.env instead of on the CLI." + ) + parser.add_argument("--library", default=_env_str("LIBRARY"), help="Root path of your photo library (env: LIBRARY; required except for --balance)") + parser.add_argument("--db", default=_env_str("DB", DB_FILE), help=f"SQLite DB path (env: DB; default: {DB_FILE})") + parser.add_argument("--dry-run", action="store_true", default=_env_bool("DRY_RUN"), help="Discover + register files, skip API and EXIF (env: DRY_RUN)") + parser.add_argument("--reanalyze", action="store_true", default=_env_bool("REANALYZE"), help="Re-analyze already-processed files (env: REANALYZE)") + parser.add_argument("--no-exif", action="store_true", default=_env_bool("NO_EXIF"), help="Skip EXIF write phase (index only) (env: NO_EXIF)") + parser.add_argument("--exif-only", action="store_true", default=_env_bool("EXIF_ONLY"), help="Skip analysis, only write EXIF for already-analyzed photos (env: EXIF_ONLY)") + parser.add_argument("--group-variants", action="store_true", default=_env_bool("GROUP_VARIANTS"), help="Analyze one photo per group of size/edit variants and copy the result to the rest — saves API calls (env: GROUP_VARIANTS)") + parser.add_argument("--backfill-phash", action="store_true", default=_env_bool("BACKFILL_PHASH"), help="Compute + store perceptual hash and SHA-1 for all photos, then exit — seeds the dedup ledger for an existing DB (env: BACKFILL_PHASH)") + parser.add_argument("--list-dupes", action="store_true", default=_env_bool("LIST_DUPES"), help="Report perceptual-duplicate clusters and exit — read-only, changes nothing (env: LIST_DUPES)") + parser.add_argument("--dedupe", action="store_true", default=_env_bool("DEDUPE"), help="Mark non-canonical duplicates status='duplicate' (skipped by analysis + excluded from upload), then exit (env: DEDUPE)") + parser.add_argument("--stats", action="store_true", default=_env_bool("STATS"), help="Print DB summary and exit (env: STATS)") + parser.add_argument("--balance", action="store_true", default=_env_bool("BALANCE"), help="Print remaining API balance and exit, if the provider exposes one (env: BALANCE)") + parser.add_argument("--quota-check", action="store_true", default=_env_bool("QUOTA_CHECK"), help="Fire ONE tiny request to test if today's quota is available, then exit (env: QUOTA_CHECK)") + parser.add_argument("--debug", action="store_true", default=_env_bool("DEBUG"), help="Show DEBUG messages on console (env: DEBUG; always written to photo_analyzer_debug.log)") + args = parser.parse_args() + + if not args.library and not args.balance and not args.quota_check: + parser.error("--library is required (set LIBRARY in photo_analyzer.env, or pass --library)") + + if args.debug: + _rich_handler.setLevel(logging.DEBUG) + + # LLM_API_KEY is the provider-agnostic name; GEMINI_API_KEY / GOOGLE_API_KEY + # are honoured as fallbacks for the default Google provider. + api_key = _env_str("LLM_API_KEY") or _env_str("GEMINI_API_KEY") or _env_str("GOOGLE_API_KEY") + if api_key == API_KEY_PLACEHOLDER: + api_key = None # they created .env but haven't pasted a real key yet + + no_api_mode = args.dry_run or args.stats or args.backfill_phash or args.list_dupes or args.dedupe + if not api_key and not no_api_mode: + log.error("No API key set (LLM_API_KEY or GEMINI_API_KEY).") + if env_file: + log.error(f"Edit {env_file} and set: LLM_API_KEY=your-key") + else: + log.error(f"Create a {ENV_FILE} file with: LLM_API_KEY=your-key") + log.error("Or export it in your shell. Get a key from your LLM provider (default: Google AI Studio, https://aistudio.google.com/apikey).") + sys.exit(1) + + if args.balance: + query_balance(api_key) + return + + conn = get_db(args.db) + client = OpenAI(api_key=api_key or "dry-run", base_url=LLM_BASE_URL) + + if args.quota_check: + sys.exit(0 if check_quota(client) else 1) + + library = Path(args.library).expanduser().resolve() + + # Follow files that moved/renamed since the last run (by SHA-1) BEFORE pruning, + # so a folder reorg updates paths in place instead of deleting + re-analyzing. + reconcile_moved(conn, library) + + # Drop DB entries for files moved/deleted since the last run, so the stats + # and album breakdown reflect what's actually on disk now. + pruned = prune_missing(conn, library) + if pruned: + log.info(f"Pruned {pruned:,} stale entries (files moved or deleted)") + + if args.stats: + print_stats(conn, library) + return + + # Content-hash / duplicate tools — operate on the DB ledger, no API needed. + if args.backfill_phash or args.list_dupes or args.dedupe: + photos = discover_photos(library) + for p in photos: + upsert_pending(conn, str(p)) + ensure_hashes(conn, photos) # only-missing; seeds the ledger + if args.backfill_phash: + return + if args.list_dupes: + list_duplicates(conn, PHASH_THRESHOLD) + return + marked, clusters = mark_duplicates(conn, PHASH_THRESHOLD) + log.info(f"Marked {marked:,} duplicate(s) across {clusters:,} cluster(s) " + f"— status='duplicate', excluded from analysis + upload.") + return + + # EXIF writing needs exiftool — verify once, up front, cleanly (instead of + # failing deep inside a worker thread or mid-backlog). + will_write_exif = not args.no_exif and not args.dry_run + if will_write_exif and shutil.which("exiftool") is None: + log.error("exiftool not found on PATH. Install it (brew install exiftool),") + log.error("or run with --no-exif to analyze into the DB only.") + sys.exit(1) + + # On every start, first flush photos that were already analyzed in a previous + # run but never had their EXIF written (e.g. analyzed before exiftool was + # installed). This writes the stored DB data into those images up front, + # independent of — and before — the long analysis queue. + run_exif_write(args, conn) + + if not args.exif_only: + run_analysis(args, conn, client) # analyzes pending; writes EXIF inline + run_exif_write(args, conn) # mop up any inline write failures + + write_throttle_summary() # append run-summary line to throttle_events.jsonl + print_stats(conn, library) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1329dc6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "photoanalyzer" +version = "0.1.0" +requires-python = ">=3.11" + +[tool.ruff] +line-length = 100 diff --git a/scripts/work-item b/scripts/work-item new file mode 100755 index 0000000..b9a0888 --- /dev/null +++ b/scripts/work-item @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu + +for python in python3.13 python3.12 python3.11; do + if command -v "$python" >/dev/null 2>&1; then + exec "$python" -m work_item "$@" + fi +done + +echo "work-item: Python 3.11 or newer is required" >&2 +exit 2 diff --git a/test_dedup.py b/test_dedup.py new file mode 100644 index 0000000..3b25afe --- /dev/null +++ b/test_dedup.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Self-check for the phash/dedup/reconcile additions. Runnable, no framework: + python test_dedup.py +Creates synthetic images in a temp dir + a temp DB, asserts behaviour, cleans up. +""" +import io +import os +import sqlite3 +import tempfile +from pathlib import Path + +import numpy as np +from PIL import Image + +import photo_analyzer as pa + + +def _make_photo(path: Path, seed: int, size=(800, 600)): + """Deterministic, structured (not pure-noise) image so phash is stable.""" + rng = np.random.default_rng(seed) + # low-frequency structure: a few coloured blocks + gradient — survives resize + base = np.zeros((size[1], size[0], 3), dtype=np.uint8) + for _ in range(6): + x0, y0 = rng.integers(0, size[0] - 100), rng.integers(0, size[1] - 100) + col = rng.integers(0, 256, 3) + base[y0:y0 + 150, x0:x0 + 150] = col + grad = np.linspace(0, 120, size[0], dtype=np.uint8) + base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255) + Image.fromarray(base).save(path, quality=95) + + +def main(): + tmp = Path(tempfile.mkdtemp(prefix="dedup_test_")) + db_path = tmp / "test.db" + try: + # ── synthetic library ──────────────────────────────────────────────── + orig = tmp / "A.jpg" + _make_photo(orig, seed=1) + + # exact byte copy (backup-of-backup) + exact = tmp / "A_backup.jpg" + exact.write_bytes(orig.read_bytes()) + + # near-dupe: same picture, resized + recompressed (different phone) + near = tmp / "A_phone.jpg" + with Image.open(orig) as im: + im.resize((400, 300)).save(near, quality=60) + + # a genuinely different photo + other = tmp / "B.jpg" + _make_photo(other, seed=999) + + # ── phash: dupes close, different far ──────────────────────────────── + def dist(p1, p2): + h1, h2 = int(pa._phash_image(p1), 16), int(pa._phash_image(p2), 16) + return bin(h1 ^ h2).count("1") + + d_exact = dist(orig, exact) + d_near = dist(orig, near) + d_other = dist(orig, other) + print(f"phash Hamming: exact={d_exact} near={d_near} other={d_other}") + assert d_exact == 0, f"exact copy should have phash distance 0, got {d_exact}" + assert d_near <= pa.PHASH_THRESHOLD, f"near-dupe {d_near} > threshold {pa.PHASH_THRESHOLD}" + assert d_other > pa.PHASH_THRESHOLD, f"different photo {d_other} <= threshold — false match" + + # ── sha1: identical bytes match, different bytes don't ─────────────── + assert pa._sha1_file(orig) == pa._sha1_file(exact), "byte-identical files must share sha1" + assert pa._sha1_file(orig) != pa._sha1_file(near), "recompressed file must differ in sha1" + + # ── DB: register, hash, cluster, mark ──────────────────────────────── + conn = pa.get_db(str(db_path)) + photos = [orig, exact, near, other] + for p in photos: + pa.upsert_pending(conn, str(p)) + n = pa.ensure_hashes(conn, photos) + assert n == 4, f"expected 4 hashed, got {n}" + + # all four should carry phash + sha1 now + rows = conn.execute("SELECT path, phash, file_sha1 FROM photos").fetchall() + assert all(r["phash"] and r["file_sha1"] for r in rows), "some hashes missing" + + clusters = pa.cluster_duplicates(conn, pa.PHASH_THRESHOLD) + # exactly one cluster: {orig, exact, near}; 'other' stands alone + assert len(clusters) == 1, f"expected 1 cluster, got {len(clusters)}" + cluster_paths = {it["path"] for it in clusters[0]} + assert cluster_paths == {str(orig), str(exact), str(near)}, cluster_paths + # canonical = largest file (orig, saved at q95 full-res) + assert clusters[0][0]["path"] == str(orig), "canonical should be the largest file" + + marked, ncl = pa.mark_duplicates(conn, pa.PHASH_THRESHOLD) + assert (marked, ncl) == (2, 1), f"expected (2 marked, 1 cluster), got ({marked}, {ncl})" + + dupes = conn.execute( + "SELECT path, dup_of FROM photos WHERE status='duplicate'" + ).fetchall() + assert {r["path"] for r in dupes} == {str(exact), str(near)} + assert all(r["dup_of"] == str(orig) for r in dupes), "dup_of must point at canonical" + + # get_pending must exclude duplicates + pending = pa.get_pending(conn, reanalyze=True) + assert str(exact) not in pending and str(near) not in pending, "dupes leaked into pending" + assert str(orig) in pending and str(other) in pending + + # ── reconcile_moved: rename a file, path follows by sha1 ───────────── + moved_dst = tmp / "B_renamed.jpg" + os.rename(other, moved_dst) # 'other' now missing at old path + n_moved = pa.reconcile_moved(conn, tmp) + assert n_moved == 1, f"expected 1 reconciled, got {n_moved}" + row = conn.execute("SELECT path FROM photos WHERE path=?", (str(moved_dst),)).fetchone() + assert row is not None, "moved file's row was not repathed" + gone = conn.execute("SELECT 1 FROM photos WHERE path=?", (str(other),)).fetchone() + assert gone is None, "old path should no longer exist in DB" + + conn.close() + print("ALL DEDUP/RECONCILE CHECKS PASSED") + finally: + import shutil + shutil.rmtree(tmp, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/test_nsfw_skip.py b/test_nsfw_skip.py new file mode 100644 index 0000000..2a7c41f --- /dev/null +++ b/test_nsfw_skip.py @@ -0,0 +1,15 @@ +"""Self-check for photo_analyzer's nsfw-skip parsing. Run: python test_nsfw_skip.py""" +from photo_analyzer import _rec_has_nsfw + +def test(): + assert _rec_has_nsfw({"Keywords": ["beach", "nsfw"]}) # list, present + assert _rec_has_nsfw({"Subject": "nsfw"}) # scalar in Subject + assert _rec_has_nsfw({"Keywords": "NSFW"}) # case-insensitive + assert _rec_has_nsfw({"Keywords": [" nsfw "]}) # whitespace + assert not _rec_has_nsfw({"Keywords": ["sfw", "beach"]}) # sfw only + assert not _rec_has_nsfw({"Keywords": ["nsfw-ish"]}) # substring != tag + assert not _rec_has_nsfw({}) # no fields + print("ok") + +if __name__ == "__main__": + test() diff --git a/tests/test_cli_e2e.py b/tests/test_cli_e2e.py new file mode 100644 index 0000000..7032db5 --- /dev/null +++ b/tests/test_cli_e2e.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + + +SOURCE_ROOT = Path(__file__).resolve().parents[1] + + +def run(args, *, cwd: Path, env=None, check=True): + result = subprocess.run(args, cwd=cwd, env=env, text=True, capture_output=True) + if check and result.returncode: + raise AssertionError(f"{args}\nstdout={result.stdout}\nstderr={result.stderr}") + return result + + +FAKE_TEA = r'''#!/usr/bin/env python3 +import json, os, subprocess, sys +from pathlib import Path + +state_path = Path(os.environ["FAKE_TEA_STATE"]) +state = json.loads(state_path.read_text()) +args = sys.argv[1:] + +def save(): + state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n") + +def out(value): + print(json.dumps(value)) + +if args[0] == "api": + endpoint = args[1] + if endpoint.endswith("/issues?state=open&type=issues&limit=100"): + out([issue for issue in state["issues"] if issue["state"] == "open"]) + elif "/dependencies" in endpoint: + issue = int(endpoint.split("/issues/")[1].split("/")[0]) + blockers = state["dependencies"].get(str(issue), []) + out([next(x for x in state["issues"] if x["number"] == number) for number in blockers]) + elif "/pulls/" in endpoint: + number = int(endpoint.rsplit("/", 1)[1]) + out(state["prs"][str(number)]) + elif "/commits/" in endpoint and endpoint.endswith("/status"): + out({"state": "success"}) + else: + raise SystemExit(f"unsupported api: {endpoint}") +elif args[:2] == ["issues", "edit"]: + issue = next(x for x in state["issues"] if x["number"] == int(args[2])) + if "--add-assignees" in args: + issue["assignees"] = [{"login": args[args.index("--add-assignees") + 1]}] + labels = {x["name"] for x in issue.get("labels", [])} + if "--remove-labels" in args: + labels -= set(args[args.index("--remove-labels") + 1].split(",")) + if "--add-labels" in args: + labels |= set(args[args.index("--add-labels") + 1].split(",")) + issue["labels"] = [{"name": x} for x in sorted(labels)] + save(); out(issue) +elif args[:2] == ["comments", "add"]: + state["comments"].append({"issue": int(args[2]), "body": args[3]}) + save(); out(state["comments"][-1]) +elif args[:2] == ["pulls", "create"]: + number = len(state["prs"]) + 1 + branch = args[args.index("--head") + 1] + sha = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + pr = {"number": number, "index": number, "url": f"https://example.test/pr/{number}", + "merged": False, "head": {"sha": sha, "ref": branch}} + state["prs"][str(number)] = pr + save(); out(pr) +elif args[:2] == ["pulls", "merge"]: + number = int(args[2]); pr = state["prs"][str(number)] + subprocess.check_call(["git", "push", "origin", f"{pr['head']['ref']}:main"]) + pr["merged"] = True + save(); out(pr) +elif args[:2] == ["issues", "close"]: + issue = next(x for x in state["issues"] if x["number"] == int(args[2])) + issue["state"] = "closed"; save(); out(issue) +else: + raise SystemExit("unsupported tea command: " + repr(args)) +''' + + +class CliEndToEndTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.base = Path(self.temp.name) + remote_parent = self.base / "domverse" + remote_parent.mkdir() + seed = self.base / "seed" + seed.mkdir() + run(["git", "init", "-b", "main"], cwd=seed) + run(["git", "config", "user.name", "Test Agent"], cwd=seed) + run(["git", "config", "user.email", "agent@example.test"], cwd=seed) + (seed / "README.md").write_text("seed\n") + run(["git", "add", "README.md"], cwd=seed) + run(["git", "commit", "-m", "seed"], cwd=seed) + self.remote = remote_parent / "photoanalyzer.git" + run(["git", "clone", "--bare", str(seed), str(self.remote)], cwd=self.base) + self.repo = self.base / "work" + run(["git", "clone", str(self.remote), str(self.repo)], cwd=self.base) + run(["git", "config", "user.name", "Test Agent"], cwd=self.repo) + run(["git", "config", "user.email", "agent@example.test"], cwd=self.repo) + + config = textwrap.dedent(f'''\ + [repository] + slug = "domverse/photoanalyzer" + login = "fake" + assignee = "agent" + remote = "origin" + main_branch = "main" + [workflow] + branch_prefix = "us" + require_ci = true + required_tests = [] + [safety] + max_file_bytes = 100000 + allow = ["tests/fixtures/**"] + deny = ["_IGNORE/**", "**/_IGNORE/**", "pictures/**", "*.env", "*.jpg"] + ''') + (self.repo / ".work-item.toml").write_text(config) + run(["git", "add", ".work-item.toml"], cwd=self.repo) + run(["git", "commit", "-m", "config"], cwd=self.repo) + run(["git", "push", "origin", "main"], cwd=self.repo) + + bin_dir = self.base / "bin" + bin_dir.mkdir() + tea = bin_dir / "tea" + tea.write_text(FAKE_TEA) + tea.chmod(0o755) + self.state_path = self.base / "tea-state.json" + self.state_path.write_text(json.dumps({ + "issues": [{ + "number": 1, + "title": "US01-01 — Implement Safe Workflow", + "state": "open", + "labels": [{"name": "status/backlog"}], + "assignees": [], + }], + "dependencies": {"1": []}, + "comments": [], + "prs": {}, + })) + self.env = os.environ.copy() + self.env["PATH"] = f"{bin_dir}{os.pathsep}{self.env['PATH']}" + self.env["FAKE_TEA_STATE"] = str(self.state_path) + self.env["PYTHONPATH"] = str(SOURCE_ROOT) + + def tearDown(self) -> None: + self.temp.cleanup() + + def cli(self, *args, check=True): + return run([sys.executable, "-m", "work_item", *args], cwd=self.repo, env=self.env, check=check) + + def state(self): + return json.loads(self.state_path.read_text()) + + def test_claim_submit_merge_complete_lifecycle(self) -> None: + next_result = self.cli("next") + self.assertEqual(json.loads(next_result.stdout)["story_id"], "US01-01") + + claim = json.loads(self.cli("claim").stdout) + self.assertEqual(claim["status"], "in-progress") + self.assertEqual(run(["git", "branch", "--show-current"], cwd=self.repo).stdout.strip(), claim["branch"]) + issue = self.state()["issues"][0] + self.assertEqual(issue["assignees"], [{"login": "agent"}]) + self.assertIn({"name": "status/in-progress"}, issue["labels"]) + + (self.repo / "private.env").write_text("not-a-real-secret\n") + unsafe = self.cli("submit", "--test", "true", "--yes", check=False) + self.assertEqual(unsafe.returncode, 2) + self.assertIn("denied path", unsafe.stderr) + (self.repo / "private.env").unlink() + + (self.repo / "feature.txt").write_text("implemented\n") + review_only = self.cli("submit", "--test", "true", check=False) + self.assertEqual(review_only.returncode, 2) + self.assertIn("rerun with --yes", review_only.stderr) + self.assertEqual(run(["git", "status", "--porcelain"], cwd=self.repo).stdout.strip(), "?? feature.txt") + + submitted = json.loads(self.cli("submit", "--test", "true", "--yes").stdout) + self.assertEqual(submitted["status"], "review") + self.assertEqual(submitted["pr"], 1) + self.assertIn("status/review", {x["name"] for x in self.state()["issues"][0]["labels"]}) + + not_merged = self.cli("complete", check=False) + self.assertEqual(not_merged.returncode, 2) + self.assertIn("not merged", not_merged.stderr) + + completed = json.loads(self.cli("complete", "--merge").stdout) + self.assertEqual(completed["status"], "done") + self.assertEqual(run(["git", "branch", "--show-current"], cwd=self.repo).stdout.strip(), "main") + self.assertEqual(run(["git", "status", "--porcelain"], cwd=self.repo).stdout.strip(), "") + issue = self.state()["issues"][0] + self.assertEqual(issue["state"], "closed") + self.assertIn("status/done", {x["name"] for x in issue["labels"]}) + self.assertFalse((self.repo / ".git" / "work-item-state.json").exists()) + + def test_dirty_tree_prevents_claim(self) -> None: + (self.repo / "unexpected.txt").write_text("dirty\n") + result = self.cli("claim", check=False) + self.assertEqual(result.returncode, 2) + self.assertIn("clean", result.stderr) + self.assertEqual(run(["git", "branch", "--show-current"], cwd=self.repo).stdout.strip(), "main") + + def test_block_requires_reason_and_records_state(self) -> None: + self.cli("claim") + result = self.cli("block", "--reason", "Provider fixture unavailable") + payload = json.loads(result.stdout) + self.assertEqual(payload["status"], "blocked") + issue = self.state()["issues"][0] + self.assertIn("status/blocked", {x["name"] for x in issue["labels"]}) + self.assertIn("Provider fixture unavailable", self.state()["comments"][-1]["body"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..1ea6d7d --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +from work_item.core import Config, GitRepo, Story, WorkItemError, Workflow + + +def run(*args: str, cwd: Path) -> str: + result = subprocess.run(args, cwd=cwd, text=True, capture_output=True, check=True) + return result.stdout.strip() + + +def config_for(slug: str, **changes) -> Config: + values = dict( + repo_slug=slug, + login="test", + assignee="agent", + remote="origin", + main_branch="main", + branch_prefix="us", + require_ci=False, + max_file_bytes=64, + required_tests=(), + denied_patterns=("_IGNORE/**", "**/_IGNORE/**", "pictures/**", "*.env", "*.jpg"), + allowed_patterns=("tests/fixtures/**",), + ) + values.update(changes) + return Config(**values) + + +class RepositoryFixture(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.base = Path(self.temp.name) + remote_parent = self.base / "domverse" + remote_parent.mkdir() + self.seed = self.base / "seed" + self.seed.mkdir() + run("git", "init", "-b", "main", cwd=self.seed) + run("git", "config", "user.name", "Test Agent", cwd=self.seed) + run("git", "config", "user.email", "agent@example.test", cwd=self.seed) + (self.seed / "README.md").write_text("seed\n", encoding="utf-8") + run("git", "add", "README.md", cwd=self.seed) + run("git", "commit", "-m", "seed", cwd=self.seed) + self.remote = remote_parent / "photoanalyzer.git" + run("git", "clone", "--bare", str(self.seed), str(self.remote), cwd=self.base) + self.repo = self.base / "work" + run("git", "clone", str(self.remote), str(self.repo), cwd=self.base) + run("git", "config", "user.name", "Test Agent", cwd=self.repo) + run("git", "config", "user.email", "agent@example.test", cwd=self.repo) + self.config = config_for("domverse/photoanalyzer") + + def tearDown(self) -> None: + self.temp.cleanup() + + +class StoryTests(unittest.TestCase): + def test_parse_and_branch_slug(self) -> None: + story = Story.from_issue({ + "number": 12, + "title": "US02-05 — Build the Static Application Shell", + "labels": [{"name": "type/feature"}], + "assignees": [{"login": "agent"}], + }) + self.assertIsNotNone(story) + assert story is not None + self.assertEqual((story.epic, story.sequence), (2, 5)) + self.assertEqual(story.branch_slug, "build-the-static-application-shell") + self.assertEqual(story.assignees, ("agent",)) + + def test_non_story_issue_is_ignored(self) -> None: + self.assertIsNone(Story.from_issue({"number": 1, "title": "Maintenance"})) + + +class GitSafetyTests(RepositoryFixture): + def test_verify_remote_and_clean_tree(self) -> None: + git = GitRepo(self.repo, self.config, Mock(wraps=None)) + # Use the real runner; assigning after construction keeps the test explicit. + from work_item.core import Runner + git.runner = Runner() + git.verify() + git.ensure_clean() + (self.repo / "change.txt").write_text("change\n", encoding="utf-8") + with self.assertRaisesRegex(WorkItemError, "clean"): + git.ensure_clean() + + def test_rejects_private_paths_large_files_and_secrets(self) -> None: + from work_item.core import Runner + git = GitRepo(self.repo, self.config, Runner()) + + (self.repo / "pictures").mkdir() + (self.repo / "pictures" / "private.jpg").write_bytes(b"x") + with self.assertRaisesRegex(WorkItemError, "denied path"): + git.assert_safe_changes() + (self.repo / "pictures" / "private.jpg").unlink() + (self.repo / "pictures").rmdir() + + (self.repo / "large.txt").write_text("x" * 65, encoding="utf-8") + with self.assertRaisesRegex(WorkItemError, "exceeds"): + git.assert_safe_changes() + (self.repo / "large.txt").unlink() + + (self.repo / "secret.txt").write_text("api_key = abcdefghijklmnopqrstuvwxyz\n", encoding="utf-8") + with self.assertRaisesRegex(WorkItemError, "possible secret"): + git.assert_safe_changes() + + def test_fixture_allowlist_overrides_image_deny(self) -> None: + from work_item.core import Runner + git = GitRepo(self.repo, self.config, Runner()) + fixture = self.repo / "tests" / "fixtures" / "synthetic.jpg" + fixture.parent.mkdir(parents=True) + fixture.write_bytes(b"synthetic") + self.assertEqual(git.assert_safe_changes(), [Path("tests/fixtures/synthetic.jpg")]) + + def test_remote_mismatch_is_rejected(self) -> None: + from work_item.core import Runner + git = GitRepo(self.repo, config_for("someone/else"), Runner()) + with self.assertRaisesRegex(WorkItemError, "does not match"): + git.verify() + + +class SelectionTests(unittest.TestCase): + def make_workflow(self) -> Workflow: + workflow = object.__new__(Workflow) + workflow.gitea = Mock() + return workflow + + def test_next_story_uses_numeric_story_order(self) -> None: + from work_item.core import Gitea + gitea = object.__new__(Gitea) + gitea.open_stories = Mock(return_value=[ + Story(2, "US01-02", "Second", 1, 2, frozenset(), (), {}), + Story(1, "US01-01", "First", 1, 1, frozenset(), (), {}), + ]) + gitea.dependencies_closed = Mock(return_value=True) + # open_stories normally sorts; make the contract explicit here too. + gitea.open_stories.return_value.sort(key=lambda x: (x.epic, x.sequence)) + self.assertEqual(gitea.next_story().story_id, "US01-01") + + def test_next_story_skips_claimed_blocked_and_open_dependencies(self) -> None: + from work_item.core import Gitea + gitea = object.__new__(Gitea) + gitea.open_stories = Mock(return_value=[ + Story(1, "US01-01", "Busy", 1, 1, frozenset({"status/in-progress"}), (), {}), + Story(2, "US01-02", "Blocked", 1, 2, frozenset({"status/blocked"}), (), {}), + Story(3, "US01-03", "Dependency", 1, 3, frozenset(), (), {}), + Story(4, "US01-04", "Ready", 1, 4, frozenset(), (), {}), + ]) + gitea.dependencies_closed = Mock(side_effect=lambda issue: issue == 4) + self.assertEqual(gitea.next_story().number, 4) + + def test_no_eligible_story_fails(self) -> None: + from work_item.core import Gitea + gitea = object.__new__(Gitea) + gitea.open_stories = Mock(return_value=[]) + with self.assertRaisesRegex(WorkItemError, "No eligible"): + gitea.next_story() + + +class StateTests(RepositoryFixture): + def test_state_is_stored_inside_git_directory(self) -> None: + workflow = Workflow(self.repo, self.config) + workflow.save_state({"issue": 1}) + self.assertTrue(workflow.state_path.is_file()) + self.assertIn(".git", workflow.state_path.parts) + self.assertEqual(json.loads(workflow.state_path.read_text()), {"issue": 1}) + self.assertEqual(run("git", "status", "--porcelain", cwd=self.repo), "") + workflow.clear_state() + self.assertFalse(workflow.state_path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/webapp/README.md b/webapp/README.md new file mode 100644 index 0000000..450a35c --- /dev/null +++ b/webapp/README.md @@ -0,0 +1,62 @@ +# webapp — Photo Analyzer web UI + +A local, browser-based UI for the `photo_analyzer.py` pipeline. Browse and +full-text-search the analyzed library, watch analysis runs live, and start/stop +runs — reusing the `nsfwtag` review-app design (dark OLED, stdlib HTTP server, one +token-injected HTML page). Nothing leaves the machine. + +```bash +python -m webapp # DB + library default from photo_analyzer.env +python -m webapp --db photo_analysis.db --library pictures/ +python -m webapp --no-open # don't auto-open a browser +``` + +See `../WEBAPP_CONCEPT.md` for the full UX concept and rationale. + +## Views + +- **Library** — FTS5 search over description/tags/mood/location + facet filters + (setting, time of day, season, people, year range, has-place, status), a folder + sidebar (album = leaf folder), an infinite-scroll grid, and a lightbox with an + **Analysis** panel (all DB fields) and a raw **EXIF** panel (incl. Google Maps + link for geotagged photos). +- **Analyze** — run controls mapping every CLI flag (dry-run / reanalyze / no-exif / + exif-only / group-variants), a live overall progress bar, per-album progress, and a + recent feed. Start/Stop drive `photo_analyzer.py` as a subprocess; Stop is the + analyzer's own SIGINT drain. Balance / quota-check buttons. A **Duplicates & + maintenance** row runs the content-hash actions (backfill hashes / find duplicates / + mark duplicates) — no API calls; output streams to the Activity log and marked + duplicates appear under the `duplicate` status filter. (Move/rename reconciliation + and perceptual hashing also run automatically at the start of every normal run.) +- **Stats** — KPI cards + CSS breakdown bars (setting/time/season/people, by-year + histogram, top tags) + an error list. No chart library. + +## Architecture + +- `server.py` — stdlib `ThreadingHTTPServer` on `127.0.0.1:`; routes below. +- `query.py` — read-only DB access (FTS5 search, facets, stats, one-photo). +- `runner.py` — drives `photo_analyzer.py` as a subprocess; DB-derived progress. +- `page.py` / `analyzer.html` — the single frontend page (inherits nsfwtag's tokens). +- SQLite stays the source of truth; the server only reads it (+ a `busy_timeout` so a + concurrent analysis write never errors a read). `/img` and `/exif` paths are + validated against the DB — the server can't be pointed at arbitrary files. + +| Route | Purpose | +|-------|---------| +| GET `/` | the page | +| GET `/search?q=&setting=&tod=&season=&people=&year_min=&year_max=&has_location=&status=&album=&sort=&offset=` | paged results (FTS5 when `q`) | +| GET `/facets` · `/stats` | filter options · aggregates | +| GET `/photo?path=` · `/exif?path=` · `/img?path=` | one record · raw EXIF · thumbnail | +| GET `/progress` · `/log` | live run state · activity log | +| POST `/run` · `/stop` | start / stop a run — body `{library, ...flags}` or `{library, action}` where action ∈ backfill-phash \| list-dupes \| dedupe | +| GET `/balance` · `/quota-check` | provider balance · one-shot quota probe | + +## Known ceilings (ponytail) + +- `/img` serves **full-resolution originals**, not resized thumbnails — a 120-card + page can move a lot of bytes. Fine locally; add a Pillow thumbnail+cache endpoint if + the grid feels slow. (Same limitation as nsfwtag.) +- HEIC files won't render in the browser (broken thumb); metadata/search still work. +- The Analyze **tokens/ETA** readout is omitted — those live only in the terminal + dashboard's memory; the subprocess model surfaces DB-derived progress instead. Parse + run stdout for token counts if you want them back. diff --git a/webapp/__init__.py b/webapp/__init__.py new file mode 100644 index 0000000..e6e8277 --- /dev/null +++ b/webapp/__init__.py @@ -0,0 +1,11 @@ +"""webapp — a local web UI for photo_analyzer. + +Browse and full-text-search the analyzed photo library, watch analysis runs live, +and start/stop runs — all in the browser, reusing the nsfwtag review-app design. +Nothing leaves the machine (stdlib HTTP server bound to 127.0.0.1). + +Entry point: python -m webapp [--db photo_analysis.db] [--library pictures/] +""" +__version__ = "0.1.0" + +PAGE_SIZE = 120 # results per /search page (infinite scroll appends) diff --git a/webapp/__main__.py b/webapp/__main__.py new file mode 100644 index 0000000..ac60942 --- /dev/null +++ b/webapp/__main__.py @@ -0,0 +1,42 @@ +"""python -m webapp — launch the Photo Analyzer web UI. + +Defaults come from photo_analyzer's env (photo_analyzer.env: DB, LIBRARY), so in +the common case you can just run `python -m webapp`. +""" +import argparse +import os +import sys +from pathlib import Path + +from .server import serve + +DEFAULT_DB = "photo_analysis.db" + + +def _env(name, fallback=None): + v = os.environ.get(name) + return v if v not in (None, "") else fallback + + +def main(): + # Reuse photo_analyzer's .env loader so DB/LIBRARY defaults match the CLI. + try: + import photo_analyzer as pa + pa.load_env_file() + except Exception: + pass + + ap = argparse.ArgumentParser(description="Web UI for the photo_analyzer pipeline.") + ap.add_argument("--db", default=_env("DB", DEFAULT_DB), help=f"SQLite DB (default {DEFAULT_DB})") + ap.add_argument("--library", default=_env("LIBRARY"), help="Photo library root (for album grouping + run default)") + ap.add_argument("--no-open", action="store_true", help="Don't open a browser") + args = ap.parse_args() + + if not Path(args.db).exists(): + sys.exit(f"DB not found: {args.db} — run photo_analyzer first, or pass --db.") + + serve(args.db, args.library, open_browser=not args.no_open) + + +if __name__ == "__main__": + main() diff --git a/webapp/analyzer.html b/webapp/analyzer.html new file mode 100644 index 0000000..d8aa7d5 --- /dev/null +++ b/webapp/analyzer.html @@ -0,0 +1,490 @@ + + +Photo Analyzer · %%TOTAL%% photos + + + +
+
+

Photo Analyzer

%%LIBRARY%%

+
+ + +
+
+
+
%%TOTAL%%Total
+
–Analyzed
+
–Errors
+
+ +
+ + +
+
+ + + + + + + + + +
+ + + +
+
+ +
+
+
+ +
+
+
+ + +
+

Run

+
+ + + + + + +
+ + + + +
+
+

Duplicates & maintenance

+
+ Content-hash tools — no API calls. + Output appears in the Activity log; photos marked as duplicates show up under the + duplicate status filter in Library. +
+ + + +
+
+

Overall

+
Idle +
+
+
+
+
+

Albums

No run in progress.

+

Recent

Analyzed photos will appear here.

+
+
+ + +
+
+
+

Setting

+

Time of day

+

Season

+

People

+
+

By year

+

Top tags

+

Errors

+
+ + +
+
+ + +
+

+
+
+ + + + + + + + diff --git a/webapp/page.py b/webapp/page.py new file mode 100644 index 0000000..9ad977a --- /dev/null +++ b/webapp/page.py @@ -0,0 +1,12 @@ +"""Render analyzer.html with %%TOKENS%% filled in. The page is otherwise static +and pulls its data from the JSON endpoints, so this only injects boot values.""" +from pathlib import Path + +_HTML = (Path(__file__).with_name("analyzer.html")).read_text(encoding="utf-8") + + +def render_page(total: int, library: str, db: str) -> str: + return (_HTML + .replace("%%TOTAL%%", f"{total:,}") + .replace("%%LIBRARY%%", library) + .replace("%%DB%%", db)) diff --git a/webapp/query.py b/webapp/query.py new file mode 100644 index 0000000..1df6b30 --- /dev/null +++ b/webapp/query.py @@ -0,0 +1,214 @@ +"""Read-only DB access for the web UI: FTS5 search, facets, stats, one-photo. + +The analysis pipeline (photo_analyzer.py) owns writes; this module only reads. +Album = leaf folder relative to the library (same rule as photo_analyzer.album_label). +""" +import json +import os +import re +import sqlite3 +from pathlib import Path + +from . import PAGE_SIZE + +# Columns sent to the browser for a card / lightbox. raw_response is intentionally +# omitted from lists (big); the lightbox fetches it via /photo if ever needed. +CARD_COLS = ("path", "status", "description", "tags", "people_count", "setting", + "time_of_day", "season", "mood", "location_hint", "approx_year") + +DONE = ("analyzed", "exif_written") + + +def connect(db_path: str) -> sqlite3.Connection: + conn = sqlite3.connect(db_path, check_same_thread=False) # server is multi-threaded + conn.row_factory = sqlite3.Row + return conn + + +def album_of(path: str, library: Path | None) -> str: + """Leaf folder relative to the library — mirrors photo_analyzer.album_label.""" + parent = Path(path).parent + if library: + try: + rel = parent.relative_to(library) + return "(root)" if str(rel) == "." else str(rel) + except ValueError: + pass + return parent.name or str(parent) + + +def _fts_query(q: str) -> str | None: + """Turn free text into a safe FTS5 MATCH string. Each word becomes a prefix + term (AND-ed), so 'beach sun' matches 'beachy sunset'. Returns None if empty + (→ caller browses instead of matching), so raw punctuation can't crash MATCH.""" + tokens = re.findall(r"\w+", q, re.UNICODE) + return " ".join(f'"{t}"*' for t in tokens) if tokens else None + + +def _where(filters: dict) -> tuple[list[str], list]: + """Build a WHERE clause fragment list + params from the facet filters.""" + clauses, params = [], [] + if v := filters.get("setting"): + clauses.append("p.setting = ?"); params.append(v) + if v := filters.get("tod"): + clauses.append("p.time_of_day = ?"); params.append(v) + if v := filters.get("season"): + clauses.append("p.season = ?"); params.append(v) + if v := filters.get("status"): + clauses.append("p.status = ?"); params.append(v) + people = filters.get("people") + if people == "3+": + clauses.append("p.people_count >= 3") + elif people in ("0", "1", "2"): + clauses.append("p.people_count = ?"); params.append(int(people)) + if filters.get("year_min"): + clauses.append("p.approx_year >= ?"); params.append(int(filters["year_min"])) + if filters.get("year_max"): + clauses.append("p.approx_year <= ?"); params.append(int(filters["year_max"])) + if filters.get("has_location"): + clauses.append("p.location_hint IS NOT NULL AND TRIM(p.location_hint) != '' " + "AND LOWER(p.location_hint) != 'null'") + if album := filters.get("album"): + # album is an absolute leaf-folder path; match it and its subfolders + clauses.append("(p.path LIKE ? OR p.path LIKE ?)") + params += [f"{album}/%", f"{album}%/%"] + return clauses, params + + +_SORTS = { + "year": "p.approx_year DESC NULLS LAST, p.path", + "people": "p.people_count DESC NULLS LAST, p.path", + "recent": "p.analyzed_at DESC NULLS LAST, p.path", + "path": "p.path", +} + + +def _row_to_card(r: sqlite3.Row) -> dict: + d = {k: r[k] for k in CARD_COLS} + try: + d["tags"] = json.loads(r["tags"]) if r["tags"] else [] + except (ValueError, TypeError): + d["tags"] = [] + return d + + +def search(conn, q="", filters=None, sort="relevance", offset=0, limit=PAGE_SIZE): + """Paged result set. FTS5 when `q` is present, plain filter/browse otherwise. + Returns {rows, total, offset, limit}.""" + filters = filters or {} + clauses, params = _where(filters) + match = _fts_query(q) if q else None + + if match: + base = ("FROM photos p JOIN photos_fts f ON p.id = f.rowid " + "WHERE photos_fts MATCH ?") + params = [match] + params + order = "f.rank" if sort in ("relevance", "") else _SORTS.get(sort, "f.rank") + else: + base = "FROM photos p WHERE 1=1" + order = _SORTS.get(sort if sort != "relevance" else "path", "p.path") + if clauses: + base += " AND " + " AND ".join(clauses) + + total = conn.execute(f"SELECT COUNT(*) {base}", params).fetchone()[0] + rows = conn.execute( + f"SELECT p.* {base} ORDER BY {order} LIMIT ? OFFSET ?", + params + [limit, offset], + ).fetchall() + return {"rows": [_row_to_card(r) for r in rows], "total": total, + "offset": offset, "limit": limit} + + +def photo(conn, path: str) -> dict | None: + r = conn.execute("SELECT * FROM photos WHERE path = ?", (path,)).fetchone() + if not r: + return None + d = {k: r[k] for k in r.keys() if k != "raw_response"} + try: + d["tags"] = json.loads(r["tags"]) if r["tags"] else [] + except (ValueError, TypeError): + d["tags"] = [] + return d + + +def _col_counts(conn, col: str) -> list[dict]: + rows = conn.execute( + f"SELECT {col} AS v, COUNT(*) AS n FROM photos " + f"WHERE {col} IS NOT NULL AND TRIM({col}) != '' GROUP BY {col} ORDER BY n DESC" + ).fetchall() + return [{"value": r["v"], "count": r["n"]} for r in rows] + + +def facets(conn, library: Path | None) -> dict: + """Distinct filter values + counts, plus the album tree, for the toolbar.""" + year = conn.execute( + "SELECT MIN(approx_year), MAX(approx_year) FROM photos WHERE approx_year IS NOT NULL" + ).fetchone() + albums: dict[str, int] = {} + for (p,) in conn.execute("SELECT path FROM photos"): + a = album_of(p, library) + albums[a] = albums.get(a, 0) + 1 + return { + "setting": _col_counts(conn, "setting"), + "tod": _col_counts(conn, "time_of_day"), + "season": _col_counts(conn, "season"), + "status": _col_counts(conn, "status"), + "year_min": year[0], "year_max": year[1], + "albums": [{"album": a, "count": n} for a, n in sorted(albums.items())], + } + + +def _people_buckets(conn) -> list[dict]: + rows = conn.execute( + "SELECT CASE WHEN people_count >= 3 THEN '3+' ELSE CAST(people_count AS TEXT) END AS b, " + "COUNT(*) AS n FROM photos WHERE people_count IS NOT NULL GROUP BY b" + ).fetchall() + return [{"value": r["b"], "count": r["n"]} for r in rows] + + +def _top_tags(conn, limit=40) -> list[dict]: + counts: dict[str, int] = {} + for (t,) in conn.execute("SELECT tags FROM photos WHERE tags IS NOT NULL"): + try: + for tag in json.loads(t): + tag = str(tag).strip() + if tag: + counts[tag] = counts.get(tag, 0) + 1 + except (ValueError, TypeError): + continue + top = sorted(counts.items(), key=lambda kv: kv[1], reverse=True)[:limit] + return [{"value": t, "count": n} for t, n in top] + + +def stats(conn, library: Path | None) -> dict: + status = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, COUNT(*) AS n FROM photos GROUP BY status")} + total = sum(status.values()) + years = [{"value": r["y"], "count": r["n"]} for r in conn.execute( + "SELECT approx_year AS y, COUNT(*) AS n FROM photos " + "WHERE approx_year IS NOT NULL GROUP BY approx_year ORDER BY approx_year")] + albums: dict[str, dict] = {} + for r in conn.execute("SELECT path, status FROM photos"): + a = album_of(r["path"], library) + d = albums.setdefault(a, {"album": a, "done": 0, "total": 0}) + d["total"] += 1 + if r["status"] in DONE: + d["done"] += 1 + errors = [{"path": r["path"], "error": r["error_message"]} for r in conn.execute( + "SELECT path, error_message FROM photos WHERE status = 'error' ORDER BY path")] + return { + "total": total, "status": status, + "setting": _col_counts(conn, "setting"), + "time_of_day": _col_counts(conn, "time_of_day"), + "season": _col_counts(conn, "season"), + "people": _people_buckets(conn), + "years": years, + "top_tags": _top_tags(conn), + "albums": sorted(albums.values(), key=lambda d: d["album"]), + "errors": errors, + } + + +def all_paths(conn) -> set: + """Every registered path — the allow-set for /img and /exif (path validation).""" + return {r[0] for r in conn.execute("SELECT path FROM photos")} diff --git a/webapp/runner.py b/webapp/runner.py new file mode 100644 index 0000000..2a7bbc4 --- /dev/null +++ b/webapp/runner.py @@ -0,0 +1,110 @@ +"""Drive photo_analyzer.py as a subprocess and expose live run-state to the web UI. + +Why a subprocess and not an in-process thread: the pipeline already is a complete, +resume-safe CLI that owns SQLite as the source of truth. Running it as `python +photo_analyzer.py …` keeps the two fully decoupled — the web server only *reads* the +same DB for progress, Stop is the analyzer's own tested SIGINT drain, and a web crash +can't corrupt a run (or vice-versa). Progress is derived from the DB, so it survives +a browser reload and matches `--stats` exactly. +""" +import signal +import subprocess +import sys +import threading +from pathlib import Path + +from . import query + +REPO = Path(__file__).resolve().parent.parent # dir holding photo_analyzer.py +_FLAGS = ("dry-run", "reanalyze", "no-exif", "exif-only", "group-variants") +# Exit-after-action modes: each runs one maintenance task and quits (no analysis). +# Passed alone — the boolean _FLAGS are ignored when an action is chosen. +_ACTIONS = ("backfill-phash", "list-dupes", "dedupe") + + +class Runner: + def __init__(self, db_path: str, log): + self.db = db_path + self.log = log + self.proc: subprocess.Popen | None = None + self.lock = threading.Lock() + + def running(self) -> bool: + return self.proc is not None and self.proc.poll() is None + + def start(self, opts: dict) -> dict: + with self.lock: + if self.running(): + return {"error": "A run is already in progress."} + lib = (opts.get("library") or "").strip() + if not lib: + return {"error": "Library path is required."} + argv = [sys.executable, str(REPO / "photo_analyzer.py"), + "--library", lib, "--db", self.db] + action = opts.get("action") + if action in _ACTIONS: + argv.append(f"--{action}") # maintenance action, runs alone + else: + argv += [f"--{f}" for f in _FLAGS if opts.get(f)] + try: + self.proc = subprocess.Popen( + argv, cwd=str(REPO), stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, bufsize=1) + except Exception as e: + return {"error": f"Could not start: {e}"} + threading.Thread(target=self._reader, args=(self.proc,), daemon=True).start() + self.log("run started: " + " ".join(argv[2:])) + return {"ok": True} + + def _reader(self, proc): + # Surface the analyzer's own log lines in the browser's Activity panel. + for line in proc.stdout: + line = line.rstrip() + if line: + self.log(line) + self.log(f"run finished (exit {proc.poll()})") + + def stop(self) -> dict: + with self.lock: + if self.running(): + self.proc.send_signal(signal.SIGINT) # reuse the tested drain path + self.log("stop requested") + return {"ok": True} + + +def progress(conn, library: Path | None, running: bool) -> dict: + """DB-derived run state for /progress. Cheap full scan (~25k rows, a few ms); + ponytail: if the library grows past ~200k, cache album counts between polls.""" + st = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, COUNT(*) AS n FROM photos GROUP BY status")} + done = st.get("analyzed", 0) + st.get("exif_written", 0) + dup = st.get("duplicate", 0) + # Duplicates are deliberately skipped, not pending work — exclude them from the + # denominator so the bar reflects real progress rather than dragging to <100%. + work_total = sum(st.values()) - dup + + albums: dict[str, dict] = {} + for r in conn.execute("SELECT path, status FROM photos"): + if r["status"] == "duplicate": + continue # not work; keep albums reaching 100% + a = query.album_of(r["path"], library) + d = albums.setdefault(a, {"album": a, "done": 0, "total": 0}) + d["total"] += 1 + if r["status"] in query.DONE: + d["done"] += 1 + # incomplete albums first — that's what a running job is working on + alist = sorted(albums.values(), key=lambda d: (d["done"] >= d["total"], d["album"])) + + feed = [] + for r in conn.execute( + "SELECT path, status, description, error_message FROM photos " + "WHERE analyzed_at IS NOT NULL ORDER BY analyzed_at DESC LIMIT 14" + ): + ok = r["status"] != "error" + feed.append({"ok": ok, "name": Path(r["path"]).name, + "msg": (r["description"] if ok else r["error_message"]) or ""}) + + return {"running": running, + "totals": {"done": done, "total": work_total, "ok": done, + "err": st.get("error", 0), "dup": dup}, + "albums": alist[:60], "feed": feed, "tokens": None} diff --git a/webapp/server.py b/webapp/server.py new file mode 100644 index 0000000..14ebef1 --- /dev/null +++ b/webapp/server.py @@ -0,0 +1,162 @@ +"""Local, stdlib-only HTTP server for the Photo Analyzer web UI. + +Serves one page + JSON endpoints, all bound to 127.0.0.1. Reads the analysis DB +(via query.py), serves thumbnails/EXIF for the lightbox (reusing nsfwtag's readers), +and drives analysis runs (runner.py). Path args on /img and /exif are validated +against the DB, so the server can't be pointed at arbitrary files. +""" +import http.server +import json +import mimetypes +import sys +import time +import webbrowser +from collections import deque +from pathlib import Path +from urllib.parse import parse_qs, unquote, urlparse + +from . import PAGE_SIZE, page, query +from .runner import Runner, progress as run_progress + +try: + from nsfwtag.exif import read_exif # reuse the lightbox EXIF reader +except Exception: # nsfwtag not importable → degrade + def read_exif(_p): return {} + + +def serve(db_path: str, library: str | None, host: str = "127.0.0.1", open_browser=True): + conn = query.connect(db_path) + conn.execute("PRAGMA busy_timeout=3000") # tolerate the analyzer writing concurrently + lib = Path(library).expanduser().resolve() if library else None + + log_entries = deque(maxlen=2000) + + def log(m): + log_entries.append({"t": time.strftime("%H:%M:%S"), "m": m}) + print(m, file=sys.stderr) + + runner = Runner(db_path, log) + + def allowed(fp: str) -> bool: + return conn.execute("SELECT 1 FROM photos WHERE path=? LIMIT 1", (fp,)).fetchone() is not None + + def filters_from(qd: dict) -> dict: + g = lambda k: qd.get(k, [""])[0] + return {k: g(k) for k in ("q", "setting", "tod", "season", "people", + "year_min", "year_max", "has_location", "status", + "album", "sort")} + + total0 = conn.execute("SELECT COUNT(*) FROM photos").fetchone()[0] + page_bytes = page.render_page(total0, str(lib) if lib else (library or ""), db_path).encode("utf-8") + log(f"web ui ready — {total0:,} photos indexed") + + class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self, code, ctype, body): + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _json(self, obj, code=200): + self._send(code, "application/json", json.dumps(obj).encode("utf-8")) + + def do_GET(self): + u = urlparse(self.path) + qd = parse_qs(u.query) + p = u.path + if p == "/": + return self._send(200, "text/html; charset=utf-8", page_bytes) + if p == "/img": + fp = unquote(qd.get("path", [""])[0]) + if allowed(fp) and Path(fp).exists(): + ctype = mimetypes.guess_type(fp)[0] or "application/octet-stream" + return self._send(200, ctype, Path(fp).read_bytes()) + return self._send(404, "text/plain", b"no image") + if p == "/exif": + fp = unquote(qd.get("path", [""])[0]) + if allowed(fp) and Path(fp).exists(): + return self._json(read_exif(fp)) + return self._json({}) + if p == "/photo": + fp = unquote(qd.get("path", [""])[0]) + return self._json(query.photo(conn, fp) or {}) + if p == "/search": + f = filters_from(qd) + try: + offset = int(qd.get("offset", ["0"])[0]) + except ValueError: + offset = 0 + return self._json(query.search( + conn, q=f.pop("q"), filters=f, sort=f.get("sort") or "relevance", + offset=offset, limit=PAGE_SIZE)) + if p == "/facets": + return self._json(query.facets(conn, lib)) + if p == "/stats": + return self._json(query.stats(conn, lib)) + if p == "/progress": + return self._json(run_progress(conn, lib, runner.running())) + if p == "/log": + return self._json(list(log_entries)) + if p == "/balance": + return self._json(_balance()) + if p == "/quota-check": + return self._json({"ok": _quota()}) + return self._send(404, "text/plain", b"not found") + + def do_POST(self): + u = urlparse(self.path) + n = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(n) if n else b"{}" + if u.path == "/run": + try: + opts = json.loads(body or b"{}") + except ValueError: + return self._json({"error": "bad request"}, 400) + return self._json(runner.start(opts)) + if u.path == "/stop": + return self._json(runner.stop()) + return self._send(404, "text/plain", b"not found") + + def _balance(): + try: + import photo_analyzer as pa + pa.load_env_file() + import os + key = os.environ.get("LLM_API_KEY") or os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY") + d = pa.fetch_balance(key) + if not d: + return {"text": ""} + return {"text": f"available {d.get('available_balance')} · " + f"cash {d.get('cash_balance')} · voucher {d.get('voucher_balance')}"} + except Exception as e: + log(f"balance error: {e}") + return {"text": ""} + + def _quota(): + try: + import os + import photo_analyzer as pa + pa.load_env_file() + from openai import OpenAI + key = os.environ.get("LLM_API_KEY") or os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY") + client = OpenAI(api_key=key or "x", base_url=pa.LLM_BASE_URL) + return pa.check_quota(client) + except Exception as e: + log(f"quota error: {e}") + return False + + srv = http.server.ThreadingHTTPServer((host, 0), H) + url = f"http://{host}:{srv.server_address[1]}/" + print(f"\nPhoto Analyzer web UI: {url}\n Ctrl-C here to stop the server.\n", file=sys.stderr) + if open_browser: + webbrowser.open(url) + try: + srv.serve_forever() + except KeyboardInterrupt: + print("\nstopped.", file=sys.stderr) + finally: + srv.server_close() diff --git a/work_item/__init__.py b/work_item/__init__.py new file mode 100644 index 0000000..a0c970f --- /dev/null +++ b/work_item/__init__.py @@ -0,0 +1,5 @@ +"""Safe Gitea-backed work-item workflow helpers.""" + +from .core import WorkItemError, Workflow + +__all__ = ["WorkItemError", "Workflow"] diff --git a/work_item/__main__.py b/work_item/__main__.py new file mode 100644 index 0000000..51b838f --- /dev/null +++ b/work_item/__main__.py @@ -0,0 +1,3 @@ +from .core import main + +raise SystemExit(main()) diff --git a/work_item/core.py b/work_item/core.py new file mode 100644 index 0000000..753dc7f --- /dev/null +++ b/work_item/core.py @@ -0,0 +1,547 @@ +from __future__ import annotations + +import argparse +import fnmatch +import json +import os +import re +import shlex +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Sequence + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python < 3.11 is unsupported + tomllib = None + + +STORY_RE = re.compile(r"^(US(?P\d{2})-(?P\d{2}))\s+—\s+(?P.+)$") +SECRET_PATTERNS = ( + re.compile(r"(?i)(api[_-]?key|access[_-]?token|client[_-]?secret|password)\s*[:=]\s*['\"]?[A-Za-z0-9_./+\-=]{12,}"), + re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), + re.compile(r"(?i)authorization:\s*(?:bearer|token)\s+[A-Za-z0-9._~+\-/=]{12,}"), +) + + +class WorkItemError(RuntimeError): + pass + + +class Runner: + def run( + self, + args: Sequence[str], + *, + cwd: Path, + check: bool = True, + input_text: str | None = None, + ) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + list(args), + cwd=cwd, + text=True, + input=input_text, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if check and result.returncode: + command = " ".join(shlex.quote(part) for part in args) + detail = (result.stderr or result.stdout).strip() + raise WorkItemError(f"Command failed ({result.returncode}): {command}\n{detail}") + return result + + def run_shell(self, command: str, *, cwd: Path) -> subprocess.CompletedProcess[str]: + result = subprocess.run(command, cwd=cwd, text=True, shell=True) + if result.returncode: + raise WorkItemError(f"Test command failed ({result.returncode}): {command}") + return result + + +@dataclass(frozen=True) +class Config: + repo_slug: str + login: str + assignee: str + remote: str = "origin" + main_branch: str = "main" + branch_prefix: str = "us" + require_ci: bool = False + max_file_bytes: int = 5_000_000 + required_tests: tuple[str, ...] = () + denied_patterns: tuple[str, ...] = () + allowed_patterns: tuple[str, ...] = () + + @classmethod + def load(cls, path: Path) -> "Config": + if tomllib is None: + raise WorkItemError("Python 3.11 or newer is required") + if not path.is_file(): + raise WorkItemError(f"Missing workflow configuration: {path}") + with path.open("rb") as handle: + raw = tomllib.load(handle) + repo = raw.get("repository", {}) + workflow = raw.get("workflow", {}) + safety = raw.get("safety", {}) + required = ("slug", "login", "assignee") + missing = [key for key in required if not repo.get(key)] + if missing: + raise WorkItemError(f"Missing repository configuration: {', '.join(missing)}") + return cls( + repo_slug=repo["slug"], + login=repo["login"], + assignee=repo["assignee"], + remote=repo.get("remote", "origin"), + main_branch=repo.get("main_branch", "main"), + branch_prefix=workflow.get("branch_prefix", "us"), + require_ci=bool(workflow.get("require_ci", False)), + max_file_bytes=int(safety.get("max_file_bytes", 5_000_000)), + required_tests=tuple(workflow.get("required_tests", ())), + denied_patterns=tuple(safety.get("deny", ())), + allowed_patterns=tuple(safety.get("allow", ())), + ) + + +@dataclass(frozen=True) +class Story: + number: int + story_id: str + title: str + epic: int + sequence: int + labels: frozenset[str] + assignees: tuple[str, ...] + raw: dict[str, Any] + + @classmethod + def from_issue(cls, issue: dict[str, Any]) -> "Story | None": + match = STORY_RE.match(issue.get("title", "")) + if not match: + return None + labels = frozenset(label["name"] for label in issue.get("labels") or ()) + assignees = tuple(user["login"] for user in issue.get("assignees") or ()) + return cls( + number=int(issue["number"]), + story_id=match.group(1), + title=match.group("title"), + epic=int(match.group("epic")), + sequence=int(match.group("story")), + labels=labels, + assignees=assignees, + raw=issue, + ) + + @property + def branch_slug(self) -> str: + value = re.sub(r"[^a-z0-9]+", "-", self.title.lower()).strip("-") + return value[:48].rstrip("-") + + +class GitRepo: + def __init__(self, root: Path, config: Config, runner: Runner) -> None: + self.root = root + self.config = config + self.runner = runner + + def git(self, *args: str, check: bool = True) -> str: + return self.runner.run(("git", *args), cwd=self.root, check=check).stdout.strip() + + def verify(self) -> None: + actual = Path(self.git("rev-parse", "--show-toplevel")).resolve() + if actual != self.root.resolve(): + raise WorkItemError(f"Run from repository root {actual}; current root is {self.root}") + remote_url = self.git("remote", "get-url", self.config.remote) + normalized = remote_url.removesuffix(".git").replace(":", "/") + if self.config.repo_slug not in normalized: + raise WorkItemError( + f"Remote {self.config.remote!r} does not match {self.config.repo_slug!r}: {remote_url}" + ) + + def ensure_clean(self) -> None: + if self.git("status", "--porcelain=v1"): + raise WorkItemError("Working tree must be clean before claiming a story") + + def update_main(self) -> None: + self.git("fetch", "--prune", self.config.remote) + self.git("switch", self.config.main_branch) + self.git("pull", "--ff-only", self.config.remote, self.config.main_branch) + + def current_branch(self) -> str: + return self.git("branch", "--show-current") + + def create_story_branch(self, story: Story) -> str: + branch = f"{self.config.branch_prefix}/{story.story_id}-{story.branch_slug}" + self.git("switch", "-c", branch) + return branch + + def changed_paths(self) -> list[Path]: + output = self.git("status", "--porcelain=v1", "--untracked-files=all", "-z") + if not output: + return [] + entries = output.split("\0") + paths: list[Path] = [] + skip_rename_source = False + for entry in entries: + if not entry: + continue + if skip_rename_source: + skip_rename_source = False + continue + if len(entry) < 4: + continue + status, name = entry[:2], entry[3:] + paths.append(Path(name)) + if "R" in status or "C" in status: + skip_rename_source = True + return paths + + def assert_safe_changes(self) -> list[Path]: + paths = self.changed_paths() + if not paths: + raise WorkItemError("No changes to submit") + violations: list[str] = [] + for relative in paths: + portable = relative.as_posix() + allowed = any(fnmatch.fnmatch(portable, pattern) for pattern in self.config.allowed_patterns) + denied = any(fnmatch.fnmatch(portable, pattern) for pattern in self.config.denied_patterns) + if denied and not allowed: + violations.append(f"denied path: {portable}") + continue + candidate = self.root / relative + if candidate.is_file() and candidate.stat().st_size > self.config.max_file_bytes: + violations.append(f"file exceeds {self.config.max_file_bytes} bytes: {portable}") + continue + if candidate.is_file() and candidate.stat().st_size <= self.config.max_file_bytes: + try: + content = candidate.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + content = "" + for pattern in SECRET_PATTERNS: + if pattern.search(content): + violations.append(f"possible secret in: {portable}") + break + if violations: + raise WorkItemError("Unsafe changes detected:\n- " + "\n- ".join(violations)) + self.git("diff", "--check") + return paths + + def stage_commit_push(self, story: Story, branch: str) -> str: + self.git("add", "--all") + staged = self.git("diff", "--cached", "--name-only") + if not staged: + raise WorkItemError("Nothing was staged") + subject = f"{story.story_id}: {story.title}" + self.git("commit", "-m", subject) + self.git("push", "--set-upstream", self.config.remote, branch) + return self.git("rev-parse", "HEAD") + + +class Gitea: + def __init__(self, root: Path, config: Config, runner: Runner) -> None: + self.root = root + self.config = config + self.runner = runner + + def tea(self, *args: str) -> str: + command = ("tea", *args, "--login", self.config.login) + return self.runner.run(command, cwd=self.root).stdout.strip() + + def api(self, endpoint: str, *, method: str = "GET", data: Any = None) -> Any: + args = ["api", endpoint, "--method", method] + if data is not None: + args.extend(("--data", json.dumps(data))) + output = self.tea(*args) + return json.loads(output) if output else None + + @property + def base(self) -> str: + return f"/repos/{self.config.repo_slug}" + + def open_stories(self) -> list[Story]: + issues = self.api(f"{self.base}/issues?state=open&type=issues&limit=100") + stories = [story for issue in issues if (story := Story.from_issue(issue))] + return sorted(stories, key=lambda story: (story.epic, story.sequence, story.number)) + + def dependencies_closed(self, issue_number: int) -> bool: + dependencies = self.api(f"{self.base}/issues/{issue_number}/dependencies?limit=100") + return all(issue.get("state") == "closed" for issue in dependencies) + + def next_story(self) -> Story: + excluded = {"status/in-progress", "status/review", "status/blocked", "status/done"} + for story in self.open_stories(): + if story.labels & excluded or story.assignees: + continue + if self.dependencies_closed(story.number): + return story + raise WorkItemError("No eligible open user story is available") + + def edit_labels(self, issue: int, *, add: Iterable[str], remove: Iterable[str]) -> None: + args = ["issues", "edit", str(issue), "--repo", self.config.repo_slug] + add_value = ",".join(add) + remove_value = ",".join(remove) + if add_value: + args.extend(("--add-labels", add_value)) + if remove_value: + args.extend(("--remove-labels", remove_value)) + self.tea(*args) + + def claim(self, story: Story) -> None: + self.tea( + "issues", "edit", str(story.number), + "--repo", self.config.repo_slug, + "--add-assignees", self.config.assignee, + "--add-labels", "status/in-progress", + "--remove-labels", "status/backlog,status/ready,status/blocked,status/review,status/done", + ) + + def comment(self, issue: int, body: str) -> None: + self.tea("comments", "add", str(issue), body, "--repo", self.config.repo_slug) + + def create_pr(self, story: Story, branch: str, tests: Sequence[str], commit: str) -> dict[str, Any]: + body = "\n".join(( + f"Implements #{story.number}", + "", + "Tests:", + *(f"- `{command}`" for command in tests), + "", + f"Commit: `{commit}`", + )) + output = self.tea( + "pulls", "create", + "--repo", self.config.repo_slug, + "--head", branch, + "--base", self.config.main_branch, + "--title", f"{story.story_id}: {story.title}", + "--description", body, + "--output", "json", + ) + try: + return json.loads(output) + except json.JSONDecodeError as error: + raise WorkItemError(f"Could not parse created pull request: {output}") from error + + def pr(self, number: int) -> dict[str, Any]: + return self.api(f"{self.base}/pulls/{number}") + + def merge_pr(self, number: int) -> None: + self.tea("pulls", "merge", str(number), "--repo", self.config.repo_slug, "--style", "squash") + + def require_ci_success(self, pr: dict[str, Any]) -> None: + if not self.config.require_ci: + return + sha = pr["head"]["sha"] + combined = self.api(f"{self.base}/commits/{sha}/status") + if combined.get("state") != "success": + raise WorkItemError(f"Required CI is not successful: {combined.get('state', 'unknown')}") + + def mark_review(self, issue: int, pr: dict[str, Any], tests: Sequence[str]) -> None: + self.edit_labels( + issue, + add=("status/review",), + remove=("status/in-progress", "status/backlog", "status/ready", "status/blocked", "status/done"), + ) + number = pr.get("index") or pr.get("number") + url = pr.get("url") or pr.get("html_url") or f"PR #{number}" + self.comment(issue, f"Submitted for review: {url}\n\nTests passed:\n" + "\n".join(f"- `{x}`" for x in tests)) + + def mark_done(self, issue: int, pr_number: int) -> None: + self.edit_labels( + issue, + add=("status/done",), + remove=("status/in-progress", "status/backlog", "status/ready", "status/blocked", "status/review"), + ) + self.comment(issue, f"Completed and merged via PR #{pr_number}.") + self.tea("issues", "close", str(issue), "--repo", self.config.repo_slug) + + +class Workflow: + STATE_NAME = "work-item-state.json" + + def __init__(self, root: Path, config: Config, runner: Runner | None = None) -> None: + self.root = root.resolve() + self.config = config + self.runner = runner or Runner() + self.git = GitRepo(self.root, config, self.runner) + self.gitea = Gitea(self.root, config, self.runner) + + @property + def state_path(self) -> Path: + git_dir = self.git.git("rev-parse", "--git-dir") + path = Path(git_dir) + if not path.is_absolute(): + path = self.root / path + return path / self.STATE_NAME + + def load_state(self) -> dict[str, Any]: + if not self.state_path.is_file(): + raise WorkItemError("No story is currently claimed") + return json.loads(self.state_path.read_text(encoding="utf-8")) + + def save_state(self, state: dict[str, Any]) -> None: + temporary = self.state_path.with_suffix(".tmp") + temporary.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(self.state_path) + + def clear_state(self) -> None: + self.state_path.unlink(missing_ok=True) + + def next(self) -> Story: + self.git.verify() + return self.gitea.next_story() + + def claim(self) -> dict[str, Any]: + self.git.verify() + self.git.ensure_clean() + if self.state_path.exists(): + raise WorkItemError("A story is already claimed; complete or block it first") + self.git.update_main() + story = self.gitea.next_story() + branch = self.git.create_story_branch(story) + self.gitea.claim(story) + state = { + "issue": story.number, + "story_id": story.story_id, + "title": story.title, + "branch": branch, + "status": "in-progress", + } + self.save_state(state) + self.gitea.comment(story.number, f"Claimed for implementation on branch `{branch}`.") + return state + + def status(self) -> dict[str, Any]: + self.git.verify() + state = self.load_state() + return {**state, "current_branch": self.git.current_branch(), "changes": [p.as_posix() for p in self.git.changed_paths()]} + + def submit(self, tests: Sequence[str], *, confirmed: bool) -> dict[str, Any]: + self.git.verify() + state = self.load_state() + if self.git.current_branch() != state["branch"]: + raise WorkItemError(f"Expected branch {state['branch']}, found {self.git.current_branch()}") + commands = tuple(dict.fromkeys((*self.config.required_tests, *tests))) + if not commands: + raise WorkItemError("At least one automated test command is required") + self.git.assert_safe_changes() + for command in commands: + self.runner.run_shell(command, cwd=self.root) + paths = self.git.assert_safe_changes() + if not confirmed: + summary = self.git.git("diff", "--stat") + raise WorkItemError( + "Review the changes and rerun with --yes to commit:\n" + f"{summary}\nFiles: {', '.join(path.as_posix() for path in paths)}" + ) + story = Story( + number=int(state["issue"]), story_id=state["story_id"], title=state["title"], + epic=int(state["story_id"][2:4]), sequence=int(state["story_id"][5:7]), + labels=frozenset(), assignees=(), raw={}, + ) + commit = self.git.stage_commit_push(story, state["branch"]) + pr = self.gitea.create_pr(story, state["branch"], commands, commit) + pr_number = int(pr.get("index") or pr.get("number")) + state.update({"status": "review", "pr": pr_number, "commit": commit, "tests": list(commands)}) + self.save_state(state) + self.gitea.mark_review(story.number, pr, commands) + return state + + def complete(self, *, merge: bool) -> dict[str, Any]: + self.git.verify() + state = self.load_state() + if state.get("status") != "review" or not state.get("pr"): + raise WorkItemError("The claimed story has not been submitted for review") + pr = self.gitea.pr(int(state["pr"])) + self.gitea.require_ci_success(pr) + if not pr.get("merged"): + if not merge: + raise WorkItemError("Pull request is not merged; rerun with --merge after review") + self.gitea.merge_pr(int(state["pr"])) + pr = self.gitea.pr(int(state["pr"])) + if not pr.get("merged"): + raise WorkItemError("Pull request merge could not be verified") + self.gitea.mark_done(int(state["issue"]), int(state["pr"])) + branch = state["branch"] + self.git.git("switch", self.config.main_branch) + self.git.git("pull", "--ff-only", self.config.remote, self.config.main_branch) + # A squash-merged PR is verified by Gitea but its feature commit is not an + # ancestor of main, so ordinary `-d` would incorrectly reject safe cleanup. + self.git.git("branch", "-D", branch) + result = {**state, "status": "done"} + self.clear_state() + return result + + def block(self, reason: str) -> dict[str, Any]: + self.git.verify() + if not reason.strip(): + raise WorkItemError("A non-empty blocking reason is required") + state = self.load_state() + self.gitea.edit_labels( + int(state["issue"]), + add=("status/blocked",), + remove=("status/in-progress", "status/ready", "status/review", "status/done"), + ) + self.gitea.comment(int(state["issue"]), f"Blocked: {reason.strip()}") + state.update({"status": "blocked", "reason": reason.strip()}) + self.save_state(state) + return state + + +def find_root(start: Path) -> Path: + result = subprocess.run( + ("git", "rev-parse", "--show-toplevel"), cwd=start, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + if result.returncode: + raise WorkItemError("This command must run inside the photoanalyzer Git repository") + return Path(result.stdout.strip()).resolve() + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(prog="work-item", description="Safe issue/Git workflow helper") + result.add_argument("--config", default=".work-item.toml") + commands = result.add_subparsers(dest="command", required=True) + commands.add_parser("next", help="Show the top eligible user story") + commands.add_parser("claim", help="Claim the top story and create its branch") + commands.add_parser("status", help="Show the current claim and Git state") + submit = commands.add_parser("submit", help="Test, commit, push, and open a pull request") + submit.add_argument("--test", action="append", default=[], help="Additional automated test command; repeatable") + submit.add_argument("--yes", action="store_true", help="Confirm the reviewed diff may be committed") + complete = commands.add_parser("complete", help="Verify merge, close the issue, and return to main") + complete.add_argument("--merge", action="store_true", help="Merge the reviewed PR before completing") + block = commands.add_parser("block", help="Mark the current story blocked") + block.add_argument("--reason", required=True) + return result + + +def main(argv: Sequence[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + root = find_root(Path.cwd()) + config_path = Path(args.config) + if not config_path.is_absolute(): + config_path = root / config_path + workflow = Workflow(root, Config.load(config_path)) + if args.command == "next": + story = workflow.next() + payload: Any = {"issue": story.number, "story_id": story.story_id, "title": story.title} + elif args.command == "claim": + payload = workflow.claim() + elif args.command == "status": + payload = workflow.status() + elif args.command == "submit": + payload = workflow.submit(args.test, confirmed=args.yes) + elif args.command == "complete": + payload = workflow.complete(merge=args.merge) + else: + payload = workflow.block(args.reason) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + except WorkItemError as error: + print(f"work-item: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main())