From ffafa308de98b2cf15639d891ea521f15c3dce60 Mon Sep 17 00:00:00 2001 From: domverse Date: Wed, 15 Jul 2026 16:13:00 +0200 Subject: [PATCH] US01-01 follow-up: replace donor characterization iteration with YAML ledger + full suite (#46) --- .../stories/US01-01-donor-characterization.md | 6 + donor_ledger.yaml | 589 ++++++++++++++++++ donor_ledger/README.md | 23 - donor_ledger/ledger.json | 180 ------ tests/characterization/conftest.py | 107 ++++ tests/characterization/test_donor_ledger.py | 88 +++ tests/characterization/test_donors.py | 295 --------- tests/characterization/test_ledger.py | 62 -- tests/characterization/test_nsfwtag.py | 95 +++ tests/characterization/test_pa_album.py | 28 + tests/characterization/test_pa_config.py | 46 ++ tests/characterization/test_pa_db.py | 68 ++ tests/characterization/test_pa_discovery.py | 27 + tests/characterization/test_pa_exif.py | 87 +++ .../characterization/test_pa_hashing_dedup.py | 109 ++++ tests/characterization/test_pa_imaging.py | 38 ++ tests/characterization/test_pa_variants.py | 45 ++ tests/characterization/test_traceability.json | 15 - tests/characterization/test_webapp_query.py | 105 ++++ .../characterization/expected_outputs.json | 15 - tests/fixtures/characterization/manifest.json | 13 - work_item/.work-item.yml | 2 +- 22 files changed, 1439 insertions(+), 604 deletions(-) create mode 100644 donor_ledger.yaml delete mode 100644 donor_ledger/README.md delete mode 100644 donor_ledger/ledger.json create mode 100644 tests/characterization/conftest.py create mode 100644 tests/characterization/test_donor_ledger.py delete mode 100644 tests/characterization/test_donors.py delete mode 100644 tests/characterization/test_ledger.py create mode 100644 tests/characterization/test_nsfwtag.py create mode 100644 tests/characterization/test_pa_album.py create mode 100644 tests/characterization/test_pa_config.py create mode 100644 tests/characterization/test_pa_db.py create mode 100644 tests/characterization/test_pa_discovery.py create mode 100644 tests/characterization/test_pa_exif.py create mode 100644 tests/characterization/test_pa_hashing_dedup.py create mode 100644 tests/characterization/test_pa_imaging.py create mode 100644 tests/characterization/test_pa_variants.py delete mode 100644 tests/characterization/test_traceability.json create mode 100644 tests/characterization/test_webapp_query.py delete mode 100644 tests/fixtures/characterization/expected_outputs.json delete mode 100644 tests/fixtures/characterization/manifest.json diff --git a/delivery_backlog/stories/US01-01-donor-characterization.md b/delivery_backlog/stories/US01-01-donor-characterization.md index 6008db5..042ad44 100644 --- a/delivery_backlog/stories/US01-01-donor-characterization.md +++ b/delivery_backlog/stories/US01-01-donor-characterization.md @@ -2,6 +2,11 @@ Epic: [E01](../E01-shared-identity-inventory.md) +**State: done (2026-07-15).** Deliverables: [`donor_ledger.yaml`](../../donor_ledger.yaml) +(44 rows, all 12 required areas), characterization suite +`tests/characterization/` (61 tests incl. ledger lint), all passing via +`python3 -m pytest tests/characterization/`. No legacy file moved. + As a maintainer, I want a donor ledger and characterization suite for the current CLI implementations so proven behavior is reused deliberately instead of rewritten. @@ -18,3 +23,4 @@ implementations so proven behavior is reused deliberately instead of rewritten. - Characterization tests run against current donor entry points and core functions. - A ledger-lint test rejects missing source references, target locations, or test IDs. + diff --git a/donor_ledger.yaml b/donor_ledger.yaml new file mode 100644 index 0000000..c69fcdd --- /dev/null +++ b/donor_ledger.yaml @@ -0,0 +1,589 @@ +# Donor ledger — US01-01 (INTEGRATED_PIPELINE_CONCEPT.md §3 "Donor-first CLI migration") +# +# Maps every relevant legacy module/function to its classification, rationale, +# target module in the future photo_pipeline package, and characterization +# test IDs. Linted by tests/characterization/test_donor_ledger.py. +# +# classification: reuse — take as-is +# extract — move behind a thin adapter, behavior unchanged +# refactor — extract while changing structure, behavior preserved +# unless an intentional delta is stated +# replace — new implementation; rationale documents why and what +# (if anything) carries over +# Every row needs either `tests` (existing test IDs, module::function) or +# `pending_story` (the backlog story that will characterize/deliver it). +# status: characterized | pending + +rows: + # ── photo_analyzer.py ────────────────────────────────────────────────────── + - id: pa-discovery + area: discovery + source: {file: photo_analyzer.py, symbols: [discover_photos, SUPPORTED_EXTENSIONS]} + classification: extract + rationale: > + Proven recursive discovery with _IGNORE/ and .@__thumb exclusion, sorted + + deduplicated. Becomes the one shared discovery used by every stage; the + exclusion predicate moves to path_policy.py. + target: photo_pipeline/services/inventory.py + photo_pipeline/path_policy.py + tests: + - test_pa_discovery::test_discover_excludes_ignore_and_thumbs + - test_pa_discovery::test_discover_finds_supported_including_uppercase + - test_pa_discovery::test_discover_sorted_and_deduplicated + - test_pa_discovery::test_supported_extensions_contract + status: characterized + + - id: pa-purge-excluded + area: discovery + source: {file: photo_analyzer.py, symbols: [purge_excluded]} + classification: refactor + rationale: > + SQL LIKE patterns duplicate the path policy; refactor to call the shared + path_policy predicate so exclusion has exactly one definition. + target: photo_pipeline/services/inventory.py + tests: [test_pa_db::test_purge_excluded_removes_ignore_and_thumb_rows] + status: characterized + + - id: pa-prune-missing + area: database + source: {file: photo_analyzer.py, symbols: [prune_missing]} + classification: replace + rationale: > + Deletes rows for missing files — incompatible with stable asset identity; + the concept keeps the asset and sets missing_at/availability_state. + The unmounted-library guard (never prune when the root is absent) carries + over as behavior. + target: photo_pipeline/services/inventory.py + tests: [test_pa_db::test_prune_missing_guards_unmounted_library] + status: characterized + + - id: pa-variants + area: discovery + source: {file: photo_analyzer.py, symbols: [_strip_variant_markers, _is_variant, _variant_key, build_variant_groups]} + classification: extract + rationale: > + Filename-based variant grouping (pixel-size suffix, 'bearbeitet' edit + marker) with unmarked-largest primary selection; feeds duplicate/variant + clustering evidence. + target: photo_pipeline/services/duplicates.py + tests: + - test_pa_variants::test_strip_variant_markers_goldens + - test_pa_variants::test_is_variant + - test_pa_variants::test_variant_key_folder_scoped_case_insensitive + - test_pa_variants::test_build_variant_groups_primary_is_largest_unmarked + status: characterized + + - id: pa-variant-propagate + area: database + source: {file: photo_analyzer.py, symbols: [propagate_variants, _row_to_result]} + classification: refactor + rationale: > + Copying a primary's analysis into variant rows survives, but keyed by + asset_id and recorded as stage state instead of raw row copies. + target: photo_pipeline/services/duplicates.py + pending_story: US01-04 + status: pending + + - id: pa-hashing + area: hashing + source: {file: photo_analyzer.py, symbols: [_sha1_file, _phash_image]} + classification: extract + rationale: > + Streamed SHA-1 plus 64-bit DCT phash (imagehash.phash recipe on + numpy/scipy, no extra dependency). Hash outputs are goldens — a change + means the decoder/algorithm changed and hash_version must bump. + target: photo_pipeline/services/inventory.py + tests: + - test_pa_hashing_dedup::test_phash_goldens + - test_pa_hashing_dedup::test_phash_survives_resize_and_recompress + - test_pa_hashing_dedup::test_phash_format_16_hex_chars + - test_pa_hashing_dedup::test_sha1_matches_hashlib + - test_pa_hashing_dedup::test_sha1_unreadable_returns_none + status: characterized + + - id: pa-ensure-hashes + area: hashing + source: {file: photo_analyzer.py, symbols: [ensure_hashes]} + classification: extract + rationale: > + Resume-safe incremental hashing (skip already-hashed, COALESCE so a failed + hash never clobbers a stored one, periodic commits, honors stop event). + target: photo_pipeline/services/inventory.py + tests: [test_pa_hashing_dedup::test_ensure_hashes_skips_hashed_and_is_resume_safe] + status: characterized + + - id: pa-cluster + area: hashing + source: {file: photo_analyzer.py, symbols: [cluster_duplicates]} + classification: extract + rationale: > + Union-find over the vectorized Hamming graph; largest-first ordering. + Known ceiling (documented in-source): O(n²) scan, BK-tree past ~100k. + target: photo_pipeline/services/duplicates.py + tests: [test_pa_hashing_dedup::test_cluster_and_mark_duplicates_largest_is_canonical] + status: characterized + + - id: pa-mark-duplicates + area: database + source: {file: photo_analyzer.py, symbols: [mark_duplicates]} + classification: refactor + rationale: > + Intentional delta — the concept forbids silently marking fuzzy matches: + largest-file canonical becomes a *recommendation*; the decision lands in + reviewable duplicate_clusters, auto-resolve only for exact matches. + The status='duplicate' exclusion from analysis/upload carries over. + target: photo_pipeline/services/duplicates.py + tests: [test_pa_hashing_dedup::test_cluster_and_mark_duplicates_largest_is_canonical] + status: characterized + + - id: pa-list-duplicates + area: ui + source: {file: photo_analyzer.py, symbols: [list_duplicates]} + classification: replace + rationale: Console report; superseded by the duplicate-review API/UI (US01-06). + target: photo_pipeline/api/routes + frontend duplicate review + pending_story: US01-06 + status: pending + + - id: pa-reconcile + area: database + source: {file: photo_analyzer.py, symbols: [reconcile_moved]} + classification: refactor + rationale: > + SHA-1-keyed move/rename reconciliation preserving analysis + EXIF state. + Refactor to update assets.current_path by asset_id and append an + asset_paths history row instead of rewriting the path in place. + target: photo_pipeline/services/inventory.py + tests: [test_pa_hashing_dedup::test_reconcile_moved_preserves_row_by_sha1] + status: characterized + + - id: pa-db-schema + area: database + source: {file: photo_analyzer.py, symbols: [SCHEMA, get_db, _migrate_schema]} + classification: refactor + rationale: > + photos table + FTS5 + idempotent ALTER migration become the migration + baseline; new schema adds assets/asset_paths/stage state per concept. + Existing data must migrate losslessly (US01-02). + target: photo_pipeline migrations (Alembic) + tests: [test_pa_db::test_schema_and_migration_idempotent] + status: characterized + + - id: pa-db-status + area: error + source: {file: photo_analyzer.py, symbols: [upsert_pending, mark_analyzed, mark_error, mark_exif_written, get_pending, get_analyzed_no_exif]} + classification: refactor + rationale: > + Status lifecycle (pending→analyzed→exif_written; error rows auto-retried + on the next run; INSERT OR IGNORE never downgrades; duplicates never + queued) is the resume-safety contract — preserved as asset_stage_states + transitions keyed by asset_id. + target: photo_pipeline/repositories + services/analysis.py + tests: + - test_pa_db::test_status_lifecycle_pending_analyzed_exif_written + - test_pa_db::test_mark_error_and_retry_via_get_pending + - test_pa_db::test_mark_analyzed_clears_error + - test_pa_db::test_upsert_pending_never_downgrades + status: characterized + + - id: pa-fts + area: database + source: {file: photo_analyzer.py, symbols: [photos_fts]} + classification: reuse + rationale: FTS5 table + sync triggers already work; carried into the new schema. + target: photo_pipeline migrations (Alembic) + tests: [test_pa_db::test_fts_kept_in_sync_by_triggers] + status: characterized + + - id: pa-imaging + area: imaging + source: {file: photo_analyzer.py, symbols: [prepare_image, MAX_LONG_EDGE]} + classification: extract + rationale: > + RGB-normalize (drops alpha, converts HEIC), LANCZOS resize to 2048px + long-edge, JPEG q85 base64 — the provider-input contract. Truncated-image + tolerance (ImageFile.LOAD_TRUNCATED_IMAGES) carries with it. + target: photo_pipeline/integrations/vision.py + tests: + - test_pa_imaging::test_prepare_image_small_passthrough_jpeg + - test_pa_imaging::test_prepare_image_resizes_to_max_long_edge + - test_pa_imaging::test_prepare_image_converts_png_alpha_to_rgb_jpeg + status: characterized + + - id: pa-repair + area: error + source: {file: photo_analyzer.py, symbols: [repair_image, _REPAIRABLE_ERRORS, _log_repair]} + classification: refactor + rationale: > + Re-encode structurally broken images via sips (macOS-only — portability + note), .orig backup, atomic replace, repairs.jsonl audit. Needs corrupt + fixtures to characterize; lands with media hardening. + target: photo_pipeline/integrations/exiftool.py + pending_story: US07-03 + status: pending + + - id: pa-vision + area: vision + source: {file: photo_analyzer.py, symbols: [analyze_image, ANALYSIS_PROMPT, LLM_BASE_URL, LLM_MODEL]} + classification: extract + rationale: > + OpenAI-compatible request construction, base64 image_url payload, JSON + response validation, 429/503 retry with exponential backoff. Prompt and + model/config version must be persisted per analysis_runs. Characterized + against a deterministic fake provider when the analysis service is ported. + target: photo_pipeline/integrations/vision.py + pending_story: US02-06 + status: pending + + - id: pa-throttle + area: logging + source: {file: photo_analyzer.py, symbols: [_record_ratelimit, _ratelimit_recent, write_throttle_summary, _rpd_load, _rpd_increment, _rpd_count]} + classification: refactor + rationale: > + Rolling throttle window + persistent throttle_events.jsonl + RPD day + counter become job metrics/events on the durable job model. + target: photo_pipeline/jobs/coordinator.py + pending_story: US02-02 + status: pending + + - id: pa-nsfw-filter + area: nsfw + source: {file: photo_analyzer.py, symbols: [_rec_has_nsfw, filter_nsfw_tagged]} + classification: replace + rationale: > + Intentional delta — donor fails OPEN (exiftool hiccup → analyze + everything), which violates the concept privacy invariant; the new gate + is the DB safety decision and fails CLOSED (only confirmed sfw reaches + the provider). EXIF keyword reading survives as evidence/verification, + not as the gate. Current behavior captured for migration. + target: photo_pipeline/services/safety.py + tests: + - test_pa_exif::test_rec_has_nsfw_list_scalar_case + - test_pa_exif::test_filter_nsfw_tagged_splits_and_normalizes + - test_pa_exif::test_filter_nsfw_tagged_empty_list_noop + status: characterized + + - id: pa-exif-caption + area: exif + source: {file: photo_analyzer.py, symbols: [build_exif_caption, build_exif_caption_from_result]} + classification: refactor + rationale: > + Caption format ('desc | Tags: … | Mood: … | Location: … | ~year') is + what 25k+ photos already carry — preserved for compatibility. Intentional + delta: new writes go into a managed 'AI:' segment so re-analysis can + replace only its own text (concept EXIF ownership). + target: photo_pipeline/services/exif projection + tests: + - test_pa_exif::test_caption_golden_full + - test_pa_exif::test_caption_golden_sparse + status: characterized + + - id: pa-exif-read + area: exif + source: {file: photo_analyzer.py, symbols: [read_existing_exif]} + classification: extract + rationale: Targeted exiftool JSON read of the four owned fields, 30s timeout, fail-empty. + target: photo_pipeline/integrations/exiftool.py + tests: [test_pa_exif::test_write_exif_roundtrip_and_idempotent] + status: characterized + + - id: pa-exif-write + area: exif + source: {file: photo_analyzer.py, symbols: [write_exif]} + classification: refactor + rationale: > + Merge-don't-overwrite semantics are the crown jewels: caption prepended to + existing text (idempotent — never appended twice), keywords merged + + order-preserving dedup, -m -overwrite_original, repair-and-retry on + structurally broken files. Refactored to add the concept's read-back + verification + non-owned-field snapshot comparison. sys.exit on missing + exiftool becomes a service error. + target: photo_pipeline/integrations/exiftool.py + services/exif checkpoints + tests: + - test_pa_exif::test_write_exif_roundtrip_and_idempotent + - test_pa_exif::test_write_exif_preserves_existing_metadata + status: characterized + + - id: pa-run-loop + area: cancellation + source: {file: photo_analyzer.py, symbols: [run_analysis, _run_folder_loop, run_exif_write, _stop, _handle_sigint]} + classification: refactor + rationale: > + Folder-grouped worker pool, per-item DB commits, cooperative stop event + checked between items, double-SIGINT force quit — becomes the durable + JobRunner worker loop with the same drain-and-resume semantics. + target: photo_pipeline/jobs/worker.py + pending_story: US02-02 + status: pending + + - id: pa-ui-terminal + area: ui + source: {file: photo_analyzer.py, symbols: [_Dashboard, _LiveRenderer, _PlainRenderer, _make_progress, _stats_panel, _feed_panel, _tokens_panel, _menu_panel, _read_key, _key_listener, print_stats, fit_label]} + classification: replace + rationale: > + Rich terminal dashboard (Live layout, hotkey menu, termios key reader) is + superseded by the web Workflow/Analyze views. The *data* it renders + (album progress, feed, tokens, ETA) survives via webapp.runner.progress + (see wa-runner). fit_label/print_stats die with it. + target: frontend Analyze view (Phase B) + tests: [test_pa_album::test_fit_label_left_truncates] + status: characterized + + - id: pa-album-label + area: ui + source: {file: photo_analyzer.py, symbols: [album_label]} + classification: extract + rationale: > + Album = leaf folder relative to library, '(root)' at root, bare parent + outside the library. The single album-identity rule for stats, proposals + and folder-as-album upload. webapp.query.album_of is a copy — consolidate + to one implementation (see wa-album-of). + target: photo_pipeline/services/albums.py + tests: + - test_pa_album::test_album_label_leaf_folder_relative + - test_pa_album::test_webapp_album_of_mirrors_album_label + status: characterized + + - id: pa-config + area: configuration + source: {file: photo_analyzer.py, symbols: [load_env_file, _env_str, _env_int, _env_bool, ENV_FILE]} + classification: refactor + rationale: > + photo_analyzer.env parsing (shell env wins, export prefix, quoted values, + inline-comment rules) feeds the typed config module; same file keeps + working so the transition needs no re-setup. + target: photo_pipeline/config.py + tests: + - test_pa_config::test_load_env_file_parsing + - test_pa_config::test_env_helpers + status: characterized + + - id: pa-logging + area: logging + source: {file: photo_analyzer.py, symbols: [log_history, _history_handler, _rich_handler]} + classification: replace + rationale: > + Module-level triple-handler logging (console + info log + debug log) and + the JSONL history logger become structured JSON logging with job_id/ + asset_id and job_events rows; per-photo history maps to job events. + target: photo_pipeline structured logging + jobs/job_events + pending_story: US02-02 + status: pending + + - id: pa-balance + area: vision + source: {file: photo_analyzer.py, symbols: [fetch_balance, query_balance, check_quota]} + classification: extract + rationale: > + Provider balance/quota probes (report 'unsupported' on providers without + the endpoint). Network-bound; characterized against the fake provider. + target: photo_pipeline/services/analysis.py + pending_story: US02-06 + status: pending + + - id: pa-cli + area: configuration + source: {file: photo_analyzer.py, symbols: [main]} + classification: replace + rationale: > + argparse surface is superseded by the API; flags map to job configs + (documented in WEBAPP_CONCEPT.md §8 parity table). Transitional CLI calls + the shared services until archival (E07). + target: photo_pipeline/api + transitional CLI + pending_story: US07-01 + status: pending + + # ── nsfwtag/ ─────────────────────────────────────────────────────────────── + - id: nt-discovery + area: discovery + source: {file: nsfwtag/scoring.py, symbols: [discover_images]} + classification: refactor + rationale: > + Merged into the shared discovery. Known divergence captured: nsfwtag EXTS + lacks .tiff/.tif, is non-recursive by default, and dedupes by resolved + path (symlink guard) — the shared service adopts the superset + symlink + dedup, always recursive, plus _IGNORE policy (donor has none). + target: photo_pipeline/services/inventory.py + tests: + - test_nsfwtag::test_exts_contract_differs_from_photo_analyzer + - test_nsfwtag::test_discover_images_flat_recursive_limit + - test_nsfwtag::test_discover_images_dedupes_symlinked_paths + status: characterized + + - id: nt-score-cache + area: nsfw + source: {file: nsfwtag/scoring.py, symbols: [load_cache, _save_cache, CSV_NAME]} + classification: replace + rationale: > + nsfw_scores.csv stops being the source of truth (concept: DB state). + Format characterized (4-decimal scores, bad rows dropped) because the + existing CSV must migrate into assets.safety_score. + target: photo_pipeline/repositories (safety), CSV import in US01-02 migration + tests: [test_nsfwtag::test_score_cache_roundtrip_and_tolerance] + status: characterized + + - id: nt-score-model + area: nsfw + source: {file: nsfwtag/scoring.py, symbols: [score_images, MODEL_ID, BATCH]} + classification: extract + rationale: > + On-device ViT scoring: MPS/CPU pick, model-defined input size (not + hardcoded — survives model swap), batch inference where one bad batch + never discards prior scores, periodic checkpoint every 500. Cache-hit + short-circuit (model never loads when nothing is new) characterized now; + inference path needs the local model + deterministic fake. + target: photo_pipeline/integrations/nsfw_model.py + tests: [test_nsfwtag::test_score_images_cache_hit_skips_model] + status: characterized + + - id: nt-exif-keyword + area: exif + source: {file: nsfwtag/exif.py, symbols: [write_keyword, remove_keyword]} + classification: extract + rationale: > + Idempotent add/remove via '-Keywords-=x -Keywords+=x' (no duplicates on + re-run), touching only Keywords+Subject. Basis of the safety EXIF + checkpoint; the mutual-exclusion write (sfw removed when nsfw added and + vice versa) is the concept's addition on top. + target: photo_pipeline/services/safety.py + integrations/exiftool.py + tests: [test_nsfwtag::test_write_remove_keyword_idempotent] + status: characterized + + - id: nt-exif-marks + area: nsfw + source: {file: nsfwtag/exif.py, symbols: [_read_keywords, read_tagged, read_marks]} + classification: extract + rationale: > + Batched exiftool read via stdin, lowercase normalization, and the safety + rule that a file carrying both keywords reads as nsfw (never as safe). + target: photo_pipeline/services/safety.py + tests: + - test_nsfwtag::test_write_remove_keyword_idempotent + - test_nsfwtag::test_read_marks_nsfw_wins_over_sfw + status: characterized + + - id: nt-exif-view + area: exif + source: {file: nsfwtag/exif.py, symbols: [read_exif, _EXIF_NOISE]} + classification: extract + rationale: > + Lightbox EXIF read: filesystem-noise filter, binary/oversize skip, signed + decimal GPS, Google-Maps link prepended. Backs the asset EXIF endpoint. + target: photo_pipeline/api (asset EXIF endpoint) + tests: [test_nsfwtag::test_read_exif_filters_noise_and_prepends_map] + status: characterized + + - id: nt-apply-list + area: nsfw + source: {file: nsfwtag/exif.py, symbols: [apply_list]} + classification: replace + rationale: > + Newline-list bulk tagging (nsfw_confirmed.txt flow) is superseded by DB + review decisions; the existing list is a one-time migration input. + target: photo_pipeline/services/safety.py (decision import in US01-02) + pending_story: US01-02 + status: pending + + - id: nt-ui + area: ui + source: {file: nsfwtag/server.py, symbols: [serve_review]} + classification: replace + rationale: > + stdlib ThreadingHTTPServer + token-injected review.html backend is + superseded by FastAPI. review.html's design tokens, folder tree, + threshold/score review flow, lightbox and keyboard model are the frontend + donor for the Safety view (preserved per concept §10; ported in US02-01). + target: photo_pipeline/api + frontend Safety view + pending_story: US02-01 + status: pending + + - id: nt-bench + area: nsfw + source: {file: nsfwtag/bench.py, symbols: [main]} + classification: replace + rationale: > + Dev-only model benchmark; archived without webapp replacement (recorded + basis of the AdamCodd model choice). No production caller. + target: none (archive as reference) + pending_story: US07-01 + status: pending + + # ── webapp/ ──────────────────────────────────────────────────────────────── + - id: wa-query-search + area: database + source: {file: webapp/query.py, symbols: [_fts_query, _where, search, _row_to_card, photo, _SORTS]} + classification: extract + rationale: > + Safe FTS5 MATCH building (tokenize → quoted prefix terms; punctuation + can't crash MATCH), facet WHERE composition, paged search with + relevance/sort modes, raw_response kept out of list payloads. + target: photo_pipeline/repositories (search) + api/routes + tests: + - test_webapp_query::test_fts_query_sanitization_goldens + - test_webapp_query::test_where_clause_goldens + - test_webapp_query::test_search_fts_prefix_match + - test_webapp_query::test_search_filters_and_browse + - test_webapp_query::test_search_paging + - test_webapp_query::test_photo_omits_raw_response + status: characterized + + - id: wa-query-aggregates + area: database + source: {file: webapp/query.py, symbols: [facets, stats, _col_counts, _people_buckets, _top_tags]} + classification: extract + rationale: > + Facet counts, year range, album tree, status/album progress (done = + analyzed + exif_written) and error listing — the Stats/Workflow data + model. + target: photo_pipeline/repositories + services/workflow.py + tests: + - test_webapp_query::test_facets_and_albums + - test_webapp_query::test_stats_album_progress_and_errors + status: characterized + + - id: wa-album-of + area: ui + source: {file: webapp/query.py, symbols: [album_of]} + classification: replace + rationale: > + Verbatim copy of photo_analyzer.album_label (equivalence characterized); + consolidated into the single albums-service implementation (pa-album-label). + target: photo_pipeline/services/albums.py + tests: [test_pa_album::test_webapp_album_of_mirrors_album_label] + status: characterized + + - id: wa-allowlist + area: ui + source: {file: webapp/query.py, symbols: [all_paths]} + classification: replace + rationale: > + Path-set allowlist for /img and /exif is superseded by asset-ID-addressed + endpoints that never accept a browser-supplied path (concept thumbnail + rules). The validate-before-serving intent carries over. + target: photo_pipeline/api (asset_id thumbnail/EXIF endpoints) + tests: [test_webapp_query::test_all_paths_is_the_image_endpoint_allowlist] + status: characterized + + - id: wa-runner + area: cancellation + source: {file: webapp/runner.py, symbols: [Runner, progress]} + classification: replace + rationale: > + Subprocess-driving-the-CLI job control is superseded by durable DB jobs + with a worker process. Two ideas carry over: progress derived from DB + counts (not job-private state) and single-mutating-job enforcement. + target: photo_pipeline/jobs/coordinator.py + pending_story: US02-02 + status: pending + + - id: wa-server + area: ui + source: {file: webapp/server.py, symbols: [serve]} + classification: replace + rationale: > + stdlib HTTP route ladder → FastAPI typed routes. Carried intents: + 127.0.0.1 bind, path validation against the DB, no secrets to the + browser. analyzer.html + page.py design (dark OLED tokens, Library/ + Analyze/Stats views) is frontend donor material per concept §10. + target: photo_pipeline/api/app.py + frontend + pending_story: US02-05 + status: pending diff --git a/donor_ledger/README.md b/donor_ledger/README.md deleted file mode 100644 index 4f84ad4..0000000 --- a/donor_ledger/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Donor ledger - -`ledger.json` is the machine-readable migration inventory for the existing Photo -Analyzer and NSFW Tagger implementations. It is intentionally kept beside the live -donors until the archival gates in `INTEGRATED_PIPELINE_CONCEPT.md` are satisfied. - -Each row records the current source symbol, the intended treatment, its destination in -the integrated application, the reason for that decision, and the stable -characterization-test IDs protecting current behavior. `migration_status` remains -`inventoried` in US01-01: this story characterizes code but does not relocate it. - -Classification meanings: - -- `reuse`: preserve the implementation essentially unchanged; -- `extract`: move the coherent behavior behind a thin service/integration adapter; -- `refactor`: preserve the useful behavior while changing an incompatible boundary; -- `replace`: intentionally supersede unsafe or architecturally incompatible behavior. - -Run the story suite with: - -```bash -work_item/scripts/python -m unittest discover -s tests/characterization -v -``` diff --git a/donor_ledger/ledger.json b/donor_ledger/ledger.json deleted file mode 100644 index 67f296f..0000000 --- a/donor_ledger/ledger.json +++ /dev/null @@ -1,180 +0,0 @@ -{ - "schema_version": 1, - "story": "US01-01", - "concept": "INTEGRATED_PIPELINE_CONCEPT.md", - "classifications": ["reuse", "extract", "refactor", "replace"], - "required_areas": [ - "discovery", "hashing", "imaging", "nsfw", "vision", "exif", - "database", "ui", "configuration", "logging", "cancellation", "error_behavior" - ], - "entries": [ - { - "id": "DONOR-001", "area": "discovery", - "sources": ["photo_analyzer.py::discover_photos", "photo_analyzer.py::SUPPORTED_EXTENSIONS"], - "classification": "extract", "target": "src/photo_pipeline/services/inventory.py", - "rationale": "Preserve sorted recursive discovery and supported formats behind the shared path policy.", - "intentional_changes": "All callers use stable asset IDs and one centralized boundary/exclusion policy.", - "test_ids": ["CHAR-001"], "migration_status": "inventoried" - }, - { - "id": "DONOR-002", "area": "discovery", - "sources": ["nsfwtag/scoring.py::discover_images", "nsfwtag/__init__.py::EXTS"], - "classification": "replace", "target": "src/photo_pipeline/services/inventory.py", - "rationale": "Its shallow/recursive switch and stable ordering are useful, but a second discovery policy and narrower formats conflict with the shared inventory.", - "intentional_changes": "Safety consumes canonical inventory assets rather than walking paths itself.", - "test_ids": ["CHAR-001"], "migration_status": "inventoried" - }, - { - "id": "DONOR-003", "area": "hashing", - "sources": ["photo_analyzer.py::_sha1_file", "photo_analyzer.py::_phash_image", "photo_analyzer.py::ensure_hashes"], - "classification": "refactor", "target": "src/photo_pipeline/services/inventory.py", - "rationale": "Retain streaming exact hashing and DCT pHash behavior while adding SHA-256, normalized pixels, versions, bounded workers, and stable IDs.", - "intentional_changes": "SHA-256 becomes byte identity; pHash is evidence rather than automatic fuzzy exclusion.", - "test_ids": ["CHAR-002"], "migration_status": "inventoried" - }, - { - "id": "DONOR-004", "area": "hashing", - "sources": ["photo_analyzer.py::cluster_duplicates", "photo_analyzer.py::mark_duplicates", "photo_analyzer.py::reconcile_moved"], - "classification": "refactor", "target": "src/photo_pipeline/services/duplicates.py", - "rationale": "Keep Hamming-distance evidence and reconciliation knowledge, but path-keyed union-find and automatic largest-file selection do not meet review and identity rules.", - "intentional_changes": "Exact matches may be recommended; fuzzy matches require durable human decisions and negative links.", - "test_ids": ["CHAR-002", "CHAR-003"], "migration_status": "inventoried" - }, - { - "id": "DONOR-005", "area": "imaging", - "sources": ["photo_analyzer.py::prepare_image"], - "classification": "extract", "target": "src/photo_pipeline/services/analysis.py", - "rationale": "The in-memory RGB conversion, long-edge resize, and JPEG normalization are proven analysis preparation behavior.", - "intentional_changes": "Orientation, color profile, pixel limits, and decoder version become explicit and tested.", - "test_ids": ["CHAR-002"], "migration_status": "inventoried" - }, - { - "id": "DONOR-006", "area": "imaging", - "sources": ["nsfwtag/server.py::serve_review", "nsfwtag/webapp.py::render_page"], - "classification": "refactor", "target": "src/photo_pipeline/services/thumbnails.py", - "rationale": "Preserve useful review presentation behavior, but arbitrary path image serving cannot be the integrated thumbnail boundary.", - "intentional_changes": "Managed oriented thumbnails are resolved only by asset ID and bounded size.", - "test_ids": ["CHAR-007"], "migration_status": "inventoried" - }, - { - "id": "DONOR-007", "area": "nsfw", - "sources": ["nsfwtag/scoring.py::score_images", "nsfwtag/__init__.py::MODEL_ID", "nsfwtag/__init__.py::DEFAULT_THRESHOLD"], - "classification": "extract", "target": "src/photo_pipeline/integrations/nsfw_model.py", - "rationale": "Retain the selected local model, batching, score interpretation, and error isolation behind an injectable integration.", - "intentional_changes": "SQLite is authoritative; CSV becomes optional export and only canonical assets are scored.", - "test_ids": ["CHAR-006"], "migration_status": "inventoried" - }, - { - "id": "DONOR-008", "area": "nsfw", - "sources": ["nsfwtag/__main__.py::main", "nsfw_tag.py::main"], - "classification": "refactor", "target": "src/photo_pipeline/services/safety.py", - "rationale": "Keep CLI compatibility temporarily while moving orchestration to durable jobs and explicit review decisions.", - "intentional_changes": "The API never invokes the legacy CLI and cloud analysis requires confirmed SFW state.", - "test_ids": ["CHAR-008"], "migration_status": "inventoried" - }, - { - "id": "DONOR-009", "area": "vision", - "sources": ["photo_analyzer.py::ANALYSIS_PROMPT", "photo_analyzer.py::analyze_image"], - "classification": "extract", "target": "src/photo_pipeline/integrations/vision.py", - "rationale": "Preserve provider-compatible image requests, prompt context, fenced-JSON handling, token accounting, and retry categories.", - "intentional_changes": "Typed validation, persisted prompt/model versions, bounded jittered retries, and stale-result fencing are added.", - "test_ids": ["CHAR-005"], "migration_status": "inventoried" - }, - { - "id": "DONOR-010", "area": "exif", - "sources": ["photo_analyzer.py::build_exif_caption_from_result", "photo_analyzer.py::write_exif", "photo_analyzer.py::read_existing_exif"], - "classification": "refactor", "target": "src/photo_pipeline/integrations/exiftool.py", - "rationale": "Retain additive caption/tag projection and exiftool knowledge while making ownership and read-back verification explicit.", - "intentional_changes": "Each metadata stage snapshots non-owned fields, verifies its projection, and refreshes post-write hashes.", - "test_ids": ["CHAR-004", "CHAR-009"], "migration_status": "inventoried" - }, - { - "id": "DONOR-011", "area": "exif", - "sources": ["nsfwtag/exif.py::write_keyword", "nsfwtag/exif.py::remove_keyword", "nsfwtag/exif.py::read_marks"], - "classification": "extract", "target": "src/photo_pipeline/integrations/exiftool.py", - "rationale": "The idempotent keyword commands and mutually exclusive review semantics are reusable in a shared adapter.", - "intentional_changes": "Both SFW and NSFW decisions are persisted and verified without removing unrelated metadata.", - "test_ids": ["CHAR-009"], "migration_status": "inventoried" - }, - { - "id": "DONOR-012", "area": "database", - "sources": ["photo_analyzer.py::SCHEMA", "photo_analyzer.py::get_db", "photo_analyzer.py::_migrate_schema"], - "classification": "refactor", "target": "src/photo_pipeline/models/", - "rationale": "Preserve historical schema and FTS migration knowledge while introducing SQLAlchemy repositories, Alembic, WAL, foreign keys, and stable asset identity.", - "intentional_changes": "Paths become occurrences, not primary identity; migrations are versioned and transactional where possible.", - "test_ids": ["CHAR-003"], "migration_status": "inventoried" - }, - { - "id": "DONOR-013", "area": "database", - "sources": ["webapp/query.py::search", "webapp/query.py::facets", "webapp/query.py::stats"], - "classification": "extract", "target": "src/photo_pipeline/repositories/assets.py", - "rationale": "FTS query sanitization, paging, facets, album labels, and stats are useful read-model behavior.", - "intentional_changes": "Repositories own SQL and return typed stable-ID projections.", - "test_ids": ["CHAR-003"], "migration_status": "inventoried" - }, - { - "id": "DONOR-014", "area": "ui", - "sources": ["nsfwtag/webapp.py::folder_tree_html", "nsfwtag/webapp.py::render_page", "nsfwtag/review.html"], - "classification": "refactor", "target": "frontend/js/views/safety.js", - "rationale": "Preserve folder navigation, score cards, review controls, lightbox interaction, and visual language.", - "intentional_changes": "Semantic static HTML/CSS/JS fetch JSON from /api/v1; Python no longer renders operational cards.", - "test_ids": ["CHAR-007"], "migration_status": "inventoried" - }, - { - "id": "DONOR-015", "area": "ui", - "sources": ["webapp/page.py::render_page", "webapp/server.py::serve"], - "classification": "replace", "target": "frontend/index.html", - "rationale": "The design is a donor, but server-generated application state and ad-hoc HTTP routing conflict with the application-shell/API architecture.", - "intentional_changes": "FastAPI serves a data-free shell and versioned JSON/SSE endpoints.", - "test_ids": ["CHAR-007"], "migration_status": "inventoried" - }, - { - "id": "DONOR-016", "area": "configuration", - "sources": ["photo_analyzer.py::load_env_file", "photo_analyzer.py::_env_int", "photo_analyzer.py::_env_bool"], - "classification": "replace", "target": "src/photo_pipeline/config.py", - "rationale": "Environment precedence is useful, but a manual untyped parser cannot validate the integrated app's paths, secrets, and worker settings.", - "intentional_changes": "Pydantic settings provide typed validation and secret references.", - "test_ids": ["CHAR-010"], "migration_status": "inventoried" - }, - { - "id": "DONOR-017", "area": "logging", - "sources": ["photo_analyzer.py::log_history", "photo_analyzer.py::_record_ratelimit"], - "classification": "refactor", "target": "src/photo_pipeline/jobs/events.py", - "rationale": "Per-item history, token evidence, and throttle diagnostics remain valuable operational events.", - "intentional_changes": "Structured durable events carry job, asset, attempt, and operation IDs with retention controls.", - "test_ids": ["CHAR-011"], "migration_status": "inventoried" - }, - { - "id": "DONOR-018", "area": "cancellation", - "sources": ["photo_analyzer.py::_handle_sigint", "webapp/runner.py::Runner"], - "classification": "replace", "target": "src/photo_pipeline/jobs/coordinator.py", - "rationale": "The cooperative stop intent is correct, but in-memory events and subprocess ownership do not survive restart or provide fencing.", - "intentional_changes": "Cancellation is a durable job transition checked between items and recovered by heartbeat/lease state.", - "test_ids": ["CHAR-012"], "migration_status": "inventoried" - }, - { - "id": "DONOR-019", "area": "error_behavior", - "sources": ["photo_analyzer.py::mark_error", "photo_analyzer.py::analyze_image", "photo_analyzer.py::write_exif"], - "classification": "refactor", "target": "src/photo_pipeline/services/analysis.py", - "rationale": "Keep item-level failures, retry categories, raw-response diagnostics, and explicit EXIF failure behavior.", - "intentional_changes": "Errors gain stable codes, attempts, retryability, and unknown/divergent states rather than broad status strings.", - "test_ids": ["CHAR-005", "CHAR-010"], "migration_status": "inventoried" - }, - { - "id": "DONOR-020", "area": "error_behavior", - "sources": ["nsfwtag/scoring.py::score_images", "nsfwtag/exif.py::_read_keywords", "webapp/runner.py::Runner"], - "classification": "refactor", "target": "src/photo_pipeline/services/safety.py", - "rationale": "Per-file inference isolation and tolerant metadata reads are useful, while process errors need durable domain outcomes.", - "intentional_changes": "Failures are persisted per attempt; unavailable integrations block precisely and never imply a safety decision.", - "test_ids": ["CHAR-006", "CHAR-010"], "migration_status": "inventoried" - }, - { - "id": "DONOR-021", "area": "configuration", - "sources": ["photo_analyzer.py::main", "nsfwtag/__main__.py::main"], - "classification": "refactor", "target": "src/photo_pipeline/__main__.py", - "rationale": "The current flags document supported workflows and compatibility needs, but long operations must become service-backed durable jobs.", - "intentional_changes": "Compatibility CLIs call shared services; API and worker management use typed commands.", - "test_ids": ["CHAR-008"], "migration_status": "inventoried" - } - ] -} diff --git a/tests/characterization/conftest.py b/tests/characterization/conftest.py new file mode 100644 index 0000000..17e6842 --- /dev/null +++ b/tests/characterization/conftest.py @@ -0,0 +1,107 @@ +"""Shared fixtures for donor characterization tests (US01-01). + +Builds a small deterministic synthetic photo library in a temp dir — never the +real library, never _IGNORE/ contents. Every image has a stable logical fixture +ID (FX_*) so captured outputs stay keyed to identity, not to paths. +""" +import shutil +import sqlite3 +import sys +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO)) + +EXIFTOOL = shutil.which("exiftool") + +# Stable logical fixture IDs → deterministic generation recipe (seed, size). +# Structured low-frequency content (blocks + gradient) so phash survives resize. +FIXTURES = { + "fx-blocks-01": {"seed": 1, "size": (800, 600)}, + "fx-blocks-02": {"seed": 2, "size": (800, 600)}, + "fx-blocks-03": {"seed": 3, "size": (640, 480)}, + "fx-blocks-04": {"seed": 4, "size": (800, 600)}, +} + + +def make_image_array(fixture_id: str) -> np.ndarray: + spec = FIXTURES[fixture_id] + w, h = spec["size"] + rng = np.random.default_rng(spec["seed"]) + base = np.zeros((h, w, 3), dtype=np.uint8) + for _ in range(6): + x0 = int(rng.integers(0, w - 150)) + y0 = int(rng.integers(0, h - 150)) + col = rng.integers(0, 256, 3) + base[y0:y0 + 150, x0:x0 + 150] = col + grad = np.linspace(0, 120, w, dtype=np.uint8) + base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255) + return base + + +def write_fixture(fixture_id: str, path: Path, quality: int = 95): + path.parent.mkdir(parents=True, exist_ok=True) + Image.fromarray(make_image_array(fixture_id)).save(path, quality=quality) + return path + + +@pytest.fixture +def img(tmp_path): + """One fx-blocks-01 JPEG in a temp dir.""" + return write_fixture("fx-blocks-01", tmp_path / "fx-blocks-01.jpg") + + +@pytest.fixture +def library(tmp_path): + """Temp library: + root/fx-blocks-01.jpg + root/album_a/fx-blocks-02.jpg + root/album_a/fx-blocks-02.PNG (uppercase ext, distinct file) + root/album_b/fx-blocks-03.jpg + root/_IGNORE/secret.jpg (must never be discovered) + root/album_b/.@__thumb/thumb.jpg (must never be discovered) + root/notes.txt (unsupported) + """ + root = tmp_path / "lib" + write_fixture("fx-blocks-01", root / "fx-blocks-01.jpg") + write_fixture("fx-blocks-02", root / "album_a" / "fx-blocks-02.jpg") + img = Image.fromarray(make_image_array("fx-blocks-02")) + (root / "album_a").mkdir(parents=True, exist_ok=True) + img.save(root / "album_a" / "fx-blocks-02.PNG") + write_fixture("fx-blocks-03", root / "album_b" / "fx-blocks-03.jpg") + write_fixture("fx-blocks-04", root / "_IGNORE" / "secret.jpg") + write_fixture("fx-blocks-04", root / "album_b" / ".@__thumb" / "thumb.jpg") + (root / "notes.txt").write_text("not an image") + return root + + +@pytest.fixture +def db(tmp_path): + import photo_analyzer as pa + conn = pa.get_db(str(tmp_path / "test.db")) + yield conn + conn.close() + + +def seed_analyzed(conn: sqlite3.Connection, path: str, **overrides): + """Insert a row and mark it analyzed with a deterministic result.""" + import photo_analyzer as pa + result = { + "description": "A red block pattern.", + "tags": ["blocks", "test"], + "people_count": 0, + "setting": "indoor", + "time_of_day": "unknown", + "season": "unknown", + "mood": "calm", + "location_hint": None, + "approx_year": None, + } + result.update(overrides) + pa.upsert_pending(conn, path) + pa.mark_analyzed(conn, path, result, raw="{}") + return result diff --git a/tests/characterization/test_donor_ledger.py b/tests/characterization/test_donor_ledger.py new file mode 100644 index 0000000..02f9cf6 --- /dev/null +++ b/tests/characterization/test_donor_ledger.py @@ -0,0 +1,88 @@ +"""Ledger lint (US01-01): every donor-ledger row must carry a real source +reference, a target location, and either existing characterization test IDs or +a real pending backlog story.""" +import re +from pathlib import Path + +import yaml + +REPO = Path(__file__).resolve().parents[2] +LEDGER = REPO / "donor_ledger.yaml" +STORIES = REPO / "delivery_backlog" / "stories" +TESTS_DIR = Path(__file__).resolve().parent + +CLASSIFICATIONS = {"reuse", "extract", "refactor", "replace"} +REQUIRED_AREAS = {"discovery", "hashing", "imaging", "nsfw", "vision", "exif", + "database", "ui", "configuration", "logging", "cancellation", + "error"} +STATUSES = {"characterized", "pending"} + + +def load_rows(): + rows = yaml.safe_load(LEDGER.read_text(encoding="utf-8"))["rows"] + assert rows, "empty ledger" + return rows + + +def collect_test_ids() -> set: + """module::function for every test in this suite.""" + ids = set() + for f in TESTS_DIR.glob("test_*.py"): + for m in re.finditer(r"^def (test_\w+)", f.read_text(encoding="utf-8"), + re.MULTILINE): + ids.add(f"{f.stem}::{m.group(1)}") + return ids + + +def test_rows_have_required_fields_and_unique_ids(): + rows = load_rows() + ids = [r["id"] for r in rows] + assert len(ids) == len(set(ids)), "duplicate row ids" + for r in rows: + for field in ("id", "area", "source", "classification", "rationale", + "target", "status"): + assert r.get(field), f"{r.get('id', '?')}: missing {field}" + assert r["classification"] in CLASSIFICATIONS, r["id"] + assert r["status"] in STATUSES, r["id"] + assert len(str(r["rationale"]).strip()) >= 20, \ + f"{r['id']}: rationale too thin to count as a documented reason" + + +def test_source_references_resolve(): + for r in load_rows(): + src = r["source"] + f = REPO / src["file"] + assert f.is_file(), f"{r['id']}: source file {src['file']} missing" + text = f.read_text(encoding="utf-8") + for sym in src["symbols"]: + assert sym in text, f"{r['id']}: symbol {sym!r} not found in {src['file']}" + + +def test_rows_have_tests_or_pending_story(): + known_tests = collect_test_ids() + for r in load_rows(): + tests = r.get("tests", []) + pending = r.get("pending_story") + assert tests or pending, f"{r['id']}: neither tests nor pending_story" + for t in tests: + assert t in known_tests, f"{r['id']}: unknown test id {t}" + if pending: + matches = list(STORIES.glob(f"{pending}-*.md")) + assert matches, f"{r['id']}: pending_story {pending} has no story file" + if r["status"] == "characterized": + assert tests, f"{r['id']}: characterized rows need test ids" + + +def test_all_required_areas_covered(): + covered = {r["area"] for r in load_rows()} + assert REQUIRED_AREAS <= covered, f"uncovered areas: {REQUIRED_AREAS - covered}" + assert covered <= REQUIRED_AREAS, f"unknown areas: {covered - REQUIRED_AREAS}" + + +def test_no_legacy_file_moved(): + # US01-01 explicitly forbids moving/archiving donors; the ledger's source + # files must all still exist at their original locations. + for donor in ("photo_analyzer.py", "nsfwtag/scoring.py", "nsfwtag/exif.py", + "nsfwtag/server.py", "webapp/query.py", "webapp/runner.py", + "webapp/server.py"): + assert (REPO / donor).is_file(), f"donor moved: {donor}" diff --git a/tests/characterization/test_donors.py b/tests/characterization/test_donors.py deleted file mode 100644 index a403489..0000000 --- a/tests/characterization/test_donors.py +++ /dev/null @@ -1,295 +0,0 @@ -from __future__ import annotations - -import base64 -import importlib -import json -import logging -import os -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest import mock - -from PIL import Image - - -REPO = Path(__file__).resolve().parents[2] -FIXTURES = REPO / "tests" / "fixtures" / "characterization" - - -class DonorCharacterizationTests(unittest.TestCase): - """Golden contracts for donor behavior; these are not target-architecture tests.""" - - @classmethod - def setUpClass(cls): - cls.import_dir = tempfile.TemporaryDirectory(prefix="donor-import-") - old_cwd = Path.cwd() - sys.path.insert(0, str(REPO)) - try: - os.chdir(cls.import_dir.name) - cls.analyzer = importlib.import_module("photo_analyzer") - cls.nsfw_scoring = importlib.import_module("nsfwtag.scoring") - cls.nsfw_exif = importlib.import_module("nsfwtag.exif") - cls.nsfw_webapp = importlib.import_module("nsfwtag.webapp") - cls.web_query = importlib.import_module("webapp.query") - finally: - os.chdir(old_cwd) - cls.manifest = json.loads((FIXTURES / "manifest.json").read_text())["fixtures"] - cls.expected = json.loads((FIXTURES / "expected_outputs.json").read_text())["outputs"] - - @classmethod - def tearDownClass(cls): - for handler in list(logging.getLogger().handlers) + list( - logging.getLogger("history").handlers - ): - handler.close() - cls.import_dir.cleanup() - - def setUp(self): - self.tmp = tempfile.TemporaryDirectory(prefix="donor-fixtures-") - self.root = Path(self.tmp.name) - - def tearDown(self): - self.analyzer._stop.clear() - self.tmp.cleanup() - - def _make_discovery_library(self) -> None: - recipes = {item["id"]: item for item in self.manifest} - colors = { - "DISC-ROOT-JPEG": (10, 20, 30), - "DISC-UPPER-JPEG": (40, 50, 60), - "DISC-NESTED-WEBP": (70, 80, 90), - "DISC-TIFF": (100, 110, 120), - } - sizes = { - "DISC-ROOT-JPEG": (12, 8), - "DISC-UPPER-JPEG": (8, 12), - "DISC-NESTED-WEBP": (9, 9), - "DISC-TIFF": (16, 4), - } - for fixture_id, color in colors.items(): - path = self.root / recipes[fixture_id]["relative_path"] - path.parent.mkdir(parents=True, exist_ok=True) - Image.new("RGB", sizes[fixture_id], color).save(path) - unsupported = self.root / recipes["DISC-UNSUPPORTED"]["relative_path"] - unsupported.write_text("not-an-image", encoding="utf-8") - - def _relative(self, paths) -> list[str]: - return [str(Path(path).relative_to(self.root)) for path in paths] - - def test_discovery_outputs(self): - self._make_discovery_library() - - analyzer_paths = self.analyzer.discover_photos(self.root) - nsfw_recursive = self.nsfw_scoring.discover_images(self.root, recursive=True) - nsfw_shallow = self.nsfw_scoring.discover_images(self.root, recursive=False) - - self.assertEqual( - self._relative(analyzer_paths), self.expected["analyzer_recursive_discovery"] - ) - self.assertEqual(self._relative(nsfw_recursive), self.expected["nsfw_recursive_discovery"]) - self.assertEqual(self._relative(nsfw_shallow), self.expected["nsfw_shallow_discovery"]) - - def test_image_preparation_and_hash_outputs(self): - raw_path = self.root / "hash.bin" - raw_path.write_bytes(b"photo-pipeline-donor") - self.assertEqual(self.analyzer._sha1_file(raw_path), self.expected["hash_bytes_sha1"]) - - gradient = Image.new("RGB", (40, 20)) - gradient.putdata( - [(x * 5 % 256, y * 11 % 256, (x + y) * 7 % 256) for y in range(20) for x in range(40)] - ) - image_path = self.root / "gradient.png" - gradient.save(image_path) - old_edge = self.analyzer.MAX_LONG_EDGE - try: - self.analyzer.MAX_LONG_EDGE = 16 - encoded, mime = self.analyzer.prepare_image(image_path) - finally: - self.analyzer.MAX_LONG_EDGE = old_edge - prepared_path = self.root / "prepared.jpg" - prepared_path.write_bytes(base64.b64decode(encoded)) - with Image.open(prepared_path) as prepared: - self.assertEqual(prepared.size, (16, 8)) - self.assertEqual(prepared.mode, "RGB") - self.assertEqual(mime, "image/jpeg") - phash = self.analyzer._phash_image(image_path) - self.assertRegex(phash or "", r"^[0-9a-f]{16}$") - self.assertEqual(phash, self.analyzer._phash_image(image_path)) - - def test_database_status_and_fts_outputs(self): - db_path = self.root / "characterization.sqlite" - conn = self.analyzer.get_db(str(db_path)) - self.addCleanup(conn.close) - photo_path = str(self.root / "lake.jpg") - self.analyzer.upsert_pending(conn, photo_path) - result = { - "description": "Three adults walk beside a lake.", - "tags": ["people", "lake", "summer"], - "people_count": 3, - "setting": "outdoor", - "time_of_day": "afternoon", - "season": "summer", - "mood": "relaxed", - "location_hint": "Como, Italy", - "approx_year": 2021, - } - self.analyzer.mark_analyzed(conn, photo_path, result, json.dumps(result)) - row = conn.execute("SELECT * FROM photos WHERE path = ?", (photo_path,)).fetchone() - self.assertEqual(row["status"], "analyzed") - self.assertEqual(json.loads(row["tags"]), result["tags"]) - found = self.web_query.search(conn, q="lake") - self.assertEqual(found["total"], 1) - self.assertEqual(found["rows"][0]["path"], photo_path) - self.analyzer.mark_exif_written(conn, photo_path) - self.assertEqual(conn.execute("SELECT status FROM photos").fetchone()[0], "exif_written") - - def test_caption_and_variant_outputs(self): - result = { - "description": "Three adults walk beside a lake.", - "tags": ["people", "lake", "summer"], - "mood": "relaxed", - "location_hint": "Como, Italy", - "approx_year": 2021, - } - self.assertEqual( - self.analyzer.build_exif_caption_from_result(result), self.expected["caption"] - ) - self.assertEqual( - self.analyzer._strip_variant_markers("IMG_0001-bearbeitet (1920x1080)"), - self.expected["variant_base"], - ) - - def test_vision_dry_run_output(self): - result, raw, tokens = self.analyzer.analyze_image( - None, self.root / "fixture.jpg", dry_run=True - ) - self.assertEqual(result["tags"], self.expected["dry_run_tags"]) - self.assertEqual(json.loads(raw), result) - self.assertEqual(tokens, {"prompt": 0, "completion": 0, "total": 0}) - - def test_nsfw_cache_output(self): - cache_path = self.root / "scores.csv" - values = self.expected["nsfw_cache"] - self.nsfw_scoring._save_cache(cache_path, values) - self.assertEqual(self.nsfw_scoring.load_cache(cache_path), values) - self.assertEqual( - cache_path.read_text(encoding="utf-8").splitlines(), - ["path,nsfw_score", "a.jpg,0.1250", "b.jpg,0.9876"], - ) - - def test_review_html_output(self): - logical = self.root / "album & one" / ".jpg" - page = self.nsfw_webapp.render_page([(logical, 0.8123), (logical, 0.7)], 0.6) - self.assertIn("<portrait>.jpg", page) - self.assertIn("album & one", page) - self.assertIn(f'data-score="{0.8123:.4f}"', page) - self.assertIn(self.expected["ui_total"], page) - self.assertIn(self.expected["ui_threshold"], page) - self.assertEqual(page.count('class="card"'), 1) - - def test_cli_entry_point_help(self): - env = os.environ.copy() - env["PYTHONPATH"] = str(REPO) - commands = [ - ([sys.executable, str(REPO / "photo_analyzer.py"), "--help"], "--group-variants"), - ([sys.executable, str(REPO / "nsfw_tag.py"), "--help"], "--review-min"), - ([sys.executable, "-m", "nsfwtag", "--help"], "--threshold"), - ] - for command, marker in commands: - with self.subTest(command=command): - result = subprocess.run( - command, - cwd=self.root, - env=env, - capture_output=True, - text=True, - timeout=30, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn(marker, result.stdout) - - def test_exif_command_contracts(self): - calls = [] - - def fake_run(command, **kwargs): - calls.append(command) - if "-json" in command: - return subprocess.CompletedProcess( - command, - 0, - stdout=json.dumps( - [{"ImageDescription": "User caption", "Keywords": ["family"]}] - ), - stderr="", - ) - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - - with mock.patch.object(self.analyzer.subprocess, "run", side_effect=fake_run): - self.assertTrue( - self.analyzer.write_exif("fixture.jpg", "AI caption", ["family", "lake"]) - ) - write_command = calls[-1] - self.assertIn("-overwrite_original", write_command) - self.assertIn("-ImageDescription=AI caption | User caption", write_command) - self.assertEqual(write_command.count("-Keywords=family"), 1) - self.assertEqual(write_command.count("-Keywords=lake"), 1) - - with mock.patch.object( - self.nsfw_exif.subprocess, - "run", - return_value=subprocess.CompletedProcess([], 0, stdout="", stderr=""), - ) as run: - self.assertTrue(self.nsfw_exif.write_keyword("fixture.jpg")) - command = run.call_args.args[0] - self.assertIn("-Keywords-=nsfw", command) - self.assertIn("-Keywords+=nsfw", command) - self.assertIn("-Subject-=nsfw", command) - self.assertIn("-Subject+=nsfw", command) - - def test_error_and_configuration_fallbacks(self): - with mock.patch.dict(os.environ, {"MAX_WORKERS": "not-an-int"}): - self.assertEqual(self.analyzer._env_int("MAX_WORKERS", 3), 3) - invalid = subprocess.CompletedProcess([], 0, stdout="not-json", stderr="") - with mock.patch.object(self.nsfw_exif.subprocess, "run", return_value=invalid): - self.assertEqual(self.nsfw_exif._read_keywords(["fixture.jpg"]), {}) - - def test_history_log_shape(self): - sink = mock.Mock() - result = { - "description": "Fixture description", - "tags": ["fixture"], - "mood": "calm", - "setting": "indoor", - "people_count": 0, - "location_hint": None, - "approx_year": None, - } - with mock.patch.object(self.analyzer.history_log, "info", sink): - self.analyzer.log_history( - "fixture.jpg", "analyzed", result=result, tokens={"total": 12} - ) - entry = json.loads(sink.call_args.args[0]) - self.assertEqual(entry["path"], "fixture.jpg") - self.assertEqual(entry["status"], "analyzed") - self.assertEqual(entry["tokens_total"], 12) - self.assertEqual(entry["tags"], ["fixture"]) - self.assertRegex(entry["ts"], r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$") - - def test_cooperative_cancellation_contract(self): - self.analyzer._stop.clear() - fake_stderr = SimpleNamespace(write=mock.Mock(), flush=mock.Mock()) - with ( - mock.patch.object(self.analyzer.sys, "__stderr__", fake_stderr), - mock.patch.object(self.analyzer.log, "warning"), - ): - self.analyzer._handle_sigint(None, None) - self.assertTrue(self.analyzer._stop.is_set()) - fake_stderr.write.assert_called_once_with("\a") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/characterization/test_ledger.py b/tests/characterization/test_ledger.py deleted file mode 100644 index f016cce..0000000 --- a/tests/characterization/test_ledger.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -import ast -import json -import unittest -from pathlib import Path - -REPO = Path(__file__).resolve().parents[2] -LEDGER = REPO / "donor_ledger" / "ledger.json" -TRACEABILITY = REPO / "tests" / "characterization" / "test_traceability.json" -DONOR_TESTS = REPO / "tests" / "characterization" / "test_donors.py" - - -def source_symbols(path: Path) -> set[str]: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - names = set() - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names.add(node.name) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names.update(t.id for t in targets if isinstance(t, ast.Name)) - elif isinstance(node, ast.ImportFrom): - names.update(alias.asname or alias.name for alias in node.names) - return names - - -class DonorLedgerLintTests(unittest.TestCase): - def test_ledger_is_complete_and_resolvable(self): - ledger = json.loads(LEDGER.read_text(encoding="utf-8")) - traceability = json.loads(TRACEABILITY.read_text(encoding="utf-8")) - entries = ledger["entries"] - donor_test_symbols = source_symbols(DONOR_TESTS) - - self.assertEqual(ledger["story"], "US01-01") - self.assertEqual(len({entry["id"] for entry in entries}), len(entries)) - self.assertEqual(set(ledger["required_areas"]), {entry["area"] for entry in entries}) - self.assertFalse((REPO / "legacy_cli_archive").exists()) - - allowed = set(ledger["classifications"]) - for entry in entries: - with self.subTest(entry=entry["id"]): - self.assertIn(entry["classification"], allowed) - self.assertTrue(entry["rationale"].strip()) - self.assertTrue(entry["target"].strip()) - self.assertTrue(entry["test_ids"]) - self.assertEqual(entry["migration_status"], "inventoried") - for test_id in entry["test_ids"]: - self.assertIn(test_id, traceability) - class_name, method_name = traceability[test_id].split(".", 1) - self.assertEqual(class_name, "DonorCharacterizationTests") - self.assertIn(method_name, donor_test_symbols) - for source in entry["sources"]: - file_name, separator, symbol = source.partition("::") - source_path = REPO / file_name - self.assertTrue(source_path.is_file(), source) - if separator: - self.assertIn(symbol, source_symbols(source_path), source) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/characterization/test_nsfwtag.py b/tests/characterization/test_nsfwtag.py new file mode 100644 index 0000000..e84081e --- /dev/null +++ b/tests/characterization/test_nsfwtag.py @@ -0,0 +1,95 @@ +"""Characterize nsfwtag donors: discovery, score cache, EXIF safety keywords. + +score_images' model inference is NOT run here (needs the ~350 MB local model); +its cache-reuse path is characterized instead — the model is only invoked for +paths missing from the cache. +""" +import subprocess + +import pytest + +import nsfwtag.exif as nexif +import nsfwtag.scoring as scoring +from nsfwtag import EXTS, KEYWORD +from conftest import EXIFTOOL, write_fixture + +needs_exiftool = pytest.mark.skipif(not EXIFTOOL, reason="exiftool not on PATH") + + +def test_exts_contract_differs_from_photo_analyzer(): + import photo_analyzer as pa + assert EXTS == {".jpg", ".jpeg", ".png", ".heic", ".heif", ".webp"} + assert pa.SUPPORTED_EXTENSIONS - EXTS == {".tiff", ".tif"}, \ + "known donor divergence: nsfwtag skips TIFF" + + +def test_discover_images_flat_recursive_limit(tmp_path): + a = write_fixture("fx-blocks-01", tmp_path / "a.jpg") + b = write_fixture("fx-blocks-02", tmp_path / "sub" / "b.jpg") + (tmp_path / "c.txt").write_text("x") + assert scoring.discover_images(tmp_path) == [a], "flat by default" + assert scoring.discover_images(tmp_path, recursive=True) == [a, b] + assert scoring.discover_images(tmp_path, recursive=True, limit=1) == [a] + + +def test_discover_images_dedupes_symlinked_paths(tmp_path): + a = write_fixture("fx-blocks-01", tmp_path / "a.jpg") + (tmp_path / "link.jpg").symlink_to(a) + found = scoring.discover_images(tmp_path) + assert len(found) == 1, "same resolved file listed once" + + +def test_score_cache_roundtrip_and_tolerance(tmp_path): + csv = tmp_path / "scores.csv" + scoring._save_cache(csv, {"/a.jpg": 0.91234, "/b.jpg": 0.1}) + cache = scoring.load_cache(csv) + assert cache == {"/a.jpg": 0.9123, "/b.jpg": 0.1}, "4-decimal persistence" + csv.write_text("path,nsfw_score\n/ok.jpg,0.5\n/bad.jpg,not-a-float\n") + assert scoring.load_cache(csv) == {"/ok.jpg": 0.5}, "bad rows dropped silently" + assert scoring.load_cache(tmp_path / "missing.csv") == {} + + +def test_score_images_cache_hit_skips_model(tmp_path): + img = write_fixture("fx-blocks-01", tmp_path / "a.jpg") + csv = tmp_path / "scores.csv" + scoring._save_cache(csv, {str(img): 0.42}) + scored, errors = scoring.score_images([img], csv) # would download a model on miss + assert scored == [(img, 0.42)] + assert errors == 0 + + +@needs_exiftool +def test_write_remove_keyword_idempotent(img): + assert nexif.write_keyword(str(img)) + assert nexif.write_keyword(str(img)), "second write is a no-op, not a dup" + r = subprocess.run(["exiftool", "-j", "-Keywords", "-Subject", str(img)], + capture_output=True, text=True) + assert r.stdout.count(KEYWORD) == 2, "once in Keywords, once in Subject" + assert nexif.read_tagged([str(img)]) == {str(img)} + assert nexif.remove_keyword(str(img)) + assert nexif.read_tagged([str(img)]) == set() + + +@needs_exiftool +def test_read_marks_nsfw_wins_over_sfw(img, tmp_path): + safe = write_fixture("fx-blocks-02", tmp_path / "safe.jpg") + subprocess.run(["exiftool", "-m", "-overwrite_original", + "-Keywords=nsfw", "-Keywords=sfw", str(img)], + check=True, capture_output=True) + subprocess.run(["exiftool", "-m", "-overwrite_original", + "-Keywords=sfw", str(safe)], check=True, capture_output=True) + marks = nexif.read_marks([str(img), str(safe)]) + assert marks["nsfw"] == {str(img)}, "both keywords → nsfw wins (never reads safe)" + assert marks["sfw"] == {str(safe)} + + +@needs_exiftool +def test_read_exif_filters_noise_and_prepends_map(img): + subprocess.run(["exiftool", "-m", "-overwrite_original", + "-GPSLatitude=48.8584", "-GPSLatitudeRef=N", + "-GPSLongitude=2.2945", "-GPSLongitudeRef=E", str(img)], + check=True, capture_output=True) + rec = nexif.read_exif(str(img)) + assert list(rec)[0] == "Map" + assert rec["Map"] == "https://www.google.com/maps?q=48.858400,2.294500" + assert "SourceFile" not in rec and "ExifToolVersion" not in rec diff --git a/tests/characterization/test_pa_album.py b/tests/characterization/test_pa_album.py new file mode 100644 index 0000000..bb23478 --- /dev/null +++ b/tests/characterization/test_pa_album.py @@ -0,0 +1,28 @@ +"""Characterize album labeling (donor: album_label, fit_label; webapp.query.album_of +mirrors album_label — asserted identical here).""" +from pathlib import Path + +import photo_analyzer as pa +from webapp import query + + +def test_album_label_leaf_folder_relative(): + lib = Path("/lib") + assert pa.album_label(Path("/lib/Urlaub/Rom/a.jpg"), lib) == "Urlaub/Rom" + assert pa.album_label(Path("/lib/Urlaub/Venedig/a.jpg"), lib) == "Urlaub/Venedig" + assert pa.album_label(Path("/lib/a.jpg"), lib) == "(root)" + assert pa.album_label(Path("/elsewhere/x/a.jpg"), lib) == "x", \ + "outside library → bare parent name" + assert pa.album_label(Path("/lib/x/a.jpg"), None) == "x" + + +def test_webapp_album_of_mirrors_album_label(): + lib = Path("/lib") + for p in ("/lib/Urlaub/Rom/a.jpg", "/lib/a.jpg", "/other/x/a.jpg"): + assert query.album_of(p, lib) == pa.album_label(Path(p), lib), p + + +def test_fit_label_left_truncates(): + assert pa.fit_label("short", 10) == "short" + assert pa.fit_label("Urlaub/Somewhere/Rom", 8) == "…ere/Rom" + assert len(pa.fit_label("Urlaub/Somewhere/Rom", 8)) == 8 diff --git a/tests/characterization/test_pa_config.py b/tests/characterization/test_pa_config.py new file mode 100644 index 0000000..aab825d --- /dev/null +++ b/tests/characterization/test_pa_config.py @@ -0,0 +1,46 @@ +"""Characterize configuration behavior (donor: load_env_file, _env_str, +_env_int, _env_bool).""" +import os + +import photo_analyzer as pa + + +def test_load_env_file_parsing(tmp_path, monkeypatch): + env = tmp_path / pa.ENV_FILE + env.write_text( + "# comment\n" + "PLAIN=value\n" + "export EXPORTED=yes\n" + 'QUOTED="hash # kept"\n' + "INLINE=val # comment stripped\n" + "TOKEN=pass#word\n" + "PRESET=file-loses\n" + "NOEQUALS\n" + ) + monkeypatch.chdir(tmp_path) + for k in ("PLAIN", "EXPORTED", "QUOTED", "INLINE", "TOKEN"): + monkeypatch.delenv(k, raising=False) + monkeypatch.setenv("PRESET", "shell-wins") + + used = pa.load_env_file() + assert used == env + assert os.environ["PLAIN"] == "value" + assert os.environ["EXPORTED"] == "yes", "'export ' prefix accepted" + assert os.environ["QUOTED"] == "hash # kept", "quoted # not a comment" + assert os.environ["INLINE"] == "val", "inline comment stripped" + assert os.environ["TOKEN"] == "pass#word", "mid-token # kept" + assert os.environ["PRESET"] == "shell-wins", "shell env always wins" + + +def test_env_helpers(monkeypatch): + monkeypatch.setenv("S", "") + assert pa._env_str("S", "fb") == "fb", "empty string falls back" + monkeypatch.setenv("I", "not-int") + assert pa._env_int("I", 7) == 7, "bad int falls back with warning" + monkeypatch.setenv("I", "42") + assert pa._env_int("I", 7) == 42 + for truthy in ("1", "true", "YES", "On"): + monkeypatch.setenv("B", truthy) + assert pa._env_bool("B") is True + monkeypatch.setenv("B", "0") + assert pa._env_bool("B") is False diff --git a/tests/characterization/test_pa_db.py b/tests/characterization/test_pa_db.py new file mode 100644 index 0000000..9e91aa0 --- /dev/null +++ b/tests/characterization/test_pa_db.py @@ -0,0 +1,68 @@ +"""Characterize DB layer (donor: get_db, _migrate_schema, status transitions, +get_pending, purge_excluded, prune_missing, FTS sync).""" +import photo_analyzer as pa +from conftest import seed_analyzed, write_fixture + + +def test_schema_and_migration_idempotent(tmp_path): + db_path = str(tmp_path / "x.db") + conn = pa.get_db(db_path) + conn.close() + conn = pa.get_db(db_path) # re-open runs migration again — must not fail + cols = {r["name"] for r in conn.execute("PRAGMA table_info(photos)")} + assert {"path", "status", "phash", "file_sha1", "dup_of", "description", + "tags", "raw_response", "analyzed_at", "exif_written_at"} <= cols + conn.close() + + +def test_status_lifecycle_pending_analyzed_exif_written(db): + seed_analyzed(db, "/x/a.jpg") + assert db.execute("SELECT status FROM photos").fetchone()["status"] == "analyzed" + pa.mark_exif_written(db, "/x/a.jpg") + assert db.execute("SELECT status FROM photos").fetchone()["status"] == "exif_written" + + +def test_mark_error_and_retry_via_get_pending(db): + pa.upsert_pending(db, "/x/a.jpg") + pa.mark_error(db, "/x/a.jpg", "boom") + row = db.execute("SELECT status, error_message FROM photos").fetchone() + assert (row["status"], row["error_message"]) == ("error", "boom") + # error rows are retried on the next run + assert pa.get_pending(db, reanalyze=False) == ["/x/a.jpg"] + + +def test_mark_analyzed_clears_error(db): + pa.upsert_pending(db, "/x/a.jpg") + pa.mark_error(db, "/x/a.jpg", "boom") + seed_analyzed(db, "/x/a.jpg") + assert db.execute("SELECT error_message FROM photos").fetchone()["error_message"] is None + + +def test_upsert_pending_never_downgrades(db): + seed_analyzed(db, "/x/a.jpg") + pa.upsert_pending(db, "/x/a.jpg") # INSERT OR IGNORE — no reset to pending + assert db.execute("SELECT status FROM photos").fetchone()["status"] == "analyzed" + + +def test_fts_kept_in_sync_by_triggers(db): + seed_analyzed(db, "/x/beach.jpg", description="A sunny beach with palm trees.") + hit = db.execute( + "SELECT path FROM photos_fts WHERE photos_fts MATCH 'beach'").fetchall() + assert [r["path"] for r in hit] == ["/x/beach.jpg"] + + +def test_purge_excluded_removes_ignore_and_thumb_rows(db): + for p in ("/lib/_IGNORE/a.jpg", "/lib/x/.@__thumb/b.jpg", "/lib/keep.jpg"): + pa.upsert_pending(db, p) + assert pa.purge_excluded(db) == 2 + assert [r["path"] for r in db.execute("SELECT path FROM photos")] == ["/lib/keep.jpg"] + + +def test_prune_missing_guards_unmounted_library(db, tmp_path): + pa.upsert_pending(db, "/gone/a.jpg") + # library root missing → refuses to prune (protects against unmounted volume) + assert pa.prune_missing(db, tmp_path / "not-there") == 0 + assert db.execute("SELECT COUNT(*) FROM photos").fetchone()[0] == 1 + # existing root → stale row is pruned + assert pa.prune_missing(db, tmp_path) == 1 + assert db.execute("SELECT COUNT(*) FROM photos").fetchone()[0] == 0 diff --git a/tests/characterization/test_pa_discovery.py b/tests/characterization/test_pa_discovery.py new file mode 100644 index 0000000..b4c51ba --- /dev/null +++ b/tests/characterization/test_pa_discovery.py @@ -0,0 +1,27 @@ +"""Characterize photo_analyzer discovery + path policy (donor: discover_photos).""" +import photo_analyzer as pa + + +def test_discover_excludes_ignore_and_thumbs(library): + found = pa.discover_photos(library) + names = [p.name for p in found] + assert "secret.jpg" not in names, "_IGNORE/ content must never be discovered" + assert "thumb.jpg" not in names, ".@__thumb/ content must never be discovered" + assert "notes.txt" not in names + + +def test_discover_finds_supported_including_uppercase(library): + names = sorted(p.name for p in pa.discover_photos(library)) + assert names == ["fx-blocks-01.jpg", "fx-blocks-02.PNG", + "fx-blocks-02.jpg", "fx-blocks-03.jpg"] + + +def test_discover_sorted_and_deduplicated(library): + found = pa.discover_photos(library) + assert found == sorted(set(found)) + + +def test_supported_extensions_contract(): + # Donor contract: these exact extensions (nsfwtag EXTS differs — no .tiff/.tif). + assert pa.SUPPORTED_EXTENSIONS == { + ".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"} diff --git a/tests/characterization/test_pa_exif.py b/tests/characterization/test_pa_exif.py new file mode 100644 index 0000000..7b0473c --- /dev/null +++ b/tests/characterization/test_pa_exif.py @@ -0,0 +1,87 @@ +"""Characterize EXIF behavior (donor: build_exif_caption_from_result, +read_existing_exif, write_exif merge/idempotency, _rec_has_nsfw, +filter_nsfw_tagged). Uses the real exiftool on synthetic temp files.""" +import subprocess + +import pytest + +import photo_analyzer as pa +from conftest import EXIFTOOL + +needs_exiftool = pytest.mark.skipif(not EXIFTOOL, reason="exiftool not on PATH") + +RESULT = { + "description": "A dog on a beach.", + "tags": ["dog", "beach"], + "mood": "joyful", + "location_hint": "Rome, Italy", + "approx_year": 2015, +} + + +def test_caption_golden_full(): + assert pa.build_exif_caption_from_result(RESULT) == ( + "A dog on a beach. | Tags: dog, beach | Mood: joyful | " + "Location: Rome, Italy | ~2015") + + +def test_caption_golden_sparse(): + assert pa.build_exif_caption_from_result( + {"description": "X.", "tags": [], "location_hint": None}) == "X." + + +def test_rec_has_nsfw_list_scalar_case(): + assert pa._rec_has_nsfw({"Keywords": ["beach", "NSFW "]}) + assert pa._rec_has_nsfw({"Subject": "nsfw"}) + assert not pa._rec_has_nsfw({"Keywords": ["sfw"], "Subject": ["beach"]}) + assert not pa._rec_has_nsfw({}) + + +@needs_exiftool +def test_write_exif_roundtrip_and_idempotent(img): + caption = pa.build_exif_caption_from_result(RESULT) + assert pa.write_exif(str(img), caption, RESULT["tags"]) + rec = pa.read_existing_exif(str(img)) + assert rec["ImageDescription"] == caption + assert rec["XPComment"] == caption + assert rec["Subject"] == RESULT["tags"] + assert rec["Keywords"] == RESULT["tags"] + + # second write with the same caption must not duplicate anything + assert pa.write_exif(str(img), caption, RESULT["tags"]) + rec2 = pa.read_existing_exif(str(img)) + assert rec2["ImageDescription"] == caption + assert rec2["Keywords"] == RESULT["tags"] + + +@needs_exiftool +def test_write_exif_preserves_existing_metadata(img): + subprocess.run( + ["exiftool", "-m", "-overwrite_original", "-Artist=Original Artist", + "-ImageDescription=User caption", "-Keywords=userkw", str(img)], + check=True, capture_output=True) + assert pa.write_exif(str(img), "AI caption", ["aitag"]) + rec = pa.read_existing_exif(str(img)) + # donor behavior: AI caption prepended, user caption kept after ' | ' + assert rec["ImageDescription"] == "AI caption | User caption" + kws = rec["Keywords"] if isinstance(rec["Keywords"], list) else [rec["Keywords"]] + assert "userkw" in kws and "aitag" in kws, "keywords merge, never replace" + full = subprocess.run(["exiftool", "-j", "-Artist", str(img)], + capture_output=True, text=True) + assert "Original Artist" in full.stdout, "non-owned field preserved" + + +@needs_exiftool +def test_filter_nsfw_tagged_splits_and_normalizes(img, tmp_path): + from conftest import write_fixture + clean = write_fixture("fx-blocks-02", tmp_path / "clean.jpg") + subprocess.run(["exiftool", "-m", "-overwrite_original", + "-Keywords=nsfw", "-Subject=nsfw", str(img)], + check=True, capture_output=True) + kept, skipped = pa.filter_nsfw_tagged([str(img), str(clean)]) + assert kept == [str(clean)] + assert skipped == [str(img)] + + +def test_filter_nsfw_tagged_empty_list_noop(): + assert pa.filter_nsfw_tagged([]) == ([], []) diff --git a/tests/characterization/test_pa_hashing_dedup.py b/tests/characterization/test_pa_hashing_dedup.py new file mode 100644 index 0000000..fac094f --- /dev/null +++ b/tests/characterization/test_pa_hashing_dedup.py @@ -0,0 +1,109 @@ +"""Characterize hashing + duplicate ledger (donor: _sha1_file, _phash_image, +ensure_hashes, cluster_duplicates, mark_duplicates, reconcile_moved). + +Complements the pre-existing root-level self-check test_dedup.py. +""" +import hashlib + +import photo_analyzer as pa +from conftest import write_fixture + +# Captured goldens (Pillow/scipy in current env; a change here means the +# decoder/algorithm changed — exactly what characterization should catch). +GOLDEN_PHASH = { + "fx-blocks-01": "851e7ae08fa4347b", + "fx-blocks-02": "84333bcce3338ce9", + "fx-blocks-03": "841f32f069cfcd38", + "fx-blocks-04": "864c3c7073b761b9", +} + + +def test_phash_goldens(tmp_path): + for fid, expected in GOLDEN_PHASH.items(): + p = write_fixture(fid, tmp_path / f"{fid}.jpg") + assert pa._phash_image(p) == expected, fid + + +def test_phash_survives_resize_and_recompress(tmp_path, img): + from PIL import Image + small = tmp_path / "small.jpg" + with Image.open(img) as im: + im.resize((400, 300)).save(small, quality=70) + a = int(pa._phash_image(img), 16) + b = int(pa._phash_image(small), 16) + assert bin(a ^ b).count("1") <= pa.PHASH_THRESHOLD + + +def test_phash_format_16_hex_chars(img): + ph = pa._phash_image(img) + assert len(ph) == 16 + int(ph, 16) + + +def test_sha1_matches_hashlib(img): + assert pa._sha1_file(img) == hashlib.sha1(img.read_bytes()).hexdigest() + + +def test_sha1_unreadable_returns_none(tmp_path): + assert pa._sha1_file(tmp_path / "missing.jpg") is None + + +def test_ensure_hashes_skips_hashed_and_is_resume_safe(db, tmp_path): + p1 = write_fixture("fx-blocks-01", tmp_path / "a.jpg") + p2 = write_fixture("fx-blocks-02", tmp_path / "b.jpg") + for p in (p1, p2): + pa.upsert_pending(db, str(p)) + assert pa.ensure_hashes(db, [p1, p2]) == 2 + # second run: nothing to do + assert pa.ensure_hashes(db, [p1, p2]) == 0 + row = db.execute("SELECT phash, file_sha1 FROM photos WHERE path=?", + (str(p1),)).fetchone() + assert row["phash"] == GOLDEN_PHASH["fx-blocks-01"] + assert row["file_sha1"] == pa._sha1_file(p1) + + +def test_cluster_and_mark_duplicates_largest_is_canonical(db, tmp_path): + from PIL import Image + big = write_fixture("fx-blocks-01", tmp_path / "big.jpg", quality=95) + small = tmp_path / "small.jpg" + with Image.open(big) as im: + im.resize((400, 300)).save(small, quality=60) + other = write_fixture("fx-blocks-02", tmp_path / "other.jpg") + for p in (big, small, other): + pa.upsert_pending(db, str(p)) + pa.ensure_hashes(db, [big, small, other]) + + clusters = pa.cluster_duplicates(db, pa.PHASH_THRESHOLD) + assert len(clusters) == 1 + assert clusters[0][0]["path"] == str(big), "canonical = largest file" + assert {c["path"] for c in clusters[0]} == {str(big), str(small)} + + marked, n = pa.mark_duplicates(db, pa.PHASH_THRESHOLD) + assert (marked, n) == (1, 1) + row = db.execute("SELECT status, dup_of FROM photos WHERE path=?", + (str(small),)).fetchone() + assert row["status"] == "duplicate" + assert row["dup_of"] == str(big) + # duplicates never re-enter the pending queue + assert str(small) not in pa.get_pending(db, reanalyze=False) + assert str(small) not in pa.get_pending(db, reanalyze=True) + + +def test_reconcile_moved_preserves_row_by_sha1(db, tmp_path): + lib = tmp_path / "lib" + old = write_fixture("fx-blocks-03", lib / "album" / "pic.jpg") + pa.upsert_pending(db, str(old)) + pa.ensure_hashes(db, [old]) + db.execute("UPDATE photos SET status='analyzed', description='kept' WHERE path=?", + (str(old),)) + db.commit() + + new = lib / "renamed" / "pic_new.jpg" + new.parent.mkdir(parents=True) + old.rename(new) + + assert pa.reconcile_moved(db, lib) == 1 + row = db.execute("SELECT path, status, description FROM photos").fetchone() + assert row["path"] == str(new) + assert row["status"] == "analyzed" + assert row["description"] == "kept", "analysis survives the move" diff --git a/tests/characterization/test_pa_imaging.py b/tests/characterization/test_pa_imaging.py new file mode 100644 index 0000000..2798c99 --- /dev/null +++ b/tests/characterization/test_pa_imaging.py @@ -0,0 +1,38 @@ +"""Characterize image preparation (donor: prepare_image).""" +import base64 +import io + +from PIL import Image + +import photo_analyzer as pa +from conftest import make_image_array + + +def _decode(b64: str) -> Image.Image: + return Image.open(io.BytesIO(base64.b64decode(b64))) + + +def test_prepare_image_small_passthrough_jpeg(img): + b64, mime = pa.prepare_image(img) + assert mime == "image/jpeg" + out = _decode(b64) + assert out.size == (800, 600), "under MAX_LONG_EDGE → no resize" + assert out.format == "JPEG" + + +def test_prepare_image_resizes_to_max_long_edge(tmp_path): + big = tmp_path / "big.jpg" + arr = make_image_array("fx-blocks-01") + Image.fromarray(arr).resize((4096, 3072)).save(big, quality=90) + b64, _ = pa.prepare_image(big) + out = _decode(b64) + assert max(out.size) == pa.MAX_LONG_EDGE + assert out.size == (2048, 1536), "aspect ratio preserved" + + +def test_prepare_image_converts_png_alpha_to_rgb_jpeg(tmp_path): + p = tmp_path / "alpha.png" + Image.new("RGBA", (100, 80), (255, 0, 0, 128)).save(p) + b64, mime = pa.prepare_image(p) + assert mime == "image/jpeg" + assert _decode(b64).mode == "RGB" diff --git a/tests/characterization/test_pa_variants.py b/tests/characterization/test_pa_variants.py new file mode 100644 index 0000000..ef4ef16 --- /dev/null +++ b/tests/characterization/test_pa_variants.py @@ -0,0 +1,45 @@ +"""Characterize variant grouping (donor: _strip_variant_markers, _is_variant, +_variant_key, build_variant_groups).""" +from pathlib import Path + +import photo_analyzer as pa +from conftest import write_fixture + + +def test_strip_variant_markers_goldens(): + cases = { + "_MG_1432": "_MG_1432", + "_MG_1432-bearbeitet": "_MG_1432", + "_MG_1432 (640x427)": "_MG_1432", + "_MG_1432 (640x427)-bearbeitet": "_MG_1432", + "IMG (1920 x 1080)": "IMG", + "party_Bearbeitet": "party", + "no-markers-here": "no-markers-here", + } + for stem, expected in cases.items(): + assert pa._strip_variant_markers(stem) == expected, stem + + +def test_is_variant(): + assert pa._is_variant(Path("/a/x (640x427).jpg")) + assert pa._is_variant(Path("/a/x-bearbeitet.jpg")) + assert not pa._is_variant(Path("/a/x.jpg")) + + +def test_variant_key_folder_scoped_case_insensitive(): + assert pa._variant_key(Path("/a/IMG (640x427).jpg")) == ("/a", "img") + assert pa._variant_key(Path("/b/IMG.jpg")) != pa._variant_key(Path("/a/IMG.jpg")) + + +def test_build_variant_groups_primary_is_largest_unmarked(tmp_path): + # base (small), edited (large): primary must be the UNMARKED base even + # though the edited file is bigger. + base = write_fixture("fx-blocks-03", tmp_path / "pic.jpg", quality=30) + edited = write_fixture("fx-blocks-01", tmp_path / "pic-bearbeitet.jpg", quality=95) + resized = write_fixture("fx-blocks-03", tmp_path / "pic (640x480).jpg", quality=30) + lone = write_fixture("fx-blocks-02", tmp_path / "other.jpg") + assert edited.stat().st_size > base.stat().st_size + + groups = pa.build_variant_groups([base, edited, resized, lone]) + assert groups == {str(base): sorted([str(edited), str(resized)])} + assert str(lone) not in groups, "standalone files form no group" diff --git a/tests/characterization/test_traceability.json b/tests/characterization/test_traceability.json deleted file mode 100644 index 00fe0e6..0000000 --- a/tests/characterization/test_traceability.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "CHAR-001": "DonorCharacterizationTests.test_discovery_outputs", - "CHAR-002": "DonorCharacterizationTests.test_image_preparation_and_hash_outputs", - "CHAR-003": "DonorCharacterizationTests.test_database_status_and_fts_outputs", - "CHAR-004": "DonorCharacterizationTests.test_caption_and_variant_outputs", - "CHAR-005": "DonorCharacterizationTests.test_vision_dry_run_output", - "CHAR-006": "DonorCharacterizationTests.test_nsfw_cache_output", - "CHAR-007": "DonorCharacterizationTests.test_review_html_output", - "CHAR-008": "DonorCharacterizationTests.test_cli_entry_point_help", - "CHAR-009": "DonorCharacterizationTests.test_exif_command_contracts", - "CHAR-010": "DonorCharacterizationTests.test_error_and_configuration_fallbacks", - "CHAR-011": "DonorCharacterizationTests.test_history_log_shape", - "CHAR-012": "DonorCharacterizationTests.test_cooperative_cancellation_contract", - "LEDGER-001": "DonorLedgerLintTests.test_ledger_is_complete_and_resolvable" -} diff --git a/tests/characterization/test_webapp_query.py b/tests/characterization/test_webapp_query.py new file mode 100644 index 0000000..0e25c54 --- /dev/null +++ b/tests/characterization/test_webapp_query.py @@ -0,0 +1,105 @@ +"""Characterize webapp read layer (donor: query._fts_query, _where, search, +facets, stats, photo, all_paths) against a seeded temp DB.""" +from pathlib import Path + +import pytest + +import photo_analyzer as pa +from webapp import query +from conftest import seed_analyzed + + +def test_fts_query_sanitization_goldens(): + assert query._fts_query("beach sun") == '"beach"* "sun"*' + assert query._fts_query('AND OR "quoted"') == '"AND"* "OR"* "quoted"*' + assert query._fts_query("!!! ???") is None, "pure punctuation can't crash MATCH" + assert query._fts_query("") is None + assert query._fts_query("Straße") == '"Straße"*', "unicode words survive" + + +def test_where_clause_goldens(): + clauses, params = query._where({ + "setting": "indoor", "people": "3+", "year_min": "2010", + "has_location": "1"}) + assert clauses == [ + "p.setting = ?", "p.people_count >= 3", "p.approx_year >= ?", + "p.location_hint IS NOT NULL AND TRIM(p.location_hint) != '' " + "AND LOWER(p.location_hint) != 'null'"] + assert params == ["indoor", 2010] + assert query._where({}) == ([], []) + + +@pytest.fixture +def seeded(db): + seed_analyzed(db, "/lib/album_a/beach.jpg", + description="A sunny beach with palm trees.", + tags=["beach", "palm"], setting="outdoor", people_count=2, + approx_year=2015, location_hint="Rome, Italy") + seed_analyzed(db, "/lib/album_a/city.jpg", + description="A night city street.", tags=["city"], + setting="outdoor", time_of_day="night", people_count=0, + approx_year=2011) + seed_analyzed(db, "/lib/album_b/dinner.jpg", + description="Family dinner at home.", tags=["family", "dinner"], + setting="indoor", people_count=4) + pa.mark_exif_written(db, "/lib/album_b/dinner.jpg") + pa.upsert_pending(db, "/lib/album_b/err.jpg") + pa.mark_error(db, "/lib/album_b/err.jpg", "timeout") + return db + + +def test_search_fts_prefix_match(seeded): + out = query.search(seeded, q="beach") + assert out["total"] == 1 + assert out["rows"][0]["path"] == "/lib/album_a/beach.jpg" + assert out["rows"][0]["tags"] == ["beach", "palm"], "tags decoded from JSON" + assert query.search(seeded, q="bea")["total"] == 1, "prefix term" + + +def test_search_filters_and_browse(seeded): + assert query.search(seeded)["total"] == 4, "empty query = browse all" + assert query.search(seeded, filters={"setting": "indoor"})["total"] == 1 + assert query.search(seeded, filters={"people": "3+"})["total"] == 1 + assert query.search(seeded, filters={"year_min": 2012})["total"] == 1 + assert query.search(seeded, filters={"has_location": "1"})["total"] == 1 + assert query.search(seeded, q="city", filters={"setting": "indoor"})["total"] == 0, \ + "search ANDs with filters" + + +def test_search_paging(seeded): + page = query.search(seeded, limit=2, offset=2) + assert page["total"] == 4 + assert len(page["rows"]) == 2 + assert page["offset"] == 2 + + +def test_photo_omits_raw_response(seeded): + d = query.photo(seeded, "/lib/album_a/beach.jpg") + assert d["description"] == "A sunny beach with palm trees." + assert "raw_response" not in d + assert query.photo(seeded, "/nope.jpg") is None + + +def test_facets_and_albums(seeded): + f = query.facets(seeded, Path("/lib")) + assert (f["year_min"], f["year_max"]) == (2011, 2015) + assert {a["album"]: a["count"] for a in f["albums"]} == { + "album_a": 2, "album_b": 2} + assert {v["value"]: v["count"] for v in f["setting"]} == { + "outdoor": 2, "indoor": 1} + + +def test_stats_album_progress_and_errors(seeded): + s = query.stats(seeded, Path("/lib")) + assert s["total"] == 4 + assert s["status"] == {"analyzed": 2, "exif_written": 1, "error": 1} + albums = {a["album"]: (a["done"], a["total"]) for a in s["albums"]} + assert albums == {"album_a": (2, 2), "album_b": (1, 2)}, \ + "done counts analyzed + exif_written" + assert s["errors"] == [{"path": "/lib/album_b/err.jpg", "error": "timeout"}] + + +def test_all_paths_is_the_image_endpoint_allowlist(seeded): + assert query.all_paths(seeded) == { + "/lib/album_a/beach.jpg", "/lib/album_a/city.jpg", + "/lib/album_b/dinner.jpg", "/lib/album_b/err.jpg"} diff --git a/tests/fixtures/characterization/expected_outputs.json b/tests/fixtures/characterization/expected_outputs.json deleted file mode 100644 index 67908b5..0000000 --- a/tests/fixtures/characterization/expected_outputs.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "schema_version": 1, - "outputs": { - "analyzer_recursive_discovery": ["album/UPPER.JPG", "album/nested.webp", "root.jpg", "wide.tiff"], - "nsfw_recursive_discovery": ["album/UPPER.JPG", "album/nested.webp", "root.jpg"], - "nsfw_shallow_discovery": ["root.jpg"], - "hash_bytes_sha1": "6a961f2bf4e8cdcd04838175b8cc98d40d729fe4", - "variant_base": "IMG_0001", - "caption": "Three adults walk beside a lake. | Tags: people, lake, summer | Mood: relaxed | Location: Como, Italy | ~2021", - "dry_run_tags": ["dry-run", "test"], - "nsfw_cache": {"a.jpg": 0.125, "b.jpg": 0.9876}, - "ui_total": "1", - "ui_threshold": "0.60" - } -} diff --git a/tests/fixtures/characterization/manifest.json b/tests/fixtures/characterization/manifest.json deleted file mode 100644 index b0f5f76..0000000 --- a/tests/fixtures/characterization/manifest.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema_version": 1, - "fixtures": [ - {"id": "DISC-ROOT-JPEG", "relative_path": "root.jpg", "recipe": "rgb:12x8:10,20,30"}, - {"id": "DISC-UPPER-JPEG", "relative_path": "album/UPPER.JPG", "recipe": "rgb:8x12:40,50,60"}, - {"id": "DISC-NESTED-WEBP", "relative_path": "album/nested.webp", "recipe": "rgb:9x9:70,80,90"}, - {"id": "DISC-TIFF", "relative_path": "wide.tiff", "recipe": "rgb:16x4:100,110,120"}, - {"id": "DISC-UNSUPPORTED", "relative_path": "notes.txt", "recipe": "text:not-an-image"}, - {"id": "HASH-BYTES", "relative_path": "hash.bin", "recipe": "bytes:photo-pipeline-donor"}, - {"id": "IMAGE-GRADIENT", "relative_path": "gradient.png", "recipe": "gradient:40x20"}, - {"id": "UI-ESCAPED", "relative_path": "album & one/.jpg", "recipe": "logical-only"} - ] -} diff --git a/work_item/.work-item.yml b/work_item/.work-item.yml index 81cbff6..1f6c9fb 100644 --- a/work_item/.work-item.yml +++ b/work_item/.work-item.yml @@ -10,7 +10,7 @@ workflow: require_ci: false required_tests: - work_item/scripts/python -m unittest discover -s work_item/tests -v - - work_item/scripts/python -m unittest discover -s tests/characterization -v + - work_item/scripts/python -m pytest tests/characterization -q safety: max_file_bytes: 5000000