US01-01 follow-up: replace donor characterization iteration with YAML ledger + full suite (#46)
This commit was merged in pull request #46.
This commit is contained in:
@@ -2,6 +2,11 @@
|
||||
|
||||
Epic: [E01](../E01-shared-identity-inventory.md)
|
||||
|
||||
**State: done (2026-07-15).** Deliverables: [`donor_ledger.yaml`](../../donor_ledger.yaml)
|
||||
(44 rows, all 12 required areas), characterization suite
|
||||
`tests/characterization/` (61 tests incl. ledger lint), all passing via
|
||||
`python3 -m pytest tests/characterization/`. No legacy file moved.
|
||||
|
||||
As a maintainer, I want a donor ledger and characterization suite for the current CLI
|
||||
implementations so proven behavior is reused deliberately instead of rewritten.
|
||||
|
||||
@@ -18,3 +23,4 @@ implementations so proven behavior is reused deliberately instead of rewritten.
|
||||
|
||||
- Characterization tests run against current donor entry points and core functions.
|
||||
- A ledger-lint test rejects missing source references, target locations, or test IDs.
|
||||
|
||||
|
||||
589
donor_ledger.yaml
Normal file
589
donor_ledger.yaml
Normal file
@@ -0,0 +1,589 @@
|
||||
# Donor ledger — US01-01 (INTEGRATED_PIPELINE_CONCEPT.md §3 "Donor-first CLI migration")
|
||||
#
|
||||
# Maps every relevant legacy module/function to its classification, rationale,
|
||||
# target module in the future photo_pipeline package, and characterization
|
||||
# test IDs. Linted by tests/characterization/test_donor_ledger.py.
|
||||
#
|
||||
# classification: reuse — take as-is
|
||||
# extract — move behind a thin adapter, behavior unchanged
|
||||
# refactor — extract while changing structure, behavior preserved
|
||||
# unless an intentional delta is stated
|
||||
# replace — new implementation; rationale documents why and what
|
||||
# (if anything) carries over
|
||||
# Every row needs either `tests` (existing test IDs, module::function) or
|
||||
# `pending_story` (the backlog story that will characterize/deliver it).
|
||||
# status: characterized | pending
|
||||
|
||||
rows:
|
||||
# ── photo_analyzer.py ──────────────────────────────────────────────────────
|
||||
- id: pa-discovery
|
||||
area: discovery
|
||||
source: {file: photo_analyzer.py, symbols: [discover_photos, SUPPORTED_EXTENSIONS]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Proven recursive discovery with _IGNORE/ and .@__thumb exclusion, sorted +
|
||||
deduplicated. Becomes the one shared discovery used by every stage; the
|
||||
exclusion predicate moves to path_policy.py.
|
||||
target: photo_pipeline/services/inventory.py + photo_pipeline/path_policy.py
|
||||
tests:
|
||||
- test_pa_discovery::test_discover_excludes_ignore_and_thumbs
|
||||
- test_pa_discovery::test_discover_finds_supported_including_uppercase
|
||||
- test_pa_discovery::test_discover_sorted_and_deduplicated
|
||||
- test_pa_discovery::test_supported_extensions_contract
|
||||
status: characterized
|
||||
|
||||
- id: pa-purge-excluded
|
||||
area: discovery
|
||||
source: {file: photo_analyzer.py, symbols: [purge_excluded]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
SQL LIKE patterns duplicate the path policy; refactor to call the shared
|
||||
path_policy predicate so exclusion has exactly one definition.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_db::test_purge_excluded_removes_ignore_and_thumb_rows]
|
||||
status: characterized
|
||||
|
||||
- id: pa-prune-missing
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [prune_missing]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Deletes rows for missing files — incompatible with stable asset identity;
|
||||
the concept keeps the asset and sets missing_at/availability_state.
|
||||
The unmounted-library guard (never prune when the root is absent) carries
|
||||
over as behavior.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_db::test_prune_missing_guards_unmounted_library]
|
||||
status: characterized
|
||||
|
||||
- id: pa-variants
|
||||
area: discovery
|
||||
source: {file: photo_analyzer.py, symbols: [_strip_variant_markers, _is_variant, _variant_key, build_variant_groups]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Filename-based variant grouping (pixel-size suffix, 'bearbeitet' edit
|
||||
marker) with unmarked-largest primary selection; feeds duplicate/variant
|
||||
clustering evidence.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
tests:
|
||||
- test_pa_variants::test_strip_variant_markers_goldens
|
||||
- test_pa_variants::test_is_variant
|
||||
- test_pa_variants::test_variant_key_folder_scoped_case_insensitive
|
||||
- test_pa_variants::test_build_variant_groups_primary_is_largest_unmarked
|
||||
status: characterized
|
||||
|
||||
- id: pa-variant-propagate
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [propagate_variants, _row_to_result]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Copying a primary's analysis into variant rows survives, but keyed by
|
||||
asset_id and recorded as stage state instead of raw row copies.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
pending_story: US01-04
|
||||
status: pending
|
||||
|
||||
- id: pa-hashing
|
||||
area: hashing
|
||||
source: {file: photo_analyzer.py, symbols: [_sha1_file, _phash_image]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Streamed SHA-1 plus 64-bit DCT phash (imagehash.phash recipe on
|
||||
numpy/scipy, no extra dependency). Hash outputs are goldens — a change
|
||||
means the decoder/algorithm changed and hash_version must bump.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests:
|
||||
- test_pa_hashing_dedup::test_phash_goldens
|
||||
- test_pa_hashing_dedup::test_phash_survives_resize_and_recompress
|
||||
- test_pa_hashing_dedup::test_phash_format_16_hex_chars
|
||||
- test_pa_hashing_dedup::test_sha1_matches_hashlib
|
||||
- test_pa_hashing_dedup::test_sha1_unreadable_returns_none
|
||||
status: characterized
|
||||
|
||||
- id: pa-ensure-hashes
|
||||
area: hashing
|
||||
source: {file: photo_analyzer.py, symbols: [ensure_hashes]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Resume-safe incremental hashing (skip already-hashed, COALESCE so a failed
|
||||
hash never clobbers a stored one, periodic commits, honors stop event).
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_hashing_dedup::test_ensure_hashes_skips_hashed_and_is_resume_safe]
|
||||
status: characterized
|
||||
|
||||
- id: pa-cluster
|
||||
area: hashing
|
||||
source: {file: photo_analyzer.py, symbols: [cluster_duplicates]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Union-find over the vectorized Hamming graph; largest-first ordering.
|
||||
Known ceiling (documented in-source): O(n²) scan, BK-tree past ~100k.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
tests: [test_pa_hashing_dedup::test_cluster_and_mark_duplicates_largest_is_canonical]
|
||||
status: characterized
|
||||
|
||||
- id: pa-mark-duplicates
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [mark_duplicates]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Intentional delta — the concept forbids silently marking fuzzy matches:
|
||||
largest-file canonical becomes a *recommendation*; the decision lands in
|
||||
reviewable duplicate_clusters, auto-resolve only for exact matches.
|
||||
The status='duplicate' exclusion from analysis/upload carries over.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
tests: [test_pa_hashing_dedup::test_cluster_and_mark_duplicates_largest_is_canonical]
|
||||
status: characterized
|
||||
|
||||
- id: pa-list-duplicates
|
||||
area: ui
|
||||
source: {file: photo_analyzer.py, symbols: [list_duplicates]}
|
||||
classification: replace
|
||||
rationale: Console report; superseded by the duplicate-review API/UI (US01-06).
|
||||
target: photo_pipeline/api/routes + frontend duplicate review
|
||||
pending_story: US01-06
|
||||
status: pending
|
||||
|
||||
- id: pa-reconcile
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [reconcile_moved]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
SHA-1-keyed move/rename reconciliation preserving analysis + EXIF state.
|
||||
Refactor to update assets.current_path by asset_id and append an
|
||||
asset_paths history row instead of rewriting the path in place.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_hashing_dedup::test_reconcile_moved_preserves_row_by_sha1]
|
||||
status: characterized
|
||||
|
||||
- id: pa-db-schema
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [SCHEMA, get_db, _migrate_schema]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
photos table + FTS5 + idempotent ALTER migration become the migration
|
||||
baseline; new schema adds assets/asset_paths/stage state per concept.
|
||||
Existing data must migrate losslessly (US01-02).
|
||||
target: photo_pipeline migrations (Alembic)
|
||||
tests: [test_pa_db::test_schema_and_migration_idempotent]
|
||||
status: characterized
|
||||
|
||||
- id: pa-db-status
|
||||
area: error
|
||||
source: {file: photo_analyzer.py, symbols: [upsert_pending, mark_analyzed, mark_error, mark_exif_written, get_pending, get_analyzed_no_exif]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Status lifecycle (pending→analyzed→exif_written; error rows auto-retried
|
||||
on the next run; INSERT OR IGNORE never downgrades; duplicates never
|
||||
queued) is the resume-safety contract — preserved as asset_stage_states
|
||||
transitions keyed by asset_id.
|
||||
target: photo_pipeline/repositories + services/analysis.py
|
||||
tests:
|
||||
- test_pa_db::test_status_lifecycle_pending_analyzed_exif_written
|
||||
- test_pa_db::test_mark_error_and_retry_via_get_pending
|
||||
- test_pa_db::test_mark_analyzed_clears_error
|
||||
- test_pa_db::test_upsert_pending_never_downgrades
|
||||
status: characterized
|
||||
|
||||
- id: pa-fts
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [photos_fts]}
|
||||
classification: reuse
|
||||
rationale: FTS5 table + sync triggers already work; carried into the new schema.
|
||||
target: photo_pipeline migrations (Alembic)
|
||||
tests: [test_pa_db::test_fts_kept_in_sync_by_triggers]
|
||||
status: characterized
|
||||
|
||||
- id: pa-imaging
|
||||
area: imaging
|
||||
source: {file: photo_analyzer.py, symbols: [prepare_image, MAX_LONG_EDGE]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
RGB-normalize (drops alpha, converts HEIC), LANCZOS resize to 2048px
|
||||
long-edge, JPEG q85 base64 — the provider-input contract. Truncated-image
|
||||
tolerance (ImageFile.LOAD_TRUNCATED_IMAGES) carries with it.
|
||||
target: photo_pipeline/integrations/vision.py
|
||||
tests:
|
||||
- test_pa_imaging::test_prepare_image_small_passthrough_jpeg
|
||||
- test_pa_imaging::test_prepare_image_resizes_to_max_long_edge
|
||||
- test_pa_imaging::test_prepare_image_converts_png_alpha_to_rgb_jpeg
|
||||
status: characterized
|
||||
|
||||
- id: pa-repair
|
||||
area: error
|
||||
source: {file: photo_analyzer.py, symbols: [repair_image, _REPAIRABLE_ERRORS, _log_repair]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Re-encode structurally broken images via sips (macOS-only — portability
|
||||
note), .orig backup, atomic replace, repairs.jsonl audit. Needs corrupt
|
||||
fixtures to characterize; lands with media hardening.
|
||||
target: photo_pipeline/integrations/exiftool.py
|
||||
pending_story: US07-03
|
||||
status: pending
|
||||
|
||||
- id: pa-vision
|
||||
area: vision
|
||||
source: {file: photo_analyzer.py, symbols: [analyze_image, ANALYSIS_PROMPT, LLM_BASE_URL, LLM_MODEL]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
OpenAI-compatible request construction, base64 image_url payload, JSON
|
||||
response validation, 429/503 retry with exponential backoff. Prompt and
|
||||
model/config version must be persisted per analysis_runs. Characterized
|
||||
against a deterministic fake provider when the analysis service is ported.
|
||||
target: photo_pipeline/integrations/vision.py
|
||||
pending_story: US02-06
|
||||
status: pending
|
||||
|
||||
- id: pa-throttle
|
||||
area: logging
|
||||
source: {file: photo_analyzer.py, symbols: [_record_ratelimit, _ratelimit_recent, write_throttle_summary, _rpd_load, _rpd_increment, _rpd_count]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Rolling throttle window + persistent throttle_events.jsonl + RPD day
|
||||
counter become job metrics/events on the durable job model.
|
||||
target: photo_pipeline/jobs/coordinator.py
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: pa-nsfw-filter
|
||||
area: nsfw
|
||||
source: {file: photo_analyzer.py, symbols: [_rec_has_nsfw, filter_nsfw_tagged]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Intentional delta — donor fails OPEN (exiftool hiccup → analyze
|
||||
everything), which violates the concept privacy invariant; the new gate
|
||||
is the DB safety decision and fails CLOSED (only confirmed sfw reaches
|
||||
the provider). EXIF keyword reading survives as evidence/verification,
|
||||
not as the gate. Current behavior captured for migration.
|
||||
target: photo_pipeline/services/safety.py
|
||||
tests:
|
||||
- test_pa_exif::test_rec_has_nsfw_list_scalar_case
|
||||
- test_pa_exif::test_filter_nsfw_tagged_splits_and_normalizes
|
||||
- test_pa_exif::test_filter_nsfw_tagged_empty_list_noop
|
||||
status: characterized
|
||||
|
||||
- id: pa-exif-caption
|
||||
area: exif
|
||||
source: {file: photo_analyzer.py, symbols: [build_exif_caption, build_exif_caption_from_result]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Caption format ('desc | Tags: … | Mood: … | Location: … | ~year') is
|
||||
what 25k+ photos already carry — preserved for compatibility. Intentional
|
||||
delta: new writes go into a managed 'AI:' segment so re-analysis can
|
||||
replace only its own text (concept EXIF ownership).
|
||||
target: photo_pipeline/services/exif projection
|
||||
tests:
|
||||
- test_pa_exif::test_caption_golden_full
|
||||
- test_pa_exif::test_caption_golden_sparse
|
||||
status: characterized
|
||||
|
||||
- id: pa-exif-read
|
||||
area: exif
|
||||
source: {file: photo_analyzer.py, symbols: [read_existing_exif]}
|
||||
classification: extract
|
||||
rationale: Targeted exiftool JSON read of the four owned fields, 30s timeout, fail-empty.
|
||||
target: photo_pipeline/integrations/exiftool.py
|
||||
tests: [test_pa_exif::test_write_exif_roundtrip_and_idempotent]
|
||||
status: characterized
|
||||
|
||||
- id: pa-exif-write
|
||||
area: exif
|
||||
source: {file: photo_analyzer.py, symbols: [write_exif]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Merge-don't-overwrite semantics are the crown jewels: caption prepended to
|
||||
existing text (idempotent — never appended twice), keywords merged +
|
||||
order-preserving dedup, -m -overwrite_original, repair-and-retry on
|
||||
structurally broken files. Refactored to add the concept's read-back
|
||||
verification + non-owned-field snapshot comparison. sys.exit on missing
|
||||
exiftool becomes a service error.
|
||||
target: photo_pipeline/integrations/exiftool.py + services/exif checkpoints
|
||||
tests:
|
||||
- test_pa_exif::test_write_exif_roundtrip_and_idempotent
|
||||
- test_pa_exif::test_write_exif_preserves_existing_metadata
|
||||
status: characterized
|
||||
|
||||
- id: pa-run-loop
|
||||
area: cancellation
|
||||
source: {file: photo_analyzer.py, symbols: [run_analysis, _run_folder_loop, run_exif_write, _stop, _handle_sigint]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Folder-grouped worker pool, per-item DB commits, cooperative stop event
|
||||
checked between items, double-SIGINT force quit — becomes the durable
|
||||
JobRunner worker loop with the same drain-and-resume semantics.
|
||||
target: photo_pipeline/jobs/worker.py
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: pa-ui-terminal
|
||||
area: ui
|
||||
source: {file: photo_analyzer.py, symbols: [_Dashboard, _LiveRenderer, _PlainRenderer, _make_progress, _stats_panel, _feed_panel, _tokens_panel, _menu_panel, _read_key, _key_listener, print_stats, fit_label]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Rich terminal dashboard (Live layout, hotkey menu, termios key reader) is
|
||||
superseded by the web Workflow/Analyze views. The *data* it renders
|
||||
(album progress, feed, tokens, ETA) survives via webapp.runner.progress
|
||||
(see wa-runner). fit_label/print_stats die with it.
|
||||
target: frontend Analyze view (Phase B)
|
||||
tests: [test_pa_album::test_fit_label_left_truncates]
|
||||
status: characterized
|
||||
|
||||
- id: pa-album-label
|
||||
area: ui
|
||||
source: {file: photo_analyzer.py, symbols: [album_label]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Album = leaf folder relative to library, '(root)' at root, bare parent
|
||||
outside the library. The single album-identity rule for stats, proposals
|
||||
and folder-as-album upload. webapp.query.album_of is a copy — consolidate
|
||||
to one implementation (see wa-album-of).
|
||||
target: photo_pipeline/services/albums.py
|
||||
tests:
|
||||
- test_pa_album::test_album_label_leaf_folder_relative
|
||||
- test_pa_album::test_webapp_album_of_mirrors_album_label
|
||||
status: characterized
|
||||
|
||||
- id: pa-config
|
||||
area: configuration
|
||||
source: {file: photo_analyzer.py, symbols: [load_env_file, _env_str, _env_int, _env_bool, ENV_FILE]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
photo_analyzer.env parsing (shell env wins, export prefix, quoted values,
|
||||
inline-comment rules) feeds the typed config module; same file keeps
|
||||
working so the transition needs no re-setup.
|
||||
target: photo_pipeline/config.py
|
||||
tests:
|
||||
- test_pa_config::test_load_env_file_parsing
|
||||
- test_pa_config::test_env_helpers
|
||||
status: characterized
|
||||
|
||||
- id: pa-logging
|
||||
area: logging
|
||||
source: {file: photo_analyzer.py, symbols: [log_history, _history_handler, _rich_handler]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Module-level triple-handler logging (console + info log + debug log) and
|
||||
the JSONL history logger become structured JSON logging with job_id/
|
||||
asset_id and job_events rows; per-photo history maps to job events.
|
||||
target: photo_pipeline structured logging + jobs/job_events
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: pa-balance
|
||||
area: vision
|
||||
source: {file: photo_analyzer.py, symbols: [fetch_balance, query_balance, check_quota]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Provider balance/quota probes (report 'unsupported' on providers without
|
||||
the endpoint). Network-bound; characterized against the fake provider.
|
||||
target: photo_pipeline/services/analysis.py
|
||||
pending_story: US02-06
|
||||
status: pending
|
||||
|
||||
- id: pa-cli
|
||||
area: configuration
|
||||
source: {file: photo_analyzer.py, symbols: [main]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
argparse surface is superseded by the API; flags map to job configs
|
||||
(documented in WEBAPP_CONCEPT.md §8 parity table). Transitional CLI calls
|
||||
the shared services until archival (E07).
|
||||
target: photo_pipeline/api + transitional CLI
|
||||
pending_story: US07-01
|
||||
status: pending
|
||||
|
||||
# ── nsfwtag/ ───────────────────────────────────────────────────────────────
|
||||
- id: nt-discovery
|
||||
area: discovery
|
||||
source: {file: nsfwtag/scoring.py, symbols: [discover_images]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Merged into the shared discovery. Known divergence captured: nsfwtag EXTS
|
||||
lacks .tiff/.tif, is non-recursive by default, and dedupes by resolved
|
||||
path (symlink guard) — the shared service adopts the superset + symlink
|
||||
dedup, always recursive, plus _IGNORE policy (donor has none).
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests:
|
||||
- test_nsfwtag::test_exts_contract_differs_from_photo_analyzer
|
||||
- test_nsfwtag::test_discover_images_flat_recursive_limit
|
||||
- test_nsfwtag::test_discover_images_dedupes_symlinked_paths
|
||||
status: characterized
|
||||
|
||||
- id: nt-score-cache
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/scoring.py, symbols: [load_cache, _save_cache, CSV_NAME]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
nsfw_scores.csv stops being the source of truth (concept: DB state).
|
||||
Format characterized (4-decimal scores, bad rows dropped) because the
|
||||
existing CSV must migrate into assets.safety_score.
|
||||
target: photo_pipeline/repositories (safety), CSV import in US01-02 migration
|
||||
tests: [test_nsfwtag::test_score_cache_roundtrip_and_tolerance]
|
||||
status: characterized
|
||||
|
||||
- id: nt-score-model
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/scoring.py, symbols: [score_images, MODEL_ID, BATCH]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
On-device ViT scoring: MPS/CPU pick, model-defined input size (not
|
||||
hardcoded — survives model swap), batch inference where one bad batch
|
||||
never discards prior scores, periodic checkpoint every 500. Cache-hit
|
||||
short-circuit (model never loads when nothing is new) characterized now;
|
||||
inference path needs the local model + deterministic fake.
|
||||
target: photo_pipeline/integrations/nsfw_model.py
|
||||
tests: [test_nsfwtag::test_score_images_cache_hit_skips_model]
|
||||
status: characterized
|
||||
|
||||
- id: nt-exif-keyword
|
||||
area: exif
|
||||
source: {file: nsfwtag/exif.py, symbols: [write_keyword, remove_keyword]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Idempotent add/remove via '-Keywords-=x -Keywords+=x' (no duplicates on
|
||||
re-run), touching only Keywords+Subject. Basis of the safety EXIF
|
||||
checkpoint; the mutual-exclusion write (sfw removed when nsfw added and
|
||||
vice versa) is the concept's addition on top.
|
||||
target: photo_pipeline/services/safety.py + integrations/exiftool.py
|
||||
tests: [test_nsfwtag::test_write_remove_keyword_idempotent]
|
||||
status: characterized
|
||||
|
||||
- id: nt-exif-marks
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/exif.py, symbols: [_read_keywords, read_tagged, read_marks]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Batched exiftool read via stdin, lowercase normalization, and the safety
|
||||
rule that a file carrying both keywords reads as nsfw (never as safe).
|
||||
target: photo_pipeline/services/safety.py
|
||||
tests:
|
||||
- test_nsfwtag::test_write_remove_keyword_idempotent
|
||||
- test_nsfwtag::test_read_marks_nsfw_wins_over_sfw
|
||||
status: characterized
|
||||
|
||||
- id: nt-exif-view
|
||||
area: exif
|
||||
source: {file: nsfwtag/exif.py, symbols: [read_exif, _EXIF_NOISE]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Lightbox EXIF read: filesystem-noise filter, binary/oversize skip, signed
|
||||
decimal GPS, Google-Maps link prepended. Backs the asset EXIF endpoint.
|
||||
target: photo_pipeline/api (asset EXIF endpoint)
|
||||
tests: [test_nsfwtag::test_read_exif_filters_noise_and_prepends_map]
|
||||
status: characterized
|
||||
|
||||
- id: nt-apply-list
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/exif.py, symbols: [apply_list]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Newline-list bulk tagging (nsfw_confirmed.txt flow) is superseded by DB
|
||||
review decisions; the existing list is a one-time migration input.
|
||||
target: photo_pipeline/services/safety.py (decision import in US01-02)
|
||||
pending_story: US01-02
|
||||
status: pending
|
||||
|
||||
- id: nt-ui
|
||||
area: ui
|
||||
source: {file: nsfwtag/server.py, symbols: [serve_review]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
stdlib ThreadingHTTPServer + token-injected review.html backend is
|
||||
superseded by FastAPI. review.html's design tokens, folder tree,
|
||||
threshold/score review flow, lightbox and keyboard model are the frontend
|
||||
donor for the Safety view (preserved per concept §10; ported in US02-01).
|
||||
target: photo_pipeline/api + frontend Safety view
|
||||
pending_story: US02-01
|
||||
status: pending
|
||||
|
||||
- id: nt-bench
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/bench.py, symbols: [main]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Dev-only model benchmark; archived without webapp replacement (recorded
|
||||
basis of the AdamCodd model choice). No production caller.
|
||||
target: none (archive as reference)
|
||||
pending_story: US07-01
|
||||
status: pending
|
||||
|
||||
# ── webapp/ ────────────────────────────────────────────────────────────────
|
||||
- id: wa-query-search
|
||||
area: database
|
||||
source: {file: webapp/query.py, symbols: [_fts_query, _where, search, _row_to_card, photo, _SORTS]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Safe FTS5 MATCH building (tokenize → quoted prefix terms; punctuation
|
||||
can't crash MATCH), facet WHERE composition, paged search with
|
||||
relevance/sort modes, raw_response kept out of list payloads.
|
||||
target: photo_pipeline/repositories (search) + api/routes
|
||||
tests:
|
||||
- test_webapp_query::test_fts_query_sanitization_goldens
|
||||
- test_webapp_query::test_where_clause_goldens
|
||||
- test_webapp_query::test_search_fts_prefix_match
|
||||
- test_webapp_query::test_search_filters_and_browse
|
||||
- test_webapp_query::test_search_paging
|
||||
- test_webapp_query::test_photo_omits_raw_response
|
||||
status: characterized
|
||||
|
||||
- id: wa-query-aggregates
|
||||
area: database
|
||||
source: {file: webapp/query.py, symbols: [facets, stats, _col_counts, _people_buckets, _top_tags]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Facet counts, year range, album tree, status/album progress (done =
|
||||
analyzed + exif_written) and error listing — the Stats/Workflow data
|
||||
model.
|
||||
target: photo_pipeline/repositories + services/workflow.py
|
||||
tests:
|
||||
- test_webapp_query::test_facets_and_albums
|
||||
- test_webapp_query::test_stats_album_progress_and_errors
|
||||
status: characterized
|
||||
|
||||
- id: wa-album-of
|
||||
area: ui
|
||||
source: {file: webapp/query.py, symbols: [album_of]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Verbatim copy of photo_analyzer.album_label (equivalence characterized);
|
||||
consolidated into the single albums-service implementation (pa-album-label).
|
||||
target: photo_pipeline/services/albums.py
|
||||
tests: [test_pa_album::test_webapp_album_of_mirrors_album_label]
|
||||
status: characterized
|
||||
|
||||
- id: wa-allowlist
|
||||
area: ui
|
||||
source: {file: webapp/query.py, symbols: [all_paths]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Path-set allowlist for /img and /exif is superseded by asset-ID-addressed
|
||||
endpoints that never accept a browser-supplied path (concept thumbnail
|
||||
rules). The validate-before-serving intent carries over.
|
||||
target: photo_pipeline/api (asset_id thumbnail/EXIF endpoints)
|
||||
tests: [test_webapp_query::test_all_paths_is_the_image_endpoint_allowlist]
|
||||
status: characterized
|
||||
|
||||
- id: wa-runner
|
||||
area: cancellation
|
||||
source: {file: webapp/runner.py, symbols: [Runner, progress]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Subprocess-driving-the-CLI job control is superseded by durable DB jobs
|
||||
with a worker process. Two ideas carry over: progress derived from DB
|
||||
counts (not job-private state) and single-mutating-job enforcement.
|
||||
target: photo_pipeline/jobs/coordinator.py
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: wa-server
|
||||
area: ui
|
||||
source: {file: webapp/server.py, symbols: [serve]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
stdlib HTTP route ladder → FastAPI typed routes. Carried intents:
|
||||
127.0.0.1 bind, path validation against the DB, no secrets to the
|
||||
browser. analyzer.html + page.py design (dark OLED tokens, Library/
|
||||
Analyze/Stats views) is frontend donor material per concept §10.
|
||||
target: photo_pipeline/api/app.py + frontend
|
||||
pending_story: US02-05
|
||||
status: pending
|
||||
@@ -1,23 +0,0 @@
|
||||
# Donor ledger
|
||||
|
||||
`ledger.json` is the machine-readable migration inventory for the existing Photo
|
||||
Analyzer and NSFW Tagger implementations. It is intentionally kept beside the live
|
||||
donors until the archival gates in `INTEGRATED_PIPELINE_CONCEPT.md` are satisfied.
|
||||
|
||||
Each row records the current source symbol, the intended treatment, its destination in
|
||||
the integrated application, the reason for that decision, and the stable
|
||||
characterization-test IDs protecting current behavior. `migration_status` remains
|
||||
`inventoried` in US01-01: this story characterizes code but does not relocate it.
|
||||
|
||||
Classification meanings:
|
||||
|
||||
- `reuse`: preserve the implementation essentially unchanged;
|
||||
- `extract`: move the coherent behavior behind a thin service/integration adapter;
|
||||
- `refactor`: preserve the useful behavior while changing an incompatible boundary;
|
||||
- `replace`: intentionally supersede unsafe or architecturally incompatible behavior.
|
||||
|
||||
Run the story suite with:
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m unittest discover -s tests/characterization -v
|
||||
```
|
||||
@@ -1,180 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"story": "US01-01",
|
||||
"concept": "INTEGRATED_PIPELINE_CONCEPT.md",
|
||||
"classifications": ["reuse", "extract", "refactor", "replace"],
|
||||
"required_areas": [
|
||||
"discovery", "hashing", "imaging", "nsfw", "vision", "exif",
|
||||
"database", "ui", "configuration", "logging", "cancellation", "error_behavior"
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
"id": "DONOR-001", "area": "discovery",
|
||||
"sources": ["photo_analyzer.py::discover_photos", "photo_analyzer.py::SUPPORTED_EXTENSIONS"],
|
||||
"classification": "extract", "target": "src/photo_pipeline/services/inventory.py",
|
||||
"rationale": "Preserve sorted recursive discovery and supported formats behind the shared path policy.",
|
||||
"intentional_changes": "All callers use stable asset IDs and one centralized boundary/exclusion policy.",
|
||||
"test_ids": ["CHAR-001"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-002", "area": "discovery",
|
||||
"sources": ["nsfwtag/scoring.py::discover_images", "nsfwtag/__init__.py::EXTS"],
|
||||
"classification": "replace", "target": "src/photo_pipeline/services/inventory.py",
|
||||
"rationale": "Its shallow/recursive switch and stable ordering are useful, but a second discovery policy and narrower formats conflict with the shared inventory.",
|
||||
"intentional_changes": "Safety consumes canonical inventory assets rather than walking paths itself.",
|
||||
"test_ids": ["CHAR-001"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-003", "area": "hashing",
|
||||
"sources": ["photo_analyzer.py::_sha1_file", "photo_analyzer.py::_phash_image", "photo_analyzer.py::ensure_hashes"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/services/inventory.py",
|
||||
"rationale": "Retain streaming exact hashing and DCT pHash behavior while adding SHA-256, normalized pixels, versions, bounded workers, and stable IDs.",
|
||||
"intentional_changes": "SHA-256 becomes byte identity; pHash is evidence rather than automatic fuzzy exclusion.",
|
||||
"test_ids": ["CHAR-002"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-004", "area": "hashing",
|
||||
"sources": ["photo_analyzer.py::cluster_duplicates", "photo_analyzer.py::mark_duplicates", "photo_analyzer.py::reconcile_moved"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/services/duplicates.py",
|
||||
"rationale": "Keep Hamming-distance evidence and reconciliation knowledge, but path-keyed union-find and automatic largest-file selection do not meet review and identity rules.",
|
||||
"intentional_changes": "Exact matches may be recommended; fuzzy matches require durable human decisions and negative links.",
|
||||
"test_ids": ["CHAR-002", "CHAR-003"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-005", "area": "imaging",
|
||||
"sources": ["photo_analyzer.py::prepare_image"],
|
||||
"classification": "extract", "target": "src/photo_pipeline/services/analysis.py",
|
||||
"rationale": "The in-memory RGB conversion, long-edge resize, and JPEG normalization are proven analysis preparation behavior.",
|
||||
"intentional_changes": "Orientation, color profile, pixel limits, and decoder version become explicit and tested.",
|
||||
"test_ids": ["CHAR-002"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-006", "area": "imaging",
|
||||
"sources": ["nsfwtag/server.py::serve_review", "nsfwtag/webapp.py::render_page"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/services/thumbnails.py",
|
||||
"rationale": "Preserve useful review presentation behavior, but arbitrary path image serving cannot be the integrated thumbnail boundary.",
|
||||
"intentional_changes": "Managed oriented thumbnails are resolved only by asset ID and bounded size.",
|
||||
"test_ids": ["CHAR-007"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-007", "area": "nsfw",
|
||||
"sources": ["nsfwtag/scoring.py::score_images", "nsfwtag/__init__.py::MODEL_ID", "nsfwtag/__init__.py::DEFAULT_THRESHOLD"],
|
||||
"classification": "extract", "target": "src/photo_pipeline/integrations/nsfw_model.py",
|
||||
"rationale": "Retain the selected local model, batching, score interpretation, and error isolation behind an injectable integration.",
|
||||
"intentional_changes": "SQLite is authoritative; CSV becomes optional export and only canonical assets are scored.",
|
||||
"test_ids": ["CHAR-006"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-008", "area": "nsfw",
|
||||
"sources": ["nsfwtag/__main__.py::main", "nsfw_tag.py::main"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/services/safety.py",
|
||||
"rationale": "Keep CLI compatibility temporarily while moving orchestration to durable jobs and explicit review decisions.",
|
||||
"intentional_changes": "The API never invokes the legacy CLI and cloud analysis requires confirmed SFW state.",
|
||||
"test_ids": ["CHAR-008"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-009", "area": "vision",
|
||||
"sources": ["photo_analyzer.py::ANALYSIS_PROMPT", "photo_analyzer.py::analyze_image"],
|
||||
"classification": "extract", "target": "src/photo_pipeline/integrations/vision.py",
|
||||
"rationale": "Preserve provider-compatible image requests, prompt context, fenced-JSON handling, token accounting, and retry categories.",
|
||||
"intentional_changes": "Typed validation, persisted prompt/model versions, bounded jittered retries, and stale-result fencing are added.",
|
||||
"test_ids": ["CHAR-005"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-010", "area": "exif",
|
||||
"sources": ["photo_analyzer.py::build_exif_caption_from_result", "photo_analyzer.py::write_exif", "photo_analyzer.py::read_existing_exif"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/integrations/exiftool.py",
|
||||
"rationale": "Retain additive caption/tag projection and exiftool knowledge while making ownership and read-back verification explicit.",
|
||||
"intentional_changes": "Each metadata stage snapshots non-owned fields, verifies its projection, and refreshes post-write hashes.",
|
||||
"test_ids": ["CHAR-004", "CHAR-009"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-011", "area": "exif",
|
||||
"sources": ["nsfwtag/exif.py::write_keyword", "nsfwtag/exif.py::remove_keyword", "nsfwtag/exif.py::read_marks"],
|
||||
"classification": "extract", "target": "src/photo_pipeline/integrations/exiftool.py",
|
||||
"rationale": "The idempotent keyword commands and mutually exclusive review semantics are reusable in a shared adapter.",
|
||||
"intentional_changes": "Both SFW and NSFW decisions are persisted and verified without removing unrelated metadata.",
|
||||
"test_ids": ["CHAR-009"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-012", "area": "database",
|
||||
"sources": ["photo_analyzer.py::SCHEMA", "photo_analyzer.py::get_db", "photo_analyzer.py::_migrate_schema"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/models/",
|
||||
"rationale": "Preserve historical schema and FTS migration knowledge while introducing SQLAlchemy repositories, Alembic, WAL, foreign keys, and stable asset identity.",
|
||||
"intentional_changes": "Paths become occurrences, not primary identity; migrations are versioned and transactional where possible.",
|
||||
"test_ids": ["CHAR-003"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-013", "area": "database",
|
||||
"sources": ["webapp/query.py::search", "webapp/query.py::facets", "webapp/query.py::stats"],
|
||||
"classification": "extract", "target": "src/photo_pipeline/repositories/assets.py",
|
||||
"rationale": "FTS query sanitization, paging, facets, album labels, and stats are useful read-model behavior.",
|
||||
"intentional_changes": "Repositories own SQL and return typed stable-ID projections.",
|
||||
"test_ids": ["CHAR-003"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-014", "area": "ui",
|
||||
"sources": ["nsfwtag/webapp.py::folder_tree_html", "nsfwtag/webapp.py::render_page", "nsfwtag/review.html"],
|
||||
"classification": "refactor", "target": "frontend/js/views/safety.js",
|
||||
"rationale": "Preserve folder navigation, score cards, review controls, lightbox interaction, and visual language.",
|
||||
"intentional_changes": "Semantic static HTML/CSS/JS fetch JSON from /api/v1; Python no longer renders operational cards.",
|
||||
"test_ids": ["CHAR-007"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-015", "area": "ui",
|
||||
"sources": ["webapp/page.py::render_page", "webapp/server.py::serve"],
|
||||
"classification": "replace", "target": "frontend/index.html",
|
||||
"rationale": "The design is a donor, but server-generated application state and ad-hoc HTTP routing conflict with the application-shell/API architecture.",
|
||||
"intentional_changes": "FastAPI serves a data-free shell and versioned JSON/SSE endpoints.",
|
||||
"test_ids": ["CHAR-007"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-016", "area": "configuration",
|
||||
"sources": ["photo_analyzer.py::load_env_file", "photo_analyzer.py::_env_int", "photo_analyzer.py::_env_bool"],
|
||||
"classification": "replace", "target": "src/photo_pipeline/config.py",
|
||||
"rationale": "Environment precedence is useful, but a manual untyped parser cannot validate the integrated app's paths, secrets, and worker settings.",
|
||||
"intentional_changes": "Pydantic settings provide typed validation and secret references.",
|
||||
"test_ids": ["CHAR-010"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-017", "area": "logging",
|
||||
"sources": ["photo_analyzer.py::log_history", "photo_analyzer.py::_record_ratelimit"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/jobs/events.py",
|
||||
"rationale": "Per-item history, token evidence, and throttle diagnostics remain valuable operational events.",
|
||||
"intentional_changes": "Structured durable events carry job, asset, attempt, and operation IDs with retention controls.",
|
||||
"test_ids": ["CHAR-011"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-018", "area": "cancellation",
|
||||
"sources": ["photo_analyzer.py::_handle_sigint", "webapp/runner.py::Runner"],
|
||||
"classification": "replace", "target": "src/photo_pipeline/jobs/coordinator.py",
|
||||
"rationale": "The cooperative stop intent is correct, but in-memory events and subprocess ownership do not survive restart or provide fencing.",
|
||||
"intentional_changes": "Cancellation is a durable job transition checked between items and recovered by heartbeat/lease state.",
|
||||
"test_ids": ["CHAR-012"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-019", "area": "error_behavior",
|
||||
"sources": ["photo_analyzer.py::mark_error", "photo_analyzer.py::analyze_image", "photo_analyzer.py::write_exif"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/services/analysis.py",
|
||||
"rationale": "Keep item-level failures, retry categories, raw-response diagnostics, and explicit EXIF failure behavior.",
|
||||
"intentional_changes": "Errors gain stable codes, attempts, retryability, and unknown/divergent states rather than broad status strings.",
|
||||
"test_ids": ["CHAR-005", "CHAR-010"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-020", "area": "error_behavior",
|
||||
"sources": ["nsfwtag/scoring.py::score_images", "nsfwtag/exif.py::_read_keywords", "webapp/runner.py::Runner"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/services/safety.py",
|
||||
"rationale": "Per-file inference isolation and tolerant metadata reads are useful, while process errors need durable domain outcomes.",
|
||||
"intentional_changes": "Failures are persisted per attempt; unavailable integrations block precisely and never imply a safety decision.",
|
||||
"test_ids": ["CHAR-006", "CHAR-010"], "migration_status": "inventoried"
|
||||
},
|
||||
{
|
||||
"id": "DONOR-021", "area": "configuration",
|
||||
"sources": ["photo_analyzer.py::main", "nsfwtag/__main__.py::main"],
|
||||
"classification": "refactor", "target": "src/photo_pipeline/__main__.py",
|
||||
"rationale": "The current flags document supported workflows and compatibility needs, but long operations must become service-backed durable jobs.",
|
||||
"intentional_changes": "Compatibility CLIs call shared services; API and worker management use typed commands.",
|
||||
"test_ids": ["CHAR-008"], "migration_status": "inventoried"
|
||||
}
|
||||
]
|
||||
}
|
||||
107
tests/characterization/conftest.py
Normal file
107
tests/characterization/conftest.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""Shared fixtures for donor characterization tests (US01-01).
|
||||
|
||||
Builds a small deterministic synthetic photo library in a temp dir — never the
|
||||
real library, never _IGNORE/ contents. Every image has a stable logical fixture
|
||||
ID (FX_*) so captured outputs stay keyed to identity, not to paths.
|
||||
"""
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(REPO))
|
||||
|
||||
EXIFTOOL = shutil.which("exiftool")
|
||||
|
||||
# Stable logical fixture IDs → deterministic generation recipe (seed, size).
|
||||
# Structured low-frequency content (blocks + gradient) so phash survives resize.
|
||||
FIXTURES = {
|
||||
"fx-blocks-01": {"seed": 1, "size": (800, 600)},
|
||||
"fx-blocks-02": {"seed": 2, "size": (800, 600)},
|
||||
"fx-blocks-03": {"seed": 3, "size": (640, 480)},
|
||||
"fx-blocks-04": {"seed": 4, "size": (800, 600)},
|
||||
}
|
||||
|
||||
|
||||
def make_image_array(fixture_id: str) -> np.ndarray:
|
||||
spec = FIXTURES[fixture_id]
|
||||
w, h = spec["size"]
|
||||
rng = np.random.default_rng(spec["seed"])
|
||||
base = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
for _ in range(6):
|
||||
x0 = int(rng.integers(0, w - 150))
|
||||
y0 = int(rng.integers(0, h - 150))
|
||||
col = rng.integers(0, 256, 3)
|
||||
base[y0:y0 + 150, x0:x0 + 150] = col
|
||||
grad = np.linspace(0, 120, w, dtype=np.uint8)
|
||||
base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255)
|
||||
return base
|
||||
|
||||
|
||||
def write_fixture(fixture_id: str, path: Path, quality: int = 95):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.fromarray(make_image_array(fixture_id)).save(path, quality=quality)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def img(tmp_path):
|
||||
"""One fx-blocks-01 JPEG in a temp dir."""
|
||||
return write_fixture("fx-blocks-01", tmp_path / "fx-blocks-01.jpg")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def library(tmp_path):
|
||||
"""Temp library:
|
||||
root/fx-blocks-01.jpg
|
||||
root/album_a/fx-blocks-02.jpg
|
||||
root/album_a/fx-blocks-02.PNG (uppercase ext, distinct file)
|
||||
root/album_b/fx-blocks-03.jpg
|
||||
root/_IGNORE/secret.jpg (must never be discovered)
|
||||
root/album_b/.@__thumb/thumb.jpg (must never be discovered)
|
||||
root/notes.txt (unsupported)
|
||||
"""
|
||||
root = tmp_path / "lib"
|
||||
write_fixture("fx-blocks-01", root / "fx-blocks-01.jpg")
|
||||
write_fixture("fx-blocks-02", root / "album_a" / "fx-blocks-02.jpg")
|
||||
img = Image.fromarray(make_image_array("fx-blocks-02"))
|
||||
(root / "album_a").mkdir(parents=True, exist_ok=True)
|
||||
img.save(root / "album_a" / "fx-blocks-02.PNG")
|
||||
write_fixture("fx-blocks-03", root / "album_b" / "fx-blocks-03.jpg")
|
||||
write_fixture("fx-blocks-04", root / "_IGNORE" / "secret.jpg")
|
||||
write_fixture("fx-blocks-04", root / "album_b" / ".@__thumb" / "thumb.jpg")
|
||||
(root / "notes.txt").write_text("not an image")
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path):
|
||||
import photo_analyzer as pa
|
||||
conn = pa.get_db(str(tmp_path / "test.db"))
|
||||
yield conn
|
||||
conn.close()
|
||||
|
||||
|
||||
def seed_analyzed(conn: sqlite3.Connection, path: str, **overrides):
|
||||
"""Insert a row and mark it analyzed with a deterministic result."""
|
||||
import photo_analyzer as pa
|
||||
result = {
|
||||
"description": "A red block pattern.",
|
||||
"tags": ["blocks", "test"],
|
||||
"people_count": 0,
|
||||
"setting": "indoor",
|
||||
"time_of_day": "unknown",
|
||||
"season": "unknown",
|
||||
"mood": "calm",
|
||||
"location_hint": None,
|
||||
"approx_year": None,
|
||||
}
|
||||
result.update(overrides)
|
||||
pa.upsert_pending(conn, path)
|
||||
pa.mark_analyzed(conn, path, result, raw="{}")
|
||||
return result
|
||||
88
tests/characterization/test_donor_ledger.py
Normal file
88
tests/characterization/test_donor_ledger.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""Ledger lint (US01-01): every donor-ledger row must carry a real source
|
||||
reference, a target location, and either existing characterization test IDs or
|
||||
a real pending backlog story."""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
LEDGER = REPO / "donor_ledger.yaml"
|
||||
STORIES = REPO / "delivery_backlog" / "stories"
|
||||
TESTS_DIR = Path(__file__).resolve().parent
|
||||
|
||||
CLASSIFICATIONS = {"reuse", "extract", "refactor", "replace"}
|
||||
REQUIRED_AREAS = {"discovery", "hashing", "imaging", "nsfw", "vision", "exif",
|
||||
"database", "ui", "configuration", "logging", "cancellation",
|
||||
"error"}
|
||||
STATUSES = {"characterized", "pending"}
|
||||
|
||||
|
||||
def load_rows():
|
||||
rows = yaml.safe_load(LEDGER.read_text(encoding="utf-8"))["rows"]
|
||||
assert rows, "empty ledger"
|
||||
return rows
|
||||
|
||||
|
||||
def collect_test_ids() -> set:
|
||||
"""module::function for every test in this suite."""
|
||||
ids = set()
|
||||
for f in TESTS_DIR.glob("test_*.py"):
|
||||
for m in re.finditer(r"^def (test_\w+)", f.read_text(encoding="utf-8"),
|
||||
re.MULTILINE):
|
||||
ids.add(f"{f.stem}::{m.group(1)}")
|
||||
return ids
|
||||
|
||||
|
||||
def test_rows_have_required_fields_and_unique_ids():
|
||||
rows = load_rows()
|
||||
ids = [r["id"] for r in rows]
|
||||
assert len(ids) == len(set(ids)), "duplicate row ids"
|
||||
for r in rows:
|
||||
for field in ("id", "area", "source", "classification", "rationale",
|
||||
"target", "status"):
|
||||
assert r.get(field), f"{r.get('id', '?')}: missing {field}"
|
||||
assert r["classification"] in CLASSIFICATIONS, r["id"]
|
||||
assert r["status"] in STATUSES, r["id"]
|
||||
assert len(str(r["rationale"]).strip()) >= 20, \
|
||||
f"{r['id']}: rationale too thin to count as a documented reason"
|
||||
|
||||
|
||||
def test_source_references_resolve():
|
||||
for r in load_rows():
|
||||
src = r["source"]
|
||||
f = REPO / src["file"]
|
||||
assert f.is_file(), f"{r['id']}: source file {src['file']} missing"
|
||||
text = f.read_text(encoding="utf-8")
|
||||
for sym in src["symbols"]:
|
||||
assert sym in text, f"{r['id']}: symbol {sym!r} not found in {src['file']}"
|
||||
|
||||
|
||||
def test_rows_have_tests_or_pending_story():
|
||||
known_tests = collect_test_ids()
|
||||
for r in load_rows():
|
||||
tests = r.get("tests", [])
|
||||
pending = r.get("pending_story")
|
||||
assert tests or pending, f"{r['id']}: neither tests nor pending_story"
|
||||
for t in tests:
|
||||
assert t in known_tests, f"{r['id']}: unknown test id {t}"
|
||||
if pending:
|
||||
matches = list(STORIES.glob(f"{pending}-*.md"))
|
||||
assert matches, f"{r['id']}: pending_story {pending} has no story file"
|
||||
if r["status"] == "characterized":
|
||||
assert tests, f"{r['id']}: characterized rows need test ids"
|
||||
|
||||
|
||||
def test_all_required_areas_covered():
|
||||
covered = {r["area"] for r in load_rows()}
|
||||
assert REQUIRED_AREAS <= covered, f"uncovered areas: {REQUIRED_AREAS - covered}"
|
||||
assert covered <= REQUIRED_AREAS, f"unknown areas: {covered - REQUIRED_AREAS}"
|
||||
|
||||
|
||||
def test_no_legacy_file_moved():
|
||||
# US01-01 explicitly forbids moving/archiving donors; the ledger's source
|
||||
# files must all still exist at their original locations.
|
||||
for donor in ("photo_analyzer.py", "nsfwtag/scoring.py", "nsfwtag/exif.py",
|
||||
"nsfwtag/server.py", "webapp/query.py", "webapp/runner.py",
|
||||
"webapp/server.py"):
|
||||
assert (REPO / donor).is_file(), f"donor moved: {donor}"
|
||||
@@ -1,295 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
FIXTURES = REPO / "tests" / "fixtures" / "characterization"
|
||||
|
||||
|
||||
class DonorCharacterizationTests(unittest.TestCase):
|
||||
"""Golden contracts for donor behavior; these are not target-architecture tests."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.import_dir = tempfile.TemporaryDirectory(prefix="donor-import-")
|
||||
old_cwd = Path.cwd()
|
||||
sys.path.insert(0, str(REPO))
|
||||
try:
|
||||
os.chdir(cls.import_dir.name)
|
||||
cls.analyzer = importlib.import_module("photo_analyzer")
|
||||
cls.nsfw_scoring = importlib.import_module("nsfwtag.scoring")
|
||||
cls.nsfw_exif = importlib.import_module("nsfwtag.exif")
|
||||
cls.nsfw_webapp = importlib.import_module("nsfwtag.webapp")
|
||||
cls.web_query = importlib.import_module("webapp.query")
|
||||
finally:
|
||||
os.chdir(old_cwd)
|
||||
cls.manifest = json.loads((FIXTURES / "manifest.json").read_text())["fixtures"]
|
||||
cls.expected = json.loads((FIXTURES / "expected_outputs.json").read_text())["outputs"]
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
for handler in list(logging.getLogger().handlers) + list(
|
||||
logging.getLogger("history").handlers
|
||||
):
|
||||
handler.close()
|
||||
cls.import_dir.cleanup()
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(prefix="donor-fixtures-")
|
||||
self.root = Path(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.analyzer._stop.clear()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _make_discovery_library(self) -> None:
|
||||
recipes = {item["id"]: item for item in self.manifest}
|
||||
colors = {
|
||||
"DISC-ROOT-JPEG": (10, 20, 30),
|
||||
"DISC-UPPER-JPEG": (40, 50, 60),
|
||||
"DISC-NESTED-WEBP": (70, 80, 90),
|
||||
"DISC-TIFF": (100, 110, 120),
|
||||
}
|
||||
sizes = {
|
||||
"DISC-ROOT-JPEG": (12, 8),
|
||||
"DISC-UPPER-JPEG": (8, 12),
|
||||
"DISC-NESTED-WEBP": (9, 9),
|
||||
"DISC-TIFF": (16, 4),
|
||||
}
|
||||
for fixture_id, color in colors.items():
|
||||
path = self.root / recipes[fixture_id]["relative_path"]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.new("RGB", sizes[fixture_id], color).save(path)
|
||||
unsupported = self.root / recipes["DISC-UNSUPPORTED"]["relative_path"]
|
||||
unsupported.write_text("not-an-image", encoding="utf-8")
|
||||
|
||||
def _relative(self, paths) -> list[str]:
|
||||
return [str(Path(path).relative_to(self.root)) for path in paths]
|
||||
|
||||
def test_discovery_outputs(self):
|
||||
self._make_discovery_library()
|
||||
|
||||
analyzer_paths = self.analyzer.discover_photos(self.root)
|
||||
nsfw_recursive = self.nsfw_scoring.discover_images(self.root, recursive=True)
|
||||
nsfw_shallow = self.nsfw_scoring.discover_images(self.root, recursive=False)
|
||||
|
||||
self.assertEqual(
|
||||
self._relative(analyzer_paths), self.expected["analyzer_recursive_discovery"]
|
||||
)
|
||||
self.assertEqual(self._relative(nsfw_recursive), self.expected["nsfw_recursive_discovery"])
|
||||
self.assertEqual(self._relative(nsfw_shallow), self.expected["nsfw_shallow_discovery"])
|
||||
|
||||
def test_image_preparation_and_hash_outputs(self):
|
||||
raw_path = self.root / "hash.bin"
|
||||
raw_path.write_bytes(b"photo-pipeline-donor")
|
||||
self.assertEqual(self.analyzer._sha1_file(raw_path), self.expected["hash_bytes_sha1"])
|
||||
|
||||
gradient = Image.new("RGB", (40, 20))
|
||||
gradient.putdata(
|
||||
[(x * 5 % 256, y * 11 % 256, (x + y) * 7 % 256) for y in range(20) for x in range(40)]
|
||||
)
|
||||
image_path = self.root / "gradient.png"
|
||||
gradient.save(image_path)
|
||||
old_edge = self.analyzer.MAX_LONG_EDGE
|
||||
try:
|
||||
self.analyzer.MAX_LONG_EDGE = 16
|
||||
encoded, mime = self.analyzer.prepare_image(image_path)
|
||||
finally:
|
||||
self.analyzer.MAX_LONG_EDGE = old_edge
|
||||
prepared_path = self.root / "prepared.jpg"
|
||||
prepared_path.write_bytes(base64.b64decode(encoded))
|
||||
with Image.open(prepared_path) as prepared:
|
||||
self.assertEqual(prepared.size, (16, 8))
|
||||
self.assertEqual(prepared.mode, "RGB")
|
||||
self.assertEqual(mime, "image/jpeg")
|
||||
phash = self.analyzer._phash_image(image_path)
|
||||
self.assertRegex(phash or "", r"^[0-9a-f]{16}$")
|
||||
self.assertEqual(phash, self.analyzer._phash_image(image_path))
|
||||
|
||||
def test_database_status_and_fts_outputs(self):
|
||||
db_path = self.root / "characterization.sqlite"
|
||||
conn = self.analyzer.get_db(str(db_path))
|
||||
self.addCleanup(conn.close)
|
||||
photo_path = str(self.root / "lake.jpg")
|
||||
self.analyzer.upsert_pending(conn, photo_path)
|
||||
result = {
|
||||
"description": "Three adults walk beside a lake.",
|
||||
"tags": ["people", "lake", "summer"],
|
||||
"people_count": 3,
|
||||
"setting": "outdoor",
|
||||
"time_of_day": "afternoon",
|
||||
"season": "summer",
|
||||
"mood": "relaxed",
|
||||
"location_hint": "Como, Italy",
|
||||
"approx_year": 2021,
|
||||
}
|
||||
self.analyzer.mark_analyzed(conn, photo_path, result, json.dumps(result))
|
||||
row = conn.execute("SELECT * FROM photos WHERE path = ?", (photo_path,)).fetchone()
|
||||
self.assertEqual(row["status"], "analyzed")
|
||||
self.assertEqual(json.loads(row["tags"]), result["tags"])
|
||||
found = self.web_query.search(conn, q="lake")
|
||||
self.assertEqual(found["total"], 1)
|
||||
self.assertEqual(found["rows"][0]["path"], photo_path)
|
||||
self.analyzer.mark_exif_written(conn, photo_path)
|
||||
self.assertEqual(conn.execute("SELECT status FROM photos").fetchone()[0], "exif_written")
|
||||
|
||||
def test_caption_and_variant_outputs(self):
|
||||
result = {
|
||||
"description": "Three adults walk beside a lake.",
|
||||
"tags": ["people", "lake", "summer"],
|
||||
"mood": "relaxed",
|
||||
"location_hint": "Como, Italy",
|
||||
"approx_year": 2021,
|
||||
}
|
||||
self.assertEqual(
|
||||
self.analyzer.build_exif_caption_from_result(result), self.expected["caption"]
|
||||
)
|
||||
self.assertEqual(
|
||||
self.analyzer._strip_variant_markers("IMG_0001-bearbeitet (1920x1080)"),
|
||||
self.expected["variant_base"],
|
||||
)
|
||||
|
||||
def test_vision_dry_run_output(self):
|
||||
result, raw, tokens = self.analyzer.analyze_image(
|
||||
None, self.root / "fixture.jpg", dry_run=True
|
||||
)
|
||||
self.assertEqual(result["tags"], self.expected["dry_run_tags"])
|
||||
self.assertEqual(json.loads(raw), result)
|
||||
self.assertEqual(tokens, {"prompt": 0, "completion": 0, "total": 0})
|
||||
|
||||
def test_nsfw_cache_output(self):
|
||||
cache_path = self.root / "scores.csv"
|
||||
values = self.expected["nsfw_cache"]
|
||||
self.nsfw_scoring._save_cache(cache_path, values)
|
||||
self.assertEqual(self.nsfw_scoring.load_cache(cache_path), values)
|
||||
self.assertEqual(
|
||||
cache_path.read_text(encoding="utf-8").splitlines(),
|
||||
["path,nsfw_score", "a.jpg,0.1250", "b.jpg,0.9876"],
|
||||
)
|
||||
|
||||
def test_review_html_output(self):
|
||||
logical = self.root / "album & one" / "<portrait>.jpg"
|
||||
page = self.nsfw_webapp.render_page([(logical, 0.8123), (logical, 0.7)], 0.6)
|
||||
self.assertIn("<portrait>.jpg", page)
|
||||
self.assertIn("album & one", page)
|
||||
self.assertIn(f'data-score="{0.8123:.4f}"', page)
|
||||
self.assertIn(self.expected["ui_total"], page)
|
||||
self.assertIn(self.expected["ui_threshold"], page)
|
||||
self.assertEqual(page.count('class="card"'), 1)
|
||||
|
||||
def test_cli_entry_point_help(self):
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = str(REPO)
|
||||
commands = [
|
||||
([sys.executable, str(REPO / "photo_analyzer.py"), "--help"], "--group-variants"),
|
||||
([sys.executable, str(REPO / "nsfw_tag.py"), "--help"], "--review-min"),
|
||||
([sys.executable, "-m", "nsfwtag", "--help"], "--threshold"),
|
||||
]
|
||||
for command, marker in commands:
|
||||
with self.subTest(command=command):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=self.root,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn(marker, result.stdout)
|
||||
|
||||
def test_exif_command_contracts(self):
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append(command)
|
||||
if "-json" in command:
|
||||
return subprocess.CompletedProcess(
|
||||
command,
|
||||
0,
|
||||
stdout=json.dumps(
|
||||
[{"ImageDescription": "User caption", "Keywords": ["family"]}]
|
||||
),
|
||||
stderr="",
|
||||
)
|
||||
return subprocess.CompletedProcess(command, 0, stdout="", stderr="")
|
||||
|
||||
with mock.patch.object(self.analyzer.subprocess, "run", side_effect=fake_run):
|
||||
self.assertTrue(
|
||||
self.analyzer.write_exif("fixture.jpg", "AI caption", ["family", "lake"])
|
||||
)
|
||||
write_command = calls[-1]
|
||||
self.assertIn("-overwrite_original", write_command)
|
||||
self.assertIn("-ImageDescription=AI caption | User caption", write_command)
|
||||
self.assertEqual(write_command.count("-Keywords=family"), 1)
|
||||
self.assertEqual(write_command.count("-Keywords=lake"), 1)
|
||||
|
||||
with mock.patch.object(
|
||||
self.nsfw_exif.subprocess,
|
||||
"run",
|
||||
return_value=subprocess.CompletedProcess([], 0, stdout="", stderr=""),
|
||||
) as run:
|
||||
self.assertTrue(self.nsfw_exif.write_keyword("fixture.jpg"))
|
||||
command = run.call_args.args[0]
|
||||
self.assertIn("-Keywords-=nsfw", command)
|
||||
self.assertIn("-Keywords+=nsfw", command)
|
||||
self.assertIn("-Subject-=nsfw", command)
|
||||
self.assertIn("-Subject+=nsfw", command)
|
||||
|
||||
def test_error_and_configuration_fallbacks(self):
|
||||
with mock.patch.dict(os.environ, {"MAX_WORKERS": "not-an-int"}):
|
||||
self.assertEqual(self.analyzer._env_int("MAX_WORKERS", 3), 3)
|
||||
invalid = subprocess.CompletedProcess([], 0, stdout="not-json", stderr="")
|
||||
with mock.patch.object(self.nsfw_exif.subprocess, "run", return_value=invalid):
|
||||
self.assertEqual(self.nsfw_exif._read_keywords(["fixture.jpg"]), {})
|
||||
|
||||
def test_history_log_shape(self):
|
||||
sink = mock.Mock()
|
||||
result = {
|
||||
"description": "Fixture description",
|
||||
"tags": ["fixture"],
|
||||
"mood": "calm",
|
||||
"setting": "indoor",
|
||||
"people_count": 0,
|
||||
"location_hint": None,
|
||||
"approx_year": None,
|
||||
}
|
||||
with mock.patch.object(self.analyzer.history_log, "info", sink):
|
||||
self.analyzer.log_history(
|
||||
"fixture.jpg", "analyzed", result=result, tokens={"total": 12}
|
||||
)
|
||||
entry = json.loads(sink.call_args.args[0])
|
||||
self.assertEqual(entry["path"], "fixture.jpg")
|
||||
self.assertEqual(entry["status"], "analyzed")
|
||||
self.assertEqual(entry["tokens_total"], 12)
|
||||
self.assertEqual(entry["tags"], ["fixture"])
|
||||
self.assertRegex(entry["ts"], r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$")
|
||||
|
||||
def test_cooperative_cancellation_contract(self):
|
||||
self.analyzer._stop.clear()
|
||||
fake_stderr = SimpleNamespace(write=mock.Mock(), flush=mock.Mock())
|
||||
with (
|
||||
mock.patch.object(self.analyzer.sys, "__stderr__", fake_stderr),
|
||||
mock.patch.object(self.analyzer.log, "warning"),
|
||||
):
|
||||
self.analyzer._handle_sigint(None, None)
|
||||
self.assertTrue(self.analyzer._stop.is_set())
|
||||
fake_stderr.write.assert_called_once_with("\a")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,62 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
LEDGER = REPO / "donor_ledger" / "ledger.json"
|
||||
TRACEABILITY = REPO / "tests" / "characterization" / "test_traceability.json"
|
||||
DONOR_TESTS = REPO / "tests" / "characterization" / "test_donors.py"
|
||||
|
||||
|
||||
def source_symbols(path: Path) -> set[str]:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
names = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
names.add(node.name)
|
||||
elif isinstance(node, (ast.Assign, ast.AnnAssign)):
|
||||
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
||||
names.update(t.id for t in targets if isinstance(t, ast.Name))
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
names.update(alias.asname or alias.name for alias in node.names)
|
||||
return names
|
||||
|
||||
|
||||
class DonorLedgerLintTests(unittest.TestCase):
|
||||
def test_ledger_is_complete_and_resolvable(self):
|
||||
ledger = json.loads(LEDGER.read_text(encoding="utf-8"))
|
||||
traceability = json.loads(TRACEABILITY.read_text(encoding="utf-8"))
|
||||
entries = ledger["entries"]
|
||||
donor_test_symbols = source_symbols(DONOR_TESTS)
|
||||
|
||||
self.assertEqual(ledger["story"], "US01-01")
|
||||
self.assertEqual(len({entry["id"] for entry in entries}), len(entries))
|
||||
self.assertEqual(set(ledger["required_areas"]), {entry["area"] for entry in entries})
|
||||
self.assertFalse((REPO / "legacy_cli_archive").exists())
|
||||
|
||||
allowed = set(ledger["classifications"])
|
||||
for entry in entries:
|
||||
with self.subTest(entry=entry["id"]):
|
||||
self.assertIn(entry["classification"], allowed)
|
||||
self.assertTrue(entry["rationale"].strip())
|
||||
self.assertTrue(entry["target"].strip())
|
||||
self.assertTrue(entry["test_ids"])
|
||||
self.assertEqual(entry["migration_status"], "inventoried")
|
||||
for test_id in entry["test_ids"]:
|
||||
self.assertIn(test_id, traceability)
|
||||
class_name, method_name = traceability[test_id].split(".", 1)
|
||||
self.assertEqual(class_name, "DonorCharacterizationTests")
|
||||
self.assertIn(method_name, donor_test_symbols)
|
||||
for source in entry["sources"]:
|
||||
file_name, separator, symbol = source.partition("::")
|
||||
source_path = REPO / file_name
|
||||
self.assertTrue(source_path.is_file(), source)
|
||||
if separator:
|
||||
self.assertIn(symbol, source_symbols(source_path), source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
95
tests/characterization/test_nsfwtag.py
Normal file
95
tests/characterization/test_nsfwtag.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""Characterize nsfwtag donors: discovery, score cache, EXIF safety keywords.
|
||||
|
||||
score_images' model inference is NOT run here (needs the ~350 MB local model);
|
||||
its cache-reuse path is characterized instead — the model is only invoked for
|
||||
paths missing from the cache.
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
import nsfwtag.exif as nexif
|
||||
import nsfwtag.scoring as scoring
|
||||
from nsfwtag import EXTS, KEYWORD
|
||||
from conftest import EXIFTOOL, write_fixture
|
||||
|
||||
needs_exiftool = pytest.mark.skipif(not EXIFTOOL, reason="exiftool not on PATH")
|
||||
|
||||
|
||||
def test_exts_contract_differs_from_photo_analyzer():
|
||||
import photo_analyzer as pa
|
||||
assert EXTS == {".jpg", ".jpeg", ".png", ".heic", ".heif", ".webp"}
|
||||
assert pa.SUPPORTED_EXTENSIONS - EXTS == {".tiff", ".tif"}, \
|
||||
"known donor divergence: nsfwtag skips TIFF"
|
||||
|
||||
|
||||
def test_discover_images_flat_recursive_limit(tmp_path):
|
||||
a = write_fixture("fx-blocks-01", tmp_path / "a.jpg")
|
||||
b = write_fixture("fx-blocks-02", tmp_path / "sub" / "b.jpg")
|
||||
(tmp_path / "c.txt").write_text("x")
|
||||
assert scoring.discover_images(tmp_path) == [a], "flat by default"
|
||||
assert scoring.discover_images(tmp_path, recursive=True) == [a, b]
|
||||
assert scoring.discover_images(tmp_path, recursive=True, limit=1) == [a]
|
||||
|
||||
|
||||
def test_discover_images_dedupes_symlinked_paths(tmp_path):
|
||||
a = write_fixture("fx-blocks-01", tmp_path / "a.jpg")
|
||||
(tmp_path / "link.jpg").symlink_to(a)
|
||||
found = scoring.discover_images(tmp_path)
|
||||
assert len(found) == 1, "same resolved file listed once"
|
||||
|
||||
|
||||
def test_score_cache_roundtrip_and_tolerance(tmp_path):
|
||||
csv = tmp_path / "scores.csv"
|
||||
scoring._save_cache(csv, {"/a.jpg": 0.91234, "/b.jpg": 0.1})
|
||||
cache = scoring.load_cache(csv)
|
||||
assert cache == {"/a.jpg": 0.9123, "/b.jpg": 0.1}, "4-decimal persistence"
|
||||
csv.write_text("path,nsfw_score\n/ok.jpg,0.5\n/bad.jpg,not-a-float\n")
|
||||
assert scoring.load_cache(csv) == {"/ok.jpg": 0.5}, "bad rows dropped silently"
|
||||
assert scoring.load_cache(tmp_path / "missing.csv") == {}
|
||||
|
||||
|
||||
def test_score_images_cache_hit_skips_model(tmp_path):
|
||||
img = write_fixture("fx-blocks-01", tmp_path / "a.jpg")
|
||||
csv = tmp_path / "scores.csv"
|
||||
scoring._save_cache(csv, {str(img): 0.42})
|
||||
scored, errors = scoring.score_images([img], csv) # would download a model on miss
|
||||
assert scored == [(img, 0.42)]
|
||||
assert errors == 0
|
||||
|
||||
|
||||
@needs_exiftool
|
||||
def test_write_remove_keyword_idempotent(img):
|
||||
assert nexif.write_keyword(str(img))
|
||||
assert nexif.write_keyword(str(img)), "second write is a no-op, not a dup"
|
||||
r = subprocess.run(["exiftool", "-j", "-Keywords", "-Subject", str(img)],
|
||||
capture_output=True, text=True)
|
||||
assert r.stdout.count(KEYWORD) == 2, "once in Keywords, once in Subject"
|
||||
assert nexif.read_tagged([str(img)]) == {str(img)}
|
||||
assert nexif.remove_keyword(str(img))
|
||||
assert nexif.read_tagged([str(img)]) == set()
|
||||
|
||||
|
||||
@needs_exiftool
|
||||
def test_read_marks_nsfw_wins_over_sfw(img, tmp_path):
|
||||
safe = write_fixture("fx-blocks-02", tmp_path / "safe.jpg")
|
||||
subprocess.run(["exiftool", "-m", "-overwrite_original",
|
||||
"-Keywords=nsfw", "-Keywords=sfw", str(img)],
|
||||
check=True, capture_output=True)
|
||||
subprocess.run(["exiftool", "-m", "-overwrite_original",
|
||||
"-Keywords=sfw", str(safe)], check=True, capture_output=True)
|
||||
marks = nexif.read_marks([str(img), str(safe)])
|
||||
assert marks["nsfw"] == {str(img)}, "both keywords → nsfw wins (never reads safe)"
|
||||
assert marks["sfw"] == {str(safe)}
|
||||
|
||||
|
||||
@needs_exiftool
|
||||
def test_read_exif_filters_noise_and_prepends_map(img):
|
||||
subprocess.run(["exiftool", "-m", "-overwrite_original",
|
||||
"-GPSLatitude=48.8584", "-GPSLatitudeRef=N",
|
||||
"-GPSLongitude=2.2945", "-GPSLongitudeRef=E", str(img)],
|
||||
check=True, capture_output=True)
|
||||
rec = nexif.read_exif(str(img))
|
||||
assert list(rec)[0] == "Map"
|
||||
assert rec["Map"] == "https://www.google.com/maps?q=48.858400,2.294500"
|
||||
assert "SourceFile" not in rec and "ExifToolVersion" not in rec
|
||||
28
tests/characterization/test_pa_album.py
Normal file
28
tests/characterization/test_pa_album.py
Normal file
@@ -0,0 +1,28 @@
|
||||
"""Characterize album labeling (donor: album_label, fit_label; webapp.query.album_of
|
||||
mirrors album_label — asserted identical here)."""
|
||||
from pathlib import Path
|
||||
|
||||
import photo_analyzer as pa
|
||||
from webapp import query
|
||||
|
||||
|
||||
def test_album_label_leaf_folder_relative():
|
||||
lib = Path("/lib")
|
||||
assert pa.album_label(Path("/lib/Urlaub/Rom/a.jpg"), lib) == "Urlaub/Rom"
|
||||
assert pa.album_label(Path("/lib/Urlaub/Venedig/a.jpg"), lib) == "Urlaub/Venedig"
|
||||
assert pa.album_label(Path("/lib/a.jpg"), lib) == "(root)"
|
||||
assert pa.album_label(Path("/elsewhere/x/a.jpg"), lib) == "x", \
|
||||
"outside library → bare parent name"
|
||||
assert pa.album_label(Path("/lib/x/a.jpg"), None) == "x"
|
||||
|
||||
|
||||
def test_webapp_album_of_mirrors_album_label():
|
||||
lib = Path("/lib")
|
||||
for p in ("/lib/Urlaub/Rom/a.jpg", "/lib/a.jpg", "/other/x/a.jpg"):
|
||||
assert query.album_of(p, lib) == pa.album_label(Path(p), lib), p
|
||||
|
||||
|
||||
def test_fit_label_left_truncates():
|
||||
assert pa.fit_label("short", 10) == "short"
|
||||
assert pa.fit_label("Urlaub/Somewhere/Rom", 8) == "…ere/Rom"
|
||||
assert len(pa.fit_label("Urlaub/Somewhere/Rom", 8)) == 8
|
||||
46
tests/characterization/test_pa_config.py
Normal file
46
tests/characterization/test_pa_config.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Characterize configuration behavior (donor: load_env_file, _env_str,
|
||||
_env_int, _env_bool)."""
|
||||
import os
|
||||
|
||||
import photo_analyzer as pa
|
||||
|
||||
|
||||
def test_load_env_file_parsing(tmp_path, monkeypatch):
|
||||
env = tmp_path / pa.ENV_FILE
|
||||
env.write_text(
|
||||
"# comment\n"
|
||||
"PLAIN=value\n"
|
||||
"export EXPORTED=yes\n"
|
||||
'QUOTED="hash # kept"\n'
|
||||
"INLINE=val # comment stripped\n"
|
||||
"TOKEN=pass#word\n"
|
||||
"PRESET=file-loses\n"
|
||||
"NOEQUALS\n"
|
||||
)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
for k in ("PLAIN", "EXPORTED", "QUOTED", "INLINE", "TOKEN"):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
monkeypatch.setenv("PRESET", "shell-wins")
|
||||
|
||||
used = pa.load_env_file()
|
||||
assert used == env
|
||||
assert os.environ["PLAIN"] == "value"
|
||||
assert os.environ["EXPORTED"] == "yes", "'export ' prefix accepted"
|
||||
assert os.environ["QUOTED"] == "hash # kept", "quoted # not a comment"
|
||||
assert os.environ["INLINE"] == "val", "inline comment stripped"
|
||||
assert os.environ["TOKEN"] == "pass#word", "mid-token # kept"
|
||||
assert os.environ["PRESET"] == "shell-wins", "shell env always wins"
|
||||
|
||||
|
||||
def test_env_helpers(monkeypatch):
|
||||
monkeypatch.setenv("S", "")
|
||||
assert pa._env_str("S", "fb") == "fb", "empty string falls back"
|
||||
monkeypatch.setenv("I", "not-int")
|
||||
assert pa._env_int("I", 7) == 7, "bad int falls back with warning"
|
||||
monkeypatch.setenv("I", "42")
|
||||
assert pa._env_int("I", 7) == 42
|
||||
for truthy in ("1", "true", "YES", "On"):
|
||||
monkeypatch.setenv("B", truthy)
|
||||
assert pa._env_bool("B") is True
|
||||
monkeypatch.setenv("B", "0")
|
||||
assert pa._env_bool("B") is False
|
||||
68
tests/characterization/test_pa_db.py
Normal file
68
tests/characterization/test_pa_db.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""Characterize DB layer (donor: get_db, _migrate_schema, status transitions,
|
||||
get_pending, purge_excluded, prune_missing, FTS sync)."""
|
||||
import photo_analyzer as pa
|
||||
from conftest import seed_analyzed, write_fixture
|
||||
|
||||
|
||||
def test_schema_and_migration_idempotent(tmp_path):
|
||||
db_path = str(tmp_path / "x.db")
|
||||
conn = pa.get_db(db_path)
|
||||
conn.close()
|
||||
conn = pa.get_db(db_path) # re-open runs migration again — must not fail
|
||||
cols = {r["name"] for r in conn.execute("PRAGMA table_info(photos)")}
|
||||
assert {"path", "status", "phash", "file_sha1", "dup_of", "description",
|
||||
"tags", "raw_response", "analyzed_at", "exif_written_at"} <= cols
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_status_lifecycle_pending_analyzed_exif_written(db):
|
||||
seed_analyzed(db, "/x/a.jpg")
|
||||
assert db.execute("SELECT status FROM photos").fetchone()["status"] == "analyzed"
|
||||
pa.mark_exif_written(db, "/x/a.jpg")
|
||||
assert db.execute("SELECT status FROM photos").fetchone()["status"] == "exif_written"
|
||||
|
||||
|
||||
def test_mark_error_and_retry_via_get_pending(db):
|
||||
pa.upsert_pending(db, "/x/a.jpg")
|
||||
pa.mark_error(db, "/x/a.jpg", "boom")
|
||||
row = db.execute("SELECT status, error_message FROM photos").fetchone()
|
||||
assert (row["status"], row["error_message"]) == ("error", "boom")
|
||||
# error rows are retried on the next run
|
||||
assert pa.get_pending(db, reanalyze=False) == ["/x/a.jpg"]
|
||||
|
||||
|
||||
def test_mark_analyzed_clears_error(db):
|
||||
pa.upsert_pending(db, "/x/a.jpg")
|
||||
pa.mark_error(db, "/x/a.jpg", "boom")
|
||||
seed_analyzed(db, "/x/a.jpg")
|
||||
assert db.execute("SELECT error_message FROM photos").fetchone()["error_message"] is None
|
||||
|
||||
|
||||
def test_upsert_pending_never_downgrades(db):
|
||||
seed_analyzed(db, "/x/a.jpg")
|
||||
pa.upsert_pending(db, "/x/a.jpg") # INSERT OR IGNORE — no reset to pending
|
||||
assert db.execute("SELECT status FROM photos").fetchone()["status"] == "analyzed"
|
||||
|
||||
|
||||
def test_fts_kept_in_sync_by_triggers(db):
|
||||
seed_analyzed(db, "/x/beach.jpg", description="A sunny beach with palm trees.")
|
||||
hit = db.execute(
|
||||
"SELECT path FROM photos_fts WHERE photos_fts MATCH 'beach'").fetchall()
|
||||
assert [r["path"] for r in hit] == ["/x/beach.jpg"]
|
||||
|
||||
|
||||
def test_purge_excluded_removes_ignore_and_thumb_rows(db):
|
||||
for p in ("/lib/_IGNORE/a.jpg", "/lib/x/.@__thumb/b.jpg", "/lib/keep.jpg"):
|
||||
pa.upsert_pending(db, p)
|
||||
assert pa.purge_excluded(db) == 2
|
||||
assert [r["path"] for r in db.execute("SELECT path FROM photos")] == ["/lib/keep.jpg"]
|
||||
|
||||
|
||||
def test_prune_missing_guards_unmounted_library(db, tmp_path):
|
||||
pa.upsert_pending(db, "/gone/a.jpg")
|
||||
# library root missing → refuses to prune (protects against unmounted volume)
|
||||
assert pa.prune_missing(db, tmp_path / "not-there") == 0
|
||||
assert db.execute("SELECT COUNT(*) FROM photos").fetchone()[0] == 1
|
||||
# existing root → stale row is pruned
|
||||
assert pa.prune_missing(db, tmp_path) == 1
|
||||
assert db.execute("SELECT COUNT(*) FROM photos").fetchone()[0] == 0
|
||||
27
tests/characterization/test_pa_discovery.py
Normal file
27
tests/characterization/test_pa_discovery.py
Normal file
@@ -0,0 +1,27 @@
|
||||
"""Characterize photo_analyzer discovery + path policy (donor: discover_photos)."""
|
||||
import photo_analyzer as pa
|
||||
|
||||
|
||||
def test_discover_excludes_ignore_and_thumbs(library):
|
||||
found = pa.discover_photos(library)
|
||||
names = [p.name for p in found]
|
||||
assert "secret.jpg" not in names, "_IGNORE/ content must never be discovered"
|
||||
assert "thumb.jpg" not in names, ".@__thumb/ content must never be discovered"
|
||||
assert "notes.txt" not in names
|
||||
|
||||
|
||||
def test_discover_finds_supported_including_uppercase(library):
|
||||
names = sorted(p.name for p in pa.discover_photos(library))
|
||||
assert names == ["fx-blocks-01.jpg", "fx-blocks-02.PNG",
|
||||
"fx-blocks-02.jpg", "fx-blocks-03.jpg"]
|
||||
|
||||
|
||||
def test_discover_sorted_and_deduplicated(library):
|
||||
found = pa.discover_photos(library)
|
||||
assert found == sorted(set(found))
|
||||
|
||||
|
||||
def test_supported_extensions_contract():
|
||||
# Donor contract: these exact extensions (nsfwtag EXTS differs — no .tiff/.tif).
|
||||
assert pa.SUPPORTED_EXTENSIONS == {
|
||||
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"}
|
||||
87
tests/characterization/test_pa_exif.py
Normal file
87
tests/characterization/test_pa_exif.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""Characterize EXIF behavior (donor: build_exif_caption_from_result,
|
||||
read_existing_exif, write_exif merge/idempotency, _rec_has_nsfw,
|
||||
filter_nsfw_tagged). Uses the real exiftool on synthetic temp files."""
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
import photo_analyzer as pa
|
||||
from conftest import EXIFTOOL
|
||||
|
||||
needs_exiftool = pytest.mark.skipif(not EXIFTOOL, reason="exiftool not on PATH")
|
||||
|
||||
RESULT = {
|
||||
"description": "A dog on a beach.",
|
||||
"tags": ["dog", "beach"],
|
||||
"mood": "joyful",
|
||||
"location_hint": "Rome, Italy",
|
||||
"approx_year": 2015,
|
||||
}
|
||||
|
||||
|
||||
def test_caption_golden_full():
|
||||
assert pa.build_exif_caption_from_result(RESULT) == (
|
||||
"A dog on a beach. | Tags: dog, beach | Mood: joyful | "
|
||||
"Location: Rome, Italy | ~2015")
|
||||
|
||||
|
||||
def test_caption_golden_sparse():
|
||||
assert pa.build_exif_caption_from_result(
|
||||
{"description": "X.", "tags": [], "location_hint": None}) == "X."
|
||||
|
||||
|
||||
def test_rec_has_nsfw_list_scalar_case():
|
||||
assert pa._rec_has_nsfw({"Keywords": ["beach", "NSFW "]})
|
||||
assert pa._rec_has_nsfw({"Subject": "nsfw"})
|
||||
assert not pa._rec_has_nsfw({"Keywords": ["sfw"], "Subject": ["beach"]})
|
||||
assert not pa._rec_has_nsfw({})
|
||||
|
||||
|
||||
@needs_exiftool
|
||||
def test_write_exif_roundtrip_and_idempotent(img):
|
||||
caption = pa.build_exif_caption_from_result(RESULT)
|
||||
assert pa.write_exif(str(img), caption, RESULT["tags"])
|
||||
rec = pa.read_existing_exif(str(img))
|
||||
assert rec["ImageDescription"] == caption
|
||||
assert rec["XPComment"] == caption
|
||||
assert rec["Subject"] == RESULT["tags"]
|
||||
assert rec["Keywords"] == RESULT["tags"]
|
||||
|
||||
# second write with the same caption must not duplicate anything
|
||||
assert pa.write_exif(str(img), caption, RESULT["tags"])
|
||||
rec2 = pa.read_existing_exif(str(img))
|
||||
assert rec2["ImageDescription"] == caption
|
||||
assert rec2["Keywords"] == RESULT["tags"]
|
||||
|
||||
|
||||
@needs_exiftool
|
||||
def test_write_exif_preserves_existing_metadata(img):
|
||||
subprocess.run(
|
||||
["exiftool", "-m", "-overwrite_original", "-Artist=Original Artist",
|
||||
"-ImageDescription=User caption", "-Keywords=userkw", str(img)],
|
||||
check=True, capture_output=True)
|
||||
assert pa.write_exif(str(img), "AI caption", ["aitag"])
|
||||
rec = pa.read_existing_exif(str(img))
|
||||
# donor behavior: AI caption prepended, user caption kept after ' | '
|
||||
assert rec["ImageDescription"] == "AI caption | User caption"
|
||||
kws = rec["Keywords"] if isinstance(rec["Keywords"], list) else [rec["Keywords"]]
|
||||
assert "userkw" in kws and "aitag" in kws, "keywords merge, never replace"
|
||||
full = subprocess.run(["exiftool", "-j", "-Artist", str(img)],
|
||||
capture_output=True, text=True)
|
||||
assert "Original Artist" in full.stdout, "non-owned field preserved"
|
||||
|
||||
|
||||
@needs_exiftool
|
||||
def test_filter_nsfw_tagged_splits_and_normalizes(img, tmp_path):
|
||||
from conftest import write_fixture
|
||||
clean = write_fixture("fx-blocks-02", tmp_path / "clean.jpg")
|
||||
subprocess.run(["exiftool", "-m", "-overwrite_original",
|
||||
"-Keywords=nsfw", "-Subject=nsfw", str(img)],
|
||||
check=True, capture_output=True)
|
||||
kept, skipped = pa.filter_nsfw_tagged([str(img), str(clean)])
|
||||
assert kept == [str(clean)]
|
||||
assert skipped == [str(img)]
|
||||
|
||||
|
||||
def test_filter_nsfw_tagged_empty_list_noop():
|
||||
assert pa.filter_nsfw_tagged([]) == ([], [])
|
||||
109
tests/characterization/test_pa_hashing_dedup.py
Normal file
109
tests/characterization/test_pa_hashing_dedup.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""Characterize hashing + duplicate ledger (donor: _sha1_file, _phash_image,
|
||||
ensure_hashes, cluster_duplicates, mark_duplicates, reconcile_moved).
|
||||
|
||||
Complements the pre-existing root-level self-check test_dedup.py.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
import photo_analyzer as pa
|
||||
from conftest import write_fixture
|
||||
|
||||
# Captured goldens (Pillow/scipy in current env; a change here means the
|
||||
# decoder/algorithm changed — exactly what characterization should catch).
|
||||
GOLDEN_PHASH = {
|
||||
"fx-blocks-01": "851e7ae08fa4347b",
|
||||
"fx-blocks-02": "84333bcce3338ce9",
|
||||
"fx-blocks-03": "841f32f069cfcd38",
|
||||
"fx-blocks-04": "864c3c7073b761b9",
|
||||
}
|
||||
|
||||
|
||||
def test_phash_goldens(tmp_path):
|
||||
for fid, expected in GOLDEN_PHASH.items():
|
||||
p = write_fixture(fid, tmp_path / f"{fid}.jpg")
|
||||
assert pa._phash_image(p) == expected, fid
|
||||
|
||||
|
||||
def test_phash_survives_resize_and_recompress(tmp_path, img):
|
||||
from PIL import Image
|
||||
small = tmp_path / "small.jpg"
|
||||
with Image.open(img) as im:
|
||||
im.resize((400, 300)).save(small, quality=70)
|
||||
a = int(pa._phash_image(img), 16)
|
||||
b = int(pa._phash_image(small), 16)
|
||||
assert bin(a ^ b).count("1") <= pa.PHASH_THRESHOLD
|
||||
|
||||
|
||||
def test_phash_format_16_hex_chars(img):
|
||||
ph = pa._phash_image(img)
|
||||
assert len(ph) == 16
|
||||
int(ph, 16)
|
||||
|
||||
|
||||
def test_sha1_matches_hashlib(img):
|
||||
assert pa._sha1_file(img) == hashlib.sha1(img.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def test_sha1_unreadable_returns_none(tmp_path):
|
||||
assert pa._sha1_file(tmp_path / "missing.jpg") is None
|
||||
|
||||
|
||||
def test_ensure_hashes_skips_hashed_and_is_resume_safe(db, tmp_path):
|
||||
p1 = write_fixture("fx-blocks-01", tmp_path / "a.jpg")
|
||||
p2 = write_fixture("fx-blocks-02", tmp_path / "b.jpg")
|
||||
for p in (p1, p2):
|
||||
pa.upsert_pending(db, str(p))
|
||||
assert pa.ensure_hashes(db, [p1, p2]) == 2
|
||||
# second run: nothing to do
|
||||
assert pa.ensure_hashes(db, [p1, p2]) == 0
|
||||
row = db.execute("SELECT phash, file_sha1 FROM photos WHERE path=?",
|
||||
(str(p1),)).fetchone()
|
||||
assert row["phash"] == GOLDEN_PHASH["fx-blocks-01"]
|
||||
assert row["file_sha1"] == pa._sha1_file(p1)
|
||||
|
||||
|
||||
def test_cluster_and_mark_duplicates_largest_is_canonical(db, tmp_path):
|
||||
from PIL import Image
|
||||
big = write_fixture("fx-blocks-01", tmp_path / "big.jpg", quality=95)
|
||||
small = tmp_path / "small.jpg"
|
||||
with Image.open(big) as im:
|
||||
im.resize((400, 300)).save(small, quality=60)
|
||||
other = write_fixture("fx-blocks-02", tmp_path / "other.jpg")
|
||||
for p in (big, small, other):
|
||||
pa.upsert_pending(db, str(p))
|
||||
pa.ensure_hashes(db, [big, small, other])
|
||||
|
||||
clusters = pa.cluster_duplicates(db, pa.PHASH_THRESHOLD)
|
||||
assert len(clusters) == 1
|
||||
assert clusters[0][0]["path"] == str(big), "canonical = largest file"
|
||||
assert {c["path"] for c in clusters[0]} == {str(big), str(small)}
|
||||
|
||||
marked, n = pa.mark_duplicates(db, pa.PHASH_THRESHOLD)
|
||||
assert (marked, n) == (1, 1)
|
||||
row = db.execute("SELECT status, dup_of FROM photos WHERE path=?",
|
||||
(str(small),)).fetchone()
|
||||
assert row["status"] == "duplicate"
|
||||
assert row["dup_of"] == str(big)
|
||||
# duplicates never re-enter the pending queue
|
||||
assert str(small) not in pa.get_pending(db, reanalyze=False)
|
||||
assert str(small) not in pa.get_pending(db, reanalyze=True)
|
||||
|
||||
|
||||
def test_reconcile_moved_preserves_row_by_sha1(db, tmp_path):
|
||||
lib = tmp_path / "lib"
|
||||
old = write_fixture("fx-blocks-03", lib / "album" / "pic.jpg")
|
||||
pa.upsert_pending(db, str(old))
|
||||
pa.ensure_hashes(db, [old])
|
||||
db.execute("UPDATE photos SET status='analyzed', description='kept' WHERE path=?",
|
||||
(str(old),))
|
||||
db.commit()
|
||||
|
||||
new = lib / "renamed" / "pic_new.jpg"
|
||||
new.parent.mkdir(parents=True)
|
||||
old.rename(new)
|
||||
|
||||
assert pa.reconcile_moved(db, lib) == 1
|
||||
row = db.execute("SELECT path, status, description FROM photos").fetchone()
|
||||
assert row["path"] == str(new)
|
||||
assert row["status"] == "analyzed"
|
||||
assert row["description"] == "kept", "analysis survives the move"
|
||||
38
tests/characterization/test_pa_imaging.py
Normal file
38
tests/characterization/test_pa_imaging.py
Normal file
@@ -0,0 +1,38 @@
|
||||
"""Characterize image preparation (donor: prepare_image)."""
|
||||
import base64
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
import photo_analyzer as pa
|
||||
from conftest import make_image_array
|
||||
|
||||
|
||||
def _decode(b64: str) -> Image.Image:
|
||||
return Image.open(io.BytesIO(base64.b64decode(b64)))
|
||||
|
||||
|
||||
def test_prepare_image_small_passthrough_jpeg(img):
|
||||
b64, mime = pa.prepare_image(img)
|
||||
assert mime == "image/jpeg"
|
||||
out = _decode(b64)
|
||||
assert out.size == (800, 600), "under MAX_LONG_EDGE → no resize"
|
||||
assert out.format == "JPEG"
|
||||
|
||||
|
||||
def test_prepare_image_resizes_to_max_long_edge(tmp_path):
|
||||
big = tmp_path / "big.jpg"
|
||||
arr = make_image_array("fx-blocks-01")
|
||||
Image.fromarray(arr).resize((4096, 3072)).save(big, quality=90)
|
||||
b64, _ = pa.prepare_image(big)
|
||||
out = _decode(b64)
|
||||
assert max(out.size) == pa.MAX_LONG_EDGE
|
||||
assert out.size == (2048, 1536), "aspect ratio preserved"
|
||||
|
||||
|
||||
def test_prepare_image_converts_png_alpha_to_rgb_jpeg(tmp_path):
|
||||
p = tmp_path / "alpha.png"
|
||||
Image.new("RGBA", (100, 80), (255, 0, 0, 128)).save(p)
|
||||
b64, mime = pa.prepare_image(p)
|
||||
assert mime == "image/jpeg"
|
||||
assert _decode(b64).mode == "RGB"
|
||||
45
tests/characterization/test_pa_variants.py
Normal file
45
tests/characterization/test_pa_variants.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""Characterize variant grouping (donor: _strip_variant_markers, _is_variant,
|
||||
_variant_key, build_variant_groups)."""
|
||||
from pathlib import Path
|
||||
|
||||
import photo_analyzer as pa
|
||||
from conftest import write_fixture
|
||||
|
||||
|
||||
def test_strip_variant_markers_goldens():
|
||||
cases = {
|
||||
"_MG_1432": "_MG_1432",
|
||||
"_MG_1432-bearbeitet": "_MG_1432",
|
||||
"_MG_1432 (640x427)": "_MG_1432",
|
||||
"_MG_1432 (640x427)-bearbeitet": "_MG_1432",
|
||||
"IMG (1920 x 1080)": "IMG",
|
||||
"party_Bearbeitet": "party",
|
||||
"no-markers-here": "no-markers-here",
|
||||
}
|
||||
for stem, expected in cases.items():
|
||||
assert pa._strip_variant_markers(stem) == expected, stem
|
||||
|
||||
|
||||
def test_is_variant():
|
||||
assert pa._is_variant(Path("/a/x (640x427).jpg"))
|
||||
assert pa._is_variant(Path("/a/x-bearbeitet.jpg"))
|
||||
assert not pa._is_variant(Path("/a/x.jpg"))
|
||||
|
||||
|
||||
def test_variant_key_folder_scoped_case_insensitive():
|
||||
assert pa._variant_key(Path("/a/IMG (640x427).jpg")) == ("/a", "img")
|
||||
assert pa._variant_key(Path("/b/IMG.jpg")) != pa._variant_key(Path("/a/IMG.jpg"))
|
||||
|
||||
|
||||
def test_build_variant_groups_primary_is_largest_unmarked(tmp_path):
|
||||
# base (small), edited (large): primary must be the UNMARKED base even
|
||||
# though the edited file is bigger.
|
||||
base = write_fixture("fx-blocks-03", tmp_path / "pic.jpg", quality=30)
|
||||
edited = write_fixture("fx-blocks-01", tmp_path / "pic-bearbeitet.jpg", quality=95)
|
||||
resized = write_fixture("fx-blocks-03", tmp_path / "pic (640x480).jpg", quality=30)
|
||||
lone = write_fixture("fx-blocks-02", tmp_path / "other.jpg")
|
||||
assert edited.stat().st_size > base.stat().st_size
|
||||
|
||||
groups = pa.build_variant_groups([base, edited, resized, lone])
|
||||
assert groups == {str(base): sorted([str(edited), str(resized)])}
|
||||
assert str(lone) not in groups, "standalone files form no group"
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"CHAR-001": "DonorCharacterizationTests.test_discovery_outputs",
|
||||
"CHAR-002": "DonorCharacterizationTests.test_image_preparation_and_hash_outputs",
|
||||
"CHAR-003": "DonorCharacterizationTests.test_database_status_and_fts_outputs",
|
||||
"CHAR-004": "DonorCharacterizationTests.test_caption_and_variant_outputs",
|
||||
"CHAR-005": "DonorCharacterizationTests.test_vision_dry_run_output",
|
||||
"CHAR-006": "DonorCharacterizationTests.test_nsfw_cache_output",
|
||||
"CHAR-007": "DonorCharacterizationTests.test_review_html_output",
|
||||
"CHAR-008": "DonorCharacterizationTests.test_cli_entry_point_help",
|
||||
"CHAR-009": "DonorCharacterizationTests.test_exif_command_contracts",
|
||||
"CHAR-010": "DonorCharacterizationTests.test_error_and_configuration_fallbacks",
|
||||
"CHAR-011": "DonorCharacterizationTests.test_history_log_shape",
|
||||
"CHAR-012": "DonorCharacterizationTests.test_cooperative_cancellation_contract",
|
||||
"LEDGER-001": "DonorLedgerLintTests.test_ledger_is_complete_and_resolvable"
|
||||
}
|
||||
105
tests/characterization/test_webapp_query.py
Normal file
105
tests/characterization/test_webapp_query.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""Characterize webapp read layer (donor: query._fts_query, _where, search,
|
||||
facets, stats, photo, all_paths) against a seeded temp DB."""
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import photo_analyzer as pa
|
||||
from webapp import query
|
||||
from conftest import seed_analyzed
|
||||
|
||||
|
||||
def test_fts_query_sanitization_goldens():
|
||||
assert query._fts_query("beach sun") == '"beach"* "sun"*'
|
||||
assert query._fts_query('AND OR "quoted"') == '"AND"* "OR"* "quoted"*'
|
||||
assert query._fts_query("!!! ???") is None, "pure punctuation can't crash MATCH"
|
||||
assert query._fts_query("") is None
|
||||
assert query._fts_query("Straße") == '"Straße"*', "unicode words survive"
|
||||
|
||||
|
||||
def test_where_clause_goldens():
|
||||
clauses, params = query._where({
|
||||
"setting": "indoor", "people": "3+", "year_min": "2010",
|
||||
"has_location": "1"})
|
||||
assert clauses == [
|
||||
"p.setting = ?", "p.people_count >= 3", "p.approx_year >= ?",
|
||||
"p.location_hint IS NOT NULL AND TRIM(p.location_hint) != '' "
|
||||
"AND LOWER(p.location_hint) != 'null'"]
|
||||
assert params == ["indoor", 2010]
|
||||
assert query._where({}) == ([], [])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def seeded(db):
|
||||
seed_analyzed(db, "/lib/album_a/beach.jpg",
|
||||
description="A sunny beach with palm trees.",
|
||||
tags=["beach", "palm"], setting="outdoor", people_count=2,
|
||||
approx_year=2015, location_hint="Rome, Italy")
|
||||
seed_analyzed(db, "/lib/album_a/city.jpg",
|
||||
description="A night city street.", tags=["city"],
|
||||
setting="outdoor", time_of_day="night", people_count=0,
|
||||
approx_year=2011)
|
||||
seed_analyzed(db, "/lib/album_b/dinner.jpg",
|
||||
description="Family dinner at home.", tags=["family", "dinner"],
|
||||
setting="indoor", people_count=4)
|
||||
pa.mark_exif_written(db, "/lib/album_b/dinner.jpg")
|
||||
pa.upsert_pending(db, "/lib/album_b/err.jpg")
|
||||
pa.mark_error(db, "/lib/album_b/err.jpg", "timeout")
|
||||
return db
|
||||
|
||||
|
||||
def test_search_fts_prefix_match(seeded):
|
||||
out = query.search(seeded, q="beach")
|
||||
assert out["total"] == 1
|
||||
assert out["rows"][0]["path"] == "/lib/album_a/beach.jpg"
|
||||
assert out["rows"][0]["tags"] == ["beach", "palm"], "tags decoded from JSON"
|
||||
assert query.search(seeded, q="bea")["total"] == 1, "prefix term"
|
||||
|
||||
|
||||
def test_search_filters_and_browse(seeded):
|
||||
assert query.search(seeded)["total"] == 4, "empty query = browse all"
|
||||
assert query.search(seeded, filters={"setting": "indoor"})["total"] == 1
|
||||
assert query.search(seeded, filters={"people": "3+"})["total"] == 1
|
||||
assert query.search(seeded, filters={"year_min": 2012})["total"] == 1
|
||||
assert query.search(seeded, filters={"has_location": "1"})["total"] == 1
|
||||
assert query.search(seeded, q="city", filters={"setting": "indoor"})["total"] == 0, \
|
||||
"search ANDs with filters"
|
||||
|
||||
|
||||
def test_search_paging(seeded):
|
||||
page = query.search(seeded, limit=2, offset=2)
|
||||
assert page["total"] == 4
|
||||
assert len(page["rows"]) == 2
|
||||
assert page["offset"] == 2
|
||||
|
||||
|
||||
def test_photo_omits_raw_response(seeded):
|
||||
d = query.photo(seeded, "/lib/album_a/beach.jpg")
|
||||
assert d["description"] == "A sunny beach with palm trees."
|
||||
assert "raw_response" not in d
|
||||
assert query.photo(seeded, "/nope.jpg") is None
|
||||
|
||||
|
||||
def test_facets_and_albums(seeded):
|
||||
f = query.facets(seeded, Path("/lib"))
|
||||
assert (f["year_min"], f["year_max"]) == (2011, 2015)
|
||||
assert {a["album"]: a["count"] for a in f["albums"]} == {
|
||||
"album_a": 2, "album_b": 2}
|
||||
assert {v["value"]: v["count"] for v in f["setting"]} == {
|
||||
"outdoor": 2, "indoor": 1}
|
||||
|
||||
|
||||
def test_stats_album_progress_and_errors(seeded):
|
||||
s = query.stats(seeded, Path("/lib"))
|
||||
assert s["total"] == 4
|
||||
assert s["status"] == {"analyzed": 2, "exif_written": 1, "error": 1}
|
||||
albums = {a["album"]: (a["done"], a["total"]) for a in s["albums"]}
|
||||
assert albums == {"album_a": (2, 2), "album_b": (1, 2)}, \
|
||||
"done counts analyzed + exif_written"
|
||||
assert s["errors"] == [{"path": "/lib/album_b/err.jpg", "error": "timeout"}]
|
||||
|
||||
|
||||
def test_all_paths_is_the_image_endpoint_allowlist(seeded):
|
||||
assert query.all_paths(seeded) == {
|
||||
"/lib/album_a/beach.jpg", "/lib/album_a/city.jpg",
|
||||
"/lib/album_b/dinner.jpg", "/lib/album_b/err.jpg"}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"outputs": {
|
||||
"analyzer_recursive_discovery": ["album/UPPER.JPG", "album/nested.webp", "root.jpg", "wide.tiff"],
|
||||
"nsfw_recursive_discovery": ["album/UPPER.JPG", "album/nested.webp", "root.jpg"],
|
||||
"nsfw_shallow_discovery": ["root.jpg"],
|
||||
"hash_bytes_sha1": "6a961f2bf4e8cdcd04838175b8cc98d40d729fe4",
|
||||
"variant_base": "IMG_0001",
|
||||
"caption": "Three adults walk beside a lake. | Tags: people, lake, summer | Mood: relaxed | Location: Como, Italy | ~2021",
|
||||
"dry_run_tags": ["dry-run", "test"],
|
||||
"nsfw_cache": {"a.jpg": 0.125, "b.jpg": 0.9876},
|
||||
"ui_total": "1",
|
||||
"ui_threshold": "0.60"
|
||||
}
|
||||
}
|
||||
13
tests/fixtures/characterization/manifest.json
vendored
13
tests/fixtures/characterization/manifest.json
vendored
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"fixtures": [
|
||||
{"id": "DISC-ROOT-JPEG", "relative_path": "root.jpg", "recipe": "rgb:12x8:10,20,30"},
|
||||
{"id": "DISC-UPPER-JPEG", "relative_path": "album/UPPER.JPG", "recipe": "rgb:8x12:40,50,60"},
|
||||
{"id": "DISC-NESTED-WEBP", "relative_path": "album/nested.webp", "recipe": "rgb:9x9:70,80,90"},
|
||||
{"id": "DISC-TIFF", "relative_path": "wide.tiff", "recipe": "rgb:16x4:100,110,120"},
|
||||
{"id": "DISC-UNSUPPORTED", "relative_path": "notes.txt", "recipe": "text:not-an-image"},
|
||||
{"id": "HASH-BYTES", "relative_path": "hash.bin", "recipe": "bytes:photo-pipeline-donor"},
|
||||
{"id": "IMAGE-GRADIENT", "relative_path": "gradient.png", "recipe": "gradient:40x20"},
|
||||
{"id": "UI-ESCAPED", "relative_path": "album & one/<portrait>.jpg", "recipe": "logical-only"}
|
||||
]
|
||||
}
|
||||
@@ -10,7 +10,7 @@ workflow:
|
||||
require_ci: false
|
||||
required_tests:
|
||||
- work_item/scripts/python -m unittest discover -s work_item/tests -v
|
||||
- work_item/scripts/python -m unittest discover -s tests/characterization -v
|
||||
- work_item/scripts/python -m pytest tests/characterization -q
|
||||
|
||||
safety:
|
||||
max_file_bytes: 5000000
|
||||
|
||||
Reference in New Issue
Block a user