US07-01: Complete Donor Migration and Freeze the CLI Archive (#83)
This commit was merged in pull request #83.
This commit is contained in:
22
legacy_cli_archive/CHECKSUMS.sha256
Normal file
22
legacy_cli_archive/CHECKSUMS.sha256
Normal file
@@ -0,0 +1,22 @@
|
||||
d098bbde13d2ebc872ca781e244cc1e48b97cc551fedaca3251d6a5278a15234 src/compare_models.py
|
||||
7d0f68cb95fbd6603e4c558620098b46929ebdb6fd91a598bbf84f26eea35e6c src/nsfw_tag.py
|
||||
35329c53570e215cef15f59429c1a5251a0448b90522b0910ab9d68efe1bc307 src/nsfwtag/__init__.py
|
||||
7f52b55e4f3b77eda3657d7cd2272c7cca8422100e241671de495603c1126ee2 src/nsfwtag/__main__.py
|
||||
6f5a97114e0d87d272ce22d065dc31fa0dd72ff8590d11ad14cb1c1f486d339c src/nsfwtag/bench.py
|
||||
29498fac1d73ba2b7420ffe1ffe49a87f684ec299e2c3b9c6e14e63e95643ba4 src/nsfwtag/exif.py
|
||||
934e82c402813ebf503e5a20eb03d5103f84df13117bc4dd1fd95deaa01ab263 src/nsfwtag/README.md
|
||||
68125e6184c7e4d2a5b0153f2155675753ab0b1329d1dbc4933d4769492ec4be src/nsfwtag/review.html
|
||||
a597eab74803dd31452b5d70abf7d6d6320eb17a61596c891b009da15d624e98 src/nsfwtag/scoring.py
|
||||
674969a18e58ee511a2abf574a875db92f2f524769b1b614d0d3662ab879a709 src/nsfwtag/server.py
|
||||
24a7c8d029da6e97d46b110a9fe9dbb3900127f8ae3c142f634fdd91d6c45243 src/nsfwtag/webapp.py
|
||||
2c2ea558f1b9095c1150f078ea29c8e0f180cc045a9d1097b83f61bfb145295b src/photo_analyzer.py
|
||||
5fb7ce2f977a17da1f501c45d7985328318ae14298a3f347eab46a6e3f02aab3 src/test_dedup.py
|
||||
1673dc76cc60aba56a5b065d9bc9f5dda00342cf2a79d000a3f74686fb7cffec src/test_nsfw_skip.py
|
||||
55aa82f348e0e90be0163ba6aa278e5fc8e445996e5335774917e5ef59ad3563 src/webapp/__init__.py
|
||||
ba4342bd0175591a2121f063f14a78a602d5679bcfcf24aa14c8d4b8cd5da1a2 src/webapp/__main__.py
|
||||
a60f24035a989909467778f888d62854c1afc2702f47b8d3cca32bf3cf04d5cf src/webapp/analyzer.html
|
||||
018bcc2f770d716444b456db58d6b4a41800138a98598b3c27ba503ab22d939e src/webapp/page.py
|
||||
5f599b107b2b117ca118b6fbdec5e5786a9ab4eea424aa9af1bfea7bf87cebab src/webapp/query.py
|
||||
875cce697caa02717c85a2707ba935c46dd9ace18428fd4abb13105b7a8e24d9 src/webapp/README.md
|
||||
af7d0d72d245b4bbb1e0e30f9708239697543212d4171defd1bfe26dd24a4f05 src/webapp/runner.py
|
||||
cddf0555b06fe7e4daa14309baaaf91130f8506922cb0ffb6507d76cbe39add4 src/webapp/server.py
|
||||
80
legacy_cli_archive/README.md
Normal file
80
legacy_cli_archive/README.md
Normal file
@@ -0,0 +1,80 @@
|
||||
# Legacy CLI archive (US07-01)
|
||||
|
||||
Frozen, read-only sources of the command-line tools this application was extracted
|
||||
from. They are **reference material and rollback evidence** — provenance for
|
||||
behavior that now lives in `photo_pipeline/`, and the only way to answer "what did
|
||||
the original actually do?" once the replacement has drifted.
|
||||
|
||||
> **Nothing here is production code.** No module under `photo_pipeline/` imports or
|
||||
> executes anything in this directory, and this directory is not on the application's
|
||||
> import path. `tests/unit/test_legacy_archive.py` enforces both, along with the
|
||||
> checksums and the redaction below.
|
||||
|
||||
## What is here
|
||||
|
||||
| Path | Role |
|
||||
|---|---|
|
||||
| `src/photo_analyzer.py` | the analysis CLI: discovery, hashing/dedup, vision analysis, EXIF writing, SQLite schema, album naming |
|
||||
| `src/nsfwtag/` | NSFW scoring, EXIF safety keywords, and the review server (`__init__` 1.1.0) |
|
||||
| `src/webapp/` | the stdlib review web app: FTS search, stats, subprocess runner, HTML shell (`__init__` 0.1.0) |
|
||||
| `src/nsfw_tag.py` | thin backwards-compatible entry point for `nsfwtag` |
|
||||
| `src/compare_models.py` | dev-only model comparison script |
|
||||
| `src/test_dedup.py`, `src/test_nsfw_skip.py` | the CLIs' own standalone self-checks (never pytest suites) |
|
||||
| `donor_ledger.yaml` | the donor ledger: every migrated behavior, its target, its tests, and every intentional delta |
|
||||
| `requirements-lock.txt` | the dependency versions the frozen sources were last verified against |
|
||||
| `photo_analyzer.env.sample` | the CLI's configuration surface, with every value replaced by a placeholder |
|
||||
| `CHECKSUMS.sha256` | SHA-256 of every archived source file |
|
||||
|
||||
`photo_analyzer.py` carries no `__version__`; its identity is its checksum, recorded
|
||||
in `CHECKSUMS.sha256` and taken at commit `9b7ee6b` (the merge of US06-06, the last
|
||||
commit before archival).
|
||||
|
||||
## Verifying the archive
|
||||
|
||||
```bash
|
||||
cd legacy_cli_archive && shasum -a 256 -c CHECKSUMS.sha256
|
||||
```
|
||||
|
||||
Any edit to an archived source must be accompanied by a regenerated checksum file
|
||||
and a note here explaining why a *frozen* archive changed — the normal answer being
|
||||
that it should not.
|
||||
|
||||
## Schema notes
|
||||
|
||||
`photo_analyzer.py` owned a path-keyed SQLite database (`SCHEMA`, near the top of the
|
||||
file):
|
||||
|
||||
- `photos(id, path UNIQUE, status, phash, file_sha1, dup_of, description, tags,
|
||||
people_count, setting, time_of_day, season, mood, location_hint, approx_year,
|
||||
raw_response, error_message, analyzed_at, exif_written_at)`;
|
||||
- `photos_fts` — an FTS5 external-content index over `path, description, tags, mood,
|
||||
location_hint`, kept in sync by insert/update/delete triggers;
|
||||
- late columns (`phash`, `file_sha1`, `dup_of`) were added by an in-code
|
||||
`_migrate_schema()` rather than a migration tool, and their indexes are created
|
||||
only after the `ALTER`.
|
||||
|
||||
The replacement keeps the same analysis fields but re-keys everything to a stable
|
||||
`assets.id` (Alembic migrations `0001`…), because a path is not an identity: the
|
||||
donor's `path UNIQUE` is exactly what broke on every move and rename.
|
||||
|
||||
`nsfwtag` kept its safety scores outside the database in `nsfw_scores.csv`
|
||||
(`path,nsfw_score`, four decimals, unreadable rows dropped). That file is no longer a
|
||||
source of truth; `photo_pipeline/services/legacy_import.py` imports it into
|
||||
`assets.safety_score` and reports exactly what matched, what did not, and why.
|
||||
|
||||
## Redaction
|
||||
|
||||
The archive contains no credentials. `photo_analyzer.env.sample` documents the
|
||||
configuration surface (`LLM_API_KEY`, `LLM_BASE_URL`, `LLM_MODEL`, and the tuning
|
||||
variables) with placeholder values only; the CLI itself never contained a key, it
|
||||
read one from `photo_analyzer.env` or the environment. No `.env`, database, log, CSV,
|
||||
or photo from the author's library is archived.
|
||||
|
||||
## Why these tools were retired
|
||||
|
||||
Each behavior's fate is recorded per row in `donor_ledger.yaml`: `reuse`, `extract`,
|
||||
`refactor`, or `replace`, with the target module, the characterization tests that
|
||||
pinned the donor's behavior, the parity tests the replacement passes, and — where the
|
||||
replacement deliberately does something else or nothing at all — a `delta` saying so.
|
||||
Rows still marked `pending` name the backlog story that will resolve them; they are
|
||||
the honest list of what has *not* been carried over yet.
|
||||
656
legacy_cli_archive/donor_ledger.yaml
Normal file
656
legacy_cli_archive/donor_ledger.yaml
Normal file
@@ -0,0 +1,656 @@
|
||||
# Donor ledger — US01-01 (INTEGRATED_PIPELINE_CONCEPT.md §3 "Donor-first CLI migration")
|
||||
#
|
||||
# Maps every relevant legacy module/function to its classification, rationale,
|
||||
# target module in the future photo_pipeline package, and characterization
|
||||
# test IDs. Linted by tests/characterization/test_donor_ledger.py.
|
||||
#
|
||||
# classification: reuse — take as-is
|
||||
# extract — move behind a thin adapter, behavior unchanged
|
||||
# refactor — extract while changing structure, behavior preserved
|
||||
# unless an intentional delta is stated
|
||||
# replace — new implementation; rationale documents why and what
|
||||
# (if anything) carries over
|
||||
# Every row needs either `tests` (existing test IDs, module::function) or
|
||||
# `pending_story` (the backlog story that will characterize/deliver it).
|
||||
# status: characterized — donor behavior pinned by characterization tests
|
||||
# resolved — replacement shipped; `parity` names the tests that prove
|
||||
# it, and `delta` states every intentional difference
|
||||
# pending — not migrated yet; `pending_story` says which story will
|
||||
# parity: test ids (path::function) in any suite, proving the replacement
|
||||
# delta: what the replacement deliberately does differently, or not at all
|
||||
#
|
||||
# Archived by US07-01: the sources referenced below now live beside this file in
|
||||
# src/ and are frozen (see README.md). Nothing in photo_pipeline imports them.
|
||||
|
||||
rows:
|
||||
# ── photo_analyzer.py ──────────────────────────────────────────────────────
|
||||
- id: pa-discovery
|
||||
area: discovery
|
||||
source: {file: photo_analyzer.py, symbols: [discover_photos, SUPPORTED_EXTENSIONS]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Proven recursive discovery with _IGNORE/ and .@__thumb exclusion, sorted +
|
||||
deduplicated. Becomes the one shared discovery used by every stage; the
|
||||
exclusion predicate moves to path_policy.py.
|
||||
target: photo_pipeline/services/inventory.py + photo_pipeline/path_policy.py
|
||||
tests:
|
||||
- test_pa_discovery::test_discover_excludes_ignore_and_thumbs
|
||||
- test_pa_discovery::test_discover_finds_supported_including_uppercase
|
||||
- test_pa_discovery::test_discover_sorted_and_deduplicated
|
||||
- test_pa_discovery::test_supported_extensions_contract
|
||||
status: characterized
|
||||
|
||||
- id: pa-purge-excluded
|
||||
area: discovery
|
||||
source: {file: photo_analyzer.py, symbols: [purge_excluded]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
SQL LIKE patterns duplicate the path policy; refactor to call the shared
|
||||
path_policy predicate so exclusion has exactly one definition.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_db::test_purge_excluded_removes_ignore_and_thumb_rows]
|
||||
status: characterized
|
||||
|
||||
- id: pa-prune-missing
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [prune_missing]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Deletes rows for missing files — incompatible with stable asset identity;
|
||||
the concept keeps the asset and sets missing_at/availability_state.
|
||||
The unmounted-library guard (never prune when the root is absent) carries
|
||||
over as behavior.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_db::test_prune_missing_guards_unmounted_library]
|
||||
status: characterized
|
||||
|
||||
- id: pa-variants
|
||||
area: discovery
|
||||
source: {file: photo_analyzer.py, symbols: [_strip_variant_markers, _is_variant, _variant_key, build_variant_groups]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Filename-based variant grouping (pixel-size suffix, 'bearbeitet' edit
|
||||
marker) with unmarked-largest primary selection; feeds duplicate/variant
|
||||
clustering evidence.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
tests:
|
||||
- test_pa_variants::test_strip_variant_markers_goldens
|
||||
- test_pa_variants::test_is_variant
|
||||
- test_pa_variants::test_variant_key_folder_scoped_case_insensitive
|
||||
- test_pa_variants::test_build_variant_groups_primary_is_largest_unmarked
|
||||
status: characterized
|
||||
|
||||
- id: pa-variant-propagate
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [propagate_variants, _row_to_result]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Copying a primary's analysis into variant rows survives, but keyed by
|
||||
asset_id and recorded as stage state instead of raw row copies.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
parity:
|
||||
- tests/integration/test_duplicate_engine.py::test_exact_copies_form_auto_decided_cluster
|
||||
- tests/integration/test_duplicate_engine.py::test_perceptual_variant_is_review_only
|
||||
delta: >
|
||||
The donor propagated variant links implicitly while writing rows; the replacement makes the canonical link a reviewable, reversible cluster decision, so a propagated link can always be undone.
|
||||
status: resolved
|
||||
|
||||
- id: pa-hashing
|
||||
area: hashing
|
||||
source: {file: photo_analyzer.py, symbols: [_sha1_file, _phash_image]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Streamed SHA-1 plus 64-bit DCT phash (imagehash.phash recipe on
|
||||
numpy/scipy, no extra dependency). Hash outputs are goldens — a change
|
||||
means the decoder/algorithm changed and hash_version must bump.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests:
|
||||
- test_pa_hashing_dedup::test_phash_goldens
|
||||
- test_pa_hashing_dedup::test_phash_survives_resize_and_recompress
|
||||
- test_pa_hashing_dedup::test_phash_format_16_hex_chars
|
||||
- test_pa_hashing_dedup::test_sha1_matches_hashlib
|
||||
- test_pa_hashing_dedup::test_sha1_unreadable_returns_none
|
||||
status: characterized
|
||||
|
||||
- id: pa-ensure-hashes
|
||||
area: hashing
|
||||
source: {file: photo_analyzer.py, symbols: [ensure_hashes]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Resume-safe incremental hashing (skip already-hashed, COALESCE so a failed
|
||||
hash never clobbers a stored one, periodic commits, honors stop event).
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_hashing_dedup::test_ensure_hashes_skips_hashed_and_is_resume_safe]
|
||||
status: characterized
|
||||
|
||||
- id: pa-cluster
|
||||
area: hashing
|
||||
source: {file: photo_analyzer.py, symbols: [cluster_duplicates]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Union-find over the vectorized Hamming graph; largest-first ordering.
|
||||
Known ceiling (documented in-source): O(n²) scan, BK-tree past ~100k.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
tests: [test_pa_hashing_dedup::test_cluster_and_mark_duplicates_largest_is_canonical]
|
||||
status: characterized
|
||||
|
||||
- id: pa-mark-duplicates
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [mark_duplicates]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Intentional delta — the concept forbids silently marking fuzzy matches:
|
||||
largest-file canonical becomes a *recommendation*; the decision lands in
|
||||
reviewable duplicate_clusters, auto-resolve only for exact matches.
|
||||
The status='duplicate' exclusion from analysis/upload carries over.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
tests: [test_pa_hashing_dedup::test_cluster_and_mark_duplicates_largest_is_canonical]
|
||||
status: characterized
|
||||
|
||||
- id: pa-list-duplicates
|
||||
area: ui
|
||||
source: {file: photo_analyzer.py, symbols: [list_duplicates]}
|
||||
classification: replace
|
||||
rationale: Console report; superseded by the duplicate-review API/UI (US01-06).
|
||||
target: photo_pipeline/api/routes + frontend duplicate review
|
||||
parity:
|
||||
- tests/integration/test_review_api.py::test_clusters_list_and_detail
|
||||
- tests/e2e/test_review_ui.py::test_fuzzy_decision_requires_confirmation
|
||||
delta: >
|
||||
The text listing became the paged cluster API and the comparison UI; no textual report is produced.
|
||||
status: resolved
|
||||
|
||||
- id: pa-reconcile
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [reconcile_moved]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
SHA-1-keyed move/rename reconciliation preserving analysis + EXIF state.
|
||||
Refactor to update assets.current_path by asset_id and append an
|
||||
asset_paths history row instead of rewriting the path in place.
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests: [test_pa_hashing_dedup::test_reconcile_moved_preserves_row_by_sha1]
|
||||
status: characterized
|
||||
|
||||
- id: pa-db-schema
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [SCHEMA, get_db, _migrate_schema]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
photos table + FTS5 + idempotent ALTER migration become the migration
|
||||
baseline; new schema adds assets/asset_paths/stage state per concept.
|
||||
Existing data must migrate losslessly (US01-02).
|
||||
target: photo_pipeline migrations (Alembic)
|
||||
tests: [test_pa_db::test_schema_and_migration_idempotent]
|
||||
status: characterized
|
||||
|
||||
- id: pa-db-status
|
||||
area: error
|
||||
source: {file: photo_analyzer.py, symbols: [upsert_pending, mark_analyzed, mark_error, mark_exif_written, get_pending, get_analyzed_no_exif]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Status lifecycle (pending→analyzed→exif_written; error rows auto-retried
|
||||
on the next run; INSERT OR IGNORE never downgrades; duplicates never
|
||||
queued) is the resume-safety contract — preserved as asset_stage_states
|
||||
transitions keyed by asset_id.
|
||||
target: photo_pipeline/repositories + services/analysis.py
|
||||
tests:
|
||||
- test_pa_db::test_status_lifecycle_pending_analyzed_exif_written
|
||||
- test_pa_db::test_mark_error_and_retry_via_get_pending
|
||||
- test_pa_db::test_mark_analyzed_clears_error
|
||||
- test_pa_db::test_upsert_pending_never_downgrades
|
||||
status: characterized
|
||||
|
||||
- id: pa-fts
|
||||
area: database
|
||||
source: {file: photo_analyzer.py, symbols: [photos_fts]}
|
||||
classification: reuse
|
||||
rationale: FTS5 table + sync triggers already work; carried into the new schema.
|
||||
target: photo_pipeline migrations (Alembic)
|
||||
tests: [test_pa_db::test_fts_kept_in_sync_by_triggers]
|
||||
status: characterized
|
||||
|
||||
- id: pa-imaging
|
||||
area: imaging
|
||||
source: {file: photo_analyzer.py, symbols: [prepare_image, MAX_LONG_EDGE]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
RGB-normalize (drops alpha, converts HEIC), LANCZOS resize to 2048px
|
||||
long-edge, JPEG q85 base64 — the provider-input contract. Truncated-image
|
||||
tolerance (ImageFile.LOAD_TRUNCATED_IMAGES) carries with it.
|
||||
target: photo_pipeline/services/analysis.py
|
||||
tests:
|
||||
- test_pa_imaging::test_prepare_image_small_passthrough_jpeg
|
||||
- test_pa_imaging::test_prepare_image_resizes_to_max_long_edge
|
||||
- test_pa_imaging::test_prepare_image_converts_png_alpha_to_rgb_jpeg
|
||||
status: characterized
|
||||
|
||||
- id: pa-repair
|
||||
area: error
|
||||
source: {file: photo_analyzer.py, symbols: [repair_image, _REPAIRABLE_ERRORS, _log_repair]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Re-encode structurally broken images via sips (macOS-only — portability
|
||||
note), .orig backup, atomic replace, repairs.jsonl audit. Needs corrupt
|
||||
fixtures to characterize; lands with media hardening.
|
||||
target: photo_pipeline/integrations/exiftool.py
|
||||
pending_story: US07-03
|
||||
status: pending
|
||||
|
||||
- id: pa-vision
|
||||
area: vision
|
||||
source: {file: photo_analyzer.py, symbols: [analyze_image, ANALYSIS_PROMPT, LLM_BASE_URL, LLM_MODEL]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
OpenAI-compatible request construction, base64 image_url payload, JSON
|
||||
response validation, 429/503 retry with exponential backoff. Prompt and
|
||||
model/config version must be persisted per analysis_runs. Characterized
|
||||
against a deterministic fake provider when the analysis service is ported.
|
||||
target: photo_pipeline/services/analysis.py
|
||||
parity:
|
||||
- tests/integration/test_safety_analysis.py::test_provider_called_only_for_confirmed_sfw
|
||||
delta: >
|
||||
The prompt and response schema carry over; the provider is an injected adapter so the privacy gate is testable, and results are keyed to asset ids rather than paths.
|
||||
status: resolved
|
||||
|
||||
- id: pa-throttle
|
||||
area: logging
|
||||
source: {file: photo_analyzer.py, symbols: [_record_ratelimit, _ratelimit_recent, write_throttle_summary, _rpd_load, _rpd_increment, _rpd_count]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Rolling throttle window + persistent throttle_events.jsonl + RPD day
|
||||
counter become job metrics/events on the durable job model.
|
||||
target: photo_pipeline/services/jobs.py + photo_pipeline/services/analysis.py
|
||||
pending_story: US07-04
|
||||
status: pending
|
||||
|
||||
- id: pa-nsfw-filter
|
||||
area: nsfw
|
||||
source: {file: photo_analyzer.py, symbols: [_rec_has_nsfw, filter_nsfw_tagged]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Intentional delta — donor fails OPEN (exiftool hiccup → analyze
|
||||
everything), which violates the concept privacy invariant; the new gate
|
||||
is the DB safety decision and fails CLOSED (only confirmed sfw reaches
|
||||
the provider). EXIF keyword reading survives as evidence/verification,
|
||||
not as the gate. Current behavior captured for migration.
|
||||
target: photo_pipeline/services/safety.py
|
||||
tests:
|
||||
- test_pa_exif::test_rec_has_nsfw_list_scalar_case
|
||||
- test_pa_exif::test_filter_nsfw_tagged_splits_and_normalizes
|
||||
- test_pa_exif::test_filter_nsfw_tagged_empty_list_noop
|
||||
status: characterized
|
||||
|
||||
- id: pa-exif-caption
|
||||
area: exif
|
||||
source: {file: photo_analyzer.py, symbols: [build_exif_caption, build_exif_caption_from_result]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Caption format ('desc | Tags: … | Mood: … | Location: … | ~year') is
|
||||
what 25k+ photos already carry — preserved for compatibility. Intentional
|
||||
delta: new writes go into a managed 'AI:' segment so re-analysis can
|
||||
replace only its own text (concept EXIF ownership).
|
||||
target: photo_pipeline/services/exif projection
|
||||
tests:
|
||||
- test_pa_exif::test_caption_golden_full
|
||||
- test_pa_exif::test_caption_golden_sparse
|
||||
status: characterized
|
||||
|
||||
- id: pa-exif-read
|
||||
area: exif
|
||||
source: {file: photo_analyzer.py, symbols: [read_existing_exif]}
|
||||
classification: extract
|
||||
rationale: Targeted exiftool JSON read of the four owned fields, 30s timeout, fail-empty.
|
||||
target: photo_pipeline/integrations/exiftool.py
|
||||
tests: [test_pa_exif::test_write_exif_roundtrip_and_idempotent]
|
||||
status: characterized
|
||||
|
||||
- id: pa-exif-write
|
||||
area: exif
|
||||
source: {file: photo_analyzer.py, symbols: [write_exif]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Merge-don't-overwrite semantics are the crown jewels: caption prepended to
|
||||
existing text (idempotent — never appended twice), keywords merged +
|
||||
order-preserving dedup, -m -overwrite_original, repair-and-retry on
|
||||
structurally broken files. Refactored to add the concept's read-back
|
||||
verification + non-owned-field snapshot comparison. sys.exit on missing
|
||||
exiftool becomes a service error.
|
||||
target: photo_pipeline/integrations/exiftool.py + services/exif checkpoints
|
||||
tests:
|
||||
- test_pa_exif::test_write_exif_roundtrip_and_idempotent
|
||||
- test_pa_exif::test_write_exif_preserves_existing_metadata
|
||||
status: characterized
|
||||
|
||||
- id: pa-run-loop
|
||||
area: cancellation
|
||||
source: {file: photo_analyzer.py, symbols: [run_analysis, _run_folder_loop, run_exif_write, _stop, _handle_sigint]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Folder-grouped worker pool, per-item DB commits, cooperative stop event
|
||||
checked between items, double-SIGINT force quit — becomes the durable
|
||||
JobRunner worker loop with the same drain-and-resume semantics.
|
||||
target: photo_pipeline/jobs/worker.py
|
||||
parity:
|
||||
- tests/integration/test_worker.py::test_worker_processes_all_items
|
||||
- tests/integration/test_worker.py::test_cooperative_cancellation_leaves_items_resumable
|
||||
delta: >
|
||||
The in-process folder loop with SIGINT handling became durable jobs claimed by a worker: cancellation is a persisted request, not a signal, and an interrupted run resumes from the database instead of restarting.
|
||||
status: resolved
|
||||
|
||||
- id: pa-ui-terminal
|
||||
area: ui
|
||||
source: {file: photo_analyzer.py, symbols: [_Dashboard, _LiveRenderer, _PlainRenderer, _make_progress, _stats_panel, _feed_panel, _tokens_panel, _menu_panel, _read_key, _key_listener, print_stats, fit_label]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Rich terminal dashboard (Live layout, hotkey menu, termios key reader) is
|
||||
superseded by the web Workflow/Analyze views. The *data* it renders
|
||||
(album progress, feed, tokens, ETA) survives via webapp.runner.progress
|
||||
(see wa-runner). fit_label/print_stats die with it.
|
||||
target: frontend Analyze view (Phase B)
|
||||
tests: [test_pa_album::test_fit_label_left_truncates]
|
||||
status: characterized
|
||||
|
||||
- id: pa-album-label
|
||||
area: ui
|
||||
source: {file: photo_analyzer.py, symbols: [album_label]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Album = leaf folder relative to library, '(root)' at root, bare parent
|
||||
outside the library. The single album-identity rule for stats, proposals
|
||||
and folder-as-album upload. webapp.query.album_of is a copy — consolidate
|
||||
to one implementation (see wa-album-of). Extracted in US03-01 as
|
||||
photo_pipeline.services.albums.album_label (parity test below); the album
|
||||
evidence aggregator uses it. LibraryService._album_of still keeps its own
|
||||
leaf-name grouping — fold it into this rule when the Library view is next
|
||||
touched.
|
||||
target: photo_pipeline/services/albums.py
|
||||
tests:
|
||||
- test_pa_album::test_album_label_leaf_folder_relative
|
||||
- test_pa_album::test_webapp_album_of_mirrors_album_label
|
||||
- test_pa_album::test_albums_service_album_label_matches_donor
|
||||
status: characterized
|
||||
|
||||
- id: pa-config
|
||||
area: configuration
|
||||
source: {file: photo_analyzer.py, symbols: [load_env_file, _env_str, _env_int, _env_bool, ENV_FILE]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
photo_analyzer.env parsing (shell env wins, export prefix, quoted values,
|
||||
inline-comment rules) feeds the typed config module; same file keeps
|
||||
working so the transition needs no re-setup.
|
||||
target: photo_pipeline/config.py
|
||||
tests:
|
||||
- test_pa_config::test_load_env_file_parsing
|
||||
- test_pa_config::test_env_helpers
|
||||
status: characterized
|
||||
|
||||
- id: pa-logging
|
||||
area: logging
|
||||
source: {file: photo_analyzer.py, symbols: [log_history, _history_handler, _rich_handler]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Module-level triple-handler logging (console + info log + debug log) and
|
||||
the JSONL history logger become structured JSON logging with job_id/
|
||||
asset_id and job_events rows; per-photo history maps to job events.
|
||||
target: photo_pipeline structured logging + jobs/job_events
|
||||
parity:
|
||||
- tests/integration/test_jobs.py::test_enqueue_persists_items_and_event
|
||||
- tests/integration/test_jobs_sse.py::test_sse_streams_all_events_then_closes
|
||||
delta: >
|
||||
The JSONL history file and rich console handler are replaced by structured JSON logs plus durable job_events; the browser reads events over SSE rather than tailing a file.
|
||||
status: resolved
|
||||
|
||||
- id: pa-balance
|
||||
area: vision
|
||||
source: {file: photo_analyzer.py, symbols: [fetch_balance, query_balance, check_quota]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Provider balance/quota probes (report 'unsupported' on providers without
|
||||
the endpoint). Network-bound; characterized against the fake provider.
|
||||
target: photo_pipeline/services/analysis.py
|
||||
parity:
|
||||
- tests/integration/test_safety_analysis.py::test_provider_called_only_for_confirmed_sfw
|
||||
delta: >
|
||||
Not carried over: balance/quota polling was provider-specific (Gemini/OpenAI billing endpoints) and key-scoped. Cost reporting, when a story asks for it, comes from the per-run usage recorded with each analysis result rather than from a vendor endpoint.
|
||||
status: resolved
|
||||
|
||||
- id: pa-cli
|
||||
area: configuration
|
||||
source: {file: photo_analyzer.py, symbols: [main]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
argparse surface is superseded by the API; flags map to job configs
|
||||
(documented in WEBAPP_CONCEPT.md §8 parity table). Transitional CLI calls
|
||||
the shared services until archival (E07).
|
||||
target: photo_pipeline/__main__.py (serve | worker | migrate | import-legacy-scores) + /api/v1
|
||||
parity:
|
||||
- tests/integration/test_app_lifecycle.py::test_restart_preserves_data_and_reruns_migrations
|
||||
- tests/unit/test_legacy_archive.py::test_production_code_never_imports_an_archived_module
|
||||
delta: >
|
||||
The argparse surface is not reproduced. Every flag that drove work became an API command or a job configuration; the CLI keeps only what an application needs to be operated (serve, worker, migrate) plus the one-off legacy CSV import.
|
||||
status: resolved
|
||||
|
||||
# ── nsfwtag/ ───────────────────────────────────────────────────────────────
|
||||
- id: nt-discovery
|
||||
area: discovery
|
||||
source: {file: nsfwtag/scoring.py, symbols: [discover_images]}
|
||||
classification: refactor
|
||||
rationale: >
|
||||
Merged into the shared discovery. Known divergence captured: nsfwtag EXTS
|
||||
lacks .tiff/.tif, is non-recursive by default, and dedupes by resolved
|
||||
path (symlink guard) — the shared service adopts the superset + symlink
|
||||
dedup, always recursive, plus _IGNORE policy (donor has none).
|
||||
target: photo_pipeline/services/inventory.py
|
||||
tests:
|
||||
- test_nsfwtag::test_exts_contract_differs_from_photo_analyzer
|
||||
- test_nsfwtag::test_discover_images_flat_recursive_limit
|
||||
- test_nsfwtag::test_discover_images_dedupes_symlinked_paths
|
||||
status: characterized
|
||||
|
||||
- id: nt-score-cache
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/scoring.py, symbols: [load_cache, _save_cache, CSV_NAME]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
nsfw_scores.csv stops being the source of truth (concept: DB state).
|
||||
Format characterized (4-decimal scores, bad rows dropped) because the
|
||||
existing CSV must migrate into assets.safety_score.
|
||||
target: photo_pipeline/services/legacy_import.py (one-off import into safety_reviews)
|
||||
tests: [test_nsfwtag::test_score_cache_roundtrip_and_tolerance]
|
||||
parity:
|
||||
- tests/integration/test_legacy_import.py::test_scores_are_imported_onto_asset_identity
|
||||
- tests/integration/test_legacy_import.py::test_a_reviewed_asset_is_never_overwritten_by_the_csv
|
||||
delta: >
|
||||
The CSV is no longer read at runtime at all: it is imported once into
|
||||
scored-but-unreviewed safety_reviews rows and left on disk untouched. A path
|
||||
that matches nothing is reported, never turned into an asset, and a human
|
||||
decision always outranks an imported score.
|
||||
status: resolved
|
||||
|
||||
- id: nt-score-model
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/scoring.py, symbols: [score_images, MODEL_ID, BATCH]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
On-device ViT scoring: MPS/CPU pick, model-defined input size (not
|
||||
hardcoded — survives model swap), batch inference where one bad batch
|
||||
never discards prior scores, periodic checkpoint every 500. Cache-hit
|
||||
short-circuit (model never loads when nothing is new) characterized now;
|
||||
inference path needs the local model + deterministic fake.
|
||||
target: photo_pipeline/integrations/nsfw_model.py
|
||||
tests: [test_nsfwtag::test_score_images_cache_hit_skips_model]
|
||||
status: characterized
|
||||
|
||||
- id: nt-exif-keyword
|
||||
area: exif
|
||||
source: {file: nsfwtag/exif.py, symbols: [write_keyword, remove_keyword]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Idempotent add/remove via '-Keywords-=x -Keywords+=x' (no duplicates on
|
||||
re-run), touching only Keywords+Subject. Basis of the safety EXIF
|
||||
checkpoint; the mutual-exclusion write (sfw removed when nsfw added and
|
||||
vice versa) is the concept's addition on top.
|
||||
target: photo_pipeline/services/safety.py + integrations/exiftool.py
|
||||
tests: [test_nsfwtag::test_write_remove_keyword_idempotent]
|
||||
status: characterized
|
||||
|
||||
- id: nt-exif-marks
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/exif.py, symbols: [_read_keywords, read_tagged, read_marks]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Batched exiftool read via stdin, lowercase normalization, and the safety
|
||||
rule that a file carrying both keywords reads as nsfw (never as safe).
|
||||
target: photo_pipeline/services/safety.py
|
||||
tests:
|
||||
- test_nsfwtag::test_write_remove_keyword_idempotent
|
||||
- test_nsfwtag::test_read_marks_nsfw_wins_over_sfw
|
||||
status: characterized
|
||||
|
||||
- id: nt-exif-view
|
||||
area: exif
|
||||
source: {file: nsfwtag/exif.py, symbols: [read_exif, _EXIF_NOISE]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Lightbox EXIF read: filesystem-noise filter, binary/oversize skip, signed
|
||||
decimal GPS, Google-Maps link prepended. Backs the asset EXIF endpoint.
|
||||
target: photo_pipeline/api (asset EXIF endpoint)
|
||||
tests: [test_nsfwtag::test_read_exif_filters_noise_and_prepends_map]
|
||||
status: characterized
|
||||
|
||||
- id: nt-apply-list
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/exif.py, symbols: [apply_list]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Newline-list bulk tagging (nsfw_confirmed.txt flow) is superseded by DB
|
||||
review decisions; the existing list is a one-time migration input.
|
||||
target: photo_pipeline/services/safety.py (decision import in US01-02)
|
||||
parity:
|
||||
- tests/integration/test_safety_parity.py::test_extracted_marks_and_partition_match_donor
|
||||
delta: >
|
||||
Bulk keyword application from a file list is replaced by decisions against asset ids; the EXIF write itself is the extracted, read-back-verified one.
|
||||
status: resolved
|
||||
|
||||
- id: nt-ui
|
||||
area: ui
|
||||
source: {file: nsfwtag/server.py, symbols: [serve_review]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
stdlib ThreadingHTTPServer + token-injected review.html backend is
|
||||
superseded by FastAPI. review.html's design tokens, folder tree,
|
||||
threshold/score review flow, lightbox and keyboard model are the frontend
|
||||
donor for the Safety view (preserved per concept §10; ported in US02-01).
|
||||
target: photo_pipeline/api + frontend Safety view
|
||||
parity:
|
||||
- tests/e2e/test_workflow_views.py::test_safety_review_decide_persists_across_reload
|
||||
delta: >
|
||||
The stdlib review server is replaced by the API plus the Safety view; the donor's layout, thresholds, and keyboard flow carry over, its embedded HTML generation does not.
|
||||
status: resolved
|
||||
|
||||
- id: nt-bench
|
||||
area: nsfw
|
||||
source: {file: nsfwtag/bench.py, symbols: [main]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Dev-only model benchmark; archived without webapp replacement (recorded
|
||||
basis of the AdamCodd model choice). No production caller.
|
||||
target: none (archive as reference)
|
||||
parity:
|
||||
- tests/unit/test_legacy_archive.py::test_every_archived_source_matches_its_checksum
|
||||
delta: >
|
||||
No replacement: a dev-only benchmark whose result (the AdamCodd model choice) is already recorded. Kept in the archive as the basis of that choice.
|
||||
status: resolved
|
||||
|
||||
# ── webapp/ ────────────────────────────────────────────────────────────────
|
||||
- id: wa-query-search
|
||||
area: database
|
||||
source: {file: webapp/query.py, symbols: [_fts_query, _where, search, _row_to_card, photo, _SORTS]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Safe FTS5 MATCH building (tokenize → quoted prefix terms; punctuation
|
||||
can't crash MATCH), facet WHERE composition, paged search with
|
||||
relevance/sort modes, raw_response kept out of list payloads.
|
||||
target: photo_pipeline/repositories (search) + api/routes
|
||||
tests:
|
||||
- test_webapp_query::test_fts_query_sanitization_goldens
|
||||
- test_webapp_query::test_where_clause_goldens
|
||||
- test_webapp_query::test_search_fts_prefix_match
|
||||
- test_webapp_query::test_search_filters_and_browse
|
||||
- test_webapp_query::test_search_paging
|
||||
- test_webapp_query::test_photo_omits_raw_response
|
||||
status: characterized
|
||||
|
||||
- id: wa-query-aggregates
|
||||
area: database
|
||||
source: {file: webapp/query.py, symbols: [facets, stats, _col_counts, _people_buckets, _top_tags]}
|
||||
classification: extract
|
||||
rationale: >
|
||||
Facet counts, year range, album tree, status/album progress (done =
|
||||
analyzed + exif_written) and error listing — the Stats/Workflow data
|
||||
model.
|
||||
target: photo_pipeline/repositories + services/workflow.py
|
||||
tests:
|
||||
- test_webapp_query::test_facets_and_albums
|
||||
- test_webapp_query::test_stats_album_progress_and_errors
|
||||
status: characterized
|
||||
|
||||
- id: wa-album-of
|
||||
area: ui
|
||||
source: {file: webapp/query.py, symbols: [album_of]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Verbatim copy of photo_analyzer.album_label (equivalence characterized);
|
||||
consolidated into the single albums-service implementation (pa-album-label),
|
||||
now photo_pipeline.services.albums.album_label (US03-01, parity test below).
|
||||
target: photo_pipeline/services/albums.py
|
||||
tests:
|
||||
- test_pa_album::test_webapp_album_of_mirrors_album_label
|
||||
- test_pa_album::test_albums_service_album_label_matches_donor
|
||||
status: characterized
|
||||
|
||||
- id: wa-allowlist
|
||||
area: ui
|
||||
source: {file: webapp/query.py, symbols: [all_paths]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Path-set allowlist for /img and /exif is superseded by asset-ID-addressed
|
||||
endpoints that never accept a browser-supplied path (concept thumbnail
|
||||
rules). The validate-before-serving intent carries over.
|
||||
target: photo_pipeline/api (asset_id thumbnail/EXIF endpoints)
|
||||
tests: [test_webapp_query::test_all_paths_is_the_image_endpoint_allowlist]
|
||||
status: characterized
|
||||
|
||||
- id: wa-runner
|
||||
area: cancellation
|
||||
source: {file: webapp/runner.py, symbols: [Runner, progress]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
Subprocess-driving-the-CLI job control is superseded by durable DB jobs
|
||||
with a worker process. Two ideas carry over: progress derived from DB
|
||||
counts (not job-private state) and single-mutating-job enforcement.
|
||||
target: photo_pipeline/services/jobs.py + photo_pipeline/jobs/worker.py
|
||||
parity:
|
||||
- tests/integration/test_jobs.py::test_idempotency_key_returns_same_job
|
||||
- tests/integration/test_worker.py::test_handler_failure_fails_the_job
|
||||
delta: >
|
||||
Subprocess supervision of a CLI is replaced by durable jobs in the same process family: there is no subprocess to supervise, and progress is persisted rather than scraped from stdout.
|
||||
status: resolved
|
||||
|
||||
- id: wa-server
|
||||
area: ui
|
||||
source: {file: webapp/server.py, symbols: [serve]}
|
||||
classification: replace
|
||||
rationale: >
|
||||
stdlib HTTP route ladder → FastAPI typed routes. Carried intents:
|
||||
127.0.0.1 bind, path validation against the DB, no secrets to the
|
||||
browser. analyzer.html + page.py design (dark OLED tokens, Library/
|
||||
Analyze/Stats views) is frontend donor material per concept §10.
|
||||
target: photo_pipeline/api/app.py + frontend
|
||||
parity:
|
||||
- tests/e2e/test_frontend_shell.py::test_shell_loads_assets_without_console_or_network_errors
|
||||
- tests/e2e/test_frontend_shell.py::test_deep_link_and_reload_restore_view_and_filters
|
||||
delta: >
|
||||
The stdlib HTTP routes become the versioned FastAPI API and the static frontend shell; Python no longer interpolates HTML.
|
||||
status: resolved
|
||||
18
legacy_cli_archive/photo_analyzer.env.sample
Normal file
18
legacy_cli_archive/photo_analyzer.env.sample
Normal file
@@ -0,0 +1,18 @@
|
||||
# Sample configuration for the archived photo_analyzer.py CLI (US07-01).
|
||||
#
|
||||
# REDACTED: no value below is real. Each line names a variable the CLI read and
|
||||
# what belongs there; the placeholders are deliberately not key-shaped, so this
|
||||
# file can never be mistaken for — or scanned as — a credential.
|
||||
|
||||
LLM_API_KEY=<paste your provider key here>
|
||||
LLM_BASE_URL=<provider base url, e.g. the OpenAI-compatible Gemini endpoint>
|
||||
LLM_MODEL=<model id, e.g. a Gemini Flash release>
|
||||
|
||||
# Optional tuning the CLI read from the same file:
|
||||
PHASH_THRESHOLD=8
|
||||
MAX_WORKERS=4
|
||||
RETRY_ATTEMPTS=3
|
||||
RPD_LIMIT=0
|
||||
|
||||
# The CLI ignored its own shipped placeholder (a literal "sk-REPLACE..." string)
|
||||
# until it was replaced, and a shell variable always won over this file.
|
||||
20
legacy_cli_archive/requirements-lock.txt
Normal file
20
legacy_cli_archive/requirements-lock.txt
Normal file
@@ -0,0 +1,20 @@
|
||||
# Final dependency lock of the archived CLIs (US07-01).
|
||||
#
|
||||
# These are the versions present in the environment the archive was taken from —
|
||||
# what the frozen sources were last verified against by the characterization
|
||||
# suite. Restoring a donor for forensics means pinning these, not "latest".
|
||||
#
|
||||
# Python 3.14.6
|
||||
|
||||
openai==3.0.0 # photo_analyzer: OpenAI-compatible vision client
|
||||
numpy==2.4.6 # photo_analyzer, nsfwtag: pixel work
|
||||
Pillow==12.3.0 # photo_analyzer, nsfwtag: decode/resize
|
||||
rich==15.0.0 # photo_analyzer: console output
|
||||
scipy==1.18.0 # photo_analyzer: perceptual-hash DCT
|
||||
PyYAML==6.0.3 # tooling that reads the donor ledger
|
||||
|
||||
# NSFW inference (nsfwtag/scoring.py, nsfwtag/bench.py) was never installed in the
|
||||
# archiving environment; the model stack is recorded here from the sources so a
|
||||
# forensic run can reproduce it, not from a resolved lock:
|
||||
# torch, transformers, timm — AdamCodd/vit-base-nsfw-detector (see nsfwtag/README.md)
|
||||
# exiftool is an external binary, not a Python package.
|
||||
179
legacy_cli_archive/src/compare_models.py
Normal file
179
legacy_cli_archive/src/compare_models.py
Normal file
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Side-by-side model comparison for photo_analyzer.
|
||||
|
||||
Reads already-analyzed photos from the DB (the stored result — no re-run, no
|
||||
extra cost on the model that produced them) and runs a second model (default:
|
||||
Gemini 2.5 Flash via its OpenAI-compatible endpoint) live on the same images,
|
||||
then prints the two results next to each other plus measured token cost.
|
||||
|
||||
Reuses photo_analyzer's prepare_image, ANALYSIS_PROMPT, get_db and env loader —
|
||||
no logic is duplicated.
|
||||
|
||||
Usage:
|
||||
GEMINI_API_KEY=... python compare_models.py -n 10
|
||||
python compare_models.py --selftest # offline check, no API call
|
||||
|
||||
Put GEMINI_API_KEY in photo_analyzer.env (the same file the main pipeline reads).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
import photo_analyzer as pa
|
||||
|
||||
# Gemini's OpenAI-compatible endpoint — the existing OpenAI client just works.
|
||||
GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"
|
||||
DEFAULT_MODEL = "gemini-2.5-flash"
|
||||
|
||||
# $/million tokens, for the live model's measured cost. Update if you change model.
|
||||
PRICES = { # (input, output)
|
||||
"gemini-2.5-flash": (0.30, 2.50),
|
||||
"gemini-2.5-flash-lite": (0.10, 0.40),
|
||||
"gemini-3.5-flash": (1.50, 9.00),
|
||||
}
|
||||
|
||||
|
||||
def _strip_json_fences(raw: str) -> str:
|
||||
"""Same fence-stripping photo_analyzer uses — models often wrap JSON in ```."""
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("```")[1]
|
||||
if raw.startswith("json"):
|
||||
raw = raw[4:]
|
||||
raw = raw.strip()
|
||||
return raw
|
||||
|
||||
|
||||
def analyze_with(client: OpenAI, model: str, path: Path) -> tuple[dict, dict]:
|
||||
"""Run `model` on one image. Returns (parsed_result, usage_dict)."""
|
||||
b64, mime = pa.prepare_image(path)
|
||||
resp = client.chat.completions.create(
|
||||
model=model,
|
||||
max_tokens=4096,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}},
|
||||
{"type": "text", "text": pa.ANALYSIS_PROMPT},
|
||||
],
|
||||
}],
|
||||
)
|
||||
result = json.loads(_strip_json_fences(resp.choices[0].message.content))
|
||||
u = resp.usage
|
||||
usage = {"prompt": u.prompt_tokens if u else 0,
|
||||
"completion": u.completion_tokens if u else 0}
|
||||
return result, usage
|
||||
|
||||
|
||||
def fetch_analyzed(conn, limit: int) -> list:
|
||||
"""
|
||||
Randomly sample already-analyzed photos, spread across folders. Groups rows by
|
||||
parent directory, shuffles, then round-robins one per folder so the sample
|
||||
covers as many different folders as possible. Only picked files are checked on
|
||||
disk (cheap), not the whole library.
|
||||
"""
|
||||
# "analyzed already" = has a description, regardless of status — once EXIF is
|
||||
# written the status flips from 'analyzed' to 'exif_written', but the result stays.
|
||||
rows = conn.execute(
|
||||
"""SELECT path, description, tags, people_count, mood, setting, approx_year
|
||||
FROM photos WHERE description IS NOT NULL AND description != ''""").fetchall()
|
||||
buckets = defaultdict(list)
|
||||
for r in rows:
|
||||
buckets[str(Path(r["path"]).parent)].append(r)
|
||||
folders = list(buckets.keys())
|
||||
random.shuffle(folders)
|
||||
for f in folders:
|
||||
random.shuffle(buckets[f])
|
||||
|
||||
out, exhausted = [], False
|
||||
while len(out) < limit and not exhausted:
|
||||
exhausted = True
|
||||
for f in folders: # one per folder per pass → spread
|
||||
if not buckets[f]:
|
||||
continue
|
||||
exhausted = False
|
||||
r = buckets[f].pop()
|
||||
if Path(r["path"]).exists():
|
||||
out.append(r)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _fmt(label: str, desc, tags, people, mood) -> str:
|
||||
tags_s = ", ".join(tags) if isinstance(tags, list) else (tags or "")
|
||||
return (f" {label}\n"
|
||||
f" desc: {desc}\n"
|
||||
f" tags: {tags_s}\n"
|
||||
f" people: {people} mood: {mood}")
|
||||
|
||||
|
||||
def selftest():
|
||||
assert _strip_json_fences('```json\n{"a":1}\n```') == '{"a":1}'
|
||||
assert _strip_json_fences('{"a":1}') == '{"a":1}'
|
||||
assert _strip_json_fences('```\n{"b":2}\n```') == '{"b":2}'
|
||||
print("selftest OK")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("-n", "--num", type=int, default=10, help="photos to compare")
|
||||
ap.add_argument("--model", default=DEFAULT_MODEL, help="live model to compare against the stored results")
|
||||
ap.add_argument("--db", default="photo_analysis.db")
|
||||
ap.add_argument("--selftest", action="store_true", help="run offline check and exit")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.selftest:
|
||||
selftest()
|
||||
return
|
||||
|
||||
pa.load_env_file() # picks up GEMINI_API_KEY from photo_analyzer.env
|
||||
key = os.environ.get("GEMINI_API_KEY")
|
||||
if not key:
|
||||
sys.exit("GEMINI_API_KEY not set (add it to photo_analyzer.env or export it).")
|
||||
|
||||
conn = pa.get_db(args.db)
|
||||
rows = fetch_analyzed(conn, args.num)
|
||||
if not rows:
|
||||
sys.exit("No analyzed photos found in the DB to compare against.")
|
||||
|
||||
client = OpenAI(api_key=key, base_url=GEMINI_BASE_URL)
|
||||
p_in, p_out = PRICES.get(args.model, (0.0, 0.0))
|
||||
tot_in = tot_out = 0
|
||||
ok = 0
|
||||
|
||||
for r in rows:
|
||||
path = Path(r["path"])
|
||||
print(f"\n=== {path.parent.name}/{path.name} ===")
|
||||
# stored result (already in the DB — no re-run)
|
||||
try:
|
||||
stored_tags = json.loads(r["tags"]) if r["tags"] else []
|
||||
except Exception:
|
||||
stored_tags = r["tags"]
|
||||
print(_fmt("stored (DB)", r["description"], stored_tags,
|
||||
r["people_count"], r["mood"]))
|
||||
# live call to the comparison model
|
||||
try:
|
||||
res, usage = analyze_with(client, args.model, path)
|
||||
tot_in += usage["prompt"]; tot_out += usage["completion"]; ok += 1
|
||||
print(_fmt(f"{args.model} (live)", res.get("description"),
|
||||
res.get("tags", []), res.get("people_count"), res.get("mood")))
|
||||
except Exception as e:
|
||||
print(f" {args.model} (live)\n ERROR: {e}")
|
||||
|
||||
if ok:
|
||||
cost = tot_in / 1e6 * p_in + tot_out / 1e6 * p_out
|
||||
print(f"\n— {args.model}: {ok} photos, {tot_in:,} in + {tot_out:,} out tokens, "
|
||||
f"measured cost ${cost:.4f} (${cost / ok * 1000:.2f}/1k photos)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
11
legacy_cli_archive/src/nsfw_tag.py
Normal file
11
legacy_cli_archive/src/nsfw_tag.py
Normal file
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backwards-compatible shim — the code now lives in the nsfwtag/ package.
|
||||
|
||||
Run either of these (identical behaviour):
|
||||
python nsfw_tag.py "<folder>" -r
|
||||
python -m nsfwtag "<folder>" -r
|
||||
"""
|
||||
from nsfwtag.__main__ import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
282
legacy_cli_archive/src/nsfwtag/README.md
Normal file
282
legacy_cli_archive/src/nsfwtag/README.md
Normal file
@@ -0,0 +1,282 @@
|
||||
# nsfwtag
|
||||
|
||||
Local, on-device NSFW image review and tagging for large photo libraries — built to feed [Immich](https://immich.app) search and to complement the `photo_analyzer.py` captioning pipeline.
|
||||
|
||||
It scores a photo library with a **local** vision model (Apple GPU via MPS), opens a fast **browser review app**, and writes an `nsfw` (or `sfw`) keyword into each photo's EXIF for the ones you confirm — **preserving every other tag**. Nothing ever leaves the machine: no API calls, no uploads, no telemetry.
|
||||
|
||||
```
|
||||
python -m nsfwtag "pictures/" -r
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Table of contents
|
||||
|
||||
- [The idea](#the-idea)
|
||||
- [Features](#features)
|
||||
- [How it works](#how-it-works)
|
||||
- [Model choice](#model-choice)
|
||||
- [Requirements & install](#requirements--install)
|
||||
- [CLI usage](#cli-usage)
|
||||
- [The review app](#the-review-app)
|
||||
- [EXIF / review-state model](#exif--review-state-model)
|
||||
- [HTTP API](#http-api)
|
||||
- [Data & caching](#data--caching)
|
||||
- [Project layout](#project-layout)
|
||||
- [Design decisions](#design-decisions)
|
||||
- [Immich integration](#immich-integration)
|
||||
|
||||
---
|
||||
|
||||
## The idea
|
||||
|
||||
A large personal photo library has a handful of sensitive images scattered through
|
||||
thousands of holiday snaps. To filter them in [Immich](https://immich.app) you need a
|
||||
searchable marker on each one — but going through every photo by hand is infeasible, and
|
||||
sending a private library to a cloud moderation API is a non-starter.
|
||||
|
||||
**The model does recall; you do precision.** A local classifier is good at narrowing
|
||||
thousands of photos down to a few dozen candidates, but it is *not* trustworthy as the
|
||||
final word — it over-flags swimwear, art, medical shots. So this is a **human-in-the-loop**
|
||||
tool: the model proposes, you confirm in a fast visual review app, and only your confirmed
|
||||
decisions are written to disk. Nothing ever leaves the machine — no API, no upload, no
|
||||
telemetry.
|
||||
|
||||
The decision lives **in the file's own EXIF** as an `nsfw`/`sfw` keyword, not in a
|
||||
side database, so it survives across tools, backups and restarts, and Immich indexes it
|
||||
for free at ingest.
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
- **On-device scoring** with `AdamCodd/vit-base-nsfw-detector` (ViT, ~350 MB, downloaded once then fully offline). Chosen over Falconsai after benchmarking against the library — better recall, fewer false positives (see `bench.py`).
|
||||
- **Resume-safe**: scores are cached to a CSV; a rerun reuses them and only scores newly-added files — skipping the model load entirely when nothing is new.
|
||||
- **Browser review app** (dark OLED UI, zero web dependencies — served by a stdlib HTTP server):
|
||||
- Per-image **checkmark** to mark NSFW / SFW; frame colors show state at a glance.
|
||||
- **Folder browser** sidebar, live search, a single **Show** filter (All / To-handle / NSFW / SFW), sort, NSFW-score threshold, and a contextual action group that appears only when images are selected.
|
||||
- **Lightbox** with prev/next, keyboard control, and an **EXIF panel** (incl. a clickable **Google Maps** link for geotagged photos).
|
||||
- **Activity log** tab with a per-file audit trail of every EXIF write.
|
||||
- Lazy, per-folder tag scan with a **progress ring**.
|
||||
- **Reversible & non-destructive**: tag, untag, or toggle at any time; only `Keywords`/`Subject` are modified, so Caption, dates, GPS, camera info, etc. are always preserved.
|
||||
- **Persistent decisions**: marking an image safe writes an `sfw` keyword, so your review survives restarts.
|
||||
|
||||
---
|
||||
|
||||
## How it works
|
||||
|
||||
```
|
||||
discover images score (cached) review in browser write EXIF
|
||||
folder ─────────────────▶ [*.jpg …] ───────────────▶ nsfw_scores.csv ─────────────────▶ exiftool: Keywords/Subject += nsfw|sfw
|
||||
(recursive, de-duped) supported local ViT on MPS (path,score) local HTTP server + review.html (all other tags preserved)
|
||||
```
|
||||
|
||||
1. **Discover** — walk the folder (optionally recursive), keep supported image types, de-duplicate by resolved path.
|
||||
2. **Score** — load cached scores; run the model only on files not in the cache; each image is resized to 224×224 and scored 0..1 for the `nsfw` class. Merge and persist.
|
||||
3. **Review** — candidates (score ≥ `--review-min`) are served as a single-page app. A lazy per-folder scan reads existing `nsfw`/`sfw` keywords and colors each card.
|
||||
4. **Tag** — your actions POST to the local server, which calls `exiftool` to add/remove the keyword. EXIF is written **before** the file is later uploaded to Immich.
|
||||
|
||||
---
|
||||
|
||||
## Model choice
|
||||
|
||||
The tool ships with **`AdamCodd/vit-base-nsfw-detector`** — a ~350 MB ViT binary
|
||||
classifier that runs offline on Apple MPS. It was picked by benchmarking candidates
|
||||
against **your own library**, using the photos you'd already hand-tagged `nsfw`/`sfw`
|
||||
as ground truth (`bench.py` reads those EXIF marks). A generic public benchmark wouldn't
|
||||
predict how a model behaves on *these* photos, so the comparison is run on the real data.
|
||||
|
||||
Candidates compared (`nsfwtag/bench.py`):
|
||||
|
||||
| Model | Notes |
|
||||
|-------|-------|
|
||||
| **AdamCodd/vit-base-nsfw-detector** ✅ | Chosen — best recall with the fewest false positives on this library |
|
||||
| Falconsai/nsfw_image_detection | Previous default; missed more true positives and needed a high `0.85` cutoff |
|
||||
| Marqo/nsfw-image-detection-384 | Tiny/fast, but noisier on borderline shots |
|
||||
| OwenElliott/image-safety-classifier-xs | 3-class (SFW/NSFW/NSFL); useful signal, weaker overall here |
|
||||
|
||||
**Selection criterion — recall first.** Because a human confirms every candidate, a false
|
||||
*positive* costs one extra glance, but a false *negative* means a sensitive photo silently
|
||||
slips through untagged. So the model is judged on catching everything at a threshold low
|
||||
enough to be safe without burying you in noise. AdamCodd's sweet spot is **`0.6`** (vs
|
||||
Falconsai's `0.85`) — that is the shipped default.
|
||||
|
||||
**Reproduce it** on any sample folder:
|
||||
|
||||
```bash
|
||||
python -m nsfwtag.bench "pictures/SomeFolder" -r
|
||||
# writes bench_scores.csv + bench_report.html (thumbnails, per-model scores,
|
||||
# disagreements highlighted) and prints precision/recall vs your manual tags.
|
||||
```
|
||||
|
||||
To switch models, change `MODEL_ID` in `nsfwtag/__init__.py` (any HF image-classification
|
||||
id) and re-tune `DEFAULT_THRESHOLD`.
|
||||
|
||||
---
|
||||
|
||||
## Requirements & install
|
||||
|
||||
- **Python 3.10+**
|
||||
- **exiftool** on `PATH` — `brew install exiftool` (macOS) / `apt install libimage-exiftool-perl` (Debian/Ubuntu)
|
||||
- Python packages: `pip install torch transformers pillow numpy`
|
||||
- A GPU is optional — uses Apple **MPS** if available, else CPU.
|
||||
|
||||
The model downloads automatically on first run (~350 MB) and is cached in `~/.cache/huggingface`; every run after that is offline (`local_files_only=True`, which also skips the slow HF Hub update check).
|
||||
|
||||
---
|
||||
|
||||
## CLI usage
|
||||
|
||||
Run from the directory that contains the `nsfwtag/` package:
|
||||
|
||||
```bash
|
||||
# Score + open the review app
|
||||
python -m nsfwtag "pictures/" # this folder only
|
||||
python -m nsfwtag "pictures/" -r # recurse into subfolders
|
||||
python nsfw_tag.py "pictures/" -r # identical (backwards-compat shim)
|
||||
|
||||
# Tag directly by score, no review UI
|
||||
python -m nsfwtag "pictures/" -r --apply
|
||||
|
||||
# Tag every path listed in a text file (one path per line)
|
||||
python -m nsfwtag confirmed.txt [more.txt ...]
|
||||
```
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `-r, --recursive` | off | Descend into every subfolder |
|
||||
| `--threshold` | `0.6` | Score ≥ this is treated as "NSFW to review" (and tagged directly in `--apply`) |
|
||||
| `--review-min` | `0.20` | Lowest score shown in the review app |
|
||||
| `--limit N` | `0` | Only the first N images (calibration) |
|
||||
| `--apply` | off | Tag score ≥ threshold directly, skip the review UI |
|
||||
| `--rescore` | off | Ignore the score cache and re-run the model on everything |
|
||||
|
||||
The argument is auto-detected: a **folder** opens the review app; one or more **`.txt`** files tag the listed paths directly.
|
||||
|
||||
---
|
||||
|
||||
## The review app
|
||||
|
||||
Served on `http://127.0.0.1:<random-free-port>/` and opened in your browser automatically. Press `Ctrl-C` in the terminal when done.
|
||||
|
||||
There are **two independent channels** per card: the **decision** (the EXIF tag,
|
||||
shown as the frame + tick color) and the **selection** (a blue ring, for bulk
|
||||
actions). A card can be both selected *and* decided.
|
||||
|
||||
### Frame colors (the decision)
|
||||
|
||||
| Color | Meaning | EXIF |
|
||||
|-------|---------|------|
|
||||
| 🩷 **pink** | tagged NSFW | `nsfw` keyword |
|
||||
| 🟢 **green** | tagged safe | `sfw` keyword |
|
||||
| 🔴 **red** | undecided **and** selected — a bulk candidate, nothing written yet | none |
|
||||
| ⚪ neutral | undecided, not selected | none |
|
||||
| 🔵 blue ring | selected for a bulk action (independent of the frame color) | — |
|
||||
|
||||
Each card has a **tick** (bottom-right circle) that makes a per-image decision and
|
||||
**writes EXIF on every click**, cycling: undecided → **NSFW** (pink) → **SFW**
|
||||
(green) → undecided. One click is enough — it no longer depends on the selection
|
||||
state. Clicking elsewhere on a card does nothing (no accidental toggles).
|
||||
|
||||
### Controls
|
||||
|
||||
The toolbar separates **viewing** (always visible) from **acting** (contextual). It reads left-to-right:
|
||||
|
||||
```
|
||||
🔍 Search… Show ▾ Sort ▾ NSFW ≥ 0.6 ●───── Select flagged Select all ⟨ N selected · Deselect · Remove · Mark safe · Mark NSFW ⟩
|
||||
```
|
||||
|
||||
- **Folder browser** (left sidebar, toggleable): filter to a folder + its subfolders; counts per folder.
|
||||
- **Search**: filter by filename or folder.
|
||||
- **Show** (single filter): **All** / **To-handle** (no keyword yet — what's left to decide) / **NSFW** (tagged nsfw) / **SFW** (tagged sfw).
|
||||
- **Sort**: score / name / folder.
|
||||
- **NSFW ≥ slider**: score threshold used by *Select flagged* and the red "identified" coloring.
|
||||
- **Select flagged**: selects visible, untagged images at/above the threshold.
|
||||
- **Select all**: selects every currently-visible image (respects the folder/filter). Pair with **Deselect** to clear. Use this to build a selection after a filter where *Select flagged* matches nothing.
|
||||
- **Per-image tick**: one click writes EXIF and cycles undecided → NSFW → SFW → undecided; the tick and frame colors show the current decision.
|
||||
- **Action group** (appears only when ≥1 image is selected): **Mark NSFW** / **Mark safe** / **Remove** (clears either keyword) / **Deselect**. Counts and actions are scoped to the current view (folder/filter), so the count matches what you see.
|
||||
- **Lightbox** (⤢ on a thumbnail): full-res image, EXIF side panel (incl. Google Maps link), `←`/`→` navigate, `Space` toggle selection, `Esc`/backdrop close.
|
||||
- **Activity log** (header button): timestamped, per-file record of every EXIF write.
|
||||
|
||||
---
|
||||
|
||||
## EXIF / review-state model
|
||||
|
||||
The review decision is stored **in the file's EXIF** as one of two mutually-exclusive keywords, written to both `Keywords` (IPTC) and `Subject` (XMP):
|
||||
|
||||
| Keyword | Meaning |
|
||||
|---------|---------|
|
||||
| `nsfw` | confirmed NSFW |
|
||||
| `sfw` | confirmed safe |
|
||||
| *(neither)* | not yet reviewed |
|
||||
|
||||
Rules:
|
||||
|
||||
- Writing `nsfw` removes any `sfw`, and vice-versa — a file can hold **at most one**.
|
||||
- On read, **`nsfw` takes precedence**: a stray file with both is reported as NSFW, never as safe.
|
||||
- **Only `Keywords` and `Subject` are ever modified.** All other metadata — `ImageDescription`/Caption, GPS, capture dates, camera make/model, ratings — is left byte-for-byte intact in both directions (add and remove).
|
||||
- `exiftool` runs with `-overwrite_original` (no sidecar `_original` files) and `-m` (ignore minor warnings so writes still apply to slightly-nonconforming files).
|
||||
|
||||
Immich indexes `Keywords`, so tagged photos become searchable there (search `nsfw`).
|
||||
|
||||
---
|
||||
|
||||
## HTTP API
|
||||
|
||||
Stdlib `ThreadingHTTPServer` bound to `127.0.0.1` on an OS-assigned free port. **Every path argument is validated against the scanned candidate set** — the server can't be pointed at arbitrary files.
|
||||
|
||||
| Method | Route | Purpose |
|
||||
|--------|-------|---------|
|
||||
| GET | `/` | The review page |
|
||||
| GET | `/img?path=` | Thumbnail bytes for a candidate image |
|
||||
| GET | `/exif?path=` | Filtered EXIF for the lightbox panel (decimal GPS + Google Maps URL) |
|
||||
| GET | `/log` | Activity-log entries `[{t, m}, …]` |
|
||||
| POST | `/tagged` | Body: `[path,…]` → `{nsfw:[…], sfw:[…]}` (batched keyword read) |
|
||||
| POST | `/apply` | Body: `[path,…]` → add `nsfw` (strips `sfw`) → `{tagged:n}` — "Mark NSFW" |
|
||||
| POST | `/mark?state=sfw` | Body: `[path,…]` → add `sfw` (strips `nsfw`) — "Mark safe" / checkmark→green |
|
||||
| POST | `/untag` | Body: `[path,…]` → remove both keywords (un-review) → `{untagged:n}` — "Remove" |
|
||||
| POST | `/mark?state=clear` | Body: `[path,…]` → remove `sfw` only (checkmark→red) |
|
||||
| POST | `/toggle` | Body: `[path,…]` → flip `nsfw` per file → `{tagged:[…]}` (endpoint retained; not used by the current UI) |
|
||||
|
||||
---
|
||||
|
||||
## Data & caching
|
||||
|
||||
- **`nsfw_scores.csv`** (written in the scanned folder): `path,nsfw_score`. This is the resume cache — reruns reuse it and only score new files. Delete it or pass `--rescore` to force a full re-score.
|
||||
- **Review state** is **not** in the CSV; it lives in each file's EXIF (see above), so it persists independently of the score cache and survives restarts.
|
||||
- The activity log is in-memory (last 2000 events) and resets when the server stops; messages are also echoed to stderr.
|
||||
|
||||
---
|
||||
|
||||
## Project layout
|
||||
|
||||
```
|
||||
nsfwtag/
|
||||
├── __init__.py # constants (model id, extensions, defaults), warning suppression
|
||||
├── exif.py # read/write/remove nsfw|sfw via exiftool; read_marks, read_exif (+GPS)
|
||||
├── scoring.py # discover images, cached model scoring
|
||||
├── webapp.py # render the page: fill review.html tokens (cards, folder tree, totals)
|
||||
├── review.html # the frontend — HTML/CSS/JS (no build step, no dependencies)
|
||||
├── server.py # local HTTP server + routes + activity log
|
||||
├── __main__.py # CLI entry point (python -m nsfwtag)
|
||||
└── README.md
|
||||
nsfw_tag.py # thin shim: `python nsfw_tag.py` == `python -m nsfwtag`
|
||||
```
|
||||
|
||||
The frontend lives in a real `review.html` (dynamic parts injected via `%%TOKEN%%` replacement) rather than a Python string, so it edits with normal HTML/CSS/JS tooling.
|
||||
|
||||
---
|
||||
|
||||
## Design decisions
|
||||
|
||||
- **No pre-selection at render.** Cards render unselected; the **lazy per-folder scan** decides each card's state from its EXIF keywords (pink/green) or score (red). This is deliberate: it guarantees a photo you marked safe never reappears pre-selected-red after a restart, and keeps startup instant (the browser opens before any `exiftool` scan runs).
|
||||
- **Lazy, chunked, per-folder scanning** with a progress ring, instead of one blocking scan at startup — so switching into a folder shows real progress and huge libraries stay responsive.
|
||||
- **Score cache first, model second** — the model (and its load) is skipped entirely on reruns with no new files.
|
||||
- **View-scoped counters/actions** — Apply/Toggle/Remove and their counts act on the folder/filter you're viewing, so the number always matches what's on screen.
|
||||
- **Stdlib-only server + dependency-free frontend** — nothing to build or install for the UI; the whole app is one HTML file plus a ~120-line Python server.
|
||||
|
||||
---
|
||||
|
||||
## Immich integration
|
||||
|
||||
Order matters: **write EXIF first, then upload.** immich-go deduplicates on the file's SHA-1 content hash, so writing EXIF after upload would create duplicates. This tool only writes metadata in place; ingest into Immich is a separate, deliberate `immich-go upload` step. Once uploaded, `Keywords` (including `nsfw`) are searchable in Immich alongside its own ML pipeline. **Do not** also point Immich at the folder as an External Library.
|
||||
25
legacy_cli_archive/src/nsfwtag/__init__.py
Normal file
25
legacy_cli_archive/src/nsfwtag/__init__.py
Normal file
@@ -0,0 +1,25 @@
|
||||
"""nsfwtag — local, on-device NSFW image tagging for Immich.
|
||||
|
||||
Score a photo library with a local ViT model (Apple GPU via MPS), review the
|
||||
flagged images in a browser, and write an 'nsfw' EXIF keyword into the ones you
|
||||
confirm — preserving every other tag. Nothing leaves the machine.
|
||||
|
||||
Entry point: `python -m nsfwtag <folder> [-r]` (or the nsfw_tag.py shim).
|
||||
"""
|
||||
import os
|
||||
import warnings
|
||||
|
||||
os.environ.setdefault("PYTHONWARNINGS", "ignore") # quiet numpy-2/-1 ABI noise from optional libs
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# AdamCodd/vit-base-nsfw-detector — chosen after benchmarking against the user's own
|
||||
# library (much better recall + fewer false positives than Falconsai). See bench.py.
|
||||
MODEL_ID = "AdamCodd/vit-base-nsfw-detector"
|
||||
EXTS = {".jpg", ".jpeg", ".png", ".heic", ".heif", ".webp"}
|
||||
CSV_NAME = "nsfw_scores.csv"
|
||||
DEFAULT_THRESHOLD = 0.6 # this model's sweet spot; Falconsai used 0.85
|
||||
DEFAULT_REVIEW_MIN = 0.2
|
||||
BATCH = 16
|
||||
KEYWORD = "nsfw"
|
||||
|
||||
__version__ = "1.1.0"
|
||||
65
legacy_cli_archive/src/nsfwtag/__main__.py
Normal file
65
legacy_cli_archive/src/nsfwtag/__main__.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""CLI entry point.
|
||||
|
||||
python -m nsfwtag "<folder>" # score + open the review app
|
||||
python -m nsfwtag "<folder>" -r # recurse into subfolders
|
||||
python -m nsfwtag list.txt [list2.txt] # tag every path in the .txt list(s)
|
||||
|
||||
The argument is auto-detected: a folder opens the review app; .txt file(s) tag
|
||||
directly. A tiny stdlib HTTP server serves the page and tags on Apply — no data
|
||||
leaves the machine.
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import CSV_NAME, DEFAULT_REVIEW_MIN, DEFAULT_THRESHOLD
|
||||
from .exif import apply_list, write_keyword
|
||||
from .scoring import discover_images, score_images
|
||||
from .server import serve_review
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(prog="nsfwtag", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("target", nargs="+", help="a FOLDER to scan (opens the review app) OR one/more .txt lists to tag")
|
||||
ap.add_argument("-r", "--recursive", action="store_true", help="descend into every subfolder")
|
||||
ap.add_argument("--apply", action="store_true", help="folder mode: tag score>=threshold directly, skip review")
|
||||
ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD, help=f"score >= this is pre-selected / tagged (default {DEFAULT_THRESHOLD})")
|
||||
ap.add_argument("--review-min", type=float, default=DEFAULT_REVIEW_MIN, help=f"lowest score shown in the review app (default {DEFAULT_REVIEW_MIN})")
|
||||
ap.add_argument("--limit", type=int, default=0, help="only the first N images (calibration)")
|
||||
ap.add_argument("--rescore", action="store_true", help="ignore cached scores and re-run the model on everything")
|
||||
args = ap.parse_args()
|
||||
|
||||
# auto-detected: .txt list(s) = apply mode; a single folder = review mode.
|
||||
targets = [Path(t).expanduser() for t in args.target]
|
||||
if targets and all(t.is_file() for t in targets):
|
||||
for t in targets:
|
||||
apply_list(str(t))
|
||||
return
|
||||
if len(targets) != 1 or not targets[0].is_dir():
|
||||
sys.exit("Pass ONE folder (review), or one or more .txt lists (apply).")
|
||||
target = targets[0]
|
||||
|
||||
imgs = discover_images(target, recursive=args.recursive, limit=args.limit)
|
||||
if not imgs:
|
||||
sys.exit("No images found.")
|
||||
|
||||
csv_path = target / CSV_NAME
|
||||
scored, errors = score_images(imgs, csv_path, rescore=args.rescore)
|
||||
print(f"\n{len(scored)} images ready, {errors} errors (cache -> {csv_path})")
|
||||
|
||||
if args.apply:
|
||||
tagged = sum(1 for p, s in scored if s >= args.threshold and write_keyword(str(p)))
|
||||
print(f" tagged 'nsfw' in EXIF: {tagged} (score >= {args.threshold})")
|
||||
return
|
||||
|
||||
# live review app: one page for ALL candidates across the tree, tag on click
|
||||
cands = sorted([(p, s) for p, s in scored if s >= args.review_min], key=lambda x: -x[1])
|
||||
if not cands:
|
||||
print(f" no candidates >= {args.review_min} — nothing to review.")
|
||||
return
|
||||
serve_review(cands, args.threshold) # tag-state is checked lazily in-browser, per folder
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
426
legacy_cli_archive/src/nsfwtag/bench.py
Normal file
426
legacy_cli_archive/src/nsfwtag/bench.py
Normal file
@@ -0,0 +1,426 @@
|
||||
"""Benchmark / compare NSFW models on YOUR OWN photos.
|
||||
|
||||
python -m nsfwtag.bench "<sample_folder>" [-r] [--models a,b,c] [--limit N] [--threshold 0.85]
|
||||
|
||||
Scores every image with each selected model and writes, into the sample folder:
|
||||
* bench_scores.csv — path + one nsfw-score column per model
|
||||
* bench_report.html — thumbnails with per-model scores, sortable, disagreements
|
||||
highlighted (open it to eyeball which model over-flags)
|
||||
…plus a console summary (flag counts, agreement, biggest disagreements).
|
||||
|
||||
Every score is normalized to nsfw = 1 - P(safe-class) so binary and multi-class
|
||||
models are directly comparable. Nothing leaves the machine.
|
||||
|
||||
Models (need `pip install timm` for the timm ones; each downloads once):
|
||||
falconsai Falconsai/nsfw_image_detection (current, transformers, binary)
|
||||
adamcodd AdamCodd/vit-base-nsfw-detector (transformers, binary)
|
||||
marqo Marqo/nsfw-image-detection-384 (timm, binary, tiny)
|
||||
owen-xs OwenElliott/image-safety-classifier-xs (timm, 3-class SFW/NSFW/NSFL)
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import gc
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from .exif import read_marks
|
||||
from .scoring import discover_images
|
||||
|
||||
BATCH = 16
|
||||
# label (lowercased) treated as "safe"; nsfw score = 1 - sum(P over these)
|
||||
SAFE_LABELS = {"sfw", "safe", "normal", "neutral"}
|
||||
|
||||
MODELS = { # name -> (kind, hf_id)
|
||||
"falconsai": ("hf", "Falconsai/nsfw_image_detection"),
|
||||
"adamcodd": ("hf", "AdamCodd/vit-base-nsfw-detector"),
|
||||
"marqo": ("timm", "Marqo/nsfw-image-detection-384"),
|
||||
"owen-xs": ("timm", "OwenElliott/image-safety-classifier-xs"),
|
||||
}
|
||||
|
||||
|
||||
def _device():
|
||||
import torch
|
||||
return "mps" if torch.backends.mps.is_available() else "cpu"
|
||||
|
||||
|
||||
def _unsafe_indices(labels):
|
||||
"""Given lowercased class labels, return (safe_idx, use_safe). If any safe
|
||||
label is present we score 1 - P(safe); otherwise P(labels that look unsafe)."""
|
||||
safe = [i for i, l in enumerate(labels) if l in SAFE_LABELS]
|
||||
if safe:
|
||||
return safe, True
|
||||
unsafe = [i for i, l in enumerate(labels)
|
||||
if any(k in l for k in ("nsfw", "nsfl", "porn", "sexy", "hentai", "explicit"))]
|
||||
return unsafe, False
|
||||
|
||||
|
||||
def load_hf(model_id):
|
||||
import torch
|
||||
from transformers import AutoImageProcessor, AutoModelForImageClassification
|
||||
dev = _device()
|
||||
proc = AutoImageProcessor.from_pretrained(model_id)
|
||||
model = AutoModelForImageClassification.from_pretrained(model_id).to(dev).eval()
|
||||
labels = [model.config.id2label[i].lower() for i in range(len(model.config.id2label))]
|
||||
idx, use_safe = _unsafe_indices(labels)
|
||||
|
||||
def score(imgs):
|
||||
inputs = proc(images=imgs, return_tensors="pt").to(dev)
|
||||
with torch.no_grad():
|
||||
p = model(**inputs).logits.softmax(-1)
|
||||
part = p[:, idx].sum(-1) if idx else p[:, :1].sum(-1) * 0
|
||||
return (1 - part if use_safe else part).cpu().tolist()
|
||||
|
||||
return score, model
|
||||
|
||||
|
||||
def load_timm(model_id):
|
||||
import timm
|
||||
import torch
|
||||
dev = _device()
|
||||
model = timm.create_model(f"hf_hub:{model_id}", pretrained=True).to(dev).eval()
|
||||
cfg = timm.data.resolve_model_data_config(model)
|
||||
tf = timm.data.create_transform(**cfg, is_training=False)
|
||||
labels = [l.lower() for l in model.pretrained_cfg["label_names"]]
|
||||
idx, use_safe = _unsafe_indices(labels)
|
||||
|
||||
def score(imgs):
|
||||
batch = torch.stack([tf(im) for im in imgs]).to(dev)
|
||||
with torch.no_grad():
|
||||
p = model(batch).softmax(-1)
|
||||
part = p[:, idx].sum(-1) if idx else p[:, :1].sum(-1) * 0
|
||||
return (1 - part if use_safe else part).cpu().tolist()
|
||||
|
||||
return score, model
|
||||
|
||||
|
||||
def score_all(names, imgs):
|
||||
"""Return {model_name: {path: nsfw_score}}; loads one model at a time."""
|
||||
from PIL import Image, ImageFile
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||
dev = _device()
|
||||
out = {}
|
||||
for name in names:
|
||||
kind, mid = MODELS[name]
|
||||
print(f"\n[{name}] loading {mid} on {dev} …", file=sys.stderr)
|
||||
try:
|
||||
score, model = (load_hf if kind == "hf" else load_timm)(mid)
|
||||
except Exception as e:
|
||||
print(f" SKIP {name}: {e}", file=sys.stderr)
|
||||
continue
|
||||
res = {}
|
||||
for i in range(0, len(imgs), BATCH):
|
||||
pil, paths = [], []
|
||||
for p in imgs[i:i + BATCH]:
|
||||
try:
|
||||
pil.append(Image.open(p).convert("RGB")); paths.append(p)
|
||||
except Exception as e:
|
||||
print(f"\n error {p}: {e}", file=sys.stderr)
|
||||
if not pil:
|
||||
continue
|
||||
for p, s in zip(paths, score(pil)):
|
||||
res[str(p)] = float(s)
|
||||
print(f"\r {min(i + BATCH, len(imgs))}/{len(imgs)}", end="", file=sys.stderr, flush=True)
|
||||
print(file=sys.stderr)
|
||||
out[name] = res
|
||||
del score, model
|
||||
gc.collect()
|
||||
return out
|
||||
|
||||
|
||||
def _chip_color(s):
|
||||
hue = round((1 - s) * 130) # green (safe) -> red (unsafe)
|
||||
return f"hsl({hue} 70% 40%)"
|
||||
|
||||
|
||||
def write_report(rows, names, target, threshold, gt):
|
||||
"""rows: list of (path, {name: score}). gt: {path: 1(nsfw)/0(sfw)}. Writes
|
||||
bench_report.html in target. Chips that disagree with your manual tag get a
|
||||
red ring; sortable by error count."""
|
||||
target = Path(target)
|
||||
cards = []
|
||||
for p, scores in rows:
|
||||
rel = quote(os.path.relpath(p, target))
|
||||
vals = [scores.get(n) for n in names if scores.get(n) is not None]
|
||||
spread = (max(vals) - min(vals)) if len(vals) > 1 else 0.0
|
||||
gtv = gt.get(p) # 1 nsfw / 0 sfw / None
|
||||
errors, chips = 0, []
|
||||
for n in names:
|
||||
s = scores.get(n)
|
||||
if s is None:
|
||||
chips.append(f'<span class="chip na">{n} —</span>'); continue
|
||||
hot = s >= threshold
|
||||
err = gtv is not None and (hot != (gtv == 1))
|
||||
errors += err
|
||||
cls = "chip" + (" hot" if hot else "") + (" err" if err else "")
|
||||
chips.append(f'<span class="{cls}" style="background:{_chip_color(s)}">{n} {s:.2f}</span>')
|
||||
gtbadge = (f'<span class="gt gt-{"nsfw" if gtv==1 else "sfw"}">you: {"nsfw" if gtv==1 else "sfw"}</span>'
|
||||
if gtv is not None else "")
|
||||
data = html.escape(json.dumps({n: scores.get(n) for n in names}), quote=True)
|
||||
cards.append(
|
||||
f'<div class="card" data-scores="{data}" data-spread="{spread:.4f}" data-errors="{errors}" '
|
||||
f'data-name="{html.escape(Path(p).name.lower(), quote=True)}">'
|
||||
f'{gtbadge}<img loading="lazy" src="{rel}" alt="">'
|
||||
f'<div class="chips">{"".join(chips)}</div>'
|
||||
f'<div class="nm" title="{html.escape(str(p), quote=True)}">{html.escape(Path(p).name)}</div></div>')
|
||||
opts = []
|
||||
if gt:
|
||||
opts.append('<option value="errors" selected>Errors vs. your tags ↓</option>')
|
||||
opts.append('<option value="spread"' + ('' if gt else ' selected') + '>Model disagreement ↓</option>')
|
||||
opts += [f'<option value="{n}">{n} ↓</option>' for n in names]
|
||||
opts.append('<option value="name">Name</option>')
|
||||
doc = _REPORT.replace("%%CARDS%%", "".join(cards)).replace("%%SORTOPTS%%", "".join(opts)) \
|
||||
.replace("%%N%%", str(len(rows))).replace("%%THR%%", f"{threshold:.2f}")
|
||||
out = target / "bench_report.html"
|
||||
out.write_text(doc, encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
_REPORT = """<!doctype html><html lang="en"><head><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1"><title>NSFW model bench (%%N%%)</title>
|
||||
<style>
|
||||
*{box-sizing:border-box}body{margin:0;background:#0b0d10;color:#e8eaee;font-size:14px;
|
||||
font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif}
|
||||
.bar{position:sticky;top:0;z-index:5;display:flex;flex-wrap:wrap;gap:14px;align-items:center;
|
||||
padding:12px 16px;background:rgba(11,13,16,.92);backdrop-filter:blur(8px);border-bottom:1px solid #262b34}
|
||||
.bar b{font-size:15px}.bar .sum{color:#9aa1ae;font-size:12.5px}
|
||||
select{background:#14171c;color:#e8eaee;border:1px solid #262b34;border-radius:8px;height:32px;padding:0 8px}
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px;padding:14px}
|
||||
.card{position:relative;background:#14171c;border:1px solid #262b34;border-radius:10px;overflow:hidden}
|
||||
.card img{width:100%;height:190px;object-fit:cover;display:block;background:#000}
|
||||
.gt{position:absolute;top:6px;left:6px;z-index:2;font-size:10px;font-weight:700;padding:2px 6px;border-radius:5px}
|
||||
.gt-nsfw{background:#e05fa8;color:#2a0a1c}.gt-sfw{background:#10b981;color:#04130c}
|
||||
.chips{display:flex;flex-wrap:wrap;gap:4px;padding:7px}
|
||||
.chip{font-size:11px;font-weight:700;color:#fff;border-radius:5px;padding:2px 6px;border:2px solid transparent;
|
||||
font-variant-numeric:tabular-nums;text-shadow:0 1px 2px rgba(0,0,0,.5)}
|
||||
.chip.hot{border-color:#fff}.chip.err{border-color:#ef4444}.chip.na{background:#20252e!important;color:#6b727f}
|
||||
.nm{font-size:11px;color:#8b90a0;padding:0 8px 8px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
</style></head><body>
|
||||
<div class="bar"><b>NSFW model bench</b><span class="sum">%%N%% images · white ring = ≥ %%THR%% · red ring = disagrees with your tag</span>
|
||||
<label style="margin-left:auto;color:#9aa1ae;font-size:12.5px">Sort
|
||||
<select id="sort">%%SORTOPTS%%</select></label></div>
|
||||
<div class="grid" id="grid">%%CARDS%%</div>
|
||||
<script>
|
||||
const grid=document.getElementById('grid'),cards=[...grid.children],sel=document.getElementById('sort');
|
||||
sel.addEventListener('change',e=>{
|
||||
const k=e.target.value;
|
||||
cards.sort((a,b)=>{
|
||||
if(k==='name')return a.dataset.name.localeCompare(b.dataset.name);
|
||||
if(k==='spread')return b.dataset.spread-a.dataset.spread;
|
||||
if(k==='errors')return b.dataset.errors-a.dataset.errors;
|
||||
const av=JSON.parse(a.dataset.scores)[k],bv=JSON.parse(b.dataset.scores)[k];
|
||||
return (bv==null?-1:bv)-(av==null?-1:av);});
|
||||
cards.forEach(c=>grid.appendChild(c));});
|
||||
sel.dispatchEvent(new Event('change')); // apply default sort on load
|
||||
</script></body></html>"""
|
||||
|
||||
|
||||
def summarize(rows, names, threshold):
|
||||
print(f"\n=== {len(rows)} images · flagged = score ≥ {threshold} ===")
|
||||
print(f"{'model':<12}{'flagged':>10}{'mean':>8}{'median':>8}")
|
||||
for n in names:
|
||||
vals = sorted(s[n] for _, s in rows if s.get(n) is not None)
|
||||
if not vals:
|
||||
print(f"{n:<12}{'(skipped)':>10}"); continue
|
||||
flagged = sum(v >= threshold for v in vals)
|
||||
mean = sum(vals) / len(vals)
|
||||
med = vals[len(vals) // 2]
|
||||
print(f"{n:<12}{f'{flagged}/{len(vals)}':>10}{mean:>8.3f}{med:>8.3f}")
|
||||
|
||||
ok = [n for n in names if any(s.get(n) is not None for _, s in rows)]
|
||||
if len(ok) > 1:
|
||||
print("\npairwise agreement (same side of threshold):")
|
||||
for i in range(len(ok)):
|
||||
for j in range(i + 1, len(ok)):
|
||||
a, b = ok[i], ok[j]
|
||||
both = [(s[a], s[b]) for _, s in rows if s.get(a) is not None and s.get(b) is not None]
|
||||
agree = sum((x >= threshold) == (y >= threshold) for x, y in both)
|
||||
print(f" {a:<10} vs {b:<10} {agree}/{len(both)} ({100*agree/max(1,len(both)):.0f}%)")
|
||||
|
||||
if len(ok) > 1:
|
||||
print("\nbiggest disagreements (open the HTML to judge):")
|
||||
spread = sorted(rows, key=lambda r: -(max(v for v in (r[1].get(n) for n in ok) if v is not None)
|
||||
- min(v for v in (r[1].get(n) for n in ok) if v is not None)))
|
||||
for p, s in spread[:8]:
|
||||
print(" " + Path(p).name + " " + " ".join(f"{n}={s[n]:.2f}" for n in ok if s.get(n) is not None))
|
||||
|
||||
|
||||
def metrics(rows, names, gt, threshold):
|
||||
"""Per-model precision/recall/F1 vs. your manual tags, at the given threshold,
|
||||
plus each model's F1-optimal threshold."""
|
||||
labs = [gt[p] for p, _ in rows if p in gt]
|
||||
npos, nneg = sum(labs), len(labs) - sum(labs)
|
||||
single = npos == 0 or nneg == 0 # a known single-class folder (--assume)
|
||||
print(f"\n=== vs. ground truth · {npos} nsfw + {nneg} sfw · flag = score ≥ {threshold} ===")
|
||||
print(f"{'model':<11}{'acc':>7}{'prec':>7}{'recall':>8}{'F1':>6}{'FP':>5}{'FN':>5}{' best-thr':>11}{'F1@best':>8}")
|
||||
for n in names:
|
||||
pairs = [(gt[p], s[n]) for p, s in rows if p in gt and s.get(n) is not None]
|
||||
if not pairs:
|
||||
print(f"{n:<11}{'(skipped)':>7}"); continue
|
||||
|
||||
def conf(t):
|
||||
tp = fp = fn = tn = 0
|
||||
for y, sc in pairs:
|
||||
pred = sc >= t
|
||||
if y == 1 and pred: tp += 1
|
||||
elif y == 0 and pred: fp += 1
|
||||
elif y == 1: fn += 1
|
||||
else: tn += 1
|
||||
return tp, fp, fn, tn
|
||||
|
||||
def f1(t):
|
||||
tp, fp, fn, _ = conf(t)
|
||||
pr = tp / (tp + fp) if tp + fp else 0
|
||||
rc = tp / (tp + fn) if tp + fn else 0
|
||||
return 2 * pr * rc / (pr + rc) if pr + rc else 0
|
||||
|
||||
tp, fp, fn, tn = conf(threshold)
|
||||
pr = tp / (tp + fp) if tp + fp else 0
|
||||
rc = tp / (tp + fn) if tp + fn else 0
|
||||
f = 2 * pr * rc / (pr + rc) if pr + rc else 0
|
||||
acc = (tp + tn) / len(pairs)
|
||||
if single:
|
||||
bt_col = f"{'—':>11}{'—':>8}" # threshold tuning needs both classes
|
||||
else:
|
||||
cand = sorted({round(sc, 3) for _, sc in pairs} | {threshold})
|
||||
bt = max(cand, key=f1)
|
||||
bt_col = f"{bt:>11.2f}{f1(bt):>8.2f}"
|
||||
print(f"{n:<11}{acc:>7.2f}{pr:>7.2f}{rc:>8.2f}{f:>6.2f}{fp:>5}{fn:>5}{bt_col}")
|
||||
if single and npos:
|
||||
print("single-class NSFW folder: recall = catch rate, FN = missed. (precision/FP need safe images too — "
|
||||
"run the bench on a mixed/SFW folder for those.)")
|
||||
elif single:
|
||||
print("single-class SFW folder: FP = false alarms, acc = correct-reject rate. (recall needs NSFW images.)")
|
||||
else:
|
||||
print("prec = of what it flagged, how much you'd tagged nsfw (fewer false alarms) · "
|
||||
"recall = of your nsfw, how much it caught · FP/FN at the current threshold")
|
||||
|
||||
|
||||
def combine(csv_paths, threshold):
|
||||
"""Pool the labeled rows from several bench_scores.csv files and print a
|
||||
precision/recall/F1 threshold sweep per model — the real cross-dataset verdict."""
|
||||
per_model, seen = {}, set()
|
||||
pos = neg = 0
|
||||
for cp in csv_paths:
|
||||
with open(cp, newline="", encoding="utf-8") as fh:
|
||||
r = csv.DictReader(fh)
|
||||
models = [c for c in r.fieldnames if c not in ("path", "manual")]
|
||||
for row in r:
|
||||
man = row.get("manual", "")
|
||||
if man not in ("nsfw", "sfw") or row["path"] in seen:
|
||||
continue
|
||||
seen.add(row["path"])
|
||||
y = 1 if man == "nsfw" else 0
|
||||
pos += y; neg += (1 - y)
|
||||
for m in models:
|
||||
v = row.get(m, "")
|
||||
if v != "":
|
||||
per_model.setdefault(m, []).append((y, float(v)))
|
||||
if not pos or not neg:
|
||||
sys.exit("Need both nsfw and sfw labels pooled — include a mixed/SFW run and an NSFW run.")
|
||||
print(f"pooled {pos} nsfw + {neg} sfw from {len(csv_paths)} file(s)")
|
||||
|
||||
def prf(pairs, t):
|
||||
tp = sum(y == 1 and s >= t for y, s in pairs)
|
||||
fp = sum(y == 0 and s >= t for y, s in pairs)
|
||||
fn = sum(y == 1 and s < t for y, s in pairs)
|
||||
pr = tp / (tp + fp) if tp + fp else 0
|
||||
rc = tp / (tp + fn) if tp + fn else 0
|
||||
return pr, rc, (2 * pr * rc / (pr + rc) if pr + rc else 0), fp, fn
|
||||
|
||||
ranking = []
|
||||
for m, pairs in per_model.items():
|
||||
cand = sorted({round(s, 3) for _, s in pairs})
|
||||
best = max(cand, key=lambda t: prf(pairs, t)[2])
|
||||
ranking.append((prf(pairs, best)[2], best, m))
|
||||
print(f"\n[{m}] best-F1 threshold = {best:.2f}")
|
||||
print(f" {'thr':>5}{'prec':>7}{'recall':>8}{'F1':>6}{'FP':>6}{'FN':>6}")
|
||||
for t in sorted(set([0.5, 0.7, 0.85, 0.9, 0.95, 0.99, round(best, 2)])):
|
||||
pr, rc, f, fp, fn = prf(pairs, t)
|
||||
print(f" {t:>5.2f}{pr:>7.2f}{rc:>8.2f}{f:>6.2f}{fp:>6}{fn:>6}{' <- best F1' if abs(t - best) < 5e-3 else ''}")
|
||||
ranking.sort(reverse=True)
|
||||
print("\nranking by best F1: " + " ".join(f"{m}={f:.2f}@{t:.2f}" for f, t, m in ranking))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(prog="nsfwtag.bench", description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("target", nargs="?", help="folder of photos to score with each model")
|
||||
ap.add_argument("--combine", nargs="+", metavar="CSV", help="pool these bench_scores.csv files (using their manual labels) and print a precision/recall/threshold sweep per model — no scoring")
|
||||
ap.add_argument("-r", "--recursive", action="store_true", help="descend into subfolders")
|
||||
ap.add_argument("--models", default=",".join(MODELS), help=f"comma list (default: all) — {', '.join(MODELS)}")
|
||||
ap.add_argument("--limit", type=int, default=0, help="cap number of images scored (quick pass)")
|
||||
ap.add_argument("--threshold", type=float, default=0.85, help="flag cutoff for the report (default 0.85)")
|
||||
ap.add_argument("--all", action="store_true", help="score ALL images, not just your manually-tagged ones")
|
||||
ap.add_argument("--assume", choices=["nsfw", "sfw"], help="treat EVERY image in the folder as this label (a known folder) instead of reading EXIF tags")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.combine:
|
||||
combine(args.combine, args.threshold)
|
||||
return
|
||||
if not args.target:
|
||||
ap.error("a target folder is required (or use --combine)")
|
||||
target = Path(args.target).expanduser()
|
||||
if not target.is_dir():
|
||||
sys.exit(f"Not a folder: {target}\n Check the path/spelling (that folder does not exist).")
|
||||
names = [n.strip() for n in args.models.split(",") if n.strip()]
|
||||
bad = [n for n in names if n not in MODELS]
|
||||
if bad:
|
||||
sys.exit(f"Unknown model(s): {bad}. Available: {list(MODELS)}")
|
||||
|
||||
imgs = discover_images(target, recursive=args.recursive)
|
||||
if not imgs:
|
||||
has_subdirs = any(p.is_dir() for p in target.iterdir())
|
||||
hint = " Images look to be in subfolders — add -r to scan recursively." if (has_subdirs and not args.recursive) else ""
|
||||
sys.exit(f"No images found directly in {target}.\n{hint}".rstrip())
|
||||
|
||||
if args.assume: # whole folder is a known class
|
||||
if args.limit:
|
||||
imgs = imgs[:args.limit]
|
||||
gt = {str(p): (1 if args.assume == "nsfw" else 0) for p in imgs}
|
||||
print(f"assuming all {len(imgs)} images are {args.assume.upper()} (folder is ground truth).", file=sys.stderr)
|
||||
else:
|
||||
# ground truth from your manual nsfw/sfw EXIF tags (one batched exiftool read)
|
||||
print(f"found {len(imgs)} images — reading your manual tags …", file=sys.stderr)
|
||||
marks = read_marks([str(p) for p in imgs])
|
||||
gt = {p: 1 for p in marks["nsfw"]}
|
||||
gt.update({p: 0 for p in marks["sfw"] if p not in marks["nsfw"]})
|
||||
if gt and not args.all:
|
||||
imgs = [p for p in imgs if str(p) in gt] # evaluate only what you've tagged
|
||||
print(f"evaluating on {len(imgs)} tagged images "
|
||||
f"({sum(gt.values())} nsfw / {len(gt) - sum(gt.values())} sfw). Use --all to score everything.", file=sys.stderr)
|
||||
elif not gt:
|
||||
print("no manual nsfw/sfw tags found — falling back to a relative model comparison.", file=sys.stderr)
|
||||
if args.limit:
|
||||
imgs = imgs[:args.limit]
|
||||
print(f"scoring {len(imgs)} images with {len(names)} model(s): {names}", file=sys.stderr)
|
||||
|
||||
scores = score_all(names, imgs)
|
||||
names = [n for n in names if n in scores] # keep only models that loaded
|
||||
if not names:
|
||||
sys.exit("No models loaded (need: pip install torch transformers timm).")
|
||||
|
||||
rows = [(str(p), {n: scores[n].get(str(p)) for n in names}) for p in imgs]
|
||||
|
||||
csv_path = target / "bench_scores.csv"
|
||||
with open(csv_path, "w", newline="", encoding="utf-8") as fh:
|
||||
w = csv.writer(fh); w.writerow(["path", "manual", *names])
|
||||
for p, s in rows:
|
||||
man = {1: "nsfw", 0: "sfw"}.get(gt.get(p), "")
|
||||
w.writerow([p, man, *[("" if s[n] is None else f"{s[n]:.4f}") for n in names]])
|
||||
|
||||
if any(p in gt for p, _ in rows):
|
||||
metrics(rows, names, gt, args.threshold)
|
||||
else:
|
||||
summarize(rows, names, args.threshold)
|
||||
report = write_report(rows, names, target, args.threshold, gt)
|
||||
print(f"\nscores -> {csv_path}\nreport -> {report}", file=sys.stderr)
|
||||
webbrowser.open(report.resolve().as_uri())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
123
legacy_cli_archive/src/nsfwtag/exif.py
Normal file
123
legacy_cli_archive/src/nsfwtag/exif.py
Normal file
@@ -0,0 +1,123 @@
|
||||
"""EXIF keyword read/write via exiftool.
|
||||
|
||||
Only the Keywords + Subject fields are ever touched, so every other tag
|
||||
(Caption, ImageDescription, dates, GPS, Artist, …) is preserved on tag/untag.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import KEYWORD
|
||||
|
||||
|
||||
def write_keyword(path: str, kw: str = KEYWORD) -> bool:
|
||||
"""Idempotently add a keyword to EXIF Keywords + Subject (no duplicates on re-run)."""
|
||||
return subprocess.run(
|
||||
["exiftool", "-m", "-overwrite_original",
|
||||
f"-Keywords-={kw}", f"-Keywords+={kw}",
|
||||
f"-Subject-={kw}", f"-Subject+={kw}", path],
|
||||
capture_output=True, text=True,
|
||||
).returncode == 0
|
||||
|
||||
|
||||
def remove_keyword(path: str, kw: str = KEYWORD) -> bool:
|
||||
"""Reverse of write_keyword: strip the keyword from EXIF Keywords + Subject."""
|
||||
return subprocess.run(
|
||||
["exiftool", "-m", "-overwrite_original",
|
||||
f"-Keywords-={kw}", f"-Subject-={kw}", path],
|
||||
capture_output=True, text=True,
|
||||
).returncode == 0
|
||||
|
||||
|
||||
def _read_keywords(paths) -> dict:
|
||||
"""{path: set(lowercased Keywords+Subject values)} via one batched exiftool
|
||||
call (paths fed via stdin). Empty dict if exiftool isn't available."""
|
||||
paths = list(paths)
|
||||
if not paths:
|
||||
return {}
|
||||
want = {os.path.normpath(p): p for p in paths}
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["exiftool", "-m", "-j", "-Keywords", "-Subject", "-@", "-"],
|
||||
input="\n".join(paths), capture_output=True, text=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
out = {}
|
||||
try:
|
||||
for obj in json.loads(r.stdout or "[]"):
|
||||
vals = []
|
||||
for f in ("Keywords", "Subject"):
|
||||
v = obj.get(f)
|
||||
if isinstance(v, list):
|
||||
vals += v
|
||||
elif v is not None:
|
||||
vals.append(v)
|
||||
key = want.get(os.path.normpath(obj.get("SourceFile", "")))
|
||||
if key is not None:
|
||||
out[key] = {str(x).strip().lower() for x in vals}
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def read_tagged(paths, kw: str = KEYWORD) -> set:
|
||||
"""Subset of `paths` whose EXIF already contains keyword `kw`."""
|
||||
kw = kw.lower()
|
||||
return {p for p, kws in _read_keywords(paths).items() if kw in kws}
|
||||
|
||||
|
||||
def read_marks(paths) -> dict:
|
||||
"""Which paths carry the 'nsfw' vs the 'sfw' review keyword, in one read:
|
||||
{'nsfw': set, 'sfw': set}. They should be mutually exclusive; if a stray file
|
||||
somehow has both, nsfw wins so it never reads as safe."""
|
||||
got = _read_keywords(paths)
|
||||
nsfw = {p for p, k in got.items() if "nsfw" in k}
|
||||
sfw = {p for p, k in got.items() if "sfw" in k and p not in nsfw}
|
||||
return {"nsfw": nsfw, "sfw": sfw}
|
||||
|
||||
|
||||
_EXIF_NOISE = {
|
||||
"SourceFile", "ExifToolVersion", "Directory", "FilePermissions",
|
||||
"FileModifyDate", "FileAccessDate", "FileInodeChangeDate", "FileTypeExtension",
|
||||
}
|
||||
|
||||
|
||||
def read_exif(path: str) -> dict:
|
||||
"""Human-relevant EXIF tags for one file (for the lightbox overlay), in
|
||||
exiftool's order. Skips file-system noise and binary/oversized values. When
|
||||
the photo is geotagged, a "Map" entry with a Google Maps URL is prepended.
|
||||
`-c %+.6f` prints GPS coordinates as signed decimals (other tags unaffected)."""
|
||||
r = subprocess.run(["exiftool", "-m", "-j", "-c", "%+.6f", path], capture_output=True, text=True)
|
||||
try:
|
||||
obj = json.loads(r.stdout or "[]")[0]
|
||||
except (ValueError, IndexError):
|
||||
return {}
|
||||
out = {}
|
||||
for k, v in obj.items():
|
||||
if k in _EXIF_NOISE:
|
||||
continue
|
||||
s = ", ".join(str(x) for x in v) if isinstance(v, list) else str(v)
|
||||
if not s or s.startswith("(Binary data") or len(s) > 300:
|
||||
continue
|
||||
out[k] = s
|
||||
lat, lon = out.get("GPSLatitude"), out.get("GPSLongitude")
|
||||
if lat and lon:
|
||||
try:
|
||||
out = {"Map": f"https://www.google.com/maps?q={float(lat):.6f},{float(lon):.6f}", **out}
|
||||
except ValueError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def apply_list(list_file: str):
|
||||
"""Write the nsfw keyword to every path in a newline-delimited file."""
|
||||
paths = [l.strip() for l in Path(list_file).read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
ok = miss = 0
|
||||
for p in paths:
|
||||
if not Path(p).exists():
|
||||
miss += 1; print(f" missing: {p}", file=sys.stderr); continue
|
||||
ok += 1 if write_keyword(p) else 0
|
||||
print(f"tagged 'nsfw' in EXIF: {ok}/{len(paths)}" + (f" ({miss} missing)" if miss else ""))
|
||||
426
legacy_cli_archive/src/nsfwtag/review.html
Normal file
426
legacy_cli_archive/src/nsfwtag/review.html
Normal file
@@ -0,0 +1,426 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>NSFW Review · %%TOTAL%% images</title>
|
||||
<style>
|
||||
*{box-sizing:border-box}
|
||||
:root{
|
||||
--bg:#0b0d10;--surface:#14171c;--surface-2:#1b1f26;--elev:#20252e;
|
||||
--border:#262b34;--border-2:#323947;--text:#e8eaee;--muted:#9aa1ae;--faint:#6b727f;
|
||||
--primary:#10b981;--primary-2:#0ea372;--primary-ink:#04130c;--danger:#ef4444;--ring:#3b82f6;--pink:#e05fa8;
|
||||
--r:12px;--r-sm:8px}
|
||||
@media (prefers-reduced-motion:reduce){*{transition:none!important;animation:none!important}}
|
||||
html,body{margin:0}
|
||||
body{background:var(--bg);color:var(--text);font-size:14px;line-height:1.5;-webkit-font-smoothing:antialiased;
|
||||
font-family:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif}
|
||||
svg{width:1em;height:1em;display:block}
|
||||
button{font:inherit;color:inherit;cursor:pointer}
|
||||
:focus-visible{outline:2px solid var(--ring);outline-offset:2px}
|
||||
.app{position:sticky;top:0;z-index:30;display:flex;align-items:center;gap:14px;min-height:58px;
|
||||
padding:11px 18px;background:rgba(11,13,16,.86);backdrop-filter:blur(12px);border-bottom:1px solid var(--border)}
|
||||
.brand{display:flex;align-items:center;gap:11px}
|
||||
.logo{display:grid;place-items:center;width:34px;height:34px;border-radius:10px;font-size:18px;
|
||||
background:linear-gradient(135deg,#1e3a5f,#10b981);color:#fff}
|
||||
.titles h1{margin:0;font-size:15px;font-weight:600;letter-spacing:-.01em}
|
||||
.titles p{margin:0;font-size:11px;color:var(--faint)}
|
||||
.grow{flex:1}
|
||||
.metrics{display:flex;gap:8px}
|
||||
.metric{display:flex;flex-direction:column;align-items:center;min-width:66px;padding:5px 12px;
|
||||
background:var(--surface);border:1px solid var(--border);border-radius:var(--r-sm)}
|
||||
.metric .num{font-size:16px;font-weight:600;font-variant-numeric:tabular-nums}
|
||||
.metric .lbl{font-size:10px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em}
|
||||
.toolbar{position:sticky;top:58px;z-index:20;display:flex;flex-wrap:wrap;align-items:center;gap:10px;
|
||||
padding:10px 18px;background:rgba(16,19,24,.92);backdrop-filter:blur(8px);border-bottom:1px solid var(--border)}
|
||||
.field{display:flex;align-items:center;gap:8px;height:36px;padding:0 10px;background:var(--surface);
|
||||
border:1px solid var(--border);border-radius:var(--r-sm)}
|
||||
.field svg{color:var(--faint);font-size:15px}
|
||||
.field input{background:none;border:0;outline:none;color:var(--text);height:100%;width:200px}
|
||||
.field input::placeholder{color:var(--faint)}
|
||||
.field:focus-within{border-color:var(--border-2);box-shadow:0 0 0 3px rgba(59,130,246,.22)}
|
||||
.seg{display:inline-flex;padding:3px;background:var(--surface);border:1px solid var(--border);border-radius:var(--r-sm)}
|
||||
.seg button{border:0;background:none;color:var(--muted);padding:5px 11px;border-radius:6px;font-size:12.5px;font-weight:500}
|
||||
.seg button.on{background:var(--elev);color:var(--text)}
|
||||
.seg button:hover:not(.on){color:var(--text)}
|
||||
.legend{display:flex;gap:8px;align-items:center}
|
||||
.chip{display:inline-flex;align-items:center;gap:6px;padding:4px 11px;border-radius:16px;font:inherit;
|
||||
font-size:12px;font-weight:500;border:1px solid var(--border);background:var(--surface);color:var(--muted);
|
||||
cursor:pointer;transition:opacity .15s,color .15s,border-color .15s}
|
||||
.chip .sw{width:10px;height:10px;border-radius:3px}
|
||||
.sw-nsfw{background:var(--pink)}
|
||||
.sw-sfw{background:var(--primary)}
|
||||
.chip.on{color:var(--text);border-color:var(--border-2)}
|
||||
.chip:not(.on){opacity:.4;text-decoration:line-through}
|
||||
.chip .cn{color:var(--faint);font-variant-numeric:tabular-nums}
|
||||
.thr,.sortf{display:flex;align-items:center;gap:8px;color:var(--muted);font-size:12.5px;white-space:nowrap}
|
||||
.thr output{min-width:2.4em;color:var(--text);font-weight:600;font-variant-numeric:tabular-nums}
|
||||
input[type=range]{-webkit-appearance:none;appearance:none;width:110px;height:4px;border-radius:3px;background:var(--border-2);outline:none}
|
||||
input[type=range]::-webkit-slider-thumb{-webkit-appearance:none;width:16px;height:16px;border-radius:50%;
|
||||
background:var(--primary);cursor:pointer;border:2px solid var(--bg);box-shadow:0 0 0 1px var(--primary)}
|
||||
select{height:36px;padding:0 8px;background:var(--surface);color:var(--text);border:1px solid var(--border);border-radius:var(--r-sm);outline:none}
|
||||
select:focus{border-color:var(--border-2);box-shadow:0 0 0 3px rgba(59,130,246,.22)}
|
||||
.btn{display:inline-flex;align-items:center;gap:7px;height:36px;padding:0 14px;border-radius:var(--r-sm);
|
||||
border:1px solid var(--border);background:var(--surface);color:var(--text);font-weight:500;
|
||||
transition:background .15s,border-color .15s,transform .05s}
|
||||
.btn svg{font-size:15px}
|
||||
.btn.ghost:hover{background:var(--elev);border-color:var(--border-2)}
|
||||
.btn:active{transform:translateY(1px)}
|
||||
.btn.primary{background:var(--primary);border-color:transparent;color:var(--primary-ink);font-weight:600}
|
||||
.btn.primary:hover{background:var(--primary-2)}
|
||||
.btn.primary:disabled{opacity:.45;cursor:not-allowed}
|
||||
.btn.danger{border-color:var(--danger);color:#fca5a5}
|
||||
.btn.danger:hover{background:rgba(239,68,68,.14)}
|
||||
.btn:disabled{opacity:.45;cursor:not-allowed}
|
||||
.btn.act-nsfw{background:var(--pink);border-color:transparent;color:#2a0a1c;font-weight:600}
|
||||
.btn.act-nsfw:hover{filter:brightness(1.08)}
|
||||
.btn.act-safe{background:var(--primary);border-color:transparent;color:var(--primary-ink);font-weight:600}
|
||||
.btn.act-safe:hover{background:var(--primary-2)}
|
||||
.ctl{display:flex;align-items:center;gap:6px;color:var(--muted);font-size:12.5px;white-space:nowrap}
|
||||
.actions{display:flex;align-items:center;gap:8px;padding-left:10px;border-left:1px solid var(--border)}
|
||||
.actions[hidden]{display:none}
|
||||
.selinfo{color:var(--muted);font-size:12.5px;white-space:nowrap}
|
||||
.selinfo b{color:var(--text);font-variant-numeric:tabular-nums}
|
||||
.btn .pill{min-width:20px;height:19px;padding:0 6px;display:grid;place-items:center;border-radius:10px;
|
||||
background:rgba(0,0,0,.22);font-size:11.5px;font-weight:700;font-variant-numeric:tabular-nums}
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(190px,1fr));gap:12px;padding:16px 18px 64px}
|
||||
.card{position:relative;overflow:hidden;background:var(--surface);border:2px solid var(--border);border-radius:var(--r);
|
||||
cursor:pointer;transition:border-color .15s,transform .12s,box-shadow .15s}
|
||||
.card:hover{transform:translateY(-2px);box-shadow:0 8px 24px rgba(0,0,0,.35)}
|
||||
.card:focus-within{outline:2px solid var(--ring);outline-offset:2px}
|
||||
.card.c-safe{border-color:var(--primary)}
|
||||
.card.c-tag{border-color:var(--pink)}
|
||||
.card.c-sel{border-color:var(--danger);box-shadow:0 0 0 1px var(--danger)}
|
||||
.card.selected{outline:3px solid var(--ring);outline-offset:-3px}
|
||||
.thumb{position:relative;aspect-ratio:1/1;background:#000}
|
||||
.thumb img{width:100%;height:100%;object-fit:cover;display:block}
|
||||
.nsfw{position:absolute;top:8px;left:8px;width:1px;height:1px;opacity:0;pointer-events:none}
|
||||
.score{position:absolute;left:8px;top:8px;padding:2px 7px;border-radius:6px;color:#fff;font-size:11.5px;
|
||||
font-weight:700;font-variant-numeric:tabular-nums;text-shadow:0 1px 2px rgba(0,0,0,.5)}
|
||||
.icon{display:grid;place-items:center;width:30px;height:30px;font-size:15px;border:0;border-radius:8px;
|
||||
color:#fff;background:rgba(6,8,12,.62);backdrop-filter:blur(3px);transition:background .15s}
|
||||
.icon:hover{background:rgba(6,8,12,.9)}
|
||||
.zoom{position:absolute;right:8px;top:8px}
|
||||
.tick{position:absolute;right:8px;bottom:8px;width:28px;height:28px;display:grid;place-items:center;border-radius:50%;
|
||||
border:2px solid rgba(255,255,255,.55);background:rgba(6,8,12,.5);color:transparent;font-size:15px;
|
||||
cursor:pointer;backdrop-filter:blur(3px);transition:background .15s,border-color .15s,color .15s,transform .1s}
|
||||
.tick:hover{border-color:#fff;transform:scale(1.08)}
|
||||
.card.t-nsfw .tick{background:var(--pink);border-color:var(--pink);color:#2a0a1c}
|
||||
.card.t-sfw .tick{background:var(--primary);border-color:var(--primary);color:#04121f}
|
||||
.tagged{position:absolute;left:8px;bottom:8px;display:inline-flex;align-items:center;gap:4px;
|
||||
padding:2px 8px 2px 5px;border-radius:6px;background:var(--pink);color:#2a0a1c;font-size:11px;font-weight:700}
|
||||
.tagged[hidden]{display:none}
|
||||
.tagged svg{font-size:13px}
|
||||
.safe{position:absolute;left:8px;bottom:8px;display:inline-flex;align-items:center;gap:4px;
|
||||
padding:2px 8px 2px 5px;border-radius:6px;background:var(--primary);color:var(--primary-ink);font-size:11px;font-weight:700}
|
||||
.safe[hidden]{display:none}
|
||||
.safe svg{font-size:13px}
|
||||
.info{padding:8px 10px 10px}
|
||||
.name{font-size:12px;font-weight:500;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.path{margin-top:2px;font-size:10.5px;color:var(--faint);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;direction:rtl;text-align:left}
|
||||
.card.hidden{display:none}
|
||||
.empty{padding:64px 20px;text-align:center;color:var(--muted)}
|
||||
.navtoggle{display:grid;place-items:center;width:36px;height:36px;font-size:18px;border-radius:var(--r-sm);
|
||||
background:var(--surface);border:1px solid var(--border);color:var(--muted);transition:background .15s,color .15s}
|
||||
.navtoggle:hover{background:var(--elev);color:var(--text)}
|
||||
.navtoggle.on{color:var(--primary);border-color:var(--border-2)}
|
||||
.shell{display:flex;align-items:flex-start}
|
||||
.sidebar{position:sticky;top:58px;align-self:flex-start;flex:0 0 252px;width:252px;height:calc(100dvh - 58px);
|
||||
overflow:auto;padding:8px;background:var(--surface);border-right:1px solid var(--border)}
|
||||
.shead{padding:8px 10px 4px;font-size:10px;font-weight:600;text-transform:uppercase;letter-spacing:.06em;color:var(--faint)}
|
||||
.content{flex:1;min-width:0}
|
||||
.shell.collapsed .sidebar{display:none}
|
||||
.fitem{display:flex;align-items:center;gap:8px;width:100%;padding:6px 10px;border:0;border-radius:6px;
|
||||
background:none;color:var(--muted);font-size:12.5px;text-align:left;transition:background .12s,color .12s}
|
||||
.fitem:hover{background:var(--elev);color:var(--text)}
|
||||
.fitem.on{background:var(--elev);color:var(--text);font-weight:600}
|
||||
.fname{flex:1;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.fcount{min-width:22px;padding:0 7px;text-align:center;font-size:11px;font-variant-numeric:tabular-nums;
|
||||
color:var(--faint);background:var(--surface-2);border:1px solid var(--border);border-radius:20px}
|
||||
.fitem.on .fcount{color:var(--text);border-color:var(--border-2)}
|
||||
@media (max-width:820px){.sidebar{position:fixed;left:0;top:58px;bottom:0;z-index:25;box-shadow:12px 0 32px rgba(0,0,0,.5)}}
|
||||
.lb{position:fixed;inset:0;z-index:50;display:flex;gap:16px;padding:20px;
|
||||
background:rgba(4,5,7,.94);backdrop-filter:blur(6px)}
|
||||
.lb[hidden]{display:none}
|
||||
.lb-fig{margin:0;flex:1;min-width:0;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:14px}
|
||||
.lb-fig img{max-width:100%;max-height:82vh;object-fit:contain;border-radius:8px;box-shadow:0 20px 60px rgba(0,0,0,.6)}
|
||||
.lb-bar{display:flex;align-items:center;gap:14px;max-width:100%}
|
||||
.lb-bar figcaption{color:var(--muted);font-size:13px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.lb-bar figcaption b{color:var(--text)}
|
||||
.lb .icon{width:40px;height:40px;font-size:18px;background:rgba(20,23,28,.82);border:1px solid var(--border)}
|
||||
.lb .icon:hover{background:var(--elev)}
|
||||
.lb-close{position:absolute;top:20px;left:20px;z-index:2}
|
||||
.lb-meta{flex:0 0 300px;width:300px;overflow:auto;background:rgba(20,23,28,.72);border:1px solid var(--border);
|
||||
border-radius:10px;padding:12px 14px;font-size:11px;line-height:1.5}
|
||||
.mrow{display:flex;gap:8px;padding:3px 0;border-bottom:1px solid rgba(255,255,255,.05)}
|
||||
.mk{flex:0 0 42%;color:var(--faint);word-break:break-word}
|
||||
.mv{flex:1;min-width:0;color:var(--text);word-break:break-word}
|
||||
.mv a{color:#6ea8ff;text-decoration:none}
|
||||
.mv a:hover{text-decoration:underline}
|
||||
.mload{color:var(--muted);padding:6px 0}
|
||||
@media (max-width:720px){.lb-meta{display:none}}
|
||||
.toast{position:fixed;left:50%;bottom:26px;z-index:60;transform:translateX(-50%) translateY(12px);
|
||||
padding:11px 18px;border-radius:10px;background:var(--elev);border:1px solid var(--border-2);color:var(--text);
|
||||
font-size:13px;font-weight:500;box-shadow:0 12px 32px rgba(0,0,0,.5);opacity:0;pointer-events:none;
|
||||
transition:opacity .2s,transform .2s}
|
||||
.toast.show{opacity:1;transform:translateX(-50%) translateY(0)}
|
||||
.toast.ok{border-color:var(--primary)}
|
||||
.toast.err{border-color:var(--danger);color:#fecaca}
|
||||
.prog{position:fixed;right:20px;bottom:20px;z-index:55;display:flex;align-items:center;gap:12px;
|
||||
padding:10px 16px 10px 10px;background:var(--elev);border:1px solid var(--border-2);border-radius:12px;
|
||||
box-shadow:0 12px 32px rgba(0,0,0,.5)}
|
||||
.prog[hidden]{display:none}
|
||||
.pring{position:relative;width:40px;height:40px;display:grid;place-items:center}
|
||||
.pring svg{position:absolute;inset:0;transform:rotate(-90deg)}
|
||||
.pring .pbg{fill:none;stroke:var(--border-2);stroke-width:3.5}
|
||||
.pring .pfg{fill:none;stroke:var(--primary);stroke-width:3.5;stroke-linecap:round;
|
||||
stroke-dasharray:100.53;stroke-dashoffset:100.53;transition:stroke-dashoffset .18s}
|
||||
.pring #ppct{font-size:10.5px;font-weight:700;font-variant-numeric:tabular-nums}
|
||||
.plbl{font-size:12px;color:var(--muted)}
|
||||
.logpanel{position:fixed;left:0;right:0;bottom:0;z-index:45;height:240px;display:flex;flex-direction:column;
|
||||
background:rgba(11,13,16,.98);border-top:1px solid var(--border-2);box-shadow:0 -12px 32px rgba(0,0,0,.5)}
|
||||
.logpanel[hidden]{display:none}
|
||||
.loghead{display:flex;align-items:center;gap:10px;padding:8px 14px;border-bottom:1px solid var(--border)}
|
||||
.loghead b{font-size:12px}
|
||||
.logbody{flex:1;overflow:auto;padding:8px 14px;line-height:1.6;
|
||||
font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:11.5px}
|
||||
.logline{display:flex;gap:10px;white-space:pre-wrap;word-break:break-word}
|
||||
.logline .lt{flex:0 0 auto;color:var(--faint);font-variant-numeric:tabular-nums}
|
||||
.logempty{color:var(--muted)}
|
||||
@media (max-width:640px){
|
||||
.metrics{display:none}.field input{width:120px}
|
||||
.grid{grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:8px;padding:12px}}
|
||||
</style></head><body>
|
||||
<header class="app">
|
||||
<button id="navtoggle" class="navtoggle on" aria-label="Toggle folder browser" aria-pressed="true" onclick="toggleSidebar()"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="2"/><path d="M9 3v18"/></svg></button>
|
||||
<div class="brand">
|
||||
<span class="logo"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10Z"/><path d="m9 12 2 2 4-4"/></svg></span>
|
||||
<div class="titles"><h1>NSFW Review</h1><p>On-device · nothing leaves this Mac</p></div>
|
||||
</div>
|
||||
<div class="grow"></div>
|
||||
<div class="metrics">
|
||||
<div class="metric"><span class="num">%%TOTAL%%</span><span class="lbl">images</span></div>
|
||||
</div>
|
||||
<button id="logbtn" class="navtoggle" aria-label="Toggle activity log" onclick="toggleLog()"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M8 6h11M8 12h11M8 18h11"/><path d="M4 6h.01M4 12h.01M4 18h.01"/></svg></button>
|
||||
</header>
|
||||
<div class="shell" id="shell">
|
||||
<aside class="sidebar" aria-label="Folder browser"><div class="shead">Folders</div>%%TREE%%</aside>
|
||||
<div class="content">
|
||||
<div class="toolbar">
|
||||
<div class="field"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="11" cy="11" r="7"/><path d="m21 21-4.3-4.3"/></svg>
|
||||
<input id="q" type="search" placeholder="Search name or folder…" aria-label="Search images"></div>
|
||||
<label class="ctl">Show
|
||||
<select id="showSel" aria-label="Show which images"><option value="all">All</option><option value="todo">To-handle</option><option value="nsfw">NSFW</option><option value="sfw">SFW</option></select></label>
|
||||
<label class="ctl">Sort
|
||||
<select id="sort" aria-label="Sort order"><option value="score">Score ↓</option><option value="name">Name</option><option value="path">Folder</option></select></label>
|
||||
<label class="thr">NSFW ≥ <output id="thrOut">%%THRESHOLD%%</output>
|
||||
<input id="thr" type="range" min="0" max="1" step="0.01" value="%%THRESHOLD%%" aria-label="NSFW score threshold"></label>
|
||||
<button class="btn ghost" onclick="selectFlagged()">Select flagged</button>
|
||||
<button class="btn ghost" onclick="setAll(true)">Select all</button>
|
||||
<div class="grow"></div>
|
||||
<div class="actions" id="actions" hidden>
|
||||
<span class="selinfo"><b id="selCount">0</b> selected</span>
|
||||
<button class="btn ghost" onclick="setAll(false)">Deselect</button>
|
||||
<button id="untagb" class="btn ghost" onclick="untag()">Remove tag</button>
|
||||
<button id="markSafeb" class="btn act-safe" onclick="markSafe()">Mark safe</button>
|
||||
<button id="applyb" class="btn act-nsfw" onclick="apply()">Mark NSFW</button>
|
||||
</div>
|
||||
</div>
|
||||
<main id="grid" class="grid">%%CARDS%%</main>
|
||||
<p id="empty" class="empty" hidden>No images match your filter.</p>
|
||||
</div></div>
|
||||
<div id="lb" class="lb" hidden>
|
||||
<button class="icon lb-close" aria-label="Close" onclick="closeLb()"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 6 6 18M6 6l12 12"/></svg></button>
|
||||
<figure class="lb-fig">
|
||||
<img id="lbImg" alt="">
|
||||
<div class="lb-bar">
|
||||
<button class="icon" aria-label="Previous" onclick="stepLb(-1)"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m15 18-6-6 6-6"/></svg></button>
|
||||
<figcaption id="lbCap"></figcaption>
|
||||
<button class="icon" aria-label="Next" onclick="stepLb(1)"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="m9 18 6-6-6-6"/></svg></button>
|
||||
</div>
|
||||
</figure>
|
||||
<aside id="lbMeta" class="lb-meta" aria-label="EXIF metadata"></aside>
|
||||
</div>
|
||||
<div id="prog" class="prog" role="status" aria-live="polite" hidden>
|
||||
<div class="pring"><svg viewBox="0 0 36 36"><circle class="pbg" cx="18" cy="18" r="16"/><circle id="pring" class="pfg" cx="18" cy="18" r="16"/></svg><span id="ppct">0%</span></div>
|
||||
<span class="plbl">Checking existing tags…</span>
|
||||
</div>
|
||||
<div id="logpanel" class="logpanel" hidden>
|
||||
<div class="loghead"><b>Activity log</b><span id="logcount" class="plbl"></span><div class="grow"></div>
|
||||
<button class="btn ghost" onclick="toggleLog()">Close</button></div>
|
||||
<div id="logbody" class="logbody"><div class="logempty">No activity yet.</div></div>
|
||||
</div>
|
||||
<div id="toast" class="toast" role="status" aria-live="polite" hidden></div>
|
||||
<script>
|
||||
const $=s=>document.querySelector(s);
|
||||
const grid=$('#grid'),q=$('#q'),thr=$('#thr'),thrOut=$('#thrOut'),sortSel=$('#sort'),showSel=$('#showSel'),
|
||||
selCount=$('#selCount'),applyb=$('#applyb'),untagb=$('#untagb'),markSafeb=$('#markSafeb'),
|
||||
actions=$('#actions'),empty=$('#empty'),toastEl=$('#toast');
|
||||
const cards=[...document.querySelectorAll('.card')];
|
||||
let folderFilter='',show='all';
|
||||
function refreshTagN(){} // (chip counts removed with the toolbar redesign)
|
||||
function pickFolder(btn){
|
||||
folderFilter=btn.dataset.folder||'';
|
||||
document.querySelectorAll('.fitem').forEach(x=>x.classList.toggle('on',x===btn));
|
||||
applyView();scanFolder();}
|
||||
// Lazily ask the server which images in the current folder already carry the
|
||||
// 'nsfw' tag, in chunks, updating the progress ring and filtering on the fly.
|
||||
const prog=$('#prog'),pring=$('#pring'),ppct=$('#ppct'),PC=100.53;
|
||||
function setProg(f){ppct.textContent=Math.round(f*100)+'%';pring.style.strokeDashoffset=PC*(1-f);}
|
||||
let scanning=false;
|
||||
async function scanFolder(){
|
||||
if(scanning)return;
|
||||
const scope=cards.filter(c=>c.dataset.tagged==='?'&&inFolder(c));
|
||||
if(!scope.length)return;
|
||||
scanning=true;prog.hidden=false;setProg(0);
|
||||
const CH=300;let done=0;
|
||||
for(let i=0;i<scope.length;i+=CH){
|
||||
const chunk=scope.slice(i,i+CH);let res={nsfw:[],sfw:[]};
|
||||
try{res=await(await fetch('/tagged',{method:'POST',body:JSON.stringify(chunk.map(c=>box(c).value))})).json();}
|
||||
catch(e){}
|
||||
const nset=new Set(res.nsfw),sset=new Set(res.sfw);
|
||||
chunk.forEach(c=>{const v=box(c).value,isN=nset.has(v),isS=sset.has(v);
|
||||
c.dataset.tagged=isN?'1':'0';c.dataset.sfw=isS?'1':'0';
|
||||
if(isN||isS)box(c).checked=false; // decided: nsfw->pink / sfw->green
|
||||
else box(c).checked=parseFloat(c.dataset.score)>=curThr(); // undecided: flag by score -> red
|
||||
const b=c.querySelector('.tagged');if(b)b.hidden=!isN;paint(c);});
|
||||
done+=chunk.length;setProg(done/scope.length);
|
||||
refreshTagN();updateCount();applyView();
|
||||
}
|
||||
scanning=false;prog.hidden=true;}
|
||||
function inFolder(c){if(!folderFilter)return true;const f=c.dataset.folder;return f===folderFilter||f.startsWith(folderFilter+'/');}
|
||||
function toggleSidebar(){
|
||||
const sh=$('#shell'),tb=$('#navtoggle'),off=sh.classList.toggle('collapsed');
|
||||
tb.classList.toggle('on',!off);tb.setAttribute('aria-pressed',String(!off));
|
||||
if(off){folderFilter='';document.querySelectorAll('.fitem').forEach((x,i)=>x.classList.toggle('on',i===0));applyView();}}
|
||||
document.querySelectorAll('.score').forEach(el=>{
|
||||
const s=parseFloat(el.dataset.s),hue=Math.round((1-s)*130);
|
||||
el.style.background='hsl('+hue+' 70% 40% / .9)';});
|
||||
const box=c=>c.querySelector('.nsfw');
|
||||
const visible=c=>!c.classList.contains('hidden');
|
||||
// Counts + actions operate on the CURRENT view (folder/search/filter) so the
|
||||
// counter tracks what you see, and Clear/Invert change it as expected.
|
||||
const selected=()=>cards.filter(c=>box(c).checked&&visible(c));
|
||||
const curThr=()=>parseFloat(thr.value);
|
||||
// Two independent channels: DECISION (the EXIF tag) colors the frame + tick;
|
||||
// SELECTION (the checkbox, for bulk actions) shows as a ring — a card can be both.
|
||||
function paint(c){const sel=box(c).checked,tag=c.dataset.tagged==='1',safe=c.dataset.sfw==='1';
|
||||
c.classList.toggle('c-tag',tag); // pink frame: nsfw in EXIF
|
||||
c.classList.toggle('c-safe',!tag&&safe); // green frame: sfw in EXIF
|
||||
c.classList.toggle('c-sel',!tag&&!safe&&sel); // red frame: undecided & selected (bulk candidate)
|
||||
c.classList.toggle('t-nsfw',tag); // tick fill mirrors the decision
|
||||
c.classList.toggle('t-sfw',safe);
|
||||
c.classList.toggle('selected',sel); // ring: selected (visible even on tagged cards)
|
||||
const tb=c.querySelector('.tagged');if(tb)tb.hidden=!tag; // pink "tagged" badge (nsfw in EXIF)
|
||||
const sb=c.querySelector('.safe');if(sb)sb.hidden=!safe;} // green "tagged" badge (sfw in EXIF)
|
||||
const paintAll=()=>cards.forEach(paint);
|
||||
// Per-image decision. One click writes EXIF and cycles: undecided -> NSFW -> SFW -> undecided.
|
||||
// Independent of selection, so it never needs a second click and always reaches every state.
|
||||
async function toggleCard(btn){
|
||||
const c=btn.closest('.card');
|
||||
const cur=c.dataset.tagged==='1'?'nsfw':(c.dataset.sfw==='1'?'sfw':'none');
|
||||
const next=cur==='none'?'nsfw':(cur==='nsfw'?'sfw':'none');
|
||||
const url=next==='nsfw'?'/apply':next==='sfw'?'/mark?state=sfw':'/untag';
|
||||
c.dataset.tagged=next==='nsfw'?'1':'0';c.dataset.sfw=next==='sfw'?'1':'0';
|
||||
paint(c);updateCount(); // optimistic
|
||||
try{await fetch(url,{method:'POST',body:JSON.stringify([box(c).value])});}
|
||||
catch(e){toast('Could not save decision — '+e,'err');}
|
||||
refreshTagN();updateCount();}
|
||||
function selectFlagged(){cards.forEach(c=>{if(!c.classList.contains('hidden')&&c.dataset.tagged!=='1'&&parseFloat(c.dataset.score)>=curThr())box(c).checked=true;});updateCount();paintAll();}
|
||||
function updateCount(){const sel=selected(),n=sel.length,nt=sel.filter(c=>c.dataset.tagged==='1'||c.dataset.sfw==='1').length;
|
||||
selCount.textContent=n;actions.hidden=n===0;
|
||||
applyb.disabled=n===0;markSafeb.disabled=n===0;untagb.disabled=nt===0;}
|
||||
function setAll(v){cards.forEach(c=>{if(!c.classList.contains('hidden'))box(c).checked=v;});updateCount();paintAll();}
|
||||
grid.addEventListener('change',e=>{if(e.target.classList.contains('nsfw')){updateCount();paint(e.target.closest('.card'));}});
|
||||
thr.addEventListener('input',()=>{thrOut.textContent=curThr().toFixed(2);paintAll();});
|
||||
showSel.addEventListener('change',()=>{show=showSel.value;applyView();});
|
||||
q.addEventListener('input',applyView);
|
||||
sortSel.addEventListener('change',applyView);
|
||||
function applyView(){
|
||||
const term=q.value.trim().toLowerCase();
|
||||
let vis=cards.filter(c=>{
|
||||
if(term&&!c.dataset.hay.includes(term))return false;
|
||||
if(folderFilter){const f=c.dataset.folder;if(f!==folderFilter&&!f.startsWith(folderFilter+'/'))return false;}
|
||||
if(show==='todo'&&(c.dataset.tagged==='1'||c.dataset.sfw==='1'))return false; // only un-decided
|
||||
if(show==='nsfw'&&c.dataset.tagged!=='1')return false; // only NSFW-tagged
|
||||
if(show==='sfw'&&c.dataset.sfw!=='1')return false; // only SFW-tagged
|
||||
return true;});
|
||||
const key=sortSel.value;
|
||||
vis.sort((a,b)=>{
|
||||
if(key==='name')return a.dataset.name.localeCompare(b.dataset.name);
|
||||
if(key==='path')return a.dataset.folder.localeCompare(b.dataset.folder);
|
||||
return parseFloat(b.dataset.score)-parseFloat(a.dataset.score);});
|
||||
cards.forEach(c=>c.classList.add('hidden'));
|
||||
const frag=document.createDocumentFragment();
|
||||
vis.forEach(c=>{c.classList.remove('hidden');frag.appendChild(c);});
|
||||
grid.appendChild(frag);
|
||||
empty.hidden=vis.length>0;
|
||||
updateCount();} // counter reflects the folder/filter you're viewing
|
||||
const lb=$('#lb'),lbImg=$('#lbImg'),lbCap=$('#lbCap'),lbMeta=$('#lbMeta');
|
||||
const esc=s=>String(s).replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c]));
|
||||
let lbList=[],lbIdx=-1,metaTok=0;
|
||||
function openLb(btn){
|
||||
lbList=cards.filter(c=>!c.classList.contains('hidden'));
|
||||
lbIdx=lbList.indexOf(btn.closest('.card'));showLb();lb.hidden=false;}
|
||||
function metaVal(v){
|
||||
if(/^https?:\/\//.test(v)){const t=v.includes('google.com/maps')?'Open in Google Maps ↗':esc(v);
|
||||
return '<a href="'+esc(v)+'" target="_blank" rel="noopener">'+t+'</a>';}
|
||||
return esc(v);}
|
||||
async function loadMeta(path){
|
||||
const tok=++metaTok;lbMeta.innerHTML='<div class="mload">Loading EXIF…</div>';
|
||||
try{
|
||||
const d=await(await fetch('/exif?path='+encodeURIComponent(path))).json();
|
||||
if(tok!==metaTok)return;
|
||||
const rows=Object.entries(d).map(([k,v])=>'<div class="mrow"><span class="mk">'+esc(k)+'</span><span class="mv">'+metaVal(v)+'</span></div>').join('');
|
||||
lbMeta.innerHTML=rows||'<div class="mload">No EXIF.</div>';
|
||||
}catch(e){if(tok===metaTok)lbMeta.innerHTML='<div class="mload">EXIF unavailable.</div>';}}
|
||||
function showLb(){
|
||||
const c=lbList[lbIdx];if(!c)return;const b=c.querySelector('.zoom');
|
||||
lbImg.src=b.dataset.src;
|
||||
lbCap.innerHTML='<b>'+b.dataset.name+'</b> · score '+b.dataset.score+' · '+(lbIdx+1)+' / '+lbList.length;
|
||||
loadMeta(box(c).value);}
|
||||
function stepLb(d){lbIdx=(lbIdx+d+lbList.length)%lbList.length;showLb();}
|
||||
function closeLb(){lb.hidden=true;lbImg.src='';metaTok++;lbMeta.innerHTML='';}
|
||||
lb.addEventListener('click',e=>{if(e.target===lb)closeLb();});
|
||||
addEventListener('keydown',e=>{
|
||||
if(lb.hidden)return;
|
||||
if(e.key==='Escape')closeLb();
|
||||
else if(e.key==='ArrowLeft')stepLb(-1);
|
||||
else if(e.key==='ArrowRight')stepLb(1);
|
||||
else if(e.key===' '){e.preventDefault();const c=lbList[lbIdx];box(c).checked=!box(c).checked;updateCount();paint(c);}});
|
||||
let toastT;
|
||||
function toast(msg,cls){
|
||||
toastEl.textContent=msg;toastEl.hidden=false;toastEl.className='toast show '+(cls||'');
|
||||
clearTimeout(toastT);toastT=setTimeout(()=>toastEl.classList.remove('show'),3500);}
|
||||
// batch action helper: run `fn` on selected paths, apply `after(card)` to each, toast, restore label
|
||||
async function runAction(btn,label,url,after,msg){
|
||||
const sel=selected(),paths=sel.map(c=>box(c).value);
|
||||
if(!paths.length)return;
|
||||
btn.disabled=true;const lbl=btn.textContent;btn.textContent=label;
|
||||
try{
|
||||
const j=await(await fetch(url,{method:'POST',body:JSON.stringify(paths)})).json();
|
||||
sel.forEach(c=>{after(c);box(c).checked=false;paint(c);});
|
||||
applyView();toast(msg(paths.length,j),'ok');
|
||||
}catch(e){toast('Error: '+e,'err');}
|
||||
btn.textContent=lbl;updateCount();}
|
||||
const apply=()=>runAction(applyb,'Marking…','/apply',
|
||||
c=>{c.dataset.tagged='1';c.dataset.sfw='0';},(n,j)=>'Tagged '+n+' NSFW');
|
||||
const markSafe=()=>runAction(markSafeb,'Marking…','/mark?state=sfw',
|
||||
c=>{c.dataset.sfw='1';c.dataset.tagged='0';},(n)=>'Marked '+n+' safe');
|
||||
const untag=()=>runAction(untagb,'Removing…','/untag',
|
||||
c=>{c.dataset.tagged='0';c.dataset.sfw='0';},(n,j)=>'Removed tag from '+(j.untagged??n));
|
||||
const logpanel=$('#logpanel'),logbody=$('#logbody'),logcount=$('#logcount');
|
||||
let logTimer=null,logLen=-1;
|
||||
function toggleLog(){
|
||||
const open=logpanel.hidden;logpanel.hidden=!open;$('#logbtn').classList.toggle('on',open);
|
||||
if(open){logLen=-1;refreshLog();logTimer=setInterval(refreshLog,2000);}
|
||||
else{clearInterval(logTimer);logTimer=null;}}
|
||||
async function refreshLog(){
|
||||
let ents;try{ents=await(await fetch('/log')).json();}catch(e){return;}
|
||||
logcount.textContent=ents.length+' event'+(ents.length===1?'':'s');
|
||||
if(ents.length===logLen)return;logLen=ents.length;
|
||||
const atBottom=logbody.scrollHeight-logbody.scrollTop-logbody.clientHeight<40;
|
||||
logbody.innerHTML=ents.length
|
||||
? ents.map(e=>'<div class="logline"><span class="lt">'+esc(e.t)+'</span><span>'+esc(e.m)+'</span></div>').join('')
|
||||
: '<div class="logempty">No activity yet.</div>';
|
||||
if(atBottom)logbody.scrollTop=logbody.scrollHeight;}
|
||||
refreshTagN();paintAll();updateCount();applyView();
|
||||
scanFolder(); // scan the whole "All folders" view on load so tag state is defined at startup
|
||||
</script></body></html>
|
||||
119
legacy_cli_archive/src/nsfwtag/scoring.py
Normal file
119
legacy_cli_archive/src/nsfwtag/scoring.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""Discover images and score them with the local NSFW model.
|
||||
|
||||
Scores are cached to a CSV next to the library, so a rerun reuses them and only
|
||||
scores newly-added files — skipping the model (and its load) entirely when
|
||||
nothing is new.
|
||||
"""
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import BATCH, CSV_NAME, EXTS, MODEL_ID
|
||||
|
||||
|
||||
def discover_images(target: Path, recursive: bool = False, limit: int = 0):
|
||||
"""All supported image files under `target`, de-duplicated by resolved path
|
||||
(guards against symlinks / duplicate paths). Sorted for stable ordering."""
|
||||
walk = target.rglob("*") if recursive else target.iterdir()
|
||||
seen, imgs = set(), []
|
||||
for p in sorted(walk):
|
||||
if not (p.is_file() and p.suffix.lower() in EXTS):
|
||||
continue
|
||||
rp = p.resolve()
|
||||
if rp in seen:
|
||||
continue
|
||||
seen.add(rp)
|
||||
imgs.append(p)
|
||||
return imgs[:limit] if limit else imgs
|
||||
|
||||
|
||||
def load_cache(csv_path: Path) -> dict:
|
||||
"""Read a scores CSV into {path: score}. Missing file -> empty dict."""
|
||||
cache = {}
|
||||
if csv_path.exists():
|
||||
with open(csv_path, newline="", encoding="utf-8") as fh:
|
||||
for row in csv.DictReader(fh):
|
||||
try:
|
||||
cache[row["path"]] = float(row["nsfw_score"])
|
||||
except (KeyError, ValueError):
|
||||
pass
|
||||
return cache
|
||||
|
||||
|
||||
def _save_cache(csv_path: Path, cache: dict):
|
||||
with open(csv_path, "w", newline="", encoding="utf-8") as fh:
|
||||
w = csv.writer(fh)
|
||||
w.writerow(["path", "nsfw_score"])
|
||||
w.writerows([(pth, f"{sc:.4f}") for pth, sc in cache.items()])
|
||||
|
||||
|
||||
def score_images(imgs, csv_path: Path, rescore: bool = False, batch: int = BATCH):
|
||||
"""Return (scored, errors) where scored is [(Path, score)] for every image in
|
||||
`imgs` that has a score. Reuses the CSV cache; only `imgs` missing from it are
|
||||
run through the model, then the merged cache is persisted."""
|
||||
cache = {} if rescore else load_cache(csv_path)
|
||||
todo = [p for p in imgs if str(p) not in cache]
|
||||
if cache:
|
||||
print(f"cache: {len(cache)} scores from {csv_path} — {len(todo)} new to score", file=sys.stderr)
|
||||
|
||||
errors = 0
|
||||
if todo:
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageFile
|
||||
from transformers import AutoModelForImageClassification
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||
|
||||
device = "mps" if torch.backends.mps.is_available() else "cpu"
|
||||
# Load from local cache without hitting the HF Hub (skips the slow per-run
|
||||
# update check). First run falls through to a one-time download, then cached.
|
||||
try:
|
||||
model = AutoModelForImageClassification.from_pretrained(MODEL_ID, local_files_only=True)
|
||||
print(f"Loaded {MODEL_ID} from cache on {device}", file=sys.stderr)
|
||||
except Exception:
|
||||
print(f"Downloading {MODEL_ID} (one-time, ~350 MB) …", file=sys.stderr)
|
||||
model = AutoModelForImageClassification.from_pretrained(MODEL_ID)
|
||||
model = model.to(device).eval()
|
||||
nsfw_idx = next(i for i, l in model.config.id2label.items() if l.lower() == "nsfw")
|
||||
|
||||
# Input size comes from the model, not a constant — AdamCodd wants 384,
|
||||
# Falconsai wanted 224. Hardcoding it silently breaks on a model swap.
|
||||
size = getattr(model.config, "image_size", 224)
|
||||
|
||||
def preprocess(img):
|
||||
img = img.convert("RGB").resize((size, size), Image.BILINEAR)
|
||||
a = (np.asarray(img, dtype="float32") / 255.0 - 0.5) / 0.5
|
||||
return torch.from_numpy(a).permute(2, 0, 1)
|
||||
|
||||
print(f"Scoring {len(todo)} images …", file=sys.stderr)
|
||||
done_since_save = 0
|
||||
for i in range(0, len(todo), batch):
|
||||
tens, paths = [], []
|
||||
for p in todo[i:i + batch]:
|
||||
try:
|
||||
tens.append(preprocess(Image.open(p))); paths.append(p)
|
||||
except Exception as e:
|
||||
errors += 1; print(f"\n error {p}: {e}", file=sys.stderr)
|
||||
if not tens:
|
||||
continue
|
||||
try:
|
||||
with torch.no_grad():
|
||||
probs = model(pixel_values=torch.stack(tens).to(device)).logits.softmax(-1)
|
||||
for p, pr in zip(paths, probs):
|
||||
cache[str(p)] = pr[nsfw_idx].item()
|
||||
except Exception as e:
|
||||
# One bad batch (e.g. MPS OOM / Metal error, common when two GPU
|
||||
# jobs run at once) must not discard everything scored so far.
|
||||
errors += len(paths)
|
||||
print(f"\n batch {i}-{i+len(paths)} failed: {e}", file=sys.stderr)
|
||||
done_since_save += len(paths)
|
||||
if done_since_save >= 500: # periodic checkpoint → crash-resumable
|
||||
_save_cache(csv_path, cache); done_since_save = 0
|
||||
print(f"\r {min(i + batch, len(todo))}/{len(todo)}", end="", file=sys.stderr, flush=True)
|
||||
print(file=sys.stderr)
|
||||
_save_cache(csv_path, cache)
|
||||
else:
|
||||
print("all images already scored — skipping model load", file=sys.stderr)
|
||||
|
||||
scored = [(p, cache[str(p)]) for p in imgs if str(p) in cache]
|
||||
return scored, errors
|
||||
126
legacy_cli_archive/src/nsfwtag/server.py
Normal file
126
legacy_cli_archive/src/nsfwtag/server.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Local HTTP server for the review app (stdlib only).
|
||||
|
||||
Routes:
|
||||
GET / the review page
|
||||
GET /img?path= a thumbnail (restricted to the scanned candidate set)
|
||||
POST /tagged -> which of the posted paths already carry the nsfw keyword
|
||||
POST /apply -> add the nsfw keyword to the posted paths
|
||||
POST /untag -> remove the nsfw keyword from the posted paths
|
||||
All POST routes are restricted to the allowed candidate set.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .exif import read_exif, read_marks, read_tagged, remove_keyword, write_keyword
|
||||
from .webapp import render_page
|
||||
|
||||
|
||||
def serve_review(cands, threshold, host="127.0.0.1"):
|
||||
"""Serve the review page and tag/untag confirmed images directly on request."""
|
||||
import http.server
|
||||
import json
|
||||
import mimetypes
|
||||
import time
|
||||
import webbrowser
|
||||
from collections import deque
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
allowed = {str(p) for p, _ in cands} # only these paths may be served or tagged
|
||||
page = render_page(cands, threshold).encode("utf-8")
|
||||
|
||||
log_entries = deque(maxlen=2000) # shown in the browser's Activity log tab
|
||||
|
||||
def log(msg):
|
||||
log_entries.append({"t": time.strftime("%H:%M:%S"), "m": msg})
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
def label(p): # folder + picture name for the log
|
||||
pp = Path(p)
|
||||
return f"{pp.parent.name}/{pp.name}" if pp.parent.name else pp.name
|
||||
|
||||
log(f"review ready — {len(cands)} candidate images")
|
||||
|
||||
class H(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *a): # quiet
|
||||
pass
|
||||
|
||||
def _send(self, code, ctype, body):
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self):
|
||||
u = urlparse(self.path)
|
||||
if u.path == "/":
|
||||
return self._send(200, "text/html; charset=utf-8", page)
|
||||
if u.path == "/img":
|
||||
fp = unquote(parse_qs(u.query).get("path", [""])[0])
|
||||
# ponytail: browsers won't render HEIC bytes; those thumbs show broken — score/tag still work.
|
||||
if fp in allowed and Path(fp).exists():
|
||||
ctype = mimetypes.guess_type(fp)[0] or "application/octet-stream"
|
||||
return self._send(200, ctype, Path(fp).read_bytes())
|
||||
if u.path == "/exif": # metadata panel for the lightbox
|
||||
fp = unquote(parse_qs(u.query).get("path", [""])[0])
|
||||
if fp in allowed and Path(fp).exists():
|
||||
return self._send(200, "application/json", json.dumps(read_exif(fp)).encode())
|
||||
if u.path == "/log": # activity log tab
|
||||
return self._send(200, "application/json", json.dumps(list(log_entries)).encode())
|
||||
return self._send(404, "text/plain", b"not found")
|
||||
|
||||
def do_POST(self):
|
||||
u = urlparse(self.path)
|
||||
route = u.path
|
||||
n = int(self.headers.get("Content-Length", 0))
|
||||
paths = [p for p in json.loads(self.rfile.read(n) or b"[]") if p in allowed]
|
||||
if route == "/tagged": # review state of these paths (lazy, chunked): nsfw + sfw
|
||||
m = read_marks(paths)
|
||||
return self._send(200, "application/json",
|
||||
json.dumps({"nsfw": list(m["nsfw"]), "sfw": list(m["sfw"])}).encode())
|
||||
if route == "/mark": # persist the manual SFW decision (opposite of nsfw)
|
||||
state = parse_qs(u.query).get("state", [""])[0]
|
||||
if state == "sfw": # mark safe: write 'sfw', drop 'nsfw'
|
||||
for p in paths:
|
||||
write_keyword(p, "sfw"); remove_keyword(p, "nsfw")
|
||||
log(f"marked sfw — {label(p)}")
|
||||
elif state == "clear": # un-mark safe: drop 'sfw'
|
||||
for p in paths:
|
||||
remove_keyword(p, "sfw")
|
||||
log(f"cleared sfw — {label(p)}")
|
||||
return self._send(200, "application/json", json.dumps({"ok": len(paths)}).encode())
|
||||
if route == "/apply":
|
||||
ok = 0
|
||||
for p in paths:
|
||||
if write_keyword(p):
|
||||
remove_keyword(p, "sfw") # nsfw and sfw are mutually exclusive
|
||||
ok += 1; log(f"tagged nsfw — {label(p)}")
|
||||
return self._send(200, "application/json", json.dumps({"tagged": ok}).encode())
|
||||
if route == "/untag": # clear the review keyword (nsfw or sfw) — back to un-decided
|
||||
ok = 0
|
||||
for p in paths:
|
||||
remove_keyword(p, "nsfw"); remove_keyword(p, "sfw")
|
||||
ok += 1; log(f"removed tag — {label(p)}")
|
||||
return self._send(200, "application/json", json.dumps({"untagged": ok}).encode())
|
||||
if route == "/toggle": # flip each path's tag both ways; other EXIF preserved
|
||||
cur = read_tagged(paths) # one batched read, then add/remove per file
|
||||
now_tagged = []
|
||||
for p in paths:
|
||||
if p in cur:
|
||||
remove_keyword(p); log(f"removed nsfw — {label(p)}")
|
||||
elif write_keyword(p):
|
||||
remove_keyword(p, "sfw") # mutually exclusive
|
||||
now_tagged.append(p); log(f"tagged nsfw — {label(p)}")
|
||||
return self._send(200, "application/json", json.dumps({"tagged": now_tagged}).encode())
|
||||
return self._send(404, "text/plain", b"not found")
|
||||
|
||||
srv = http.server.ThreadingHTTPServer((host, 0), H) # port 0 = OS picks a free one
|
||||
url = f"http://{host}:{srv.server_address[1]}/"
|
||||
print(f"\nReview app: {url}\n Tick images, click 'Apply' — writes the 'nsfw' EXIF tag directly (idempotent, safe to re-apply).\n Press Ctrl-C here when done.", file=sys.stderr)
|
||||
webbrowser.open(url)
|
||||
try:
|
||||
srv.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nstopped.", file=sys.stderr)
|
||||
finally:
|
||||
srv.server_close()
|
||||
91
legacy_cli_archive/src/nsfwtag/webapp.py
Normal file
91
legacy_cli_archive/src/nsfwtag/webapp.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""Render the review page.
|
||||
|
||||
The HTML/CSS/JS lives in review.html (loaded once at import). This module fills
|
||||
in the %%TOKENS%%: the folder-tree sidebar, the image cards, totals. The server
|
||||
contract is preserved: each .nsfw checkbox's value is the image's absolute path,
|
||||
thumbnails load via GET /img?path=, actions POST to /apply /untag /tagged.
|
||||
"""
|
||||
import html
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
# Inline Lucide-style SVG icons (no emoji, theme-able via currentColor).
|
||||
_SVG_EXPAND = '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3M8 21H5a2 2 0 0 1-2-2v-3m18 0v3a2 2 0 0 1-2 2h-3"/></svg>'
|
||||
_SVG_CHECK = '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><path d="M20 6 9 17l-5-5"/></svg>'
|
||||
|
||||
_PAGE = (Path(__file__).parent / "review.html").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def folder_tree_html(cands):
|
||||
"""Indented folder list for the sidebar file browser, built from the candidate
|
||||
paths. Each row filters the grid to that folder + its subfolders (prefix match);
|
||||
'All folders' resets. Counts are images directly in that folder."""
|
||||
folders = {}
|
||||
for p, _ in cands:
|
||||
folders[str(p.parent)] = folders.get(str(p.parent), 0) + 1
|
||||
if not folders:
|
||||
return ""
|
||||
paths = sorted(folders)
|
||||
root = os.path.commonpath(paths) if len(paths) > 1 else paths[0]
|
||||
rows = [
|
||||
f'<button type="button" class="fitem on" data-folder="" onclick="pickFolder(this)">'
|
||||
f'<span class="fname">All folders</span><span class="fcount">{sum(folders.values())}</span></button>'
|
||||
]
|
||||
for folder in paths:
|
||||
rel = os.path.relpath(folder, root)
|
||||
depth = 0 if rel == "." else rel.count(os.sep) + 1
|
||||
name = os.path.basename(folder) or folder
|
||||
rows.append(
|
||||
f'<button type="button" class="fitem" data-folder="{html.escape(folder.lower(), quote=True)}" '
|
||||
f'style="padding-left:{depth * 14 + 12}px" title="{html.escape(folder, quote=True)}" '
|
||||
f'onclick="pickFolder(this)"><span class="fname">{html.escape(name)}</span>'
|
||||
f'<span class="fcount">{folders[folder]}</span></button>'
|
||||
)
|
||||
return "".join(rows)
|
||||
|
||||
|
||||
def render_page(cands, threshold):
|
||||
"""Build the review page. cands: list of (Path, score) sorted desc."""
|
||||
cards, seen = [], set()
|
||||
for p, score in cands:
|
||||
sp = str(p)
|
||||
if sp in seen: # never list the same path twice
|
||||
continue
|
||||
seen.add(sp)
|
||||
# No pre-selection at render. The per-folder scan decides state so persisted
|
||||
# marks win: nsfw -> pink, sfw -> green, undecided high-score -> red.
|
||||
# (A dismissed/sfw image must never come back pre-selected red on reload.)
|
||||
checked = ""
|
||||
src = f"/img?path={quote(sp)}"
|
||||
esc = html.escape(sp, quote=True)
|
||||
name = html.escape(p.name)
|
||||
folder = html.escape(str(p.parent))
|
||||
folder_attr = html.escape(str(p.parent), quote=True)
|
||||
name_low = html.escape(p.name.lower(), quote=True)
|
||||
folder_low = html.escape(str(p.parent).lower(), quote=True)
|
||||
cards.append(
|
||||
f'<div class="card" data-score="{score:.4f}" data-name="{name_low}" '
|
||||
f'data-folder="{folder_low}" data-hay="{name_low} {folder_low}" '
|
||||
f'data-tagged="?" data-sfw="?">'
|
||||
f'<div class="thumb">'
|
||||
f'<img loading="lazy" src="{src}" alt="">'
|
||||
f'<span class="score" data-s="{score:.4f}">{score:.2f}</span>'
|
||||
f'<button type="button" class="icon zoom" aria-label="View full size" '
|
||||
f'data-src="{src}" data-name="{name}" data-score="{score:.2f}" '
|
||||
f'onclick="openLb(this)">{_SVG_EXPAND}</button>'
|
||||
f'<span class="tagged" hidden>{_SVG_CHECK}<span>tagged</span></span>'
|
||||
f'<span class="safe" hidden>{_SVG_CHECK}<span>tagged</span></span>'
|
||||
f'<button type="button" class="tick" aria-label="Mark {name} NSFW or SFW" '
|
||||
f'onclick="toggleCard(this)">{_SVG_CHECK}</button>'
|
||||
f'<input type="checkbox" class="nsfw" value="{esc}" {checked} tabindex="-1" aria-hidden="true">'
|
||||
f'</div>'
|
||||
f'<div class="info"><div class="name" title="{esc}">{name}</div>'
|
||||
f'<div class="path" title="{folder_attr}">{folder}</div></div>'
|
||||
f'</div>'
|
||||
)
|
||||
return (_PAGE
|
||||
.replace("%%TREE%%", folder_tree_html(cands))
|
||||
.replace("%%CARDS%%", "".join(cards))
|
||||
.replace("%%TOTAL%%", str(len(cards)))
|
||||
.replace("%%THRESHOLD%%", f"{threshold:.2f}"))
|
||||
2454
legacy_cli_archive/src/photo_analyzer.py
Normal file
2454
legacy_cli_archive/src/photo_analyzer.py
Normal file
File diff suppressed because it is too large
Load Diff
122
legacy_cli_archive/src/test_dedup.py
Normal file
122
legacy_cli_archive/src/test_dedup.py
Normal file
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Self-check for the phash/dedup/reconcile additions. Runnable, no framework:
|
||||
python test_dedup.py
|
||||
Creates synthetic images in a temp dir + a temp DB, asserts behaviour, cleans up.
|
||||
"""
|
||||
import io
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
import photo_analyzer as pa
|
||||
|
||||
|
||||
def _make_photo(path: Path, seed: int, size=(800, 600)):
|
||||
"""Deterministic, structured (not pure-noise) image so phash is stable."""
|
||||
rng = np.random.default_rng(seed)
|
||||
# low-frequency structure: a few coloured blocks + gradient — survives resize
|
||||
base = np.zeros((size[1], size[0], 3), dtype=np.uint8)
|
||||
for _ in range(6):
|
||||
x0, y0 = rng.integers(0, size[0] - 100), rng.integers(0, size[1] - 100)
|
||||
col = rng.integers(0, 256, 3)
|
||||
base[y0:y0 + 150, x0:x0 + 150] = col
|
||||
grad = np.linspace(0, 120, size[0], dtype=np.uint8)
|
||||
base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255)
|
||||
Image.fromarray(base).save(path, quality=95)
|
||||
|
||||
|
||||
def main():
|
||||
tmp = Path(tempfile.mkdtemp(prefix="dedup_test_"))
|
||||
db_path = tmp / "test.db"
|
||||
try:
|
||||
# ── synthetic library ────────────────────────────────────────────────
|
||||
orig = tmp / "A.jpg"
|
||||
_make_photo(orig, seed=1)
|
||||
|
||||
# exact byte copy (backup-of-backup)
|
||||
exact = tmp / "A_backup.jpg"
|
||||
exact.write_bytes(orig.read_bytes())
|
||||
|
||||
# near-dupe: same picture, resized + recompressed (different phone)
|
||||
near = tmp / "A_phone.jpg"
|
||||
with Image.open(orig) as im:
|
||||
im.resize((400, 300)).save(near, quality=60)
|
||||
|
||||
# a genuinely different photo
|
||||
other = tmp / "B.jpg"
|
||||
_make_photo(other, seed=999)
|
||||
|
||||
# ── phash: dupes close, different far ────────────────────────────────
|
||||
def dist(p1, p2):
|
||||
h1, h2 = int(pa._phash_image(p1), 16), int(pa._phash_image(p2), 16)
|
||||
return bin(h1 ^ h2).count("1")
|
||||
|
||||
d_exact = dist(orig, exact)
|
||||
d_near = dist(orig, near)
|
||||
d_other = dist(orig, other)
|
||||
print(f"phash Hamming: exact={d_exact} near={d_near} other={d_other}")
|
||||
assert d_exact == 0, f"exact copy should have phash distance 0, got {d_exact}"
|
||||
assert d_near <= pa.PHASH_THRESHOLD, f"near-dupe {d_near} > threshold {pa.PHASH_THRESHOLD}"
|
||||
assert d_other > pa.PHASH_THRESHOLD, f"different photo {d_other} <= threshold — false match"
|
||||
|
||||
# ── sha1: identical bytes match, different bytes don't ───────────────
|
||||
assert pa._sha1_file(orig) == pa._sha1_file(exact), "byte-identical files must share sha1"
|
||||
assert pa._sha1_file(orig) != pa._sha1_file(near), "recompressed file must differ in sha1"
|
||||
|
||||
# ── DB: register, hash, cluster, mark ────────────────────────────────
|
||||
conn = pa.get_db(str(db_path))
|
||||
photos = [orig, exact, near, other]
|
||||
for p in photos:
|
||||
pa.upsert_pending(conn, str(p))
|
||||
n = pa.ensure_hashes(conn, photos)
|
||||
assert n == 4, f"expected 4 hashed, got {n}"
|
||||
|
||||
# all four should carry phash + sha1 now
|
||||
rows = conn.execute("SELECT path, phash, file_sha1 FROM photos").fetchall()
|
||||
assert all(r["phash"] and r["file_sha1"] for r in rows), "some hashes missing"
|
||||
|
||||
clusters = pa.cluster_duplicates(conn, pa.PHASH_THRESHOLD)
|
||||
# exactly one cluster: {orig, exact, near}; 'other' stands alone
|
||||
assert len(clusters) == 1, f"expected 1 cluster, got {len(clusters)}"
|
||||
cluster_paths = {it["path"] for it in clusters[0]}
|
||||
assert cluster_paths == {str(orig), str(exact), str(near)}, cluster_paths
|
||||
# canonical = largest file (orig, saved at q95 full-res)
|
||||
assert clusters[0][0]["path"] == str(orig), "canonical should be the largest file"
|
||||
|
||||
marked, ncl = pa.mark_duplicates(conn, pa.PHASH_THRESHOLD)
|
||||
assert (marked, ncl) == (2, 1), f"expected (2 marked, 1 cluster), got ({marked}, {ncl})"
|
||||
|
||||
dupes = conn.execute(
|
||||
"SELECT path, dup_of FROM photos WHERE status='duplicate'"
|
||||
).fetchall()
|
||||
assert {r["path"] for r in dupes} == {str(exact), str(near)}
|
||||
assert all(r["dup_of"] == str(orig) for r in dupes), "dup_of must point at canonical"
|
||||
|
||||
# get_pending must exclude duplicates
|
||||
pending = pa.get_pending(conn, reanalyze=True)
|
||||
assert str(exact) not in pending and str(near) not in pending, "dupes leaked into pending"
|
||||
assert str(orig) in pending and str(other) in pending
|
||||
|
||||
# ── reconcile_moved: rename a file, path follows by sha1 ─────────────
|
||||
moved_dst = tmp / "B_renamed.jpg"
|
||||
os.rename(other, moved_dst) # 'other' now missing at old path
|
||||
n_moved = pa.reconcile_moved(conn, tmp)
|
||||
assert n_moved == 1, f"expected 1 reconciled, got {n_moved}"
|
||||
row = conn.execute("SELECT path FROM photos WHERE path=?", (str(moved_dst),)).fetchone()
|
||||
assert row is not None, "moved file's row was not repathed"
|
||||
gone = conn.execute("SELECT 1 FROM photos WHERE path=?", (str(other),)).fetchone()
|
||||
assert gone is None, "old path should no longer exist in DB"
|
||||
|
||||
conn.close()
|
||||
print("ALL DEDUP/RECONCILE CHECKS PASSED")
|
||||
finally:
|
||||
import shutil
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
15
legacy_cli_archive/src/test_nsfw_skip.py
Normal file
15
legacy_cli_archive/src/test_nsfw_skip.py
Normal file
@@ -0,0 +1,15 @@
|
||||
"""Self-check for photo_analyzer's nsfw-skip parsing. Run: python test_nsfw_skip.py"""
|
||||
from photo_analyzer import _rec_has_nsfw
|
||||
|
||||
def test():
|
||||
assert _rec_has_nsfw({"Keywords": ["beach", "nsfw"]}) # list, present
|
||||
assert _rec_has_nsfw({"Subject": "nsfw"}) # scalar in Subject
|
||||
assert _rec_has_nsfw({"Keywords": "NSFW"}) # case-insensitive
|
||||
assert _rec_has_nsfw({"Keywords": [" nsfw "]}) # whitespace
|
||||
assert not _rec_has_nsfw({"Keywords": ["sfw", "beach"]}) # sfw only
|
||||
assert not _rec_has_nsfw({"Keywords": ["nsfw-ish"]}) # substring != tag
|
||||
assert not _rec_has_nsfw({}) # no fields
|
||||
print("ok")
|
||||
|
||||
if __name__ == "__main__":
|
||||
test()
|
||||
62
legacy_cli_archive/src/webapp/README.md
Normal file
62
legacy_cli_archive/src/webapp/README.md
Normal file
@@ -0,0 +1,62 @@
|
||||
# webapp — Photo Analyzer web UI
|
||||
|
||||
A local, browser-based UI for the `photo_analyzer.py` pipeline. Browse and
|
||||
full-text-search the analyzed library, watch analysis runs live, and start/stop
|
||||
runs — reusing the `nsfwtag` review-app design (dark OLED, stdlib HTTP server, one
|
||||
token-injected HTML page). Nothing leaves the machine.
|
||||
|
||||
```bash
|
||||
python -m webapp # DB + library default from photo_analyzer.env
|
||||
python -m webapp --db photo_analysis.db --library pictures/
|
||||
python -m webapp --no-open # don't auto-open a browser
|
||||
```
|
||||
|
||||
See `../WEBAPP_CONCEPT.md` for the full UX concept and rationale.
|
||||
|
||||
## Views
|
||||
|
||||
- **Library** — FTS5 search over description/tags/mood/location + facet filters
|
||||
(setting, time of day, season, people, year range, has-place, status), a folder
|
||||
sidebar (album = leaf folder), an infinite-scroll grid, and a lightbox with an
|
||||
**Analysis** panel (all DB fields) and a raw **EXIF** panel (incl. Google Maps
|
||||
link for geotagged photos).
|
||||
- **Analyze** — run controls mapping every CLI flag (dry-run / reanalyze / no-exif /
|
||||
exif-only / group-variants), a live overall progress bar, per-album progress, and a
|
||||
recent feed. Start/Stop drive `photo_analyzer.py` as a subprocess; Stop is the
|
||||
analyzer's own SIGINT drain. Balance / quota-check buttons. A **Duplicates &
|
||||
maintenance** row runs the content-hash actions (backfill hashes / find duplicates /
|
||||
mark duplicates) — no API calls; output streams to the Activity log and marked
|
||||
duplicates appear under the `duplicate` status filter. (Move/rename reconciliation
|
||||
and perceptual hashing also run automatically at the start of every normal run.)
|
||||
- **Stats** — KPI cards + CSS breakdown bars (setting/time/season/people, by-year
|
||||
histogram, top tags) + an error list. No chart library.
|
||||
|
||||
## Architecture
|
||||
|
||||
- `server.py` — stdlib `ThreadingHTTPServer` on `127.0.0.1:<free-port>`; routes below.
|
||||
- `query.py` — read-only DB access (FTS5 search, facets, stats, one-photo).
|
||||
- `runner.py` — drives `photo_analyzer.py` as a subprocess; DB-derived progress.
|
||||
- `page.py` / `analyzer.html` — the single frontend page (inherits nsfwtag's tokens).
|
||||
- SQLite stays the source of truth; the server only reads it (+ a `busy_timeout` so a
|
||||
concurrent analysis write never errors a read). `/img` and `/exif` paths are
|
||||
validated against the DB — the server can't be pointed at arbitrary files.
|
||||
|
||||
| Route | Purpose |
|
||||
|-------|---------|
|
||||
| GET `/` | the page |
|
||||
| GET `/search?q=&setting=&tod=&season=&people=&year_min=&year_max=&has_location=&status=&album=&sort=&offset=` | paged results (FTS5 when `q`) |
|
||||
| GET `/facets` · `/stats` | filter options · aggregates |
|
||||
| GET `/photo?path=` · `/exif?path=` · `/img?path=` | one record · raw EXIF · thumbnail |
|
||||
| GET `/progress` · `/log` | live run state · activity log |
|
||||
| POST `/run` · `/stop` | start / stop a run — body `{library, ...flags}` or `{library, action}` where action ∈ backfill-phash \| list-dupes \| dedupe |
|
||||
| GET `/balance` · `/quota-check` | provider balance · one-shot quota probe |
|
||||
|
||||
## Known ceilings (ponytail)
|
||||
|
||||
- `/img` serves **full-resolution originals**, not resized thumbnails — a 120-card
|
||||
page can move a lot of bytes. Fine locally; add a Pillow thumbnail+cache endpoint if
|
||||
the grid feels slow. (Same limitation as nsfwtag.)
|
||||
- HEIC files won't render in the browser (broken thumb); metadata/search still work.
|
||||
- The Analyze **tokens/ETA** readout is omitted — those live only in the terminal
|
||||
dashboard's memory; the subprocess model surfaces DB-derived progress instead. Parse
|
||||
run stdout for token counts if you want them back.
|
||||
11
legacy_cli_archive/src/webapp/__init__.py
Normal file
11
legacy_cli_archive/src/webapp/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""webapp — a local web UI for photo_analyzer.
|
||||
|
||||
Browse and full-text-search the analyzed photo library, watch analysis runs live,
|
||||
and start/stop runs — all in the browser, reusing the nsfwtag review-app design.
|
||||
Nothing leaves the machine (stdlib HTTP server bound to 127.0.0.1).
|
||||
|
||||
Entry point: python -m webapp [--db photo_analysis.db] [--library pictures/]
|
||||
"""
|
||||
__version__ = "0.1.0"
|
||||
|
||||
PAGE_SIZE = 120 # results per /search page (infinite scroll appends)
|
||||
42
legacy_cli_archive/src/webapp/__main__.py
Normal file
42
legacy_cli_archive/src/webapp/__main__.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""python -m webapp — launch the Photo Analyzer web UI.
|
||||
|
||||
Defaults come from photo_analyzer's env (photo_analyzer.env: DB, LIBRARY), so in
|
||||
the common case you can just run `python -m webapp`.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .server import serve
|
||||
|
||||
DEFAULT_DB = "photo_analysis.db"
|
||||
|
||||
|
||||
def _env(name, fallback=None):
|
||||
v = os.environ.get(name)
|
||||
return v if v not in (None, "") else fallback
|
||||
|
||||
|
||||
def main():
|
||||
# Reuse photo_analyzer's .env loader so DB/LIBRARY defaults match the CLI.
|
||||
try:
|
||||
import photo_analyzer as pa
|
||||
pa.load_env_file()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ap = argparse.ArgumentParser(description="Web UI for the photo_analyzer pipeline.")
|
||||
ap.add_argument("--db", default=_env("DB", DEFAULT_DB), help=f"SQLite DB (default {DEFAULT_DB})")
|
||||
ap.add_argument("--library", default=_env("LIBRARY"), help="Photo library root (for album grouping + run default)")
|
||||
ap.add_argument("--no-open", action="store_true", help="Don't open a browser")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not Path(args.db).exists():
|
||||
sys.exit(f"DB not found: {args.db} — run photo_analyzer first, or pass --db.")
|
||||
|
||||
serve(args.db, args.library, open_browser=not args.no_open)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
490
legacy_cli_archive/src/webapp/analyzer.html
Normal file
490
legacy_cli_archive/src/webapp/analyzer.html
Normal file
@@ -0,0 +1,490 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Photo Analyzer · %%TOTAL%% photos</title>
|
||||
<style>
|
||||
*{box-sizing:border-box}
|
||||
:root{
|
||||
--bg:#0b0d10;--surface:#14171c;--surface-2:#1b1f26;--elev:#20252e;
|
||||
--border:#262b34;--border-2:#323947;--text:#e8eaee;--muted:#9aa1ae;--faint:#6b727f;
|
||||
--primary:#10b981;--primary-2:#0ea372;--primary-ink:#04130c;--danger:#ef4444;
|
||||
--warn:#f59e0b;--ring:#3b82f6;--pink:#e05fa8;
|
||||
--r:12px;--r-sm:8px}
|
||||
@media (prefers-reduced-motion:reduce){*{transition:none!important;animation:none!important}}
|
||||
html,body{margin:0}
|
||||
body{background:var(--bg);color:var(--text);font-size:14px;line-height:1.5;-webkit-font-smoothing:antialiased;
|
||||
font-family:'Inter',-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Helvetica,Arial,sans-serif}
|
||||
svg{width:1em;height:1em;display:block}
|
||||
button{font:inherit;color:inherit;cursor:pointer}
|
||||
a{color:var(--ring)}
|
||||
:focus-visible{outline:2px solid var(--ring);outline-offset:2px}
|
||||
.tabnum{font-variant-numeric:tabular-nums}
|
||||
/* ── header ── */
|
||||
.app{position:sticky;top:0;z-index:30;display:flex;align-items:center;gap:14px;min-height:58px;
|
||||
padding:11px 18px;background:rgba(11,13,16,.86);backdrop-filter:blur(12px);border-bottom:1px solid var(--border)}
|
||||
.brand{display:flex;align-items:center;gap:11px}
|
||||
.logo{display:grid;place-items:center;width:34px;height:34px;border-radius:10px;font-size:18px;
|
||||
background:linear-gradient(135deg,#1e3a5f,#10b981);color:#fff}
|
||||
.titles h1{margin:0;font-size:15px;font-weight:600;letter-spacing:-.01em}
|
||||
.titles p{margin:0;font-size:11px;color:var(--faint)}
|
||||
.grow{flex:1}
|
||||
.metrics{display:flex;gap:8px}
|
||||
.metric{display:flex;flex-direction:column;align-items:center;min-width:70px;padding:5px 12px;
|
||||
background:var(--surface);border:1px solid var(--border);border-radius:var(--r-sm)}
|
||||
.metric .num{font-size:16px;font-weight:600;font-variant-numeric:tabular-nums}
|
||||
.metric .lbl{font-size:10px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em}
|
||||
.metric.err .num{color:var(--danger)}
|
||||
/* ── segmented view switcher ── */
|
||||
.seg{display:inline-flex;padding:3px;background:var(--surface);border:1px solid var(--border);border-radius:var(--r-sm)}
|
||||
.seg button{border:0;background:none;color:var(--muted);padding:6px 14px;border-radius:6px;font-size:13px;font-weight:500}
|
||||
.seg button.on{background:var(--elev);color:var(--text)}
|
||||
.seg button:hover:not(.on){color:var(--text)}
|
||||
.navtoggle{display:grid;place-items:center;width:36px;height:36px;font-size:18px;border-radius:var(--r-sm);
|
||||
background:var(--surface);border:1px solid var(--border);color:var(--muted)}
|
||||
.navtoggle:hover{background:var(--elev);color:var(--text)}
|
||||
.navtoggle.on{color:var(--primary);border-color:var(--border-2)}
|
||||
/* ── toolbar ── */
|
||||
.toolbar{position:sticky;top:58px;z-index:20;display:flex;flex-wrap:wrap;align-items:center;gap:10px;
|
||||
padding:10px 18px;background:rgba(16,19,24,.92);backdrop-filter:blur(8px);border-bottom:1px solid var(--border)}
|
||||
.field{display:flex;align-items:center;gap:8px;height:36px;padding:0 10px;background:var(--surface);
|
||||
border:1px solid var(--border);border-radius:var(--r-sm)}
|
||||
.field svg{color:var(--faint);font-size:15px}
|
||||
.field input{background:none;border:0;outline:none;color:var(--text);height:100%;width:230px}
|
||||
.field input::placeholder{color:var(--faint)}
|
||||
.field:focus-within{border-color:var(--border-2);box-shadow:0 0 0 3px rgba(59,130,246,.22)}
|
||||
.ctl{display:flex;align-items:center;gap:6px;color:var(--muted);font-size:12.5px;white-space:nowrap}
|
||||
select{height:36px;padding:0 8px;background:var(--surface);color:var(--text);border:1px solid var(--border);border-radius:var(--r-sm);outline:none}
|
||||
select:focus{border-color:var(--border-2);box-shadow:0 0 0 3px rgba(59,130,246,.22)}
|
||||
input[type=number]{width:64px;height:36px;padding:0 8px;background:var(--surface);color:var(--text);
|
||||
border:1px solid var(--border);border-radius:var(--r-sm);outline:none;font-variant-numeric:tabular-nums}
|
||||
input[type=checkbox]{accent-color:var(--primary);width:16px;height:16px}
|
||||
.btn{display:inline-flex;align-items:center;gap:7px;height:36px;padding:0 14px;border-radius:var(--r-sm);
|
||||
border:1px solid var(--border);background:var(--surface);color:var(--text);font-weight:500;
|
||||
transition:background .15s,border-color .15s,transform .05s}
|
||||
.btn svg{font-size:15px}
|
||||
.btn.ghost:hover{background:var(--elev);border-color:var(--border-2)}
|
||||
.btn:active{transform:translateY(1px)}
|
||||
.btn.primary{background:var(--primary);border-color:transparent;color:var(--primary-ink);font-weight:600}
|
||||
.btn.primary:hover{background:var(--primary-2)}
|
||||
.btn.danger{border-color:var(--danger);color:#fca5a5}
|
||||
.btn.danger:hover{background:rgba(239,68,68,.14)}
|
||||
.btn:disabled{opacity:.45;cursor:not-allowed}
|
||||
.count{color:var(--faint);font-size:12.5px;font-variant-numeric:tabular-nums;white-space:nowrap}
|
||||
/* ── layout shell ── */
|
||||
.view{display:none}
|
||||
.view.on{display:block}
|
||||
.shell{display:flex;align-items:flex-start}
|
||||
.sidebar{position:sticky;top:104px;align-self:flex-start;flex:0 0 250px;width:250px;
|
||||
height:calc(100dvh - 104px);overflow:auto;border-right:1px solid var(--border);padding:12px 8px}
|
||||
.shell.collapsed .sidebar{display:none}
|
||||
.fitem{display:flex;justify-content:space-between;gap:8px;width:100%;text-align:left;border:0;background:none;
|
||||
color:var(--muted);padding:7px 10px;border-radius:var(--r-sm);font-size:12.5px}
|
||||
.fitem:hover{background:var(--surface);color:var(--text)}
|
||||
.fitem.on{background:var(--elev);color:var(--text)}
|
||||
.fitem .cn{color:var(--faint);font-variant-numeric:tabular-nums}
|
||||
.main{flex:1;min-width:0}
|
||||
/* ── grid + cards ── */
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(190px,1fr));gap:12px;padding:16px 18px 80px}
|
||||
.card{position:relative;overflow:hidden;background:var(--surface);border:2px solid var(--border);border-radius:var(--r);
|
||||
cursor:pointer;transition:border-color .15s,transform .12s,box-shadow .15s}
|
||||
.card:hover{transform:translateY(-2px);box-shadow:0 8px 24px rgba(0,0,0,.35)}
|
||||
.card.err{border-color:var(--danger)}
|
||||
.thumb{position:relative;aspect-ratio:1/1;background:#000}
|
||||
.thumb img{width:100%;height:100%;object-fit:cover;display:block}
|
||||
.badges{position:absolute;left:8px;top:8px;display:flex;gap:6px}
|
||||
.badge{padding:2px 7px;border-radius:6px;background:rgba(6,8,12,.72);color:#fff;font-size:11.5px;
|
||||
font-weight:700;font-variant-numeric:tabular-nums;backdrop-filter:blur(3px)}
|
||||
.badge.warnb{background:var(--warn);color:#241a02}
|
||||
.dot{position:absolute;right:8px;top:8px;width:10px;height:10px;border-radius:50%;box-shadow:0 0 0 2px rgba(0,0,0,.4)}
|
||||
.dot.err{background:var(--danger)}.dot.pending{background:var(--faint)}
|
||||
.info{padding:8px 10px}
|
||||
.desc{font-size:12px;color:var(--text);display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}
|
||||
.pathline{font-size:11px;color:var(--faint);margin-top:3px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.empty{padding:80px 20px;text-align:center;color:var(--muted)}
|
||||
.empty h3{color:var(--text);margin:0 0 6px}
|
||||
/* ── progress bar ── */
|
||||
.bar{position:relative;height:8px;border-radius:5px;background:var(--border-2);overflow:hidden}
|
||||
.bar>span{position:absolute;inset:0 auto 0 0;background:var(--primary);border-radius:5px;transition:width .3s}
|
||||
.bar.run>span{background:var(--warn)}
|
||||
/* ── analyze view ── */
|
||||
.panel{margin:14px 18px;background:var(--surface);border:1px solid var(--border);border-radius:var(--r);padding:16px}
|
||||
.panel h2{margin:0 0 12px;font-size:13px;font-weight:600;color:var(--muted);text-transform:uppercase;letter-spacing:.05em}
|
||||
.runbar{display:flex;flex-wrap:wrap;align-items:center;gap:14px}
|
||||
.runbar label{display:flex;align-items:center;gap:6px;color:var(--muted);font-size:12.5px}
|
||||
.overall .big{display:flex;align-items:baseline;gap:12px;margin-bottom:8px}
|
||||
.overall .pct{font-size:26px;font-weight:700;font-variant-numeric:tabular-nums}
|
||||
.overall .sub{color:var(--muted);font-size:12.5px;font-variant-numeric:tabular-nums}
|
||||
.cols{display:grid;grid-template-columns:1fr 1fr;gap:14px}
|
||||
@media(max-width:820px){.cols{grid-template-columns:1fr}}
|
||||
.albrow{display:grid;grid-template-columns:1fr auto 70px;align-items:center;gap:10px;padding:5px 0;font-size:12.5px}
|
||||
.albrow .nm{white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.albrow .ct{color:var(--faint);font-variant-numeric:tabular-nums}
|
||||
.feed{max-height:420px;overflow:auto}
|
||||
.feedline{display:flex;gap:9px;padding:7px 0;border-bottom:1px solid var(--border);font-size:12.5px}
|
||||
.feedline .ic{flex:0 0 auto}.feedline .ok{color:var(--primary)}.feedline .bad{color:var(--danger)}
|
||||
.feedline .fn{font-weight:600}.feedline .dz{color:var(--muted)}
|
||||
/* ── stats view ── */
|
||||
.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;padding:16px 18px 4px}
|
||||
.kpi{background:var(--surface);border:1px solid var(--border);border-radius:var(--r);padding:14px 16px}
|
||||
.kpi .num{font-size:24px;font-weight:700;font-variant-numeric:tabular-nums}
|
||||
.kpi .lbl{color:var(--muted);font-size:12px;margin-top:2px}
|
||||
.statbar{display:grid;grid-template-columns:120px 1fr 64px;align-items:center;gap:10px;padding:4px 0;font-size:12.5px}
|
||||
.statbar .lab{color:var(--muted);white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.statbar .val{text-align:right;color:var(--faint);font-variant-numeric:tabular-nums}
|
||||
.statbar .track{height:14px;border-radius:4px;background:var(--surface-2);overflow:hidden}
|
||||
.statbar .track>span{display:block;height:100%;background:var(--primary);opacity:.85}
|
||||
.years{display:flex;align-items:flex-end;gap:3px;height:90px;padding-top:8px}
|
||||
.years .yb{flex:1;min-width:4px;background:var(--primary);opacity:.8;border-radius:2px 2px 0 0}
|
||||
.years .yb:hover{opacity:1}
|
||||
.tags{display:flex;flex-wrap:wrap;gap:7px}
|
||||
.tag{padding:4px 10px;border-radius:16px;border:1px solid var(--border);background:var(--surface);
|
||||
color:var(--muted);font-size:12px;cursor:pointer}
|
||||
.tag:hover{color:var(--text);border-color:var(--border-2)}
|
||||
.tag .cn{color:var(--faint)}
|
||||
.errline{display:flex;justify-content:space-between;gap:10px;padding:7px 0;border-bottom:1px solid var(--border);font-size:12.5px}
|
||||
.errline .msg{color:#fca5a5}
|
||||
/* ── lightbox ── */
|
||||
.lb{position:fixed;inset:0;z-index:50;display:none;background:rgba(6,8,11,.9);backdrop-filter:blur(4px)}
|
||||
.lb.on{display:flex}
|
||||
.lb .stage{flex:1;display:flex;align-items:center;justify-content:center;padding:24px;min-width:0}
|
||||
.lb img{max-width:100%;max-height:100%;object-fit:contain;border-radius:var(--r)}
|
||||
.lb .side{flex:0 0 340px;max-width:44vw;background:var(--surface);border-left:1px solid var(--border);
|
||||
padding:18px;overflow:auto}
|
||||
.lb .side h3{margin:0 0 4px;font-size:14px}
|
||||
.lb .side .fn2{color:var(--faint);font-size:12px;margin-bottom:14px;word-break:break-all}
|
||||
.kv{margin:0 0 14px}
|
||||
.kv dt{color:var(--faint);font-size:10.5px;text-transform:uppercase;letter-spacing:.05em;margin-top:8px}
|
||||
.kv dd{margin:1px 0 0;font-size:13px}
|
||||
.lbbtns{position:absolute;top:16px;right:16px;display:flex;gap:8px}
|
||||
.icon{display:grid;place-items:center;width:36px;height:36px;border:0;border-radius:8px;color:#fff;
|
||||
background:rgba(6,8,12,.62);backdrop-filter:blur(3px)}
|
||||
.icon:hover{background:rgba(6,8,12,.9)}
|
||||
.lbnav{position:absolute;top:50%;transform:translateY(-50%)}
|
||||
.lbnav.prev{left:16px}.lbnav.next{right:calc(340px + 16px)}
|
||||
@media(max-width:700px){.lb .side{display:none}.lbnav.next{right:16px}}
|
||||
/* ── log panel + toast ── */
|
||||
.logpanel{position:fixed;top:0;right:0;bottom:0;width:380px;max-width:92vw;z-index:60;background:var(--surface);
|
||||
border-left:1px solid var(--border);display:flex;flex-direction:column;transform:translateX(100%);transition:transform .2s}
|
||||
.logpanel:not([hidden]){transform:none}
|
||||
.loghead{display:flex;align-items:center;justify-content:space-between;padding:14px 16px;border-bottom:1px solid var(--border)}
|
||||
.logbody{flex:1;overflow:auto;padding:8px 16px;font-size:12.5px}
|
||||
.logline{display:flex;gap:10px;padding:5px 0;border-bottom:1px solid var(--border)}
|
||||
.logline .lt{color:var(--faint);font-variant-numeric:tabular-nums;flex:0 0 auto}
|
||||
.toast{position:fixed;left:50%;bottom:24px;transform:translateX(-50%) translateY(20px);z-index:70;opacity:0;
|
||||
padding:10px 16px;background:var(--elev);border:1px solid var(--border-2);border-radius:var(--r-sm);
|
||||
pointer-events:none;transition:opacity .2s,transform .2s}
|
||||
.toast.show{opacity:1;transform:translateX(-50%)}
|
||||
.toast.err{border-color:var(--danger)}.toast.ok{border-color:var(--primary)}
|
||||
</style></head>
|
||||
<body>
|
||||
|
||||
<header class="app">
|
||||
<div class="brand"><div class="logo">◧</div>
|
||||
<div class="titles"><h1>Photo Analyzer</h1><p>%%LIBRARY%%</p></div></div>
|
||||
<div class="seg" id="viewseg" role="tablist">
|
||||
<button data-view="library" class="on">Library</button>
|
||||
<button data-view="analyze">Analyze</button>
|
||||
<button data-view="stats">Stats</button></div>
|
||||
<div class="grow"></div>
|
||||
<div class="metrics">
|
||||
<div class="metric"><span class="num tabnum" id="mTotal">%%TOTAL%%</span><span class="lbl">Total</span></div>
|
||||
<div class="metric"><span class="num tabnum" id="mDone">–</span><span class="lbl">Analyzed</span></div>
|
||||
<div class="metric err"><span class="num tabnum" id="mErr">–</span><span class="lbl">Errors</span></div>
|
||||
</div>
|
||||
<button class="navtoggle" id="logbtn" aria-label="Activity log" onclick="toggleLog()">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M8 6h11M8 12h11M8 18h11"/><path d="M4 6h.01M4 12h.01M4 18h.01"/></svg></button>
|
||||
</header>
|
||||
|
||||
<!-- ══ LIBRARY ══ -->
|
||||
<section id="view-library" class="view on">
|
||||
<div class="toolbar">
|
||||
<button class="navtoggle on" id="sidetoggle" aria-label="Toggle folders" onclick="toggleSidebar()">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><rect x="3" y="3" width="18" height="18" rx="2"/><path d="M9 3v18"/></svg></button>
|
||||
<label class="field"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><circle cx="11" cy="11" r="7"/><path d="m21 21-4.3-4.3"/></svg>
|
||||
<input id="q" type="search" placeholder="Search description, tags, mood, place…" autocomplete="off"></label>
|
||||
<label class="ctl">Setting <select id="fSetting"><option value="">any</option></select></label>
|
||||
<label class="ctl">Time <select id="fTod"><option value="">any</option></select></label>
|
||||
<label class="ctl">Season <select id="fSeason"><option value="">any</option></select></label>
|
||||
<label class="ctl">People <select id="fPeople"><option value="">any</option><option value="0">0</option><option value="1">1</option><option value="2">2</option><option value="3+">3+</option></select></label>
|
||||
<label class="ctl">Year <input id="fYmin" type="number" placeholder="min"> – <input id="fYmax" type="number" placeholder="max"></label>
|
||||
<label class="ctl"><input id="fLoc" type="checkbox"> has place</label>
|
||||
<label class="ctl">Status <select id="fStatus"><option value="">any</option></select></label>
|
||||
<div class="grow"></div>
|
||||
<label class="ctl">Sort <select id="sort">
|
||||
<option value="relevance">Relevance</option><option value="year">Year ↓</option>
|
||||
<option value="people">People ↓</option><option value="recent">Recent</option><option value="path">Folder</option></select></label>
|
||||
<button class="btn ghost" onclick="resetFilters()">Clear</button>
|
||||
<span class="count" id="resCount"></span>
|
||||
</div>
|
||||
<div class="shell" id="shell">
|
||||
<aside class="sidebar" id="folders"></aside>
|
||||
<div class="main">
|
||||
<div class="grid" id="grid"></div>
|
||||
<div id="sentinel" style="height:1px"></div>
|
||||
<div class="empty" id="emptyLib" hidden><h3>No matches</h3><p>Try a broader term or clear the filters.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ══ ANALYZE ══ -->
|
||||
<section id="view-analyze" class="view">
|
||||
<div class="panel"><h2>Run</h2>
|
||||
<div class="runbar">
|
||||
<label class="ctl">Library <input id="runLib" type="text" value="%%LIBRARY%%" style="width:260px"></label>
|
||||
<label><input type="checkbox" id="opt-dry-run"> Dry-run</label>
|
||||
<label><input type="checkbox" id="opt-reanalyze"> Re-analyze</label>
|
||||
<label><input type="checkbox" id="opt-no-exif"> No-EXIF</label>
|
||||
<label><input type="checkbox" id="opt-exif-only"> EXIF-only</label>
|
||||
<label><input type="checkbox" id="opt-group-variants"> Variants</label>
|
||||
<div class="grow"></div>
|
||||
<button class="btn ghost" onclick="apiBalance()">Balance</button>
|
||||
<button class="btn ghost" onclick="apiQuota()">Quota check</button>
|
||||
<button class="btn primary" id="startBtn" onclick="startRun()">▶ Start</button>
|
||||
<button class="btn danger" id="stopBtn" onclick="stopRun()" hidden>■ Stop</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel"><h2>Duplicates & maintenance</h2>
|
||||
<div class="runbar">
|
||||
<span class="ctl" style="max-width:560px;white-space:normal">Content-hash tools — no API calls.
|
||||
Output appears in the Activity log; photos marked as duplicates show up under the
|
||||
<b>duplicate</b> status filter in Library.</span>
|
||||
<div class="grow"></div>
|
||||
<button class="btn ghost" id="act-backfill-phash" onclick="runAction('backfill-phash')">⧉ Backfill hashes</button>
|
||||
<button class="btn ghost" id="act-list-dupes" onclick="runAction('list-dupes')">🔍 Find duplicates</button>
|
||||
<button class="btn ghost" id="act-dedupe" onclick="runAction('dedupe')">✓ Mark duplicates</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel overall"><h2>Overall</h2>
|
||||
<div class="big"><span class="pct" id="ovPct">Idle</span>
|
||||
<span class="sub" id="ovCount"></span></div>
|
||||
<div class="bar" id="ovBar"><span style="width:0"></span></div>
|
||||
<div class="sub" id="ovTokens" style="margin-top:8px"></div>
|
||||
</div>
|
||||
<div class="cols">
|
||||
<div class="panel"><h2>Albums</h2><div id="albums"><p class="count">No run in progress.</p></div></div>
|
||||
<div class="panel"><h2>Recent</h2><div class="feed" id="feed"><p class="count">Analyzed photos will appear here.</p></div></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ══ STATS ══ -->
|
||||
<section id="view-stats" class="view">
|
||||
<div class="kpis" id="kpis"></div>
|
||||
<div class="cols">
|
||||
<div class="panel"><h2>Setting</h2><div id="stSetting"></div></div>
|
||||
<div class="panel"><h2>Time of day</h2><div id="stTod"></div></div>
|
||||
<div class="panel"><h2>Season</h2><div id="stSeason"></div></div>
|
||||
<div class="panel"><h2>People</h2><div id="stPeople"></div></div>
|
||||
</div>
|
||||
<div class="panel"><h2>By year</h2><div class="years" id="stYears"></div></div>
|
||||
<div class="panel"><h2>Top tags</h2><div class="tags" id="stTags"></div></div>
|
||||
<div class="panel"><h2>Errors</h2><div id="stErrors"></div></div>
|
||||
</section>
|
||||
|
||||
<!-- lightbox -->
|
||||
<div class="lb" id="lb">
|
||||
<div class="lbbtns"><button class="icon" onclick="closeLb()" aria-label="Close">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M18 6 6 18M6 6l12 12"/></svg></button></div>
|
||||
<button class="icon lbnav prev" onclick="stepLb(-1)" aria-label="Previous">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="m15 18-6-6 6-6"/></svg></button>
|
||||
<button class="icon lbnav next" onclick="stepLb(1)" aria-label="Next">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="m9 18 6-6-6-6"/></svg></button>
|
||||
<div class="stage"><img id="lbImg" alt=""></div>
|
||||
<div class="side"><h3 id="lbName"></h3><div class="fn2" id="lbPath"></div>
|
||||
<dl class="kv" id="lbAnalysis"></dl><dl class="kv" id="lbExif"></dl></div>
|
||||
</div>
|
||||
|
||||
<!-- activity log -->
|
||||
<aside class="logpanel" id="logpanel" hidden>
|
||||
<div class="loghead"><b>Activity</b><button class="btn ghost" onclick="toggleLog()">Close</button></div>
|
||||
<div class="logbody" id="logbody"></div>
|
||||
</aside>
|
||||
|
||||
<div class="toast" id="toast" hidden></div>
|
||||
|
||||
<script>
|
||||
const $=s=>document.querySelector(s), $$=s=>[...document.querySelectorAll(s)];
|
||||
const esc=s=>String(s??'').replace(/[&<>"]/g,c=>({'&':'&','<':'<','>':'>','"':'"'}[c]));
|
||||
const enc=encodeURIComponent;
|
||||
function toast(m,c){const t=$('#toast');t.textContent=m;t.hidden=false;t.className='toast show '+(c||'');
|
||||
clearTimeout(t._t);t._t=setTimeout(()=>t.classList.remove('show'),3200);}
|
||||
|
||||
/* ── view switching (deep-linked via hash) ── */
|
||||
function setView(v){
|
||||
if(!['library','analyze','stats'].includes(v))v='library';
|
||||
$$('.view').forEach(s=>s.classList.toggle('on',s.id==='view-'+v));
|
||||
$$('#viewseg button').forEach(b=>b.classList.toggle('on',b.dataset.view===v));
|
||||
if(location.hash.slice(1).split('?')[0]!==v)history.replaceState(null,'','#'+v);
|
||||
if(v==='stats')loadStats();
|
||||
if(v==='analyze')pollProgress();
|
||||
}
|
||||
$$('#viewseg button').forEach(b=>b.onclick=()=>setView(b.dataset.view));
|
||||
addEventListener('hashchange',()=>setView(location.hash.slice(1).split('?')[0]));
|
||||
|
||||
/* ── LIBRARY ── */
|
||||
let offset=0,total=0,loading=false,done=true,curAlbum='';
|
||||
const filt=()=>({q:$('#q').value.trim(),setting:$('#fSetting').value,tod:$('#fTod').value,
|
||||
season:$('#fSeason').value,people:$('#fPeople').value,year_min:$('#fYmin').value,
|
||||
year_max:$('#fYmax').value,has_location:$('#fLoc').checked?'1':'',status:$('#fStatus').value,
|
||||
album:curAlbum,sort:$('#sort').value});
|
||||
function qs(o){return Object.entries(o).filter(([,v])=>v!==''&&v!=null).map(([k,v])=>k+'='+enc(v)).join('&');}
|
||||
|
||||
async function runSearch(reset=true){
|
||||
if(loading)return; loading=true;
|
||||
if(reset){offset=0;$('#grid').innerHTML='';done=false;}
|
||||
const r=await fetch('/search?'+qs({...filt(),offset})).then(r=>r.json());
|
||||
total=r.total; $('#resCount').textContent=total.toLocaleString()+' photo'+(total===1?'':'s');
|
||||
$('#emptyLib').hidden=total>0;
|
||||
$('#grid').insertAdjacentHTML('beforeend',r.rows.map(card).join(''));
|
||||
offset+=r.rows.length; done=offset>=total || r.rows.length===0;
|
||||
loading=false;
|
||||
}
|
||||
function card(p){
|
||||
const yr=p.approx_year?`<span class="badge">${p.approx_year}</span>`:'';
|
||||
const pe=(p.people_count>0)?`<span class="badge">👤 ${p.people_count}</span>`:'';
|
||||
const du=p.status==='duplicate'?`<span class="badge warnb" title="perceptual duplicate">dup</span>`:'';
|
||||
const dot=p.status==='error'?'<span class="dot err" title="error"></span>'
|
||||
:(p.status==='pending'?'<span class="dot pending" title="not analyzed"></span>':'');
|
||||
return `<div class="card ${p.status==='error'?'err':''}" data-path="${esc(p.path)}" onclick="openLb(this)">
|
||||
<div class="thumb"><img loading="lazy" src="/img?path=${enc(p.path)}" alt="">
|
||||
<div class="badges">${yr}${pe}${du}</div>${dot}</div>
|
||||
<div class="info"><div class="desc">${esc(p.description||(p.status==='error'?'—':'(not analyzed)'))}</div>
|
||||
<div class="pathline">${esc(p.path.split('/').slice(-2).join('/'))}</div></div></div>`;
|
||||
}
|
||||
let searchT;
|
||||
function debouncedSearch(){clearTimeout(searchT);searchT=setTimeout(()=>runSearch(true),250);}
|
||||
['#q','#fYmin','#fYmax'].forEach(s=>$(s).addEventListener('input',debouncedSearch));
|
||||
['#fSetting','#fTod','#fSeason','#fPeople','#fStatus','#sort'].forEach(s=>$(s).addEventListener('change',()=>runSearch(true)));
|
||||
$('#fLoc').addEventListener('change',()=>runSearch(true));
|
||||
function resetFilters(){['#q','#fYmin','#fYmax'].forEach(s=>$(s).value='');
|
||||
['#fSetting','#fTod','#fSeason','#fPeople','#fStatus'].forEach(s=>$(s).value='');
|
||||
$('#fLoc').checked=false;$('#sort').value='relevance';curAlbum='';
|
||||
$$('.fitem').forEach(x=>x.classList.remove('on'));runSearch(true);}
|
||||
new IntersectionObserver(es=>{if(es[0].isIntersecting&&!done&&!loading)runSearch(false);})
|
||||
.observe($('#sentinel'));
|
||||
function toggleSidebar(){const c=$('#shell').classList.toggle('collapsed');
|
||||
$('#sidetoggle').classList.toggle('on',!c);}
|
||||
|
||||
function opt(sel,items,none='any'){
|
||||
$(sel).innerHTML=`<option value="">${none}</option>`+items.map(i=>
|
||||
`<option value="${esc(i.value)}">${esc(i.value)} (${i.count.toLocaleString()})</option>`).join('');}
|
||||
async function loadFacets(){
|
||||
const f=await fetch('/facets').then(r=>r.json());
|
||||
opt('#fSetting',f.setting);opt('#fTod',f.tod);opt('#fSeason',f.season);opt('#fStatus',f.status);
|
||||
if(f.year_min){$('#fYmin').placeholder=f.year_min;$('#fYmax').placeholder=f.year_max;}
|
||||
$('#folders').innerHTML=`<button class="fitem on" onclick="pickAlbum(this,'')"><span>All folders</span></button>`+
|
||||
f.albums.map(a=>`<button class="fitem" onclick="pickAlbum(this,'${esc(a.album)}')">
|
||||
<span>${esc(a.album)}</span><span class="cn">${a.count.toLocaleString()}</span></button>`).join('');
|
||||
// album buttons carry the folder name; map to absolute path server-side via album= param
|
||||
}
|
||||
function pickAlbum(btn,album){curAlbum=album?(LIBRARY.replace(/\/$/,'')+'/'+album):'';
|
||||
$$('.fitem').forEach(x=>x.classList.toggle('on',x===btn));runSearch(true);}
|
||||
const LIBRARY='%%LIBRARY%%';
|
||||
|
||||
/* ── LIGHTBOX ── */
|
||||
let lbCards=[],lbIdx=0;
|
||||
function openLb(el){lbCards=$$('#grid .card');lbIdx=lbCards.indexOf(el);showLb();$('#lb').classList.add('on');}
|
||||
function closeLb(){$('#lb').classList.remove('on');}
|
||||
function stepLb(d){lbIdx=(lbIdx+d+lbCards.length)%lbCards.length;showLb();}
|
||||
async function showLb(){
|
||||
const el=lbCards[lbIdx],path=el.dataset.path;
|
||||
$('#lbImg').src='/img?path='+enc(path);
|
||||
$('#lbName').textContent=path.split('/').pop();$('#lbPath').textContent=path;
|
||||
$('#lbAnalysis').innerHTML='<dt>Analysis</dt><dd class="count">loading…</dd>';
|
||||
$('#lbExif').innerHTML='';
|
||||
const p=await fetch('/photo?path='+enc(path)).then(r=>r.json()).catch(()=>null);
|
||||
$('#lbAnalysis').innerHTML=p?analysisDl(p):'<dt>Analysis</dt><dd>—</dd>';
|
||||
const ex=await fetch('/exif?path='+enc(path)).then(r=>r.json()).catch(()=>({}));
|
||||
$('#lbExif').innerHTML='<dt style="margin-top:16px">EXIF</dt>'+Object.entries(ex).map(([k,v])=>
|
||||
k==='Map'?`<dd><a href="${esc(v)}" target="_blank" rel="noopener">Open in Google Maps ↗</a></dd>`
|
||||
:`<dt>${esc(k)}</dt><dd>${esc(v)}</dd>`).join('');
|
||||
}
|
||||
function row(k,v){return v==null||v===''?'':`<dt>${k}</dt><dd>${esc(v)}</dd>`;}
|
||||
function analysisDl(p){
|
||||
return row('Description',p.description)+
|
||||
(p.tags&&p.tags.length?`<dt>Tags</dt><dd>${p.tags.map(esc).join(', ')}</dd>`:'')+
|
||||
row('Mood',p.mood)+row('Setting',p.setting)+row('Time of day',p.time_of_day)+
|
||||
row('Season',p.season)+row('People',p.people_count)+row('Approx. year',p.approx_year)+
|
||||
row('Location',p.location_hint)+row('Status',p.status)+row('Duplicate of',p.dup_of)+row('Error',p.error_message);
|
||||
}
|
||||
addEventListener('keydown',e=>{if(!$('#lb').classList.contains('on'))return;
|
||||
if(e.key==='Escape')closeLb();else if(e.key==='ArrowLeft')stepLb(-1);else if(e.key==='ArrowRight')stepLb(1);});
|
||||
$('#lb').addEventListener('click',e=>{if(e.target.id==='lb')closeLb();});
|
||||
|
||||
/* ── STATS ── */
|
||||
function bars(el,items,max){const m=max||Math.max(1,...items.map(i=>i.count));
|
||||
$(el).innerHTML=items.map(i=>`<div class="statbar"><span class="lab">${esc(i.value)}</span>
|
||||
<span class="track"><span style="width:${i.count/m*100}%"></span></span>
|
||||
<span class="val">${i.count.toLocaleString()}</span></div>`).join('')||'<p class="count">no data</p>';}
|
||||
async function loadStats(){
|
||||
const s=await fetch('/stats').then(r=>r.json());
|
||||
const st=s.status||{};
|
||||
const done=(st.analyzed||0)+(st.exif_written||0);
|
||||
$('#mDone').textContent=done.toLocaleString();$('#mErr').textContent=(st.error||0).toLocaleString();
|
||||
const kpi=[['Total',s.total],['Analyzed',done],['EXIF written',st.exif_written||0],
|
||||
['Duplicates',st.duplicate||0],['Errors',st.error||0],['Pending',st.pending||0]];
|
||||
$('#kpis').innerHTML=kpi.map(([l,n])=>`<div class="kpi"><div class="num tabnum">${(n||0).toLocaleString()}</div><div class="lbl">${l}</div></div>`).join('');
|
||||
bars('#stSetting',s.setting);bars('#stTod',s.time_of_day);bars('#stSeason',s.season);bars('#stPeople',s.people);
|
||||
const ym=Math.max(1,...s.years.map(y=>y.count));
|
||||
$('#stYears').innerHTML=s.years.map(y=>`<div class="yb" style="height:${y.count/ym*100}%" title="${y.value}: ${y.count}"></div>`).join('')||'<p class="count">no dated photos</p>';
|
||||
$('#stTags').innerHTML=s.top_tags.map(t=>`<button class="tag" onclick="searchTag('${esc(t.value)}')">${esc(t.value)} <span class="cn">${t.count}</span></button>`).join('');
|
||||
$('#stErrors').innerHTML=s.errors.length?s.errors.map(e=>`<div class="errline"><span>${esc(e.path.split('/').pop())}</span><span class="msg">${esc(e.error||'')}</span></div>`).join(''):'<p class="count">No errors 🎉</p>';
|
||||
}
|
||||
function searchTag(t){$('#q').value=t;setView('library');runSearch(true);}
|
||||
|
||||
/* ── ANALYZE (progress polling; run controls wired to server) ── */
|
||||
let progT=null;
|
||||
async function pollProgress(){
|
||||
clearTimeout(progT);
|
||||
let s;try{s=await fetch('/progress').then(r=>r.json());}catch(e){progT=setTimeout(pollProgress,2000);return;}
|
||||
renderProgress(s);
|
||||
if($('#view-analyze').classList.contains('on'))progT=setTimeout(pollProgress,s.running?1500:4000);
|
||||
}
|
||||
function renderProgress(s){
|
||||
$('#startBtn').hidden=!!s.running;$('#stopBtn').hidden=!s.running;
|
||||
['act-backfill-phash','act-list-dupes','act-dedupe'].forEach(id=>{const b=$('#'+id);if(b)b.disabled=!!s.running;});
|
||||
if(!s.running&&!s.totals){$('#ovPct').textContent='Idle';$('#ovCount').textContent='';$('#ovBar').firstElementChild.style.width='0';return;}
|
||||
const t=s.totals||{done:0,total:0,ok:0,err:0};
|
||||
const pct=t.total?Math.round(t.done/t.total*100):0;
|
||||
$('#ovPct').textContent=pct+'%';
|
||||
$('#ovCount').textContent=`${(t.done||0).toLocaleString()} / ${(t.total||0).toLocaleString()} · ok ${t.ok||0} · err ${t.err||0}`+(t.dup?` · dup ${t.dup.toLocaleString()}`:'');
|
||||
const bar=$('#ovBar');bar.classList.toggle('run',!!s.running);bar.firstElementChild.style.width=pct+'%';
|
||||
if(s.tokens!=null)$('#ovTokens').textContent=`tokens ${(s.tokens||0).toLocaleString()}`+(s.eta?` · ETA ${s.eta}`:'');
|
||||
$('#albums').innerHTML=(s.albums||[]).map(a=>{const p=a.total?Math.round(a.done/a.total*100):0;
|
||||
return `<div class="albrow"><span class="nm">${esc(a.album)}</span>
|
||||
<span class="bar" style="width:120px"><span style="width:${p}%"></span></span>
|
||||
<span class="ct">${a.done===a.total?'✓':p+'%'}</span></div>`;}).join('')||'<p class="count">No run in progress.</p>';
|
||||
$('#feed').innerHTML=(s.feed||[]).map(f=>`<div class="feedline"><span class="ic ${f.ok?'ok':'bad'}">${f.ok?'✓':'✗'}</span>
|
||||
<span><span class="fn">${esc(f.name)}</span><div class="dz">${esc(f.msg||'')}</div></span></div>`).join('')||'<p class="count">Analyzed photos will appear here.</p>';
|
||||
}
|
||||
async function startRun(){
|
||||
const opts={};['dry-run','reanalyze','no-exif','exif-only','group-variants'].forEach(k=>opts[k]=$('#opt-'+k).checked);
|
||||
opts.library=$('#runLib').value;
|
||||
const r=await fetch('/run',{method:'POST',body:JSON.stringify(opts)}).then(r=>r.json()).catch(e=>({error:''+e}));
|
||||
if(r.error)toast(r.error,'err');else{toast('Run started','ok');pollProgress();}
|
||||
}
|
||||
async function stopRun(){await fetch('/stop',{method:'POST'});toast('Stopping…');pollProgress();}
|
||||
const ACTION_LABELS={'backfill-phash':'Backfilling hashes','list-dupes':'Finding duplicates','dedupe':'Marking duplicates'};
|
||||
function ensureLogOpen(){if($('#logpanel').hidden)toggleLog();}
|
||||
async function runAction(action){
|
||||
const r=await fetch('/run',{method:'POST',body:JSON.stringify({action,library:$('#runLib').value})})
|
||||
.then(r=>r.json()).catch(e=>({error:''+e}));
|
||||
if(r.error){toast(r.error,'err');return;}
|
||||
toast(ACTION_LABELS[action]+'…','ok');ensureLogOpen();pollProgress(); // output streams to Activity log
|
||||
}
|
||||
async function apiBalance(){const r=await fetch('/balance').then(r=>r.json()).catch(()=>({}));
|
||||
toast(r.text||'Balance unavailable',r.text?'ok':'err');}
|
||||
async function apiQuota(){toast('Checking quota…');const r=await fetch('/quota-check').then(r=>r.json()).catch(()=>({ok:false}));
|
||||
toast(r.ok?'Quota available ✓':'Quota check failed',r.ok?'ok':'err');}
|
||||
|
||||
/* ── activity log ── */
|
||||
let logT=null;
|
||||
function toggleLog(){const p=$('#logpanel'),open=p.hidden;p.hidden=!open;$('#logbtn').classList.toggle('on',open);
|
||||
if(open){refreshLog();logT=setInterval(refreshLog,2500);}else clearInterval(logT);}
|
||||
async function refreshLog(){const es=await fetch('/log').then(r=>r.json()).catch(()=>[]);
|
||||
$('#logbody').innerHTML=es.length?es.slice().reverse().map(e=>`<div class="logline"><span class="lt">${esc(e.t)}</span><span>${esc(e.m)}</span></div>`).join(''):'<p class="count">No activity yet.</p>';}
|
||||
|
||||
/* ── boot ── */
|
||||
loadFacets().then(()=>runSearch(true));
|
||||
loadStats();
|
||||
setView(location.hash.slice(1).split('?')[0]||'library');
|
||||
</script>
|
||||
</body></html>
|
||||
12
legacy_cli_archive/src/webapp/page.py
Normal file
12
legacy_cli_archive/src/webapp/page.py
Normal file
@@ -0,0 +1,12 @@
|
||||
"""Render analyzer.html with %%TOKENS%% filled in. The page is otherwise static
|
||||
and pulls its data from the JSON endpoints, so this only injects boot values."""
|
||||
from pathlib import Path
|
||||
|
||||
_HTML = (Path(__file__).with_name("analyzer.html")).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def render_page(total: int, library: str, db: str) -> str:
|
||||
return (_HTML
|
||||
.replace("%%TOTAL%%", f"{total:,}")
|
||||
.replace("%%LIBRARY%%", library)
|
||||
.replace("%%DB%%", db))
|
||||
214
legacy_cli_archive/src/webapp/query.py
Normal file
214
legacy_cli_archive/src/webapp/query.py
Normal file
@@ -0,0 +1,214 @@
|
||||
"""Read-only DB access for the web UI: FTS5 search, facets, stats, one-photo.
|
||||
|
||||
The analysis pipeline (photo_analyzer.py) owns writes; this module only reads.
|
||||
Album = leaf folder relative to the library (same rule as photo_analyzer.album_label).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from . import PAGE_SIZE
|
||||
|
||||
# Columns sent to the browser for a card / lightbox. raw_response is intentionally
|
||||
# omitted from lists (big); the lightbox fetches it via /photo if ever needed.
|
||||
CARD_COLS = ("path", "status", "description", "tags", "people_count", "setting",
|
||||
"time_of_day", "season", "mood", "location_hint", "approx_year")
|
||||
|
||||
DONE = ("analyzed", "exif_written")
|
||||
|
||||
|
||||
def connect(db_path: str) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(db_path, check_same_thread=False) # server is multi-threaded
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def album_of(path: str, library: Path | None) -> str:
|
||||
"""Leaf folder relative to the library — mirrors photo_analyzer.album_label."""
|
||||
parent = Path(path).parent
|
||||
if library:
|
||||
try:
|
||||
rel = parent.relative_to(library)
|
||||
return "(root)" if str(rel) == "." else str(rel)
|
||||
except ValueError:
|
||||
pass
|
||||
return parent.name or str(parent)
|
||||
|
||||
|
||||
def _fts_query(q: str) -> str | None:
|
||||
"""Turn free text into a safe FTS5 MATCH string. Each word becomes a prefix
|
||||
term (AND-ed), so 'beach sun' matches 'beachy sunset'. Returns None if empty
|
||||
(→ caller browses instead of matching), so raw punctuation can't crash MATCH."""
|
||||
tokens = re.findall(r"\w+", q, re.UNICODE)
|
||||
return " ".join(f'"{t}"*' for t in tokens) if tokens else None
|
||||
|
||||
|
||||
def _where(filters: dict) -> tuple[list[str], list]:
|
||||
"""Build a WHERE clause fragment list + params from the facet filters."""
|
||||
clauses, params = [], []
|
||||
if v := filters.get("setting"):
|
||||
clauses.append("p.setting = ?"); params.append(v)
|
||||
if v := filters.get("tod"):
|
||||
clauses.append("p.time_of_day = ?"); params.append(v)
|
||||
if v := filters.get("season"):
|
||||
clauses.append("p.season = ?"); params.append(v)
|
||||
if v := filters.get("status"):
|
||||
clauses.append("p.status = ?"); params.append(v)
|
||||
people = filters.get("people")
|
||||
if people == "3+":
|
||||
clauses.append("p.people_count >= 3")
|
||||
elif people in ("0", "1", "2"):
|
||||
clauses.append("p.people_count = ?"); params.append(int(people))
|
||||
if filters.get("year_min"):
|
||||
clauses.append("p.approx_year >= ?"); params.append(int(filters["year_min"]))
|
||||
if filters.get("year_max"):
|
||||
clauses.append("p.approx_year <= ?"); params.append(int(filters["year_max"]))
|
||||
if filters.get("has_location"):
|
||||
clauses.append("p.location_hint IS NOT NULL AND TRIM(p.location_hint) != '' "
|
||||
"AND LOWER(p.location_hint) != 'null'")
|
||||
if album := filters.get("album"):
|
||||
# album is an absolute leaf-folder path; match it and its subfolders
|
||||
clauses.append("(p.path LIKE ? OR p.path LIKE ?)")
|
||||
params += [f"{album}/%", f"{album}%/%"]
|
||||
return clauses, params
|
||||
|
||||
|
||||
_SORTS = {
|
||||
"year": "p.approx_year DESC NULLS LAST, p.path",
|
||||
"people": "p.people_count DESC NULLS LAST, p.path",
|
||||
"recent": "p.analyzed_at DESC NULLS LAST, p.path",
|
||||
"path": "p.path",
|
||||
}
|
||||
|
||||
|
||||
def _row_to_card(r: sqlite3.Row) -> dict:
|
||||
d = {k: r[k] for k in CARD_COLS}
|
||||
try:
|
||||
d["tags"] = json.loads(r["tags"]) if r["tags"] else []
|
||||
except (ValueError, TypeError):
|
||||
d["tags"] = []
|
||||
return d
|
||||
|
||||
|
||||
def search(conn, q="", filters=None, sort="relevance", offset=0, limit=PAGE_SIZE):
|
||||
"""Paged result set. FTS5 when `q` is present, plain filter/browse otherwise.
|
||||
Returns {rows, total, offset, limit}."""
|
||||
filters = filters or {}
|
||||
clauses, params = _where(filters)
|
||||
match = _fts_query(q) if q else None
|
||||
|
||||
if match:
|
||||
base = ("FROM photos p JOIN photos_fts f ON p.id = f.rowid "
|
||||
"WHERE photos_fts MATCH ?")
|
||||
params = [match] + params
|
||||
order = "f.rank" if sort in ("relevance", "") else _SORTS.get(sort, "f.rank")
|
||||
else:
|
||||
base = "FROM photos p WHERE 1=1"
|
||||
order = _SORTS.get(sort if sort != "relevance" else "path", "p.path")
|
||||
if clauses:
|
||||
base += " AND " + " AND ".join(clauses)
|
||||
|
||||
total = conn.execute(f"SELECT COUNT(*) {base}", params).fetchone()[0]
|
||||
rows = conn.execute(
|
||||
f"SELECT p.* {base} ORDER BY {order} LIMIT ? OFFSET ?",
|
||||
params + [limit, offset],
|
||||
).fetchall()
|
||||
return {"rows": [_row_to_card(r) for r in rows], "total": total,
|
||||
"offset": offset, "limit": limit}
|
||||
|
||||
|
||||
def photo(conn, path: str) -> dict | None:
|
||||
r = conn.execute("SELECT * FROM photos WHERE path = ?", (path,)).fetchone()
|
||||
if not r:
|
||||
return None
|
||||
d = {k: r[k] for k in r.keys() if k != "raw_response"}
|
||||
try:
|
||||
d["tags"] = json.loads(r["tags"]) if r["tags"] else []
|
||||
except (ValueError, TypeError):
|
||||
d["tags"] = []
|
||||
return d
|
||||
|
||||
|
||||
def _col_counts(conn, col: str) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
f"SELECT {col} AS v, COUNT(*) AS n FROM photos "
|
||||
f"WHERE {col} IS NOT NULL AND TRIM({col}) != '' GROUP BY {col} ORDER BY n DESC"
|
||||
).fetchall()
|
||||
return [{"value": r["v"], "count": r["n"]} for r in rows]
|
||||
|
||||
|
||||
def facets(conn, library: Path | None) -> dict:
|
||||
"""Distinct filter values + counts, plus the album tree, for the toolbar."""
|
||||
year = conn.execute(
|
||||
"SELECT MIN(approx_year), MAX(approx_year) FROM photos WHERE approx_year IS NOT NULL"
|
||||
).fetchone()
|
||||
albums: dict[str, int] = {}
|
||||
for (p,) in conn.execute("SELECT path FROM photos"):
|
||||
a = album_of(p, library)
|
||||
albums[a] = albums.get(a, 0) + 1
|
||||
return {
|
||||
"setting": _col_counts(conn, "setting"),
|
||||
"tod": _col_counts(conn, "time_of_day"),
|
||||
"season": _col_counts(conn, "season"),
|
||||
"status": _col_counts(conn, "status"),
|
||||
"year_min": year[0], "year_max": year[1],
|
||||
"albums": [{"album": a, "count": n} for a, n in sorted(albums.items())],
|
||||
}
|
||||
|
||||
|
||||
def _people_buckets(conn) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
"SELECT CASE WHEN people_count >= 3 THEN '3+' ELSE CAST(people_count AS TEXT) END AS b, "
|
||||
"COUNT(*) AS n FROM photos WHERE people_count IS NOT NULL GROUP BY b"
|
||||
).fetchall()
|
||||
return [{"value": r["b"], "count": r["n"]} for r in rows]
|
||||
|
||||
|
||||
def _top_tags(conn, limit=40) -> list[dict]:
|
||||
counts: dict[str, int] = {}
|
||||
for (t,) in conn.execute("SELECT tags FROM photos WHERE tags IS NOT NULL"):
|
||||
try:
|
||||
for tag in json.loads(t):
|
||||
tag = str(tag).strip()
|
||||
if tag:
|
||||
counts[tag] = counts.get(tag, 0) + 1
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
top = sorted(counts.items(), key=lambda kv: kv[1], reverse=True)[:limit]
|
||||
return [{"value": t, "count": n} for t, n in top]
|
||||
|
||||
|
||||
def stats(conn, library: Path | None) -> dict:
|
||||
status = {r["status"]: r["n"] for r in conn.execute(
|
||||
"SELECT status, COUNT(*) AS n FROM photos GROUP BY status")}
|
||||
total = sum(status.values())
|
||||
years = [{"value": r["y"], "count": r["n"]} for r in conn.execute(
|
||||
"SELECT approx_year AS y, COUNT(*) AS n FROM photos "
|
||||
"WHERE approx_year IS NOT NULL GROUP BY approx_year ORDER BY approx_year")]
|
||||
albums: dict[str, dict] = {}
|
||||
for r in conn.execute("SELECT path, status FROM photos"):
|
||||
a = album_of(r["path"], library)
|
||||
d = albums.setdefault(a, {"album": a, "done": 0, "total": 0})
|
||||
d["total"] += 1
|
||||
if r["status"] in DONE:
|
||||
d["done"] += 1
|
||||
errors = [{"path": r["path"], "error": r["error_message"]} for r in conn.execute(
|
||||
"SELECT path, error_message FROM photos WHERE status = 'error' ORDER BY path")]
|
||||
return {
|
||||
"total": total, "status": status,
|
||||
"setting": _col_counts(conn, "setting"),
|
||||
"time_of_day": _col_counts(conn, "time_of_day"),
|
||||
"season": _col_counts(conn, "season"),
|
||||
"people": _people_buckets(conn),
|
||||
"years": years,
|
||||
"top_tags": _top_tags(conn),
|
||||
"albums": sorted(albums.values(), key=lambda d: d["album"]),
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def all_paths(conn) -> set:
|
||||
"""Every registered path — the allow-set for /img and /exif (path validation)."""
|
||||
return {r[0] for r in conn.execute("SELECT path FROM photos")}
|
||||
110
legacy_cli_archive/src/webapp/runner.py
Normal file
110
legacy_cli_archive/src/webapp/runner.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Drive photo_analyzer.py as a subprocess and expose live run-state to the web UI.
|
||||
|
||||
Why a subprocess and not an in-process thread: the pipeline already is a complete,
|
||||
resume-safe CLI that owns SQLite as the source of truth. Running it as `python
|
||||
photo_analyzer.py …` keeps the two fully decoupled — the web server only *reads* the
|
||||
same DB for progress, Stop is the analyzer's own tested SIGINT drain, and a web crash
|
||||
can't corrupt a run (or vice-versa). Progress is derived from the DB, so it survives
|
||||
a browser reload and matches `--stats` exactly.
|
||||
"""
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from . import query
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent # dir holding photo_analyzer.py
|
||||
_FLAGS = ("dry-run", "reanalyze", "no-exif", "exif-only", "group-variants")
|
||||
# Exit-after-action modes: each runs one maintenance task and quits (no analysis).
|
||||
# Passed alone — the boolean _FLAGS are ignored when an action is chosen.
|
||||
_ACTIONS = ("backfill-phash", "list-dupes", "dedupe")
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(self, db_path: str, log):
|
||||
self.db = db_path
|
||||
self.log = log
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def running(self) -> bool:
|
||||
return self.proc is not None and self.proc.poll() is None
|
||||
|
||||
def start(self, opts: dict) -> dict:
|
||||
with self.lock:
|
||||
if self.running():
|
||||
return {"error": "A run is already in progress."}
|
||||
lib = (opts.get("library") or "").strip()
|
||||
if not lib:
|
||||
return {"error": "Library path is required."}
|
||||
argv = [sys.executable, str(REPO / "photo_analyzer.py"),
|
||||
"--library", lib, "--db", self.db]
|
||||
action = opts.get("action")
|
||||
if action in _ACTIONS:
|
||||
argv.append(f"--{action}") # maintenance action, runs alone
|
||||
else:
|
||||
argv += [f"--{f}" for f in _FLAGS if opts.get(f)]
|
||||
try:
|
||||
self.proc = subprocess.Popen(
|
||||
argv, cwd=str(REPO), stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, text=True, bufsize=1)
|
||||
except Exception as e:
|
||||
return {"error": f"Could not start: {e}"}
|
||||
threading.Thread(target=self._reader, args=(self.proc,), daemon=True).start()
|
||||
self.log("run started: " + " ".join(argv[2:]))
|
||||
return {"ok": True}
|
||||
|
||||
def _reader(self, proc):
|
||||
# Surface the analyzer's own log lines in the browser's Activity panel.
|
||||
for line in proc.stdout:
|
||||
line = line.rstrip()
|
||||
if line:
|
||||
self.log(line)
|
||||
self.log(f"run finished (exit {proc.poll()})")
|
||||
|
||||
def stop(self) -> dict:
|
||||
with self.lock:
|
||||
if self.running():
|
||||
self.proc.send_signal(signal.SIGINT) # reuse the tested drain path
|
||||
self.log("stop requested")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def progress(conn, library: Path | None, running: bool) -> dict:
|
||||
"""DB-derived run state for /progress. Cheap full scan (~25k rows, a few ms);
|
||||
ponytail: if the library grows past ~200k, cache album counts between polls."""
|
||||
st = {r["status"]: r["n"] for r in conn.execute(
|
||||
"SELECT status, COUNT(*) AS n FROM photos GROUP BY status")}
|
||||
done = st.get("analyzed", 0) + st.get("exif_written", 0)
|
||||
dup = st.get("duplicate", 0)
|
||||
# Duplicates are deliberately skipped, not pending work — exclude them from the
|
||||
# denominator so the bar reflects real progress rather than dragging to <100%.
|
||||
work_total = sum(st.values()) - dup
|
||||
|
||||
albums: dict[str, dict] = {}
|
||||
for r in conn.execute("SELECT path, status FROM photos"):
|
||||
if r["status"] == "duplicate":
|
||||
continue # not work; keep albums reaching 100%
|
||||
a = query.album_of(r["path"], library)
|
||||
d = albums.setdefault(a, {"album": a, "done": 0, "total": 0})
|
||||
d["total"] += 1
|
||||
if r["status"] in query.DONE:
|
||||
d["done"] += 1
|
||||
# incomplete albums first — that's what a running job is working on
|
||||
alist = sorted(albums.values(), key=lambda d: (d["done"] >= d["total"], d["album"]))
|
||||
|
||||
feed = []
|
||||
for r in conn.execute(
|
||||
"SELECT path, status, description, error_message FROM photos "
|
||||
"WHERE analyzed_at IS NOT NULL ORDER BY analyzed_at DESC LIMIT 14"
|
||||
):
|
||||
ok = r["status"] != "error"
|
||||
feed.append({"ok": ok, "name": Path(r["path"]).name,
|
||||
"msg": (r["description"] if ok else r["error_message"]) or ""})
|
||||
|
||||
return {"running": running,
|
||||
"totals": {"done": done, "total": work_total, "ok": done,
|
||||
"err": st.get("error", 0), "dup": dup},
|
||||
"albums": alist[:60], "feed": feed, "tokens": None}
|
||||
162
legacy_cli_archive/src/webapp/server.py
Normal file
162
legacy_cli_archive/src/webapp/server.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""Local, stdlib-only HTTP server for the Photo Analyzer web UI.
|
||||
|
||||
Serves one page + JSON endpoints, all bound to 127.0.0.1. Reads the analysis DB
|
||||
(via query.py), serves thumbnails/EXIF for the lightbox (reusing nsfwtag's readers),
|
||||
and drives analysis runs (runner.py). Path args on /img and /exif are validated
|
||||
against the DB, so the server can't be pointed at arbitrary files.
|
||||
"""
|
||||
import http.server
|
||||
import json
|
||||
import mimetypes
|
||||
import sys
|
||||
import time
|
||||
import webbrowser
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from . import PAGE_SIZE, page, query
|
||||
from .runner import Runner, progress as run_progress
|
||||
|
||||
try:
|
||||
from nsfwtag.exif import read_exif # reuse the lightbox EXIF reader
|
||||
except Exception: # nsfwtag not importable → degrade
|
||||
def read_exif(_p): return {}
|
||||
|
||||
|
||||
def serve(db_path: str, library: str | None, host: str = "127.0.0.1", open_browser=True):
|
||||
conn = query.connect(db_path)
|
||||
conn.execute("PRAGMA busy_timeout=3000") # tolerate the analyzer writing concurrently
|
||||
lib = Path(library).expanduser().resolve() if library else None
|
||||
|
||||
log_entries = deque(maxlen=2000)
|
||||
|
||||
def log(m):
|
||||
log_entries.append({"t": time.strftime("%H:%M:%S"), "m": m})
|
||||
print(m, file=sys.stderr)
|
||||
|
||||
runner = Runner(db_path, log)
|
||||
|
||||
def allowed(fp: str) -> bool:
|
||||
return conn.execute("SELECT 1 FROM photos WHERE path=? LIMIT 1", (fp,)).fetchone() is not None
|
||||
|
||||
def filters_from(qd: dict) -> dict:
|
||||
g = lambda k: qd.get(k, [""])[0]
|
||||
return {k: g(k) for k in ("q", "setting", "tod", "season", "people",
|
||||
"year_min", "year_max", "has_location", "status",
|
||||
"album", "sort")}
|
||||
|
||||
total0 = conn.execute("SELECT COUNT(*) FROM photos").fetchone()[0]
|
||||
page_bytes = page.render_page(total0, str(lib) if lib else (library or ""), db_path).encode("utf-8")
|
||||
log(f"web ui ready — {total0:,} photos indexed")
|
||||
|
||||
class H(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
def _send(self, code, ctype, body):
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _json(self, obj, code=200):
|
||||
self._send(code, "application/json", json.dumps(obj).encode("utf-8"))
|
||||
|
||||
def do_GET(self):
|
||||
u = urlparse(self.path)
|
||||
qd = parse_qs(u.query)
|
||||
p = u.path
|
||||
if p == "/":
|
||||
return self._send(200, "text/html; charset=utf-8", page_bytes)
|
||||
if p == "/img":
|
||||
fp = unquote(qd.get("path", [""])[0])
|
||||
if allowed(fp) and Path(fp).exists():
|
||||
ctype = mimetypes.guess_type(fp)[0] or "application/octet-stream"
|
||||
return self._send(200, ctype, Path(fp).read_bytes())
|
||||
return self._send(404, "text/plain", b"no image")
|
||||
if p == "/exif":
|
||||
fp = unquote(qd.get("path", [""])[0])
|
||||
if allowed(fp) and Path(fp).exists():
|
||||
return self._json(read_exif(fp))
|
||||
return self._json({})
|
||||
if p == "/photo":
|
||||
fp = unquote(qd.get("path", [""])[0])
|
||||
return self._json(query.photo(conn, fp) or {})
|
||||
if p == "/search":
|
||||
f = filters_from(qd)
|
||||
try:
|
||||
offset = int(qd.get("offset", ["0"])[0])
|
||||
except ValueError:
|
||||
offset = 0
|
||||
return self._json(query.search(
|
||||
conn, q=f.pop("q"), filters=f, sort=f.get("sort") or "relevance",
|
||||
offset=offset, limit=PAGE_SIZE))
|
||||
if p == "/facets":
|
||||
return self._json(query.facets(conn, lib))
|
||||
if p == "/stats":
|
||||
return self._json(query.stats(conn, lib))
|
||||
if p == "/progress":
|
||||
return self._json(run_progress(conn, lib, runner.running()))
|
||||
if p == "/log":
|
||||
return self._json(list(log_entries))
|
||||
if p == "/balance":
|
||||
return self._json(_balance())
|
||||
if p == "/quota-check":
|
||||
return self._json({"ok": _quota()})
|
||||
return self._send(404, "text/plain", b"not found")
|
||||
|
||||
def do_POST(self):
|
||||
u = urlparse(self.path)
|
||||
n = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(n) if n else b"{}"
|
||||
if u.path == "/run":
|
||||
try:
|
||||
opts = json.loads(body or b"{}")
|
||||
except ValueError:
|
||||
return self._json({"error": "bad request"}, 400)
|
||||
return self._json(runner.start(opts))
|
||||
if u.path == "/stop":
|
||||
return self._json(runner.stop())
|
||||
return self._send(404, "text/plain", b"not found")
|
||||
|
||||
def _balance():
|
||||
try:
|
||||
import photo_analyzer as pa
|
||||
pa.load_env_file()
|
||||
import os
|
||||
key = os.environ.get("LLM_API_KEY") or os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
|
||||
d = pa.fetch_balance(key)
|
||||
if not d:
|
||||
return {"text": ""}
|
||||
return {"text": f"available {d.get('available_balance')} · "
|
||||
f"cash {d.get('cash_balance')} · voucher {d.get('voucher_balance')}"}
|
||||
except Exception as e:
|
||||
log(f"balance error: {e}")
|
||||
return {"text": ""}
|
||||
|
||||
def _quota():
|
||||
try:
|
||||
import os
|
||||
import photo_analyzer as pa
|
||||
pa.load_env_file()
|
||||
from openai import OpenAI
|
||||
key = os.environ.get("LLM_API_KEY") or os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
|
||||
client = OpenAI(api_key=key or "x", base_url=pa.LLM_BASE_URL)
|
||||
return pa.check_quota(client)
|
||||
except Exception as e:
|
||||
log(f"quota error: {e}")
|
||||
return False
|
||||
|
||||
srv = http.server.ThreadingHTTPServer((host, 0), H)
|
||||
url = f"http://{host}:{srv.server_address[1]}/"
|
||||
print(f"\nPhoto Analyzer web UI: {url}\n Ctrl-C here to stop the server.\n", file=sys.stderr)
|
||||
if open_browser:
|
||||
webbrowser.open(url)
|
||||
try:
|
||||
srv.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nstopped.", file=sys.stderr)
|
||||
finally:
|
||||
srv.server_close()
|
||||
Reference in New Issue
Block a user