US07-03: Harden Media and Metadata Edge Cases (#85)
This commit was merged in pull request #85.
This commit is contained in:
@@ -41,6 +41,7 @@ import photo_pipeline.jobs.domain_handlers # noqa: F401
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.logging import configure_logging
|
||||
from photo_pipeline.services.thumbnails import ThumbnailService
|
||||
from photo_pipeline.services.upload_batches import UploadBatchService
|
||||
|
||||
FRONTEND_DIR = Path(__file__).resolve().parents[2] / "frontend"
|
||||
@@ -96,6 +97,9 @@ def create_app(config: Config | None = None) -> FastAPI:
|
||||
# An upload whose process died left no outcome behind; resolve it now so the
|
||||
# uploader lane is free and the uncertain batch is visible (US05-02).
|
||||
UploadBatchService(app.state.session_factory, config=config).recover()
|
||||
# A render killed mid-write leaves its temporary beside the cache entry;
|
||||
# remove those recognized leftovers, and only those (US07-03).
|
||||
ThumbnailService(app.state.session_factory, config).cleanup_temp_files()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
|
||||
119
photo_pipeline/imaging.py
Normal file
119
photo_pipeline/imaging.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""Bounded, defensive image decoding — one door for every pixel this app reads.
|
||||
|
||||
A photo library contains files nobody planned for: truncated downloads, zero-byte
|
||||
placeholders, a PNG whose header claims 200000×200000, a TIFF with a broken ICC
|
||||
profile, an extension that lies about its content. None of them may take down a
|
||||
request or a worker, and none may decode more pixels than the caller allowed
|
||||
(concept §17: decoded pixels, not file size, are what exhausts memory).
|
||||
|
||||
``open_image`` is that single door:
|
||||
|
||||
* the declared dimensions are checked **before** a pixel is decoded;
|
||||
* Pillow's decompression-bomb *warning* is promoted to an error, because the
|
||||
warning band (between Pillow's limit and twice it) still decodes the image;
|
||||
* every decoder failure — at open time or during the caller's decode — becomes one
|
||||
of two typed errors, so callers map them to their own item state instead of
|
||||
catching ``Exception``;
|
||||
* error text names no path: it reaches API responses, and the full reason goes to
|
||||
the server log instead (US07-02).
|
||||
|
||||
``to_srgb`` and ``draft`` are the other two bounded-decode helpers: colour-manage a
|
||||
profile-bearing image into sRGB, and let JPEG decode straight to a size near the
|
||||
requested one rather than at full resolution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import warnings
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageCms, ImageFile, UnidentifiedImageError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Matches ``Config.thumbnail_max_pixels``; used where no configuration is at hand
|
||||
# (hashing runs inside discovery, which takes no config).
|
||||
DEFAULT_MAX_PIXELS = 100_000_000
|
||||
|
||||
|
||||
class MediaError(Exception):
|
||||
"""A file could not be turned into pixels safely."""
|
||||
|
||||
|
||||
class UndecodableImage(MediaError):
|
||||
"""Corrupt, truncated, empty, or not an image at all."""
|
||||
|
||||
|
||||
class ImageTooLarge(MediaError):
|
||||
"""More pixels than this operation is allowed to decode."""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def open_image(path: Path | str, *, max_pixels: int = DEFAULT_MAX_PIXELS):
|
||||
"""Yield an open :class:`PIL.Image.Image`, bounded and with typed failures.
|
||||
|
||||
Decoder errors raised inside the ``with`` body are translated too — a truncated
|
||||
JPEG only fails when its pixels are actually pulled, which is the caller's line,
|
||||
not this one.
|
||||
"""
|
||||
# Pillow's truncation tolerance is a process-global switch that any library in
|
||||
# the process can flip (the donor CLI did). This door decides the policy for its
|
||||
# own callers: half a file is not a picture.
|
||||
tolerated = ImageFile.LOAD_TRUNCATED_IMAGES
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
with warnings.catch_warnings():
|
||||
# The warning band is not a warning for us: it means Pillow was willing to
|
||||
# decode an image large enough to be a denial-of-service.
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
width, height = image.size
|
||||
if width * height > max_pixels:
|
||||
raise ImageTooLarge(f"{width}x{height} exceeds the {max_pixels} pixel limit")
|
||||
yield image
|
||||
except MediaError:
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error:
|
||||
log.info("refused oversized image %s: %s", path, error)
|
||||
raise ImageTooLarge("image exceeds the decompression-bomb limit") from error
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, MemoryError) as error:
|
||||
log.info("cannot decode %s: %s", path, error)
|
||||
raise UndecodableImage(f"cannot decode image ({type(error).__name__})") from error
|
||||
finally:
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = tolerated
|
||||
|
||||
|
||||
def draft(image: Image.Image, size: int) -> None:
|
||||
"""Ask the decoder for a smaller image where the format allows it (JPEG).
|
||||
|
||||
This is the difference between decoding a 40-megapixel JPEG and decoding the
|
||||
roughly 1-megapixel version a 1280px preview needs.
|
||||
"""
|
||||
try:
|
||||
image.draft(None, (size, size))
|
||||
except (AttributeError, ValueError, OSError): # not a draft-capable format
|
||||
pass
|
||||
|
||||
|
||||
def to_srgb(image: Image.Image, *, mode: str) -> Image.Image:
|
||||
"""Convert into ``mode``, colour-managing through an embedded ICC profile.
|
||||
|
||||
Without this a wide-gamut original renders with visibly wrong colours, because
|
||||
its numbers are interpreted as sRGB. A broken or unreadable profile is not a
|
||||
reason to fail a preview — the plain conversion is still a correct picture.
|
||||
"""
|
||||
profile = image.info.get("icc_profile")
|
||||
if profile:
|
||||
try:
|
||||
return ImageCms.profileToProfile(
|
||||
image,
|
||||
ImageCms.ImageCmsProfile(io.BytesIO(profile)),
|
||||
ImageCms.createProfile("sRGB"),
|
||||
outputMode=mode,
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 - any ICC failure falls back
|
||||
log.info("ignoring unusable ICC profile on %s: %s", getattr(image, "filename", "?"), error)
|
||||
return image.convert(mode)
|
||||
@@ -56,6 +56,30 @@ def read_keyword_sets(paths: Iterable[str]) -> dict[str, set[str]]:
|
||||
return out
|
||||
|
||||
|
||||
def read_all(path: str) -> dict | None:
|
||||
"""Every tag exiftool can read from ``path``, or ``None`` when it cannot answer.
|
||||
|
||||
This is the snapshot an EXIF checkpoint compares against: proving that a write
|
||||
preserved the fields it does not own requires knowing all of them, not just the
|
||||
ones being written (US07-03). ``None`` (exiftool missing, unreadable file,
|
||||
unparsable output) is not an empty snapshot — a caller must not read it as
|
||||
"nothing was there".
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["exiftool", "-m", "-j", "-G0:1", path], capture_output=True, text=True
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
try:
|
||||
records = json.loads(result.stdout or "[]")
|
||||
except ValueError:
|
||||
return None
|
||||
if not records:
|
||||
return None
|
||||
return {k: v for k, v in records[0].items() if k != "SourceFile"}
|
||||
|
||||
|
||||
def apply_keywords(path: str, *, add: Iterable[str] = (), remove: Iterable[str] = ()) -> bool:
|
||||
"""Idempotently add/remove keywords in Keywords + Subject; preserve all else."""
|
||||
args = ["exiftool", "-m", "-overwrite_original"]
|
||||
|
||||
@@ -15,6 +15,8 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from photo_pipeline import imaging
|
||||
|
||||
MODEL_ID = "AdamCodd/vit-base-nsfw-detector"
|
||||
BATCH = 16
|
||||
|
||||
@@ -54,13 +56,17 @@ class NsfwModel:
|
||||
self._ensure_loaded()
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageFile
|
||||
from PIL import Image
|
||||
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||
|
||||
def preprocess(image):
|
||||
image = image.convert("RGB").resize((self._size, self._size), Image.BILINEAR)
|
||||
array = (np.asarray(image, dtype="float32") / 255.0 - 0.5) / 0.5
|
||||
# The donor set ``ImageFile.LOAD_TRUNCATED_IMAGES = True`` here. That flag is
|
||||
# process-global: in this application the same process also hashes files and
|
||||
# renders previews, and those must keep failing loudly on a truncated file
|
||||
# rather than quietly working on half of one (US07-03). An unreadable image
|
||||
# is skipped instead — it stays unscored, and therefore visibly undecided.
|
||||
def preprocess(path):
|
||||
with imaging.open_image(path) as image:
|
||||
small = image.convert("RGB").resize((self._size, self._size), Image.BILINEAR)
|
||||
array = (np.asarray(small, dtype="float32") / 255.0 - 0.5) / 0.5
|
||||
return torch.from_numpy(array).permute(2, 0, 1)
|
||||
|
||||
results: list[tuple[str, float]] = []
|
||||
@@ -69,9 +75,10 @@ class NsfwModel:
|
||||
tensors, batch_paths = [], []
|
||||
for path in items[start : start + self.batch]:
|
||||
try:
|
||||
tensors.append(preprocess(Image.open(path)))
|
||||
tensors.append(preprocess(path))
|
||||
batch_paths.append(path)
|
||||
except Exception:
|
||||
except (imaging.MediaError, OSError, ValueError):
|
||||
# One bad file must not cost the batch its other fifteen.
|
||||
continue
|
||||
if not tensors:
|
||||
continue
|
||||
|
||||
@@ -7,6 +7,7 @@ Alembic environment relies on.
|
||||
from photo_pipeline.models.albums import AlbumProposal
|
||||
from photo_pipeline.models.archives import ArchiveLocation, ArchiveOperation, ArchivePlan
|
||||
from photo_pipeline.models.assets import Asset, AssetPath
|
||||
from photo_pipeline.models.exif import ExifProjection
|
||||
from photo_pipeline.models.duplicates import (
|
||||
DuplicateCluster,
|
||||
DuplicateMember,
|
||||
@@ -28,6 +29,7 @@ __all__ = [
|
||||
"DuplicateCluster",
|
||||
"DuplicateMember",
|
||||
"DuplicateNegativeLink",
|
||||
"ExifProjection",
|
||||
"Job",
|
||||
"JobItem",
|
||||
"JobEvent",
|
||||
|
||||
33
photo_pipeline/models/exif.py
Normal file
33
photo_pipeline/models/exif.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""The durable EXIF projection per asset and stage (concept §3, US07-03).
|
||||
|
||||
One row per ``(asset_id, stage)``: what the stage wanted written, what the file
|
||||
looked like afterwards, and whether anything outside the stage's ownership moved.
|
||||
``state = divergent`` is the whole point of the table — it survives restarts, keeps
|
||||
the asset out of stages that require verified metadata, and gives a human something
|
||||
to look at instead of a silent repair.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from photo_pipeline.db import Base
|
||||
|
||||
|
||||
class ExifProjection(Base):
|
||||
__tablename__ = "exif_projections"
|
||||
|
||||
asset_id: Mapped[str] = mapped_column(ForeignKey("assets.id"), primary_key=True)
|
||||
stage: Mapped[str] = mapped_column(String, primary_key=True) # safety | analysis
|
||||
id: Mapped[str] = mapped_column(String, nullable=False)
|
||||
projection_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
desired_json: Mapped[str | None] = mapped_column(String)
|
||||
divergent_fields: Mapped[str | None] = mapped_column(String) # JSON array
|
||||
result_file_sha256: Mapped[str | None] = mapped_column(String)
|
||||
state: Mapped[str] = mapped_column(String, nullable=False) # verified|divergent|failed
|
||||
error_code: Mapped[str | None] = mapped_column(String)
|
||||
verified_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
@@ -26,9 +26,8 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline import path_policy
|
||||
from photo_pipeline.integrations import exiftool
|
||||
from photo_pipeline.models import AnalysisResult, Asset, SafetyReview
|
||||
from photo_pipeline.services import hashing
|
||||
from photo_pipeline.services import exif_checkpoint
|
||||
from photo_pipeline.services.safety import SFW
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
@@ -170,7 +169,7 @@ class AnalysisService:
|
||||
tokens=int(result.get("_tokens", 0)) if isinstance(result, dict) else 0,
|
||||
raw=json.dumps(result, ensure_ascii=False),
|
||||
)
|
||||
_write_analysis_exif(path, result)
|
||||
self._write_analysis_exif(asset_id, path, result)
|
||||
analyzed += 1
|
||||
return {"analyzed": analyzed, "skipped": skipped, "errors": errors}
|
||||
|
||||
@@ -195,10 +194,42 @@ class AnalysisService:
|
||||
row.location_hint = result.get("location_hint")
|
||||
row.approx_year = result.get("approx_year")
|
||||
row.analyzed_at = now
|
||||
row.exif_written_at = now
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
def _write_analysis_exif(self, asset_id: str, path: str, result: dict) -> None:
|
||||
"""The analysis EXIF checkpoint: additive keywords, then prove the rest held.
|
||||
|
||||
Additive by design — safety keywords and the user's own keywords are merged
|
||||
with, never replaced (concept §3). ``exif_written_at`` is set only when the
|
||||
read-back verified both the new keywords and every field this stage does not
|
||||
own; a divergent result is recorded and left for a human (US07-03).
|
||||
ponytail: the managed ``AI:`` caption segment, once captions are owned here.
|
||||
"""
|
||||
tags = tuple(str(tag) for tag in (result.get("tags") or []))
|
||||
if not tags:
|
||||
return
|
||||
checkpoint = exif_checkpoint.run(path, add=tags)
|
||||
exif_checkpoint.record(
|
||||
self._session_factory,
|
||||
asset_id=asset_id,
|
||||
stage="analysis",
|
||||
result=checkpoint,
|
||||
add=tags,
|
||||
)
|
||||
if not checkpoint.verified:
|
||||
return
|
||||
with self._session_factory() as session:
|
||||
row = session.get(AnalysisResult, asset_id)
|
||||
if row is not None:
|
||||
row.exif_written_at = checkpoint.verified_at
|
||||
asset = session.get(Asset, asset_id)
|
||||
if asset is not None and checkpoint.sha256:
|
||||
# The bytes changed when the container was rewritten; upload must use
|
||||
# the hash of what is actually on disk now (concept §3).
|
||||
asset.current_sha256 = checkpoint.sha256
|
||||
session.commit()
|
||||
|
||||
def get(self, asset_id: str) -> dict | None:
|
||||
with self._session_factory() as session:
|
||||
row = session.get(AnalysisResult, asset_id)
|
||||
@@ -211,17 +242,6 @@ def _album_hint(path: str) -> str:
|
||||
return Path(path).parent.name
|
||||
|
||||
|
||||
def _write_analysis_exif(path: str, result: dict) -> None:
|
||||
"""Additive analysis keywords into EXIF (Keywords/Subject), preserving safety
|
||||
and user keywords. The donor also wrote a managed caption; only keywords are
|
||||
written here via the shared adapter.
|
||||
ponytail: add the managed ``AI:`` caption segment + read-back preservation check
|
||||
when the analysis EXIF checkpoint is hardened."""
|
||||
tags = result.get("tags") if isinstance(result, dict) else None
|
||||
if tags:
|
||||
exiftool.apply_keywords(path, add=[str(t) for t in tags])
|
||||
|
||||
|
||||
def _result_dict(row: AnalysisResult) -> dict:
|
||||
data = {field: getattr(row, field) for field in RESULT_FIELDS}
|
||||
data["tags"] = json.loads(row.tags) if row.tags else []
|
||||
|
||||
173
photo_pipeline/services/exif_checkpoint.py
Normal file
173
photo_pipeline/services/exif_checkpoint.py
Normal file
@@ -0,0 +1,173 @@
|
||||
"""The EXIF checkpoint every metadata stage ends with (concept §3, US07-03).
|
||||
|
||||
A stage does not own a file's metadata; it owns a few fields in it. So writing is
|
||||
never "set these tags" — it is:
|
||||
|
||||
snapshot everything → write only the owned fields → read everything back
|
||||
→ prove the owned fields landed → prove nothing else moved
|
||||
→ refresh the file hash → record the projection
|
||||
|
||||
Non-destructive here means *semantic* preservation: exiftool may rewrite the whole
|
||||
container, so the file's bytes, size, and timestamps legitimately change. What may
|
||||
not change is any field this stage does not own. When one does, the checkpoint is
|
||||
``divergent``: the result is recorded, the stage is **not** marked verified, and
|
||||
nothing is silently repaired — a later stage that needs verified metadata (upload)
|
||||
therefore stays blocked until a human looks.
|
||||
|
||||
``failed`` is the third outcome and is deliberately distinct: exiftool missing, an
|
||||
unreadable file, or a write that did not take is not evidence that metadata is fine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from photo_pipeline.integrations import exiftool
|
||||
from photo_pipeline.models import ExifProjection
|
||||
from photo_pipeline.services import hashing
|
||||
|
||||
# The tags the safety and analysis stages may change. Matched on the tag name, so
|
||||
# every group carries the same rule (IPTC:Keywords, XMP:XMP-dc:Subject, ...).
|
||||
OWNED_TAGS = frozenset({"Keywords", "Subject"})
|
||||
|
||||
# Not metadata about the picture: filesystem facts, the digest that necessarily
|
||||
# moves whenever IPTC does, and the structural tags exiftool has to create the first
|
||||
# time it writes an IPTC or XMP block. Comparing these would report every write as
|
||||
# divergent and make the signal worthless.
|
||||
VOLATILE_PREFIXES = ("File:System:", "ExifTool:")
|
||||
VOLATILE_KEYS = frozenset(
|
||||
{
|
||||
"File:CurrentIPTCDigest",
|
||||
"IPTC:ApplicationRecordVersion",
|
||||
"XMP:XMP-x:XMPToolkit",
|
||||
"XMP:XMP-xmp:MetadataDate",
|
||||
}
|
||||
)
|
||||
|
||||
VERIFIED = "verified"
|
||||
DIVERGENT = "divergent"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CheckpointResult:
|
||||
state: str # verified | divergent | failed
|
||||
changed_fields: tuple[str, ...] = ()
|
||||
sha256: str | None = None
|
||||
verified_at: datetime | None = None
|
||||
reason: str | None = None
|
||||
|
||||
@property
|
||||
def verified(self) -> bool:
|
||||
return self.state == VERIFIED
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def is_owned(key: str) -> bool:
|
||||
return key.rsplit(":", 1)[-1] in OWNED_TAGS
|
||||
|
||||
|
||||
def is_volatile(key: str) -> bool:
|
||||
return key in VOLATILE_KEYS or key.startswith(VOLATILE_PREFIXES)
|
||||
|
||||
|
||||
def compare(before: dict, after: dict) -> tuple[str, ...]:
|
||||
"""Fields outside this stage's ownership whose value did not survive the write.
|
||||
|
||||
Additions count: a tag that appears out of nowhere is as much a divergence as a
|
||||
tag that disappeared — both mean the write did more than it was asked to.
|
||||
"""
|
||||
keys = set(before) | set(after)
|
||||
return tuple(
|
||||
sorted(
|
||||
key
|
||||
for key in keys
|
||||
if not is_owned(key)
|
||||
and not is_volatile(key)
|
||||
and before.get(key) != after.get(key)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def owned_values(snapshot: dict) -> set[str]:
|
||||
"""Lowercased Keywords/Subject values across every group in the snapshot."""
|
||||
values: set[str] = set()
|
||||
for key, value in snapshot.items():
|
||||
if not is_owned(key):
|
||||
continue
|
||||
items = value if isinstance(value, list) else [value]
|
||||
values.update(str(item).strip().lower() for item in items if item is not None)
|
||||
return values
|
||||
|
||||
|
||||
def run(
|
||||
path: str,
|
||||
*,
|
||||
add: tuple[str, ...] = (),
|
||||
remove: tuple[str, ...] = (),
|
||||
) -> CheckpointResult:
|
||||
"""Write the owned keywords for one asset and verify the whole file around them."""
|
||||
before = exiftool.read_all(path)
|
||||
if before is None:
|
||||
return CheckpointResult(FAILED, reason="metadata_unreadable")
|
||||
|
||||
if not exiftool.apply_keywords(path, add=add, remove=remove):
|
||||
return CheckpointResult(FAILED, reason="write_failed")
|
||||
|
||||
after = exiftool.read_all(path)
|
||||
if after is None:
|
||||
return CheckpointResult(FAILED, reason="readback_unreadable")
|
||||
|
||||
present = owned_values(after)
|
||||
wanted = {value.strip().lower() for value in add}
|
||||
unwanted = {value.strip().lower() for value in remove}
|
||||
if not wanted <= present or (unwanted & present):
|
||||
return CheckpointResult(FAILED, reason="owned_fields_not_written")
|
||||
|
||||
changed = compare(before, after)
|
||||
sha256 = hashing.sha256_file(path)
|
||||
if changed:
|
||||
return CheckpointResult(DIVERGENT, changed_fields=changed, sha256=sha256)
|
||||
return CheckpointResult(VERIFIED, sha256=sha256, verified_at=_now())
|
||||
|
||||
|
||||
def record(
|
||||
session_factory,
|
||||
*,
|
||||
asset_id: str,
|
||||
stage: str,
|
||||
result: CheckpointResult,
|
||||
add: tuple[str, ...] = (),
|
||||
remove: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
"""Persist the projection for ``(asset_id, stage)`` — one current row per pair.
|
||||
|
||||
The row is what makes divergence durable and reviewable rather than a log line
|
||||
that scrolled away.
|
||||
"""
|
||||
with session_factory() as session:
|
||||
row = session.get(ExifProjection, (asset_id, stage))
|
||||
if row is None:
|
||||
row = ExifProjection(asset_id=asset_id, stage=stage, id=str(uuid.uuid4()))
|
||||
session.add(row)
|
||||
row.projection_version = (row.projection_version or 0) + 1
|
||||
row.desired_json = json.dumps({"add": list(add), "remove": list(remove)})
|
||||
row.divergent_fields = json.dumps(list(result.changed_fields))
|
||||
row.result_file_sha256 = result.sha256
|
||||
row.state = result.state
|
||||
row.error_code = result.reason
|
||||
row.verified_at = result.verified_at
|
||||
row.updated_at = _now()
|
||||
session.commit()
|
||||
|
||||
|
||||
def state_for(session_factory, asset_id: str, stage: str) -> str | None:
|
||||
with session_factory() as session:
|
||||
row = session.get(ExifProjection, (asset_id, stage))
|
||||
return row.state if row else None
|
||||
@@ -16,7 +16,9 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
from PIL import ImageOps
|
||||
|
||||
from photo_pipeline import imaging
|
||||
|
||||
PIXEL_HASH_VERSION = 1
|
||||
PHASH_VERSION = 1
|
||||
@@ -41,7 +43,9 @@ def _digest_file(path: Path | str, digest) -> str:
|
||||
|
||||
|
||||
def pixel_sha256(path: Path | str) -> str:
|
||||
with Image.open(path) as image:
|
||||
# Bounded decode: a header claiming a billion pixels is refused before it is
|
||||
# ever turned into memory (US07-03).
|
||||
with imaging.open_image(path) as image:
|
||||
oriented = ImageOps.exif_transpose(image)
|
||||
rgb = oriented.convert("RGB")
|
||||
header = f"{PIXEL_HASH_VERSION}:{rgb.width}x{rgb.height}:".encode()
|
||||
@@ -68,7 +72,9 @@ def phash(path: Path | str) -> str:
|
||||
import numpy as np
|
||||
from scipy.fftpack import dct
|
||||
|
||||
with Image.open(path) as image:
|
||||
from PIL import Image
|
||||
|
||||
with imaging.open_image(path) as image:
|
||||
small = image.convert("L").resize((32, 32), Image.LANCZOS)
|
||||
matrix = np.asarray(small, dtype=np.float64)
|
||||
transformed = dct(dct(matrix, axis=0), axis=1)
|
||||
|
||||
@@ -108,9 +108,8 @@ from datetime import datetime, timezone
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.integrations import exiftool
|
||||
from photo_pipeline.models import Asset, SafetyReview
|
||||
from photo_pipeline.services import hashing
|
||||
from photo_pipeline.models import Asset, ExifProjection, SafetyReview
|
||||
from photo_pipeline.services import exif_checkpoint
|
||||
|
||||
DECISIONS = {SFW, NSFW, "deferred"}
|
||||
|
||||
@@ -174,6 +173,14 @@ class SafetyService:
|
||||
with self._session_factory() as session:
|
||||
assets = list(session.scalars(_eligible_assets_query().order_by(Asset.current_path)))
|
||||
latest = self._latest_by_asset(session)
|
||||
# One query, not one per asset: the reviewer needs to see a divergent
|
||||
# checkpoint, which is neither "verified" nor a plain failure (US07-03).
|
||||
projections = {
|
||||
row.asset_id: row.state
|
||||
for row in session.scalars(
|
||||
select(ExifProjection).where(ExifProjection.stage == "safety")
|
||||
)
|
||||
}
|
||||
rows = []
|
||||
for asset in assets:
|
||||
review = latest.get(asset.id)
|
||||
@@ -189,6 +196,7 @@ class SafetyService:
|
||||
"decision": decision,
|
||||
"suggested": classify(review.score) if review and review.score is not None else None,
|
||||
"exif_verified": bool(review and review.exif_verified_at),
|
||||
"exif_state": projections.get(asset.id),
|
||||
}
|
||||
)
|
||||
return {"total": len(rows), "items": rows[offset : offset + limit]}
|
||||
@@ -247,13 +255,24 @@ class SafetyService:
|
||||
result_sha256 = None
|
||||
if write_exif and decision in (SFW, NSFW) and path:
|
||||
ops = exif_projection(decision)
|
||||
if exiftool.apply_keywords(path, add=ops["add"], remove=ops["remove"]):
|
||||
# Read back: the chosen keyword present, the opposite absent.
|
||||
keywords = exiftool.read_keyword_sets([path]).get(path, set())
|
||||
opposite = NSFW if decision == SFW else SFW
|
||||
if decision in keywords and opposite not in keywords:
|
||||
exif_verified_at = _now()
|
||||
result_sha256 = hashing.sha256_file(path)
|
||||
# The full checkpoint: write the owned keyword, read the whole file back,
|
||||
# and prove every field this stage does not own survived. A divergent
|
||||
# result is recorded and left alone — it must not count as verified, so
|
||||
# upload stays blocked until a human decides (US07-03).
|
||||
result = exif_checkpoint.run(
|
||||
path, add=tuple(ops["add"]), remove=tuple(ops["remove"])
|
||||
)
|
||||
exif_checkpoint.record(
|
||||
self._session_factory,
|
||||
asset_id=asset_id,
|
||||
stage="safety",
|
||||
result=result,
|
||||
add=tuple(ops["add"]),
|
||||
remove=tuple(ops["remove"]),
|
||||
)
|
||||
if result.verified:
|
||||
exif_verified_at = result.verified_at
|
||||
result_sha256 = result.sha256
|
||||
|
||||
now = _now()
|
||||
with self._session_factory() as session:
|
||||
|
||||
@@ -20,15 +20,16 @@ pa-imaging).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from PIL import Image, ImageOps
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline import path_policy
|
||||
from photo_pipeline import imaging, path_policy
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.models import Asset, Thumbnail
|
||||
from photo_pipeline.services import availability
|
||||
@@ -42,6 +43,9 @@ except Exception: # pragma: no cover
|
||||
pass
|
||||
|
||||
SIZES = (256, 512, 1280)
|
||||
# Every in-flight render writes one of these beside its destination; startup
|
||||
# cleanup recognises exactly this pattern and nothing else.
|
||||
TEMP_SUFFIX = ".tmp"
|
||||
THUMB_VERSION = 1
|
||||
THUMB_FORMAT = "webp"
|
||||
# The size kept as durable comparison evidence for archived assets (concept §9).
|
||||
@@ -222,28 +226,33 @@ class ThumbnailService:
|
||||
return self._cache_dir / safe[:2] / f"{safe}.{THUMB_FORMAT}"
|
||||
|
||||
def _render(self, source: str, size: int, cache_key: str) -> dict:
|
||||
"""Decode bounded, orient, colour-manage, resize, and write atomically.
|
||||
|
||||
The temporary file is removed on every failure path: a decoder that dies
|
||||
halfway through ``save`` would otherwise leave a stray ``.tmp`` in the cache
|
||||
forever (US07-03).
|
||||
"""
|
||||
destination = self._cache_path(cache_key)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}{TEMP_SUFFIX}")
|
||||
try:
|
||||
with Image.open(source) as image:
|
||||
width, height = image.size
|
||||
if width * height > self._config.thumbnail_max_pixels:
|
||||
raise ImageTooLarge(
|
||||
f"{width}x{height} exceeds {self._config.thumbnail_max_pixels} px"
|
||||
)
|
||||
with imaging.open_image(source, max_pixels=self._config.thumbnail_max_pixels) as image:
|
||||
imaging.draft(image, size) # JPEG decodes near the target size
|
||||
oriented = ImageOps.exif_transpose(image)
|
||||
mode = "RGBA" if _has_alpha(oriented) else "RGB"
|
||||
converted = oriented.convert(mode)
|
||||
converted = imaging.to_srgb(oriented, mode=mode)
|
||||
converted.thumbnail((size, size), Image.LANCZOS)
|
||||
out_width, out_height = converted.size
|
||||
destination = self._cache_path(cache_key)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
|
||||
converted.save(tmp, format="WEBP", quality=82, method=4)
|
||||
except ImageTooLarge:
|
||||
except imaging.ImageTooLarge as error:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise ImageTooLarge(str(error)) from None
|
||||
except imaging.UndecodableImage as error:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise UnsupportedImage(str(error)) from None
|
||||
except Exception:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
except (UnidentifiedImageError, OSError, ValueError) as error:
|
||||
raise UnsupportedImage(f"cannot decode {source}: {error}") from error
|
||||
|
||||
import os
|
||||
|
||||
os.replace(tmp, destination)
|
||||
return {
|
||||
@@ -253,6 +262,28 @@ class ThumbnailService:
|
||||
"format": THUMB_FORMAT,
|
||||
}
|
||||
|
||||
def cleanup_temp_files(self) -> int:
|
||||
"""Remove leftover render temporaries, and only those.
|
||||
|
||||
Runs at startup, where the concept is explicit: clean *recognized* stale
|
||||
temporary files, never arbitrary ones. The pattern is this service's own —
|
||||
a dot-prefixed name inside the managed cache directory ending in
|
||||
``.tmp`` — so nothing outside the cache and nothing a user put there can
|
||||
match. Returns how many were removed.
|
||||
"""
|
||||
if not self._cache_dir.is_dir():
|
||||
return 0
|
||||
removed = 0
|
||||
for leftover in self._cache_dir.rglob(f".*{TEMP_SUFFIX}"):
|
||||
if not leftover.is_file() or leftover.is_symlink():
|
||||
continue
|
||||
try:
|
||||
leftover.unlink()
|
||||
removed += 1
|
||||
except OSError: # pragma: no cover - a racing render already won
|
||||
pass
|
||||
return removed
|
||||
|
||||
# ── persistence ────────────────────────────────────────────────────────────
|
||||
def _record_ready(
|
||||
self, cache_key: str, asset_id: str, size: int, rendered: dict, *, protected: bool = False
|
||||
|
||||
Reference in New Issue
Block a user