170 lines
6.5 KiB
Python
170 lines
6.5 KiB
Python
"""Application management CLI:
|
|
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics}``.
|
|
|
|
``serve`` and ``worker`` take the library process lock for their role (US07-05):
|
|
two workers, or the frozen CLI running beside the app, would each be safe on their
|
|
own and destructive together. ``restore`` is here rather than in the API because it
|
|
replaces the state of an installation and belongs to a stopped one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from typing import Sequence
|
|
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
|
|
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(prog="photo_pipeline")
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
serve_cmd = commands.add_parser("serve", help="Run the API server")
|
|
commands.add_parser("migrate", help="Upgrade the database to the latest revision")
|
|
worker_cmd = commands.add_parser("worker", help="Run a durable-job worker")
|
|
worker_cmd.add_argument("--id", default="worker-1", help="Worker id (lease owner)")
|
|
for locked in (serve_cmd, worker_cmd):
|
|
locked.add_argument(
|
|
"--allow-legacy",
|
|
action="store_true",
|
|
help="Start even though a legacy CLI looks active (unsafe; you own the outcome)",
|
|
)
|
|
import_cmd = commands.add_parser(
|
|
"import-legacy-scores",
|
|
help="Import the archived CLI's nsfw_scores.csv into the database (US07-01)",
|
|
)
|
|
import_cmd.add_argument("csv", help="Path to nsfw_scores.csv")
|
|
import_cmd.add_argument(
|
|
"--overwrite", action="store_true", help="Replace differing imported scores"
|
|
)
|
|
import_cmd.add_argument(
|
|
"--dry-run", action="store_true", help="Report what would happen and change nothing"
|
|
)
|
|
|
|
backup_cmd = commands.add_parser("backup", help="Take an online database backup")
|
|
backup_cmd.add_argument("--reason", default="manual", help="Why (part of the directory name)")
|
|
backup_cmd.add_argument("--keep", type=int, default=7, help="How many backups to retain")
|
|
verify_cmd = commands.add_parser("verify-backup", help="Check a backup is intact and readable")
|
|
verify_cmd.add_argument("backup", help="Path to the backup directory")
|
|
restore_cmd = commands.add_parser(
|
|
"restore", help="Restore a verified backup into a fresh data directory"
|
|
)
|
|
restore_cmd.add_argument("backup", help="Path to the backup directory")
|
|
restore_cmd.add_argument("--into", required=True, help="Fresh data directory to restore into")
|
|
commands.add_parser("diagnostics", help="Report sizes, disk headroom, locks, and warnings")
|
|
|
|
args = parser.parse_args(argv)
|
|
|
|
config = Config.from_env()
|
|
config.database_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
if args.command == "migrate":
|
|
manifest = migrate_with_backup(config)
|
|
if manifest:
|
|
print(json.dumps({"pre_migration_backup": manifest["name"]}, indent=2))
|
|
return 0
|
|
|
|
if args.command == "backup":
|
|
try:
|
|
manifest = BackupService(config).create(reason=args.reason, keep=args.keep)
|
|
except BackupError as error:
|
|
print(str(error))
|
|
return 1
|
|
print(json.dumps(manifest, indent=2))
|
|
return 0
|
|
|
|
if args.command == "verify-backup":
|
|
result = BackupService(config).verify(args.backup)
|
|
print(json.dumps(result.as_dict(), indent=2))
|
|
return 0 if result.ok else 1
|
|
|
|
if args.command == "restore":
|
|
try:
|
|
report = BackupService(config).restore(args.backup, args.into)
|
|
except BackupError as error:
|
|
print(str(error))
|
|
return 1
|
|
print(json.dumps(report, indent=2))
|
|
return 0
|
|
|
|
if args.command == "diagnostics":
|
|
from photo_pipeline.services import diagnostics
|
|
|
|
print(json.dumps(diagnostics.report(config), indent=2))
|
|
return 0
|
|
|
|
if args.command == "import-legacy-scores":
|
|
from photo_pipeline.db import create_db_engine, create_session_factory
|
|
from photo_pipeline.services.legacy_import import LegacyImportService, write_report
|
|
|
|
migrate_with_backup(config)
|
|
engine = create_db_engine(config.database_url)
|
|
service = LegacyImportService(create_session_factory(engine))
|
|
report = service.import_nsfw_scores(
|
|
args.csv, overwrite=args.overwrite, dry_run=args.dry_run
|
|
)
|
|
# The report is the point: an import nobody can audit is not a migration.
|
|
if not args.dry_run:
|
|
write_report(report, config.data_dir)
|
|
print(json.dumps(report.counts, indent=2))
|
|
return 0
|
|
|
|
if args.command == "worker":
|
|
from photo_pipeline.db import create_db_engine, create_session_factory
|
|
|
|
# Importing this registers the safety/analysis job handlers into the shared
|
|
# REGISTRY the Worker defaults to; without it a standalone worker process
|
|
# claims nothing because it knows no job types.
|
|
import photo_pipeline.jobs.domain_handlers # noqa: F401
|
|
from photo_pipeline.jobs.worker import Worker
|
|
|
|
lock = LibraryLock(config, "worker")
|
|
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
|
|
return held
|
|
try:
|
|
migrate_with_backup(config)
|
|
engine = create_db_engine(config.database_url)
|
|
Worker(
|
|
create_session_factory(engine), worker_id=args.id, config=config
|
|
).run_forever()
|
|
finally:
|
|
lock.release()
|
|
return 0
|
|
|
|
import uvicorn
|
|
|
|
from photo_pipeline.api.app import create_app
|
|
|
|
lock = LibraryLock(config, "api")
|
|
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
|
|
return held
|
|
try:
|
|
uvicorn.run(create_app(config), host=config.host, port=config.port)
|
|
finally:
|
|
lock.release()
|
|
return 0
|
|
|
|
|
|
def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
|
|
"""Take the lock, or explain on stderr why this process must not start.
|
|
|
|
Returns an exit code to return, or ``None`` when the lock was acquired.
|
|
"""
|
|
import sys
|
|
|
|
try:
|
|
lock.acquire(allow_legacy=allow_legacy)
|
|
except LockHeld as error:
|
|
print(str(error), file=sys.stderr)
|
|
return 2
|
|
except LegacyProcessActive as error:
|
|
print(f"{error} (override with --allow-legacy)", file=sys.stderr)
|
|
return 3
|
|
return None
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|