Files
photoanalyzer/work_item/tests/test_cli_e2e.py

527 lines
21 KiB
Python

from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
import textwrap
import unittest
from pathlib import Path
SOURCE_ROOT = Path(__file__).resolve().parents[1]
PACKAGE_ROOT = SOURCE_ROOT / "src"
def run(args, *, cwd: Path, env=None, check=True):
result = subprocess.run(args, cwd=cwd, env=env, text=True, capture_output=True)
if check and result.returncode:
raise AssertionError(f"{args}\nstdout={result.stdout}\nstderr={result.stderr}")
return result
FAKE_TEA = r"""#!/usr/bin/env python3
import json, os, subprocess, sys
from pathlib import Path
state_path = Path(os.environ["FAKE_TEA_STATE"])
state = json.loads(state_path.read_text())
args = sys.argv[1:]
def save():
state_path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
def out(value):
print(json.dumps(value))
if args[0] == "api":
endpoint = args[1]
if endpoint.endswith("/issues?state=open&type=issues&limit=100"):
out([issue for issue in state["issues"] if issue["state"] == "open"])
elif "/dependencies" in endpoint:
issue = int(endpoint.split("/issues/")[1].split("/")[0])
blockers = state["dependencies"].get(str(issue), [])
out([next(x for x in state["issues"] if x["number"] == number) for number in blockers])
elif endpoint.endswith("/pulls?state=open&limit=50"):
out([pr for pr in state["prs"].values() if not pr["merged"]])
elif "/pulls/" in endpoint:
number = int(endpoint.rsplit("/", 1)[1])
out(state["prs"][str(number)])
elif "/commits/" in endpoint and endpoint.endswith("/status"):
out({"state": state.get("ci_state", "success")})
elif "--method" in args and args[args.index("--method") + 1] == "PATCH" and "/issues/" in endpoint:
number = int(endpoint.rsplit("/", 1)[1])
issue = next(x for x in state["issues"] if x["number"] == number)
payload = json.loads(args[args.index("--data") + 1])
if "assignees" in payload:
issue["assignees"] = [{"login": login} for login in payload["assignees"]]
save(); out(issue)
else:
raise SystemExit(f"unsupported api: {endpoint}")
elif args[:2] == ["issues", "edit"]:
issue = next(x for x in state["issues"] if x["number"] == int(args[2]))
if "--add-assignees" in args:
issue["assignees"] = [{"login": args[args.index("--add-assignees") + 1]}]
labels = {x["name"] for x in issue.get("labels", [])}
if "--remove-labels" in args:
labels -= set(args[args.index("--remove-labels") + 1].split(","))
if "--add-labels" in args:
labels |= set(args[args.index("--add-labels") + 1].split(","))
issue["labels"] = [{"name": x} for x in sorted(labels)]
save(); out(issue)
elif args[:2] == ["comments", "add"]:
state["comments"].append({"issue": int(args[2]), "body": args[3]})
save(); out(state["comments"][-1])
elif args[:2] == ["pulls", "create"]:
number = len(state["prs"]) + 1
branch = args[args.index("--head") + 1]
sha = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
pr = {"number": number, "index": number, "url": f"https://example.test/pr/{number}",
"merged": False, "head": {"sha": sha, "ref": branch}}
state["prs"][str(number)] = pr
save(); out(pr)
elif args[:2] == ["pulls", "merge"]:
number = int(args[2]); pr = state["prs"][str(number)]
if state.get("merge_succeeds", True):
subprocess.check_call(["git", "push", "origin", f"{pr['head']['ref']}:main"])
pr["merged"] = True
save(); out(pr)
elif args[:2] == ["issues", "close"]:
issue = next(x for x in state["issues"] if x["number"] == int(args[2]))
issue["state"] = "closed"; save(); out(issue)
else:
raise SystemExit("unsupported tea command: " + repr(args))
"""
class CliEndToEndTests(unittest.TestCase):
def setUp(self) -> None:
self.temp = tempfile.TemporaryDirectory()
self.base = Path(self.temp.name)
remote_parent = self.base / "domverse"
remote_parent.mkdir()
seed = self.base / "seed"
seed.mkdir()
run(["git", "init", "-b", "main"], cwd=seed)
run(["git", "config", "user.name", "Test Agent"], cwd=seed)
run(["git", "config", "user.email", "agent@example.test"], cwd=seed)
(seed / "README.md").write_text("seed\n")
run(["git", "add", "README.md"], cwd=seed)
run(["git", "commit", "-m", "seed"], cwd=seed)
self.remote = remote_parent / "photoanalyzer.git"
run(["git", "clone", "--bare", str(seed), str(self.remote)], cwd=self.base)
self.repo = self.base / "work"
run(["git", "clone", str(self.remote), str(self.repo)], cwd=self.base)
run(["git", "config", "user.name", "Test Agent"], cwd=self.repo)
run(["git", "config", "user.email", "agent@example.test"], cwd=self.repo)
config = textwrap.dedent("""\
[repository]
slug = "domverse/photoanalyzer"
login = "fake"
assignee = "agent"
remote = "origin"
main_branch = "main"
[workflow]
branch_prefix = "us"
require_ci = true
required_tests = []
[safety]
max_file_bytes = 100000
allow = ["tests/fixtures/**"]
deny = ["_IGNORE/**", "**/_IGNORE/**", "pictures/**", "*.env", "*.jpg"]
""")
(self.repo / ".work-item.toml").write_text(config)
run(["git", "add", ".work-item.toml"], cwd=self.repo)
run(["git", "commit", "-m", "config"], cwd=self.repo)
run(["git", "push", "origin", "main"], cwd=self.repo)
bin_dir = self.base / "bin"
bin_dir.mkdir()
tea = bin_dir / "tea"
tea.write_text(FAKE_TEA)
tea.chmod(0o755)
self.state_path = self.base / "tea-state.json"
self.state_path.write_text(
json.dumps(
{
"issues": [
{
"number": 1,
"title": "US01-01 — Implement Safe Workflow",
"state": "open",
"labels": [{"name": "status/backlog"}],
"assignees": [],
}
],
"dependencies": {"1": []},
"comments": [],
"prs": {},
}
)
)
self.env = os.environ.copy()
self.env["PATH"] = f"{bin_dir}{os.pathsep}{self.env['PATH']}"
self.env["FAKE_TEA_STATE"] = str(self.state_path)
self.env["PYTHONPATH"] = str(PACKAGE_ROOT)
def tearDown(self) -> None:
self.temp.cleanup()
def cli(self, *args, check=True):
return run(
[sys.executable, "-m", "work_item", *args], cwd=self.repo, env=self.env, check=check
)
def state(self):
return json.loads(self.state_path.read_text())
def save_state(self, state):
self.state_path.write_text(json.dumps(state))
def test_claim_submit_merge_complete_lifecycle(self) -> None:
next_result = self.cli("next")
self.assertEqual(json.loads(next_result.stdout)["story_id"], "US01-01")
claim = json.loads(self.cli("claim").stdout)
self.assertEqual(claim["status"], "in-progress")
self.assertEqual(
run(["git", "branch", "--show-current"], cwd=self.repo).stdout.strip(), claim["branch"]
)
issue = self.state()["issues"][0]
self.assertEqual(issue["assignees"], [{"login": "agent"}])
self.assertIn({"name": "status/in-progress"}, issue["labels"])
(self.repo / "private.env").write_text("not-a-real-secret\n")
unsafe = self.cli("submit", "--test", "true", "--yes", check=False)
self.assertEqual(unsafe.returncode, 2)
self.assertIn("denied path", unsafe.stderr)
(self.repo / "private.env").unlink()
(self.repo / "feature.txt").write_text("implemented\n")
review_only = self.cli("submit", "--test", "true", check=False)
self.assertEqual(review_only.returncode, 2)
self.assertIn("rerun with --yes", review_only.stderr)
self.assertEqual(
run(["git", "status", "--porcelain"], cwd=self.repo).stdout.strip(), "?? feature.txt"
)
submitted = json.loads(self.cli("submit", "--test", "true", "--yes").stdout)
self.assertEqual(submitted["status"], "review")
self.assertEqual(submitted["pr"], 1)
self.assertIn("status/review", {x["name"] for x in self.state()["issues"][0]["labels"]})
not_merged = self.cli("complete", check=False)
self.assertEqual(not_merged.returncode, 2)
self.assertIn("not merged", not_merged.stderr)
completed = json.loads(self.cli("complete", "--merge").stdout)
self.assertEqual(completed["status"], "done")
self.assertEqual(
run(["git", "branch", "--show-current"], cwd=self.repo).stdout.strip(), "main"
)
self.assertEqual(run(["git", "status", "--porcelain"], cwd=self.repo).stdout.strip(), "")
issue = self.state()["issues"][0]
self.assertEqual(issue["state"], "closed")
self.assertIn("status/done", {x["name"] for x in issue["labels"]})
self.assertFalse((self.repo / ".git" / "work-item-state.json").exists())
def test_dirty_tree_prevents_claim(self) -> None:
(self.repo / "unexpected.txt").write_text("dirty\n")
result = self.cli("claim", check=False)
self.assertEqual(result.returncode, 2)
self.assertIn("clean", result.stderr)
self.assertEqual(
run(["git", "branch", "--show-current"], cwd=self.repo).stdout.strip(), "main"
)
def test_status_reports_changes_and_claim_resumes_dirty_work(self) -> None:
claimed = json.loads(self.cli("claim").stdout)
(self.repo / "work.txt").write_text("pending\n")
status = json.loads(self.cli("status").stdout)
self.assertEqual(status["branch"], claimed["branch"])
self.assertEqual(status["current_branch"], claimed["branch"])
self.assertEqual(status["changes"], ["work.txt"])
resumed = json.loads(self.cli("claim").stdout)
self.assertEqual(resumed, claimed)
self.assertEqual((self.repo / "work.txt").read_text(), "pending\n")
def test_claim_switches_back_to_claimed_branch_when_clean(self) -> None:
claimed = json.loads(self.cli("claim").stdout)
run(["git", "switch", "main"], cwd=self.repo)
resumed = json.loads(self.cli("claim").stdout)
self.assertEqual(resumed, claimed)
self.assertEqual(
run(["git", "branch", "--show-current"], cwd=self.repo).stdout.strip(),
claimed["branch"],
)
def test_next_selection_skips_ineligible_issues(self) -> None:
state = self.state()
state["issues"] = [
{"number": 10, "title": "Maintenance", "state": "open", "labels": [], "assignees": []},
{
"number": 1,
"title": "US01-01 — Busy",
"state": "open",
"labels": [{"name": "status/in-progress"}],
"assignees": [],
},
{
"number": 2,
"title": "US01-02 — Assigned",
"state": "open",
"labels": [],
"assignees": [{"login": "someone"}],
},
{
"number": 3,
"title": "US01-03 — Has dependency",
"state": "open",
"labels": [],
"assignees": [],
},
{
"number": 4,
"title": "US01-04 — Ready",
"state": "open",
"labels": [{"name": "status/backlog"}],
"assignees": [],
},
{
"number": 5,
"title": "US02-01 — Later epic",
"state": "open",
"labels": [],
"assignees": [],
},
{"number": 99, "title": "Blocker", "state": "open", "labels": [], "assignees": []},
]
state["dependencies"] = {"3": [99], "4": [], "5": []}
self.save_state(state)
payload = json.loads(self.cli("next").stdout)
self.assertEqual(payload["story_id"], "US01-04")
def test_next_fails_when_no_story_is_eligible(self) -> None:
state = self.state()
state["issues"][0]["labels"] = [{"name": "status/blocked"}]
self.save_state(state)
result = self.cli("next", check=False)
self.assertEqual(result.returncode, 2)
self.assertIn("No eligible", result.stderr)
def test_commands_requiring_a_claim_fail_without_state(self) -> None:
cases = (
("status",),
("submit", "--test", "true"),
("complete",),
("block", "--reason", "blocked"),
)
for args in cases:
with self.subTest(command=args[0]):
result = self.cli(*args, check=False)
self.assertEqual(result.returncode, 2)
self.assertIn("No story is currently claimed", result.stderr)
def test_submit_requires_changes_and_passing_tests(self) -> None:
self.cli("claim")
no_changes = self.cli("submit", "--test", "true", "--yes", check=False)
self.assertEqual(no_changes.returncode, 2)
self.assertIn("No changes", no_changes.stderr)
(self.repo / "feature.txt").write_text("implemented\n")
failed_test = self.cli("submit", "--test", "false", "--yes", check=False)
self.assertEqual(failed_test.returncode, 2)
self.assertIn("Test command failed", failed_test.stderr)
self.assertEqual(
run(["git", "status", "--porcelain"], cwd=self.repo).stdout.strip(), "?? feature.txt"
)
def test_submit_rejects_wrong_branch(self) -> None:
claimed = json.loads(self.cli("claim").stdout)
run(["git", "switch", "main"], cwd=self.repo)
result = self.cli("submit", "--test", "true", check=False)
self.assertEqual(result.returncode, 2)
self.assertIn(f"Expected branch {claimed['branch']}", result.stderr)
def test_complete_requires_review_and_successful_ci(self) -> None:
self.cli("claim")
before_review = self.cli("complete", "--merge", check=False)
self.assertEqual(before_review.returncode, 2)
self.assertIn("not been submitted", before_review.stderr)
(self.repo / "feature.txt").write_text("implemented\n")
self.cli("submit", "--test", "true", "--yes")
state = self.state()
state["ci_state"] = "pending"
self.save_state(state)
pending = self.cli("complete", "--merge", check=False)
self.assertEqual(pending.returncode, 2)
self.assertIn("CI is not successful: pending", pending.stderr)
def test_complete_detects_unverified_merge(self) -> None:
self.cli("claim")
(self.repo / "feature.txt").write_text("implemented\n")
self.cli("submit", "--test", "true", "--yes")
state = self.state()
state["merge_succeeds"] = False
self.save_state(state)
result = self.cli("complete", "--merge", check=False)
self.assertEqual(result.returncode, 2)
self.assertIn("merge could not be verified", result.stderr)
def test_complete_rejects_new_dirty_work_after_submission(self) -> None:
self.cli("claim")
(self.repo / "feature.txt").write_text("implemented\n")
self.cli("submit", "--test", "true", "--yes")
(self.repo / "unexpected.txt").write_text("do not carry to main\n")
result = self.cli("complete", "--merge", check=False)
self.assertEqual(result.returncode, 2)
self.assertIn("Working tree must be clean", result.stderr)
self.assertTrue((self.repo / "unexpected.txt").is_file())
def test_whitespace_only_block_reason_is_rejected(self) -> None:
self.cli("claim")
result = self.cli("block", "--reason", " ", check=False)
self.assertEqual(result.returncode, 2)
self.assertIn("non-empty blocking reason", result.stderr)
def test_blocked_story_can_be_resumed(self) -> None:
claimed = json.loads(self.cli("claim").stdout)
blocked = json.loads(self.cli("block", "--reason", "Waiting for a fixture").stdout)
self.assertEqual(blocked["status"], "blocked")
resumed = json.loads(self.cli("claim").stdout)
self.assertEqual(resumed["status"], "in-progress")
self.assertNotIn("reason", resumed)
issue = self.state()["issues"][0]
labels = {label["name"] for label in issue["labels"]}
self.assertIn("status/in-progress", labels)
self.assertNotIn("status/blocked", labels)
self.assertIn(claimed["branch"], self.state()["comments"][-1]["body"])
def test_review_story_cannot_be_reclaimed(self) -> None:
self.cli("claim")
(self.repo / "feature.txt").write_text("implemented\n")
self.cli("submit", "--test", "true", "--yes")
result = self.cli("claim", check=False)
self.assertEqual(result.returncode, 2)
self.assertIn("finish its review/completion", result.stderr)
def test_command_works_from_repository_subdirectory(self) -> None:
subdirectory = self.repo / "nested"
subdirectory.mkdir()
result = run(
[sys.executable, "-m", "work_item", "next"],
cwd=subdirectory,
env=self.env,
)
self.assertEqual(json.loads(result.stdout)["story_id"], "US01-01")
def test_cli_discovers_yaml_configuration(self) -> None:
(self.repo / ".work-item.toml").unlink()
(self.repo / ".work-item.yml").write_text(
textwrap.dedent(
"""\
repository:
slug: domverse/photoanalyzer
login: fake
assignee: agent
remote: origin
main_branch: main
workflow:
branch_prefix: story
require_ci: false
required_tests: []
safety:
max_file_bytes: 100000
allow: ["tests/fixtures/**"]
deny: ["*.env", "*.jpg"]
"""
),
encoding="utf-8",
)
payload = json.loads(self.cli("next").stdout)
self.assertEqual(payload["story_id"], "US01-01")
def test_cli_discovers_dotenv_and_honors_environment_override(self) -> None:
(self.repo / ".work-item.toml").unlink()
(self.repo / ".work-item.env").write_text(
textwrap.dedent(
"""\
WORK_ITEM_REPO_SLUG=wrong/repository
WORK_ITEM_LOGIN=fake
WORK_ITEM_ASSIGNEE=agent
WORK_ITEM_REQUIRED_TESTS=[]
WORK_ITEM_DENY=[]
WORK_ITEM_ALLOW=[]
"""
),
encoding="utf-8",
)
self.env["WORK_ITEM_REPO_SLUG"] = "domverse/photoanalyzer"
payload = json.loads(self.cli("next").stdout)
self.assertEqual(payload["story_id"], "US01-01")
def test_block_requires_reason_and_records_state(self) -> None:
self.cli("claim")
result = self.cli("block", "--reason", "Provider fixture unavailable")
payload = json.loads(result.stdout)
self.assertEqual(payload["status"], "blocked")
issue = self.state()["issues"][0]
self.assertIn("status/blocked", {x["name"] for x in issue["labels"]})
self.assertIn("Provider fixture unavailable", self.state()["comments"][-1]["body"])
class InterpreterWrapperTests(unittest.TestCase):
def test_wrapper_selects_supported_python(self) -> None:
result = run(
[
str(SOURCE_ROOT / "scripts" / "python"),
"-c",
"import sys; print(sys.version_info[:2])",
],
cwd=SOURCE_ROOT,
)
major, minor = eval(result.stdout.strip(), {"__builtins__": {}})
self.assertEqual(major, 3)
self.assertGreaterEqual(minor, 12)
def test_wrapper_falls_back_when_conda_python_is_too_old(self) -> None:
with tempfile.TemporaryDirectory() as directory:
bin_dir = Path(directory)
conda = bin_dir / "conda"
conda.write_text("#!/bin/sh\necho 3.11\n")
conda.chmod(0o755)
fallback = bin_dir / "python3.13"
fallback.symlink_to(sys.executable)
env = os.environ.copy()
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
result = run(
[str(SOURCE_ROOT / "scripts" / "python"), "-c", "print('fallback')"],
cwd=SOURCE_ROOT,
env=env,
)
self.assertEqual(result.stdout.strip(), "fallback")
def test_wrapper_reports_when_no_supported_python_exists(self) -> None:
with tempfile.TemporaryDirectory() as directory:
bin_dir = Path(directory)
conda = bin_dir / "conda"
conda.write_text("#!/bin/sh\necho 3.11\n")
conda.chmod(0o755)
env = os.environ.copy()
env["PATH"] = f"{bin_dir}{os.pathsep}/bin:/usr/bin"
result = run(
[str(SOURCE_ROOT / "scripts" / "python"), "-c", "print('never')"],
cwd=SOURCE_ROOT,
env=env,
check=False,
)
self.assertEqual(result.returncode, 2)
self.assertIn("Python 3.12 or newer is required", result.stderr)
if __name__ == "__main__":
unittest.main()