This commit is contained in:
@@ -0,0 +1,401 @@
|
||||
"""P4 (2026-05-19) — audit-only mode verification.
|
||||
|
||||
Covers:
|
||||
- _is_audit_issue: title pattern detection (positive + negative)
|
||||
- _audit_mode: title-based + CLI override (AUDIT_ONLY_OVERRIDE)
|
||||
- _check_audit_only_violations: forbidden prefix detection via mocked git status
|
||||
- AUDIT_ONLY_NOTE injection into context pack (via build_context_pack contract)
|
||||
|
||||
Run: pytest -q tests/orchestrator_unit/test_audit_mode.py
|
||||
"""
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
import orchestrator # noqa: E402
|
||||
from orchestrator import ( # noqa: E402
|
||||
_is_audit_issue,
|
||||
_audit_mode,
|
||||
_check_audit_only_violations,
|
||||
_check_audit_commit_scope,
|
||||
_ensure_audit_baseline,
|
||||
_load_audit_baseline,
|
||||
_audit_baseline_path,
|
||||
AUDIT_ONLY_FORBIDDEN_PREFIXES,
|
||||
AUDIT_ONLY_NOTE,
|
||||
AUDIT_ALLOWED_COMMIT_GLOBS,
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# _is_audit_issue — title detection
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestIsAuditIssue:
|
||||
def test_integration_audit_bracket(self):
|
||||
assert _is_audit_issue("[INTEGRATION-AUDIT-01] cumulative review") is True
|
||||
assert _is_audit_issue("[INTEGRATION-AUDIT-02] something") is True
|
||||
assert _is_audit_issue("[INTEGRATION-AUDIT] no number") is True
|
||||
|
||||
def test_audit_only_bracket(self):
|
||||
assert _is_audit_issue("[AUDIT-ONLY] doc consistency check") is True
|
||||
|
||||
def test_case_insensitive(self):
|
||||
assert _is_audit_issue("[integration-audit-03] foo") is True
|
||||
assert _is_audit_issue("[Audit-Only] bar") is True
|
||||
|
||||
def test_plain_integration_audit_phrase(self):
|
||||
assert _is_audit_issue("Quarterly integration audit for closed issues") is True
|
||||
assert _is_audit_issue("Integration Audit Q2") is True
|
||||
|
||||
def test_execution_issue_not_audit(self):
|
||||
"""execution sub-issue 가 audit 로 잘못 감지되면 안 됨."""
|
||||
assert _is_audit_issue("[IMP-15 실행-1] image_aspect_mismatch") is False
|
||||
assert _is_audit_issue("[IMP-15 exec-2] table overflow") is False
|
||||
|
||||
def test_unrelated_issues(self):
|
||||
assert _is_audit_issue("IMP-19 I4 zone 비중 분배") is False
|
||||
assert _is_audit_issue("Fix overflow bug") is False
|
||||
assert _is_audit_issue("docs(IMP-06): Stage 4 fix") is False
|
||||
|
||||
def test_empty_or_none(self):
|
||||
assert _is_audit_issue("") is False
|
||||
assert _is_audit_issue(None) is False
|
||||
|
||||
def test_audit_word_in_random_position_no_match(self):
|
||||
"""'audit' 가 단독으로 나오는 건 안 잡아야 함 — 'integration audit' 만."""
|
||||
assert _is_audit_issue("audit some code") is False
|
||||
assert _is_audit_issue("security audit") is False
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# _audit_mode — combination with CLI override
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestAuditMode:
|
||||
def setup_method(self):
|
||||
# 각 테스트 전에 override 리셋.
|
||||
orchestrator.AUDIT_ONLY_OVERRIDE = False
|
||||
|
||||
def teardown_method(self):
|
||||
orchestrator.AUDIT_ONLY_OVERRIDE = False
|
||||
|
||||
def test_title_based_only(self):
|
||||
assert _audit_mode("[INTEGRATION-AUDIT-01] foo") is True
|
||||
assert _audit_mode("IMP-19 zone") is False
|
||||
|
||||
def test_cli_override_forces_audit(self):
|
||||
"""title 에 marker 없어도 CLI flag 가 audit mode 강제."""
|
||||
orchestrator.AUDIT_ONLY_OVERRIDE = True
|
||||
assert _audit_mode("IMP-19 zone") is True
|
||||
assert _audit_mode("any title") is True
|
||||
assert _audit_mode("") is True
|
||||
|
||||
def test_override_off_falls_back_to_title(self):
|
||||
orchestrator.AUDIT_ONLY_OVERRIDE = False
|
||||
assert _audit_mode("IMP-19 zone") is False
|
||||
assert _audit_mode("[INTEGRATION-AUDIT-01]") is True
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# _check_audit_only_violations — git status parsing
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class _FakeCompleted:
|
||||
def __init__(self, stdout, returncode=0):
|
||||
self.stdout = stdout
|
||||
self.stderr = ""
|
||||
self.returncode = returncode
|
||||
|
||||
|
||||
class TestCheckAuditOnlyViolations:
|
||||
"""subprocess.run 을 monkeypatch 해서 다양한 git status 출력 시나리오 검증."""
|
||||
|
||||
def test_clean_tree(self, monkeypatch):
|
||||
def fake_run(*args, **kwargs):
|
||||
return _FakeCompleted(stdout="")
|
||||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||
assert _check_audit_only_violations() == []
|
||||
|
||||
def test_only_allowed_changes(self, monkeypatch):
|
||||
"""docs/architecture 변경만 있으면 violation 0."""
|
||||
stdout = (
|
||||
" M docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
|
||||
"?? docs/architecture/INTEGRATION-AUDIT-01-MATRIX.md\n"
|
||||
" M docs/architecture/PHASE-Z-IMPLEMENTATION-ISSUE-BACKLOG.md\n"
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
assert _check_audit_only_violations() == []
|
||||
|
||||
def test_src_change_detected(self, monkeypatch):
|
||||
stdout = (
|
||||
" M src/phase_z2_pipeline.py\n"
|
||||
" M docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations()
|
||||
assert v == ["src/phase_z2_pipeline.py"]
|
||||
|
||||
def test_templates_change_detected(self, monkeypatch):
|
||||
stdout = " M templates/phase_z2/families/something.html\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations()
|
||||
assert v == ["templates/phase_z2/families/something.html"]
|
||||
|
||||
def test_tests_change_detected(self, monkeypatch):
|
||||
stdout = " M tests/phase_z2/test_overflow.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations()
|
||||
assert v == ["tests/phase_z2/test_overflow.py"]
|
||||
|
||||
def test_multiple_violations(self, monkeypatch):
|
||||
stdout = (
|
||||
" M src/a.py\n"
|
||||
"?? src/b.py\n"
|
||||
" M templates/c.html\n"
|
||||
" M tests/d.py\n"
|
||||
" M docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n" # allowed
|
||||
" M data/runs/run123.json\n" # allowed (not in forbidden)
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations()
|
||||
assert set(v) == {"src/a.py", "src/b.py", "templates/c.html", "tests/d.py"}
|
||||
|
||||
def test_renamed_file_destination_checked(self, monkeypatch):
|
||||
"""rename 의 경우 destination 만 검사."""
|
||||
stdout = "R docs/old.md -> src/new.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations()
|
||||
assert v == ["src/new.py"]
|
||||
|
||||
def test_windows_backslash_path(self, monkeypatch):
|
||||
"""Windows backslash path 도 forward-slash 로 정규화돼서 매치."""
|
||||
stdout = " M src\\phase_z2_pipeline.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations()
|
||||
assert v == ["src/phase_z2_pipeline.py"]
|
||||
|
||||
def test_quoted_path_with_spaces(self, monkeypatch):
|
||||
"""공백/특수문자 포함 path 는 quoted — quote strip 후 검사."""
|
||||
stdout = ' M "src/some file.py"\n'
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations()
|
||||
assert v == ["src/some file.py"]
|
||||
|
||||
def test_git_error_fails_open(self, monkeypatch):
|
||||
"""git 자체 실패 → 가드 false positive 안 만들고 빈 list 반환."""
|
||||
monkeypatch.setattr(subprocess, "run",
|
||||
lambda *a, **kw: _FakeCompleted(stdout="", returncode=128))
|
||||
assert _check_audit_only_violations() == []
|
||||
|
||||
def test_subprocess_exception_fails_open(self, monkeypatch):
|
||||
"""subprocess.run 자체가 raise 해도 가드 false positive X."""
|
||||
def boom(*a, **kw): raise RuntimeError("git missing")
|
||||
monkeypatch.setattr(subprocess, "run", boom)
|
||||
assert _check_audit_only_violations() == []
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# AUDIT_ONLY_NOTE constants — sanity
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# P4a: baseline-aware violations
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestBaselineAwareViolations:
|
||||
def test_baseline_subtraction_removes_preexisting(self, monkeypatch):
|
||||
"""pre-existing forbidden path 는 baseline 에 있으면 violation 에서 제외."""
|
||||
stdout = (
|
||||
" M src/already_dirty.py\n" # baseline 안에 있음 — 제외돼야 함
|
||||
" M src/new_violation.py\n" # baseline 밖 — violation
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
baseline = {"src/already_dirty.py"}
|
||||
v = _check_audit_only_violations(baseline=baseline)
|
||||
assert v == ["src/new_violation.py"]
|
||||
|
||||
def test_baseline_none_keeps_all(self, monkeypatch):
|
||||
"""baseline=None 이면 기존 동작 — 모든 forbidden 잡음."""
|
||||
stdout = " M src/a.py\n M src/b.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations(baseline=None)
|
||||
assert set(v) == {"src/a.py", "src/b.py"}
|
||||
|
||||
def test_baseline_empty_set_keeps_all(self, monkeypatch):
|
||||
"""baseline=set() 이면 모두 새 violation 으로 잡음."""
|
||||
stdout = " M src/a.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_only_violations(baseline=set())
|
||||
assert v == ["src/a.py"]
|
||||
|
||||
def test_baseline_filters_all_violations(self, monkeypatch):
|
||||
"""모든 violation 이 baseline 에 있으면 빈 list 반환 — clean 으로 판정."""
|
||||
stdout = " M src/a.py\n M templates/b.html\n M tests/c.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
baseline = {"src/a.py", "templates/b.html", "tests/c.py"}
|
||||
v = _check_audit_only_violations(baseline=baseline)
|
||||
assert v == []
|
||||
|
||||
def test_baseline_path_normalized_match(self, monkeypatch):
|
||||
"""baseline 의 path 는 forward-slash 정규화 형태. Windows backslash 도 매치."""
|
||||
stdout = " M src\\windows_path.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
baseline = {"src/windows_path.py"} # baseline 도 forward-slash 형태로 저장
|
||||
v = _check_audit_only_violations(baseline=baseline)
|
||||
assert v == []
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# P4a: _ensure_audit_baseline / _load_audit_baseline
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestAuditBaselinePersist:
|
||||
def test_save_and_load_roundtrip(self, monkeypatch, tmp_path):
|
||||
"""baseline 저장 → 로드 → 동일 path set 반환."""
|
||||
# Redirect ORCH_DIR to tmp_path for isolation.
|
||||
monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
|
||||
# Mock git status output.
|
||||
stdout = " M src/a.py\n?? src/b.py\n M docs/c.md\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
_ensure_audit_baseline(999)
|
||||
loaded = _load_audit_baseline(999)
|
||||
assert loaded == {"src/a.py", "src/b.py", "docs/c.md"}
|
||||
|
||||
def test_ensure_does_not_overwrite_existing(self, monkeypatch, tmp_path):
|
||||
"""이미 baseline 파일 있으면 덮어쓰지 않음 — resumed run 의 가드 일관성."""
|
||||
monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
|
||||
# First save with one set.
|
||||
stdout1 = " M src/original.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout1))
|
||||
_ensure_audit_baseline(999)
|
||||
# Second call with DIFFERENT git status — should NOT overwrite.
|
||||
stdout2 = " M src/different.py\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout2))
|
||||
_ensure_audit_baseline(999)
|
||||
loaded = _load_audit_baseline(999)
|
||||
# Original baseline preserved.
|
||||
assert loaded == {"src/original.py"}
|
||||
|
||||
def test_load_missing_returns_empty_set(self, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
|
||||
assert _load_audit_baseline(8888) == set()
|
||||
|
||||
def test_load_corrupt_returns_empty_set(self, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
|
||||
# Manually write corrupt JSON.
|
||||
p = tmp_path / "audit_baseline_7777.json"
|
||||
p.write_text("not valid json {{{", encoding="utf-8")
|
||||
assert _load_audit_baseline(7777) == set()
|
||||
|
||||
def test_load_non_list_returns_empty_set(self, monkeypatch, tmp_path):
|
||||
"""baseline 파일이 list 가 아닌 다른 JSON (예: dict) 이면 empty set."""
|
||||
monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
|
||||
p = tmp_path / "audit_baseline_6666.json"
|
||||
p.write_text('{"unexpected": "shape"}', encoding="utf-8")
|
||||
assert _load_audit_baseline(6666) == set()
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# P4a: _check_audit_commit_scope — Stage 5 guard
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestAuditCommitScope:
|
||||
def test_clean_commit_audit_report_only(self, monkeypatch):
|
||||
"""audit report 파일만 commit 되면 통과."""
|
||||
stdout = (
|
||||
"docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
|
||||
"docs/architecture/INTEGRATION-AUDIT-01-MATRIX.md\n"
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
assert _check_audit_commit_scope() == []
|
||||
|
||||
def test_backlog_update_allowed(self, monkeypatch):
|
||||
stdout = (
|
||||
"docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
|
||||
"docs/architecture/PHASE-Z-IMPLEMENTATION-ISSUE-BACKLOG.md\n"
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
assert _check_audit_commit_scope() == []
|
||||
|
||||
def test_src_file_in_commit_detected(self, monkeypatch):
|
||||
"""audit commit 에 src/ 파일이 끼면 violation."""
|
||||
stdout = (
|
||||
"docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
|
||||
"src/phase_z2_pipeline.py\n"
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_commit_scope()
|
||||
assert v == ["src/phase_z2_pipeline.py"]
|
||||
|
||||
def test_unrelated_doc_detected(self, monkeypatch):
|
||||
"""docs/ 라도 audit 관련 아닌 doc 은 violation."""
|
||||
stdout = (
|
||||
"docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
|
||||
"docs/some_other_doc.md\n" # 다른 doc
|
||||
"docs/architecture/PHASE-Z-PIPELINE-OVERVIEW.md\n" # audit 와 무관한 doc
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
v = _check_audit_commit_scope()
|
||||
assert set(v) == {"docs/some_other_doc.md",
|
||||
"docs/architecture/PHASE-Z-PIPELINE-OVERVIEW.md"}
|
||||
|
||||
def test_git_error_fails_open(self, monkeypatch):
|
||||
"""git show 자체 실패 → 빈 list (가드가 false positive 만들지 않음)."""
|
||||
monkeypatch.setattr(subprocess, "run",
|
||||
lambda *a, **kw: _FakeCompleted(stdout="", returncode=128))
|
||||
assert _check_audit_commit_scope() == []
|
||||
|
||||
def test_windows_backslash_normalized(self, monkeypatch):
|
||||
"""Windows backslash path 도 forward-slash 정규화 후 glob 매치."""
|
||||
stdout = "docs\\architecture\\INTEGRATION-AUDIT-01-REPORT.md\n"
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
|
||||
assert _check_audit_commit_scope() == []
|
||||
|
||||
def test_empty_commit_passes(self, monkeypatch):
|
||||
"""commit 에 파일 변경 없음 (보통 안 일어나지만) — 위반 없음."""
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=""))
|
||||
assert _check_audit_commit_scope() == []
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# P4a: allowed-glob shape sanity
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestAuditCommitAllowedGlobs:
|
||||
def test_globs_have_audit_marker(self):
|
||||
"""모든 allowed glob 에 INTEGRATION-AUDIT 또는 BACKLOG 마커 존재."""
|
||||
for g in AUDIT_ALLOWED_COMMIT_GLOBS:
|
||||
assert ("INTEGRATION-AUDIT" in g) or ("BACKLOG" in g)
|
||||
|
||||
def test_globs_under_docs_architecture(self):
|
||||
"""모든 allowed path 가 docs/architecture/ 산하 — src/ 등 우발적 허용 차단."""
|
||||
for g in AUDIT_ALLOWED_COMMIT_GLOBS:
|
||||
assert g.startswith("docs/architecture/"), f"glob escapes docs/architecture/: {g}"
|
||||
|
||||
|
||||
class TestAuditOnlyConstants:
|
||||
def test_note_mentions_forbidden_prefixes(self):
|
||||
for p in AUDIT_ONLY_FORBIDDEN_PREFIXES:
|
||||
assert p in AUDIT_ONLY_NOTE, f"AUDIT_ONLY_NOTE missing prefix mention: {p}"
|
||||
|
||||
def test_note_mentions_allowed_paths(self):
|
||||
assert "INTEGRATION-AUDIT-*.md" in AUDIT_ONLY_NOTE
|
||||
assert "PHASE-Z-IMPLEMENTATION-ISSUE-BACKLOG.md" in AUDIT_ONLY_NOTE
|
||||
|
||||
def test_note_states_no_code_edit(self):
|
||||
# "report" 또는 "NOT code" 표현 명시 확인 (LLM 가독성 가드).
|
||||
lower = AUDIT_ONLY_NOTE.lower()
|
||||
assert "audit report" in lower or "report writing" in lower
|
||||
assert "not code" in lower or "no production" in lower
|
||||
|
||||
def test_forbidden_prefixes_no_trailing_slash_issues(self):
|
||||
"""블랙리스트는 startswith 매치 — 'src' (slash 없음) 면 'srcfoo.py' 도 매칭돼서 false positive.
|
||||
모든 prefix 가 '/' 로 끝나야 함."""
|
||||
for p in AUDIT_ONLY_FORBIDDEN_PREFIXES:
|
||||
assert p.endswith("/"), f"prefix '{p}' must end with '/' to avoid false matches"
|
||||
@@ -0,0 +1,492 @@
|
||||
"""P5 (2026-05-20) — Dormant trigger guard tests (issue #58, unit u4).
|
||||
|
||||
Covers the L3 dormant trigger layer end-to-end:
|
||||
|
||||
- u1 — docs/architecture/DORMANT-TRIGGERS.yaml schema + content for the
|
||||
IMP-16 / IMP-17 / IMP-18 / IMP-19 / IMP-20 axes.
|
||||
- u2 — scripts/check_dormant_triggers.py file-pattern + content-pattern
|
||||
matching, manual-evidence skip, followup-linked skip,
|
||||
false-positive guards, exit-0 standalone invocation.
|
||||
- u3 — orchestrator._check_dormant_triggers() helper fail-open contract
|
||||
and the Stage 4→5 _audit_mode() bypass predicate.
|
||||
- u5 — DORMANT-TRIGGERS.yaml self-documenting header
|
||||
(governance doc cross-reference test runs after u5 lands).
|
||||
|
||||
Each test names the IMP-# trigger it exercises (scope-qualified verification
|
||||
per the work-principles lock).
|
||||
|
||||
Run: pytest -q tests/orchestrator_unit/test_dormant_triggers.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
REGISTRY_PATH = ROOT / "docs" / "architecture" / "DORMANT-TRIGGERS.yaml"
|
||||
GOVERNANCE_PATH = ROOT / "docs" / "architecture" / "PROJECT-INTENT-AND-GOVERNANCE.md"
|
||||
CHECKER_PATH = ROOT / "scripts" / "check_dormant_triggers.py"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# Shared fixtures
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def registry_entries() -> list[dict]:
|
||||
"""Parsed registry — used across schema + content tests."""
|
||||
with REGISTRY_PATH.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
assert isinstance(data, list), "registry root must be a YAML list"
|
||||
return data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chkmod():
|
||||
"""Fresh import of the standalone checker module (function-scoped so
|
||||
monkeypatched module attributes do not leak across tests)."""
|
||||
if "check_dormant_triggers" in sys.modules:
|
||||
return importlib.reload(sys.modules["check_dormant_triggers"])
|
||||
import check_dormant_triggers as m # noqa: WPS433 — runtime import by design
|
||||
return m
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def orch():
|
||||
"""Orchestrator module under test (u3 helper)."""
|
||||
import orchestrator as m # noqa: WPS433
|
||||
return m
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# u1 — registry yaml schema + content
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestRegistrySchema:
|
||||
"""u1 — DORMANT-TRIGGERS.yaml exists, parses, and shapes the 5 dormant axes."""
|
||||
|
||||
def test_registry_yaml_parses_with_pyyaml(self):
|
||||
"""Schema sanity (covers all of IMP-16/17/18/19/20): file parses as YAML list."""
|
||||
assert REGISTRY_PATH.exists(), f"registry missing: {REGISTRY_PATH}"
|
||||
with REGISTRY_PATH.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
assert isinstance(data, list), "registry root must be a YAML list"
|
||||
|
||||
def test_registry_has_5_entries_4_active_plus_1_followup_linked(self, registry_entries):
|
||||
"""Stage 1/2 scope-lock: IMP-16 / IMP-17 / IMP-18 / IMP-19 + IMP-20 followup-linked."""
|
||||
assert len(registry_entries) == 5
|
||||
issues = sorted(e["issue"] for e in registry_entries)
|
||||
assert issues == [16, 17, 18, 19, 20]
|
||||
|
||||
def test_registry_required_fields_present(self, registry_entries):
|
||||
"""Every entry has issue / title / doc / status / trigger / on_trigger (covers IMP-16~20)."""
|
||||
for e in registry_entries:
|
||||
assert isinstance(e.get("issue"), int)
|
||||
assert isinstance(e.get("title"), str) and e["title"]
|
||||
assert isinstance(e.get("doc"), str) and e["doc"]
|
||||
assert "status" in e
|
||||
assert isinstance(e.get("trigger"), dict)
|
||||
assert isinstance(e.get("on_trigger"), dict)
|
||||
trig = e["trigger"]
|
||||
assert "description" in trig
|
||||
assert isinstance(trig.get("manual_evidence_required"), bool)
|
||||
|
||||
|
||||
class TestImp16Entry:
|
||||
"""IMP-16 (issue 16) — active watch on src/** reverse-path adapter."""
|
||||
|
||||
def test_imp16_active_src_glob_and_reverse_path_content_pattern(self, registry_entries):
|
||||
"""IMP-16 trigger: src/**/*.py glob + reverse_path / html_to_slide_mdx content."""
|
||||
e = next(x for x in registry_entries if x["issue"] == 16)
|
||||
assert e["status"] == "documented:dormant"
|
||||
assert not e.get("followup_issue"), "IMP-16 is active, not followup-linked"
|
||||
t = e["trigger"]
|
||||
assert t["manual_evidence_required"] is False
|
||||
assert any("src/**" in p for p in t["file_patterns"])
|
||||
cps = t["content_patterns"]
|
||||
assert any("reverse_path" in p or "html_to_slide_mdx" in p for p in cps)
|
||||
|
||||
|
||||
class TestImp17Entry:
|
||||
"""IMP-17 (issue 17) — manual-evidence gate (3-cond User-GO AND)."""
|
||||
|
||||
def test_imp17_manual_evidence_required_true(self, registry_entries):
|
||||
"""IMP-17 trigger gate: manual_evidence_required=true (User GO + B4 + IMP-04/05 live)."""
|
||||
e = next(x for x in registry_entries if x["issue"] == 17)
|
||||
assert e["trigger"]["manual_evidence_required"] is True
|
||||
|
||||
|
||||
class TestImp18Entry:
|
||||
"""IMP-18 (issue 18) — SVG partial under templates/phase_z2/."""
|
||||
|
||||
def test_imp18_active_watch_on_phase_z2_templates_with_svg_content(self, registry_entries):
|
||||
"""IMP-18 trigger: templates/phase_z2/{families,frames}/*.html + SVG signature."""
|
||||
e = next(x for x in registry_entries if x["issue"] == 18)
|
||||
assert e["trigger"]["manual_evidence_required"] is False
|
||||
fps = e["trigger"]["file_patterns"]
|
||||
assert any("templates/phase_z2/" in p for p in fps)
|
||||
cps = e["trigger"]["content_patterns"]
|
||||
assert any("svg" in p.lower() or "viewBox" in p for p in cps)
|
||||
|
||||
|
||||
class TestImp19Entry:
|
||||
"""IMP-19 (issue 19) — manual-evidence gate (IMP-09 owner sign-off)."""
|
||||
|
||||
def test_imp19_manual_evidence_required_true(self, registry_entries):
|
||||
"""IMP-19 trigger gate: manual_evidence_required=true (failing-case + IMP-09 sign-off)."""
|
||||
e = next(x for x in registry_entries if x["issue"] == 19)
|
||||
assert e["trigger"]["manual_evidence_required"] is True
|
||||
|
||||
|
||||
class TestImp20Entry:
|
||||
"""IMP-20 (issue 20) — followup-linked to open issue #55, note-only."""
|
||||
|
||||
def test_imp20_followup_linked_to_55_with_note_only_action(self, registry_entries):
|
||||
"""IMP-20 status: followup_issue=55, on_trigger.action=note_only (no checker watch)."""
|
||||
e = next(x for x in registry_entries if x["issue"] == 20)
|
||||
assert e.get("followup_issue") == 55
|
||||
assert e["on_trigger"]["action"] == "note_only"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# u2 — check_dormant_triggers.py
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestCheckerMatching:
|
||||
|
||||
def test_checker_clean_tree_no_alerts_covers_imp16_18(self, chkmod, monkeypatch):
|
||||
"""False-positive guard: empty change surface → no IMP-16 / IMP-18 alerts."""
|
||||
monkeypatch.setattr(chkmod, "collect_changed_files", lambda: [])
|
||||
entries = chkmod.load_registry()
|
||||
alerts = [a for a in (chkmod.check_entry(e, []) for e in entries) if a]
|
||||
assert alerts == []
|
||||
|
||||
def test_checker_imp16_alert_on_src_reverse_path_adapter(
|
||||
self, chkmod, monkeypatch, tmp_path
|
||||
):
|
||||
"""IMP-16 positive: src/foo/adapter.py with `reverse_path` content → alert."""
|
||||
fake_path = "src/foo/adapter.py"
|
||||
(tmp_path / "src" / "foo").mkdir(parents=True)
|
||||
(tmp_path / fake_path).write_text("def reverse_path(): pass\n", encoding="utf-8")
|
||||
monkeypatch.setattr(chkmod, "REPO_ROOT", tmp_path)
|
||||
entries = chkmod.load_registry()
|
||||
imp16 = next(e for e in entries if e["issue"] == 16)
|
||||
result = chkmod.check_entry(imp16, [fake_path])
|
||||
assert result is not None
|
||||
assert result["issue"] == 16
|
||||
assert fake_path in result["match"]["files"]
|
||||
|
||||
def test_checker_imp16_no_alert_on_tests_path_false_positive_guard(
|
||||
self, chkmod, monkeypatch, tmp_path
|
||||
):
|
||||
"""False-positive guard: IMP-16 must NOT fire on tests/foo.py even with matching content."""
|
||||
fake_path = "tests/foo.py"
|
||||
(tmp_path / "tests").mkdir()
|
||||
(tmp_path / fake_path).write_text("def reverse_path(): pass\n", encoding="utf-8")
|
||||
monkeypatch.setattr(chkmod, "REPO_ROOT", tmp_path)
|
||||
entries = chkmod.load_registry()
|
||||
imp16 = next(e for e in entries if e["issue"] == 16)
|
||||
assert chkmod.check_entry(imp16, [fake_path]) is None
|
||||
|
||||
def test_checker_imp18_alert_on_phase_z2_family_svg(
|
||||
self, chkmod, monkeypatch, tmp_path
|
||||
):
|
||||
"""IMP-18 positive: templates/phase_z2/families/new.html with <svg viewBox> → alert."""
|
||||
fake_path = "templates/phase_z2/families/new_partial.html"
|
||||
(tmp_path / "templates" / "phase_z2" / "families").mkdir(parents=True)
|
||||
(tmp_path / fake_path).write_text(
|
||||
'<svg viewBox="0 0 100 100"></svg>\n', encoding="utf-8"
|
||||
)
|
||||
monkeypatch.setattr(chkmod, "REPO_ROOT", tmp_path)
|
||||
entries = chkmod.load_registry()
|
||||
imp18 = next(e for e in entries if e["issue"] == 18)
|
||||
result = chkmod.check_entry(imp18, [fake_path])
|
||||
assert result is not None
|
||||
assert result["issue"] == 18
|
||||
|
||||
def test_checker_imp18_flat_glob_boundary_nested_path_skipped(
|
||||
self, chkmod, monkeypatch, tmp_path
|
||||
):
|
||||
"""False-positive guard: IMP-18 flat glob `families/*.html` skips nested family path."""
|
||||
fake_path = "templates/phase_z2/families/nested/inner.html"
|
||||
(tmp_path / "templates" / "phase_z2" / "families" / "nested").mkdir(parents=True)
|
||||
(tmp_path / fake_path).write_text(
|
||||
'<svg viewBox="0 0 100 100"></svg>\n', encoding="utf-8"
|
||||
)
|
||||
monkeypatch.setattr(chkmod, "REPO_ROOT", tmp_path)
|
||||
entries = chkmod.load_registry()
|
||||
imp18 = next(e for e in entries if e["issue"] == 18)
|
||||
assert chkmod.check_entry(imp18, [fake_path]) is None
|
||||
|
||||
def test_checker_skips_imp17_manual_evidence(self, chkmod):
|
||||
"""Guardrail: IMP-17 manual_evidence_required skips even with broad change surface."""
|
||||
entries = chkmod.load_registry()
|
||||
imp17 = next(e for e in entries if e["issue"] == 17)
|
||||
assert chkmod.check_entry(imp17, ["src/foo.py", "anything.html"]) is None
|
||||
|
||||
def test_checker_skips_imp19_manual_evidence(self, chkmod):
|
||||
"""Guardrail: IMP-19 manual_evidence_required skips even with broad change surface."""
|
||||
entries = chkmod.load_registry()
|
||||
imp19 = next(e for e in entries if e["issue"] == 19)
|
||||
assert chkmod.check_entry(imp19, ["src/foo.py"]) is None
|
||||
|
||||
def test_checker_skips_imp20_followup_linked(self, chkmod):
|
||||
"""Guardrail: IMP-20 followup_issue=55 skips (open issue #55 owns the watch)."""
|
||||
entries = chkmod.load_registry()
|
||||
imp20 = next(e for e in entries if e["issue"] == 20)
|
||||
assert chkmod.check_entry(imp20, ["anything", "src/a.py"]) is None
|
||||
|
||||
def test_checker_standalone_invocation_exit_0(self):
|
||||
"""Guardrail: standalone `python scripts/check_dormant_triggers.py` exits 0 always.
|
||||
|
||||
Exercises the IMP-16~20 informational-only contract — checker never blocks
|
||||
the orchestrator regardless of working-tree state.
|
||||
"""
|
||||
r = subprocess.run(
|
||||
[sys.executable, str(CHECKER_PATH)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
cwd=str(ROOT),
|
||||
timeout=30,
|
||||
)
|
||||
assert r.returncode == 0, f"checker exit={r.returncode} stderr={r.stderr!r}"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# u3 — orchestrator helper + Stage 4→5 hook
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestOrchestratorDormantHook:
|
||||
|
||||
def test_orchestrator_helper_returns_list_on_clean_tree(self, orch):
|
||||
"""u3 helper returns list[dict] (possibly empty) — never raises (IMP-16~20 informational)."""
|
||||
result = orch._check_dormant_triggers()
|
||||
assert isinstance(result, list)
|
||||
|
||||
def test_orchestrator_helper_fail_open_on_subprocess_error(self, orch, monkeypatch):
|
||||
"""u3 helper: subprocess raise → [] (fail-open, no false positives across all IMPs)."""
|
||||
def boom(*a, **kw):
|
||||
raise RuntimeError("subprocess unavailable")
|
||||
monkeypatch.setattr(orch.subprocess, "run", boom)
|
||||
assert orch._check_dormant_triggers() == []
|
||||
|
||||
def test_orchestrator_helper_fail_open_on_nonzero_exit(self, orch, monkeypatch):
|
||||
"""u3 helper: subprocess returncode != 0 → [] (fail-open)."""
|
||||
class _Err:
|
||||
returncode = 1
|
||||
stdout = ""
|
||||
stderr = "boom"
|
||||
monkeypatch.setattr(orch.subprocess, "run", lambda *a, **kw: _Err())
|
||||
assert orch._check_dormant_triggers() == []
|
||||
|
||||
def test_orchestrator_helper_fail_open_on_missing_alerts_file(
|
||||
self, orch, monkeypatch, tmp_path
|
||||
):
|
||||
"""u3 helper: subprocess OK but alert file absent → [] (fail-open)."""
|
||||
class _OK:
|
||||
returncode = 0
|
||||
stdout = ""
|
||||
stderr = ""
|
||||
monkeypatch.setattr(orch.subprocess, "run", lambda *a, **kw: _OK())
|
||||
monkeypatch.setattr(orch, "ORCH_DIR", tmp_path)
|
||||
assert orch._check_dormant_triggers() == []
|
||||
|
||||
def test_orchestrator_helper_parses_alerts_payload(self, orch, monkeypatch, tmp_path):
|
||||
"""u3 helper: reads `alerts` list out of .orchestrator/dormant_alerts.json payload."""
|
||||
class _OK:
|
||||
returncode = 0
|
||||
stdout = ""
|
||||
stderr = ""
|
||||
monkeypatch.setattr(orch.subprocess, "run", lambda *a, **kw: _OK())
|
||||
monkeypatch.setattr(orch, "ORCH_DIR", tmp_path)
|
||||
payload = {
|
||||
"alerts": [
|
||||
{"issue": 16, "title": "IMP-16 sample", "on_trigger": {"action": "create_runtime_issue"}},
|
||||
],
|
||||
}
|
||||
(tmp_path / "dormant_alerts.json").write_text(
|
||||
json.dumps(payload), encoding="utf-8"
|
||||
)
|
||||
result = orch._check_dormant_triggers()
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 1
|
||||
assert result[0]["issue"] == 16
|
||||
|
||||
def test_orchestrator_helper_handles_non_list_alerts_payload(
|
||||
self, orch, monkeypatch, tmp_path
|
||||
):
|
||||
"""u3 helper: malformed `alerts` (non-list) → [] (fail-open, IMP-16~20 informational)."""
|
||||
class _OK:
|
||||
returncode = 0
|
||||
stdout = ""
|
||||
stderr = ""
|
||||
monkeypatch.setattr(orch.subprocess, "run", lambda *a, **kw: _OK())
|
||||
monkeypatch.setattr(orch, "ORCH_DIR", tmp_path)
|
||||
(tmp_path / "dormant_alerts.json").write_text(
|
||||
json.dumps({"alerts": "oops"}), encoding="utf-8"
|
||||
)
|
||||
assert orch._check_dormant_triggers() == []
|
||||
|
||||
def test_stage_4_to_5_hook_predicate_audit_bypass(self, orch):
|
||||
"""u3 Stage 4→5 hook guard: _audit_mode(title) True → dormant checker bypassed.
|
||||
|
||||
Mirrors P4a placement — the dormant hook condition is
|
||||
`sid == "test-verify" and not _audit_mode(title)`. This test asserts
|
||||
the gating predicate for the IMP-16/18 active watches.
|
||||
"""
|
||||
assert orch._audit_mode("[INTEGRATION-AUDIT-02] cumulative review") is True
|
||||
assert orch._audit_mode("[AUDIT-ONLY] doc consistency") is True
|
||||
assert orch._audit_mode("[P5][DORMANT-TRIGGER-GUARD] hook") is False
|
||||
assert orch._audit_mode("IMP-16 U2 wiring") is False
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# u3 — Stage 4→5 hook integration (focused static assertions on run_stage)
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestRunStageDormantHookIntegration:
|
||||
"""u3 — Stage 4→5 hook must be wired into ``orchestrator.run_stage``.
|
||||
|
||||
Codex #5 rewind: testing ``_audit_mode()`` in isolation is insufficient.
|
||||
These focused static-source assertions on ``inspect.getsource(run_stage)``
|
||||
fail if the dormant hook is silently removed or its audit-bypass predicate
|
||||
is weakened — guarding the IMP-16/17/18/19/20 informational alert wiring.
|
||||
"""
|
||||
|
||||
def _run_stage_src(self, orch) -> str:
|
||||
import inspect
|
||||
return inspect.getsource(orch.run_stage)
|
||||
|
||||
def test_run_stage_invokes_check_dormant_triggers_on_test_verify_non_audit(self, orch):
|
||||
"""u3 contract: run_stage body calls _check_dormant_triggers().
|
||||
|
||||
Exercises the Stage 4 (test-verify) non-audit invocation path for
|
||||
IMP-16/18 active watches. If the call disappears from run_stage,
|
||||
this test fails (catches the regression that Codex #5 flagged).
|
||||
"""
|
||||
src = self._run_stage_src(orch)
|
||||
assert "_check_dormant_triggers()" in src, (
|
||||
"run_stage must invoke _check_dormant_triggers() — the L3 wiring "
|
||||
"for IMP-16/17/18/19/20 dormant alerts. Silent removal breaks the "
|
||||
"Stage 4→5 informational hook."
|
||||
)
|
||||
|
||||
def test_run_stage_dormant_hook_gated_by_test_verify_and_non_audit(self, orch):
|
||||
"""u3 contract: dormant hook is gated by both sid==test-verify AND not _audit_mode(title).
|
||||
|
||||
The predicate is what makes audit-only Stage 4 bypass the IMP-16~20
|
||||
checker (Stage 1 scope-lock guardrail). Removing either conjunct
|
||||
would silently re-enable the checker on audit-only issues whose
|
||||
change surface is restricted to audit-report docs.
|
||||
"""
|
||||
import re
|
||||
src = self._run_stage_src(orch)
|
||||
m = re.search(
|
||||
r'sid\s*==\s*"test-verify"\s+and\s+not\s+_audit_mode\(title\)\s*:\s*\n'
|
||||
r'\s+alerts\s*=\s*_check_dormant_triggers\(\)',
|
||||
src,
|
||||
)
|
||||
assert m is not None, (
|
||||
"Stage 4→5 dormant hook must be gated by "
|
||||
'`sid == "test-verify" and not _audit_mode(title):` immediately '
|
||||
"before `alerts = _check_dormant_triggers()`. Audit-only bypass "
|
||||
"and Stage 4 placement are part of the IMP-16~20 contract."
|
||||
)
|
||||
|
||||
def test_run_stage_dormant_hook_is_informational_no_continue(self, orch):
|
||||
"""u3 contract: dormant hook block must NOT contain a `continue` statement.
|
||||
|
||||
IMP-16~20 alerts are informational only (Stage 1 guardrail). The hook
|
||||
block between the _check_dormant_triggers() call and the next
|
||||
Stage 4 PASS log line ("YES (evidence verified)") must not short-
|
||||
circuit Stage 5 entry via `continue`. Comments mentioning the word
|
||||
"continue" are allowed (they document the contract).
|
||||
"""
|
||||
import re
|
||||
src = self._run_stage_src(orch)
|
||||
start = src.find("_check_dormant_triggers()")
|
||||
assert start >= 0
|
||||
end = src.find("YES (evidence verified)", start)
|
||||
assert end > start, (
|
||||
"could not locate Stage 4 PASS log line after dormant hook — "
|
||||
"run_stage shape may have shifted; re-examine integration."
|
||||
)
|
||||
hook_block = src[start:end]
|
||||
# Strip line comments before checking for the `continue` keyword as
|
||||
# an actual statement — the source intentionally documents the
|
||||
# informational-only contract with a `# Never continue — ...` comment.
|
||||
stripped_lines = []
|
||||
for line in hook_block.splitlines():
|
||||
code = line.split("#", 1)[0]
|
||||
stripped_lines.append(code)
|
||||
code_only = "\n".join(stripped_lines)
|
||||
assert not re.search(r'(^|\s)continue\b', code_only), (
|
||||
"dormant hook block contains a `continue` statement — IMP-16~20 "
|
||||
"alerts must never block Stage 5 entry (informational-only contract)."
|
||||
)
|
||||
|
||||
def test_run_stage_dormant_hook_positioned_before_stage_pass_return(self, orch):
|
||||
"""u3 contract: dormant hook is positioned within the Stage 4 YES PASS path.
|
||||
|
||||
Verifies the hook sits between the P4a audit commit-scope guard and
|
||||
the Stage 4 success log+return (i.e., on the PASS path), not in an
|
||||
unreachable branch. Protects against accidental relocation during
|
||||
future refactors.
|
||||
"""
|
||||
src = self._run_stage_src(orch)
|
||||
hook_pos = src.find("_check_dormant_triggers()")
|
||||
pass_log_pos = src.find("YES (evidence verified)", hook_pos)
|
||||
return_true_pos = src.find("return True", hook_pos)
|
||||
assert hook_pos >= 0
|
||||
assert 0 < pass_log_pos - hook_pos < 4000, (
|
||||
"dormant hook is not adjacent to the Stage 4 PASS log line — "
|
||||
"run_stage shape may have shifted away from PASS-path placement."
|
||||
)
|
||||
assert return_true_pos > pass_log_pos, (
|
||||
"Stage 4 `return True` should follow the PASS log line; "
|
||||
"dormant hook must precede both."
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# u5 — registry self-documentation + governance cross-reference
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestRegistryHeaderAndGovernanceRef:
|
||||
|
||||
def test_registry_header_explains_schema_and_l3_purpose(self):
|
||||
"""u5 acceptance: yaml header explains schema + L3 informational-only purpose.
|
||||
|
||||
The header is the durable self-documentation surface that anchors the
|
||||
IMP-16/17/18/19/20 registry (per Stage 1 exit unit u5).
|
||||
"""
|
||||
text = REGISTRY_PATH.read_text(encoding="utf-8")
|
||||
assert "Schema" in text or "schema" in text
|
||||
assert "dormant" in text.lower()
|
||||
assert "L3" in text or "machine-readable" in text.lower()
|
||||
assert "Guardrails" in text or "informational" in text.lower()
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not GOVERNANCE_PATH.exists()
|
||||
or "DORMANT-TRIGGERS.yaml" not in GOVERNANCE_PATH.read_text(encoding="utf-8"),
|
||||
reason="u5 governance-doc reference line not yet appended (passes after u5 lands).",
|
||||
)
|
||||
def test_governance_doc_references_registry(self):
|
||||
"""u5 deliverable: PROJECT-INTENT-AND-GOVERNANCE.md cites DORMANT-TRIGGERS.yaml as L3.
|
||||
|
||||
Runs after u5 lands — IMP-16~20 registry surfaces in the governance
|
||||
anti-patterns row so future maintainers find it without rediscovery.
|
||||
"""
|
||||
text = GOVERNANCE_PATH.read_text(encoding="utf-8")
|
||||
assert "DORMANT-TRIGGERS.yaml" in text
|
||||
@@ -0,0 +1,33 @@
|
||||
"""IMP-17 u1 (2026-05-19) — comment anchor for src/phase_z2_pipeline.py route hint table.
|
||||
|
||||
Stage 1 finding: line 564 previously referenced a non-existent ID ("IMP-31").
|
||||
The legitimate slot is IMP-17 (Gitea #17, carve-out — AI fallback only, normal path 밖).
|
||||
Line 565 (IMP-29 frontend zone-level override) must remain untouched.
|
||||
|
||||
Anchor re-pin (2026-05-20, IMP-30 u1 follow-up): V4Match.provisional field added at
|
||||
src/phase_z2_pipeline.py:179-184 shifted the route-hint table down by six lines.
|
||||
Pinned line numbers updated from 564/565 → 570/571 to track the actual anchor location.
|
||||
|
||||
Run: pytest -q tests/orchestrator_unit/test_imp17_comment_anchor.py
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent
|
||||
PIPELINE = ROOT / "src" / "phase_z2_pipeline.py"
|
||||
|
||||
|
||||
def _lines() -> list[str]:
|
||||
return PIPELINE.read_text(encoding="utf-8").splitlines()
|
||||
|
||||
|
||||
def test_line_570_references_imp17_not_imp31():
|
||||
line = _lines()[569] # 1-indexed line 570
|
||||
assert "restructure" in line, f"line 570 anchor drifted: {line!r}"
|
||||
assert "IMP-17" in line, f"line 570 must reference IMP-17 (carve-out): {line!r}"
|
||||
assert "IMP-31" not in line, f"line 570 must not reference non-existent IMP-31: {line!r}"
|
||||
|
||||
|
||||
def test_line_571_still_references_imp29():
|
||||
line = _lines()[570] # 1-indexed line 571
|
||||
assert "reject" in line, f"line 571 anchor drifted: {line!r}"
|
||||
assert "IMP-29" in line, f"line 571 must still reference IMP-29 frontend override: {line!r}"
|
||||
@@ -0,0 +1,348 @@
|
||||
"""P0-3 (2026-05-18) — orchestrator self-test minimum set.
|
||||
|
||||
Covers detect_agent (the bug that caused #45 infinite loop), parse_consensus,
|
||||
parse_remaining_units, IMPLEMENTATION_UNITS parsing, dual-write normalize.
|
||||
|
||||
Run: pytest -q tests/orchestrator_unit/
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add design_agent root to sys.path so we can import orchestrator.py
|
||||
ROOT = Path(__file__).parent.parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from orchestrator import (
|
||||
detect_agent,
|
||||
parse_consensus,
|
||||
parse_remaining_units,
|
||||
_is_execution_issue,
|
||||
)
|
||||
import re
|
||||
|
||||
|
||||
class TestExecutionIssueDetection:
|
||||
"""P1-4 — execution sub-issue title detection."""
|
||||
|
||||
def test_execution_korean_pattern(self):
|
||||
assert _is_execution_issue("[IMP-15 실행-1] image_aspect_mismatch") is True
|
||||
assert _is_execution_issue("[IMP-15 실행-2] table overflow") is True
|
||||
assert _is_execution_issue("[IMP-15 실행 3] something") is True
|
||||
|
||||
def test_execution_english_pattern(self):
|
||||
assert _is_execution_issue("[IMP-15 exec-1] image") is True
|
||||
assert _is_execution_issue("[IMP-15 EXEC 2] table") is True
|
||||
|
||||
def test_non_execution_title(self):
|
||||
assert _is_execution_issue("IMP-15 Step 14 visual_check 보강") is False
|
||||
assert _is_execution_issue("IMP-09 B-4 다른 layout zone-geometry") is False
|
||||
|
||||
def test_empty_title(self):
|
||||
assert _is_execution_issue("") is False
|
||||
assert _is_execution_issue(None) is False
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# detect_agent — the bug that caused #45 infinite loop
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestDetectAgent:
|
||||
def test_claude_header(self):
|
||||
assert detect_agent("[Claude #1] Stage 1 ...") == "claude"
|
||||
|
||||
def test_codex_header(self):
|
||||
assert detect_agent("[Codex #1] Stage 1 review") == "codex"
|
||||
|
||||
def test_codex_body_with_claude_citation(self):
|
||||
"""The exact bug from #45 — Codex body contains [Claude #N] citation in
|
||||
EVIDENCE section. Old detect_agent returned 'claude' (wrong)."""
|
||||
body = """[Codex #2] Stage 2 Round #1 simulation-plan verification
|
||||
|
||||
Verdict: NO.
|
||||
|
||||
=== EVIDENCE ===
|
||||
- Read current-stage Gitea comment `[Claude #2] Stage 2 Round #1 - Plan` only
|
||||
"""
|
||||
assert detect_agent(body) == "codex", \
|
||||
"Codex body containing [Claude #N] citation must still detect as codex"
|
||||
|
||||
def test_claude_body_with_codex_citation(self):
|
||||
body = """[Claude #3] Stage 2 Round #2 - Plan
|
||||
|
||||
Addressing [Codex #2] findings ...
|
||||
"""
|
||||
assert detect_agent(body) == "claude"
|
||||
|
||||
def test_empty_body(self):
|
||||
assert detect_agent("") is None
|
||||
assert detect_agent(None) is None
|
||||
assert detect_agent(" \n ") is None
|
||||
|
||||
def test_no_agent_header(self):
|
||||
assert detect_agent("This is some random text without any agent marker") is None
|
||||
|
||||
def test_leading_whitespace_before_header(self):
|
||||
body = " \n[Codex #1] header after whitespace"
|
||||
assert detect_agent(body) == "codex"
|
||||
|
||||
def test_header_must_be_at_start(self):
|
||||
"""Body that doesn't start with [Agent header should return None."""
|
||||
body = "Some intro text.\n[Codex #1] header on second line"
|
||||
# P0-1 fix: only first non-empty line is checked.
|
||||
# First line = "Some intro text." → no match → None
|
||||
assert detect_agent(body) is None
|
||||
|
||||
def test_header_with_hash_immediately(self):
|
||||
"""[Codex#1] (no space) should still match per regex \\[Codex[\\s#]."""
|
||||
assert detect_agent("[Codex#1] hello") == "codex"
|
||||
assert detect_agent("[Claude#5] hi") == "claude"
|
||||
|
||||
def test_audit_anchor_preface_breaks_detection(self):
|
||||
"""P5 (2026-05-20) — regression: AUDIT-ONLY mode 의 'Audit anchor:' preface 가
|
||||
첫 줄에 박히면 detect_agent 는 None 반환 (P0-1 strict 의도된 동작).
|
||||
이게 #56 (INTEGRATION-AUDIT-02) 의 Stage 4 Round #14 infinite loop 의 직접 원인.
|
||||
해결책 = detect_agent 완화 X, AUDIT_ONLY_NOTE 가 agent header 를 first line 으로 강제."""
|
||||
body_anchor_first = (
|
||||
"Audit anchor: This audit verifies pipeline contracts...\n"
|
||||
"It does not implement runtime code.\n"
|
||||
"\n"
|
||||
"[Codex #14] Stage 4 (test-verify) Round #14 - INTEGRATION-AUDIT-02\n"
|
||||
"\n"
|
||||
"Verdict: PASS. Stage 3 satisfies all criteria.\n"
|
||||
"FINAL_CONSENSUS: YES\n"
|
||||
)
|
||||
assert detect_agent(body_anchor_first) is None, (
|
||||
"audit anchor preface as first line MUST cause detect_agent None "
|
||||
"(P0-1 strict). Fix path: comment format, not detect_agent."
|
||||
)
|
||||
|
||||
def test_audit_anchor_after_header_works(self):
|
||||
"""P5 (2026-05-20) — 올바른 format: agent header first line, anchor line 2+."""
|
||||
body_header_first = (
|
||||
"[Codex #14] Stage 4 (test-verify) Round #14 - INTEGRATION-AUDIT-02\n"
|
||||
"\n"
|
||||
"Audit anchor: This audit verifies pipeline contracts...\n"
|
||||
"\n"
|
||||
"Verdict: PASS.\n"
|
||||
"FINAL_CONSENSUS: YES\n"
|
||||
)
|
||||
assert detect_agent(body_header_first) == "codex"
|
||||
|
||||
# P5b (2026-05-20) — Stage 2 compact-plan first-line conflict regression.
|
||||
# #24 IMP-24 K6: Codex r1~r3 가 첫 줄을 '=== IMPLEMENTATION_UNITS ===' 로 시작 →
|
||||
# detect_agent None → orchestrator silent loop. fix path = comment format strict,
|
||||
# NOT detect_agent 완화 (P0-1 강화 그대로 유지).
|
||||
|
||||
def test_implementation_units_first_line_breaks_detection(self):
|
||||
"""=== IMPLEMENTATION_UNITS === 가 첫 줄이면 detect_agent None (P0-1 strict 정상 동작)."""
|
||||
body = (
|
||||
"=== IMPLEMENTATION_UNITS ===\n"
|
||||
"- id: u1\n"
|
||||
" summary: ...\n"
|
||||
" files:\n"
|
||||
" - docs/architecture/PHASE-Q-AUDIT.md\n"
|
||||
" tests:\n"
|
||||
" - pytest -q tests\n"
|
||||
" estimate_lines: 1\n"
|
||||
"\n"
|
||||
"FINAL_CONSENSUS: YES\n"
|
||||
)
|
||||
assert detect_agent(body) is None, (
|
||||
"=== IMPLEMENTATION_UNITS === as first line MUST cause detect_agent None "
|
||||
"(P0-1 strict). Fix path: enforce agent header first-line in prompt, not relax detect_agent."
|
||||
)
|
||||
|
||||
def test_compact_plan_with_header_first_works(self):
|
||||
"""올바른 Stage 2 compact format: [Codex #N] 첫 줄 → === IMPLEMENTATION_UNITS === 둘째 줄+."""
|
||||
body = (
|
||||
"[Codex #4] Stage 2 simulation-plan review - IMP-24 K6\n"
|
||||
"\n"
|
||||
"=== IMPLEMENTATION_UNITS ===\n"
|
||||
"- id: u1\n"
|
||||
" summary: ...\n"
|
||||
" tests:\n"
|
||||
" - pytest -q tests\n"
|
||||
"\n"
|
||||
"FINAL_CONSENSUS: YES\n"
|
||||
)
|
||||
assert detect_agent(body) == "codex"
|
||||
|
||||
def test_markdown_prefix_breaks_detection(self):
|
||||
"""P5b — `## [Codex #N]` 같은 markdown header prefix 도 detect_agent None.
|
||||
(#21 Stage 4 에서 관찰된 latent silent loop 원인.)"""
|
||||
body_hash = "## [Codex #1] Stage 4 test-verify Round #1\n\nVerdict: PASS\n"
|
||||
body_emoji = "📌 **[Claude #1] Stage 2 plan**\n\nbody\n"
|
||||
body_bold = "**[Codex #1] Stage 4**\n\nbody\n"
|
||||
assert detect_agent(body_hash) is None
|
||||
assert detect_agent(body_emoji) is None
|
||||
assert detect_agent(body_bold) is None
|
||||
|
||||
|
||||
class TestRulesAndCompactPlanFirstLineContract:
|
||||
"""P5b (2026-05-20) — RULES 와 COMPACT_PLAN_RULE 둘 다 first-line agent header
|
||||
rule 을 명시해야 함. wording 검증."""
|
||||
|
||||
def test_rules_has_first_line_strict(self):
|
||||
from orchestrator import RULES
|
||||
# RULES 안에 first-line strict + 모든 stage 적용 명시 있어야 함.
|
||||
assert "FIRST non-empty line" in RULES
|
||||
assert "[Claude #N]" in RULES and "[Codex #N]" in RULES
|
||||
# P5b OVERRIDES 키워드 — body rule 들이 first-line rule 보다 우선하지 않음을 강조
|
||||
assert "OVERRIDES" in RULES or "overrides" in RULES.lower()
|
||||
|
||||
def test_compact_plan_rule_carves_out_first_line(self):
|
||||
from orchestrator import COMPACT_PLAN_RULE
|
||||
# "body" 는 first-line agent header 다음부터 시작한다고 명시
|
||||
assert "FIRST non-empty line" in COMPACT_PLAN_RULE or "first-line agent header" in COMPACT_PLAN_RULE
|
||||
# "after the first-line" 같은 carve-out wording 검증
|
||||
body_lower = COMPACT_PLAN_RULE.lower()
|
||||
assert "after the first" in body_lower or "after the agent header" in body_lower
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# parse_consensus — YES/NO + rewind_target
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseConsensus:
|
||||
def test_yes_only(self):
|
||||
body = "Some text.\nFINAL_CONSENSUS: YES"
|
||||
assert parse_consensus(body) == ("YES", None)
|
||||
|
||||
def test_no_with_rewind_target(self):
|
||||
body = "Some text.\nrewind_target: stage_2_plan\nFINAL_CONSENSUS: NO"
|
||||
assert parse_consensus(body) == ("NO", "stage_2_plan")
|
||||
|
||||
def test_no_with_continue_same(self):
|
||||
body = "blah\nrewind_target: continue_same\nFINAL_CONSENSUS: NO"
|
||||
assert parse_consensus(body) == ("NO", "continue_same")
|
||||
|
||||
def test_no_target_only_in_last_10_lines(self):
|
||||
"""parse_consensus only scans last 10 lines."""
|
||||
body = "rewind_target: stage_1_review\n" + "\n".join(["filler"] * 20) + "\nFINAL_CONSENSUS: NO"
|
||||
status, target = parse_consensus(body)
|
||||
assert status == "NO"
|
||||
assert target is None # too far from end to be picked up
|
||||
|
||||
def test_no_consensus_marker(self):
|
||||
assert parse_consensus("just text, no marker") == (None, None)
|
||||
|
||||
def test_empty_body(self):
|
||||
assert parse_consensus("") == (None, None)
|
||||
assert parse_consensus(None) == (None, None)
|
||||
|
||||
def test_unknown_rewind_target_ignored(self):
|
||||
body = "rewind_target: bogus_target\nFINAL_CONSENSUS: NO"
|
||||
status, target = parse_consensus(body)
|
||||
assert status == "NO"
|
||||
assert target is None # bogus is not in REWIND_TARGET_TO_SID
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# parse_remaining_units — Stage 3 continue_same progress detection
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestParseRemainingUnits:
|
||||
def test_bracketed_list(self):
|
||||
body = "Remaining units: [u2, u3, u4]"
|
||||
assert parse_remaining_units(body) == {"u2", "u3", "u4"}
|
||||
|
||||
def test_comma_list_no_brackets(self):
|
||||
body = "Remaining units: u5, u6, u7"
|
||||
assert parse_remaining_units(body) == {"u5", "u6", "u7"}
|
||||
|
||||
def test_none_explicit(self):
|
||||
assert parse_remaining_units("Remaining units: none") == set()
|
||||
assert parse_remaining_units("Remaining units: []") == set()
|
||||
assert parse_remaining_units("Remaining units: (none)") == set()
|
||||
assert parse_remaining_units("Remaining units: -") == set()
|
||||
|
||||
def test_line_not_present(self):
|
||||
assert parse_remaining_units("no remaining units mentioned here") is None
|
||||
|
||||
def test_case_insensitive(self):
|
||||
body = "REMAINING UNITS: [U1, U2]"
|
||||
assert parse_remaining_units(body) == {"u1", "u2"}
|
||||
|
||||
def test_only_u_prefixed_digits(self):
|
||||
"""Sentence noise ignored — only u\\d+ pattern matched."""
|
||||
body = "Remaining units: I still need to do u3 and u7 work"
|
||||
assert parse_remaining_units(body) == {"u3", "u7"}
|
||||
|
||||
def test_empty_body(self):
|
||||
assert parse_remaining_units("") is None
|
||||
assert parse_remaining_units(None) is None
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# IMPLEMENTATION_UNITS block parsing (used in Stage 2 YES guard)
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestImplementationUnitsBlock:
|
||||
"""Reproduces the parser in run_stage Stage 2 YES guard (line ~810)."""
|
||||
|
||||
def _parse(self, body):
|
||||
iu_block_pat = re.compile(
|
||||
r"===\s*IMPLEMENTATION_UNITS\s*===\s*\n(.*?)(?=\n===\s|\Z)",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
iu_unit_pat = re.compile(r"^\s*-\s*id:\s*u\d+", re.IGNORECASE | re.MULTILINE)
|
||||
m = iu_block_pat.search(body or "")
|
||||
return bool(m and iu_unit_pat.search(m.group(1)))
|
||||
|
||||
def test_valid_block(self):
|
||||
body = """text
|
||||
|
||||
=== IMPLEMENTATION_UNITS ===
|
||||
- id: u1
|
||||
summary: ...
|
||||
- id: u2
|
||||
summary: ...
|
||||
"""
|
||||
assert self._parse(body) is True
|
||||
|
||||
def test_empty_block(self):
|
||||
body = "=== IMPLEMENTATION_UNITS ===\n(no entries)\n"
|
||||
assert self._parse(body) is False # header but no - id: uN entry
|
||||
|
||||
def test_block_missing(self):
|
||||
body = "just text, no implementation_units"
|
||||
assert self._parse(body) is False
|
||||
|
||||
def test_block_with_only_non_u_entries(self):
|
||||
body = """=== IMPLEMENTATION_UNITS ===
|
||||
- id: alpha
|
||||
summary: ...
|
||||
"""
|
||||
assert self._parse(body) is False # 'alpha' is not 'u\\d+'
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# Direct integration check — the #45 bug case
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestRegressionForIssue45Bug:
|
||||
"""Verify the exact body shape that caused #45 infinite loop is now handled."""
|
||||
|
||||
def test_codex_no_with_claude_citation_full_flow(self):
|
||||
body = """[Codex #3] Stage 2 Round #2 simulation-plan verification for issue #45
|
||||
|
||||
Verdict: NO. The plan covers main axes but violates two Stage 2 requirements.
|
||||
|
||||
Findings:
|
||||
- Unit u1 declares tests: [] in === IMPLEMENTATION_UNITS ===
|
||||
- xfail-strict mechanism unclear
|
||||
|
||||
=== EVIDENCE ===
|
||||
Commands run:
|
||||
- git rev-parse HEAD
|
||||
- Read current-stage Gitea comment `[Claude #3] Stage 2 Round #2 - Plan`
|
||||
|
||||
rewind_target: stage_2_plan
|
||||
FINAL_CONSENSUS: NO
|
||||
"""
|
||||
# P0-1 fix: detect_agent reads only first line → "[Codex #3]" → codex
|
||||
assert detect_agent(body) == "codex", "P0-1 regression test"
|
||||
# parse_consensus: NO + rewind_target stage_2_plan
|
||||
status, target = parse_consensus(body)
|
||||
assert status == "NO"
|
||||
assert target == "stage_2_plan"
|
||||
@@ -0,0 +1,310 @@
|
||||
"""P3-5 (2026-05-18) — subprocess cleanup hardening verification.
|
||||
|
||||
Covers:
|
||||
C1: 정상 종료 → tree 잔류 0
|
||||
C2: timeout → TimeoutExpired raise + 자손 0
|
||||
C3: grandchild spawn 후 parent timeout → grandchild 정리
|
||||
C4: 외부 (orchestrator 가 spawn 안한) 프로세스 보호
|
||||
C5: _kill_process_tree(self.pid) 호출해도 orchestrator 자살 안 함
|
||||
C6 (CORE): parent 정상 종료 후 grandchild orphan 정리 — PID 2780 regression
|
||||
|
||||
Run: pytest -q tests/orchestrator_unit/test_subprocess_cleanup.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from orchestrator import (
|
||||
_kill_process_tree,
|
||||
_kill_tracked,
|
||||
_run_with_tree_kill,
|
||||
_proc_signature,
|
||||
_is_same_process,
|
||||
_SPAWNED,
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _py():
|
||||
"""Path to current Python interpreter — used to spawn dummy subprocesses."""
|
||||
return sys.executable
|
||||
|
||||
def _alive(pid):
|
||||
try:
|
||||
return psutil.Process(pid).is_running() and psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
|
||||
except psutil.NoSuchProcess:
|
||||
return False
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# Signature helpers
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestSignatureHelpers:
|
||||
def test_proc_signature_alive(self):
|
||||
p = psutil.Process(os.getpid())
|
||||
sig = _proc_signature(p)
|
||||
assert sig is not None
|
||||
assert sig[0] == os.getpid()
|
||||
assert isinstance(sig[1], float)
|
||||
|
||||
def test_is_same_process_orch_self_blocked(self):
|
||||
"""C5 prep — orchestrator 자기 자신은 절대 same-process true 안 됨."""
|
||||
p = psutil.Process(os.getpid())
|
||||
sig = _proc_signature(p)
|
||||
# _is_same_process 가 _ORCH_PID 체크로 False 반환해야 함.
|
||||
assert _is_same_process(sig[0], sig[1]) is False
|
||||
|
||||
def test_is_same_process_dead_pid(self):
|
||||
# 사용 가능성 낮은 PID 999999 — 거의 확실히 죽음.
|
||||
assert _is_same_process(999999, time.time()) is False
|
||||
|
||||
def test_is_same_process_wrong_create_time(self):
|
||||
"""PID 재사용 회피 검증 — 같은 PID 라도 create_time 안 맞으면 False."""
|
||||
# 살아있는 외부 프로세스 빌려서 일부러 어긋난 create_time 으로 호출.
|
||||
# System Idle 같은 특수 프로세스 (create_time=0) 회피 — 우리가 띄운 dummy 사용.
|
||||
dummy = subprocess.Popen(
|
||||
[_py(), "-c", "import time; time.sleep(5)"],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
# 실제 create_time 보다 1 년 전 시각 → 명백한 mismatch.
|
||||
far_past = time.time() - 365 * 24 * 3600
|
||||
assert _is_same_process(dummy.pid, far_past) is False
|
||||
# 맞는 create_time 으로는 True 여야 함 (sanity).
|
||||
real_ct = psutil.Process(dummy.pid).create_time()
|
||||
assert _is_same_process(dummy.pid, real_ct) is True
|
||||
finally:
|
||||
dummy.kill()
|
||||
dummy.wait(timeout=5)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# C1: 정상 종료 — tree 잔류 0
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestC1_NormalExit:
|
||||
def test_dummy_short_run_no_residue(self):
|
||||
r = _run_with_tree_kill(
|
||||
[_py(), "-c", "import time; time.sleep(0.3)"],
|
||||
timeout=10,
|
||||
)
|
||||
assert r.returncode == 0
|
||||
# 호출 후 _SPAWNED 에 우리 호출 잔재가 남으면 안 됨 (wrapper 가 discard).
|
||||
# 다른 테스트 영향 가능성 있어서 set 전체가 0 이 아니어도 됨, 단 우리 잔재 없으면 OK.
|
||||
# 보수적으로 — 우리 호출 직전에 _SPAWNED 가 비어있었으면 직후에도 비어있어야 함.
|
||||
assert len(_SPAWNED) == 0
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# C2: Timeout — TimeoutExpired raise + 자손 정리
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestC2_Timeout:
|
||||
def test_dummy_long_sleep_times_out(self):
|
||||
with pytest.raises(subprocess.TimeoutExpired):
|
||||
_run_with_tree_kill(
|
||||
[_py(), "-c", "import time; time.sleep(60)"],
|
||||
timeout=1.5,
|
||||
)
|
||||
# raise 후에도 _SPAWNED 우리 잔재 없어야 함 (wrapper finally 가 discard).
|
||||
assert len(_SPAWNED) == 0
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# C3: grandchild orphan 정리 — parent timeout path
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestC3_GrandchildTimeoutPath:
|
||||
def test_grandchild_killed_on_parent_timeout(self):
|
||||
# parent 가 grandchild 띄우고 자기는 sleep — timeout 으로 강제 종료.
|
||||
# grandchild 도 정리돼야 함.
|
||||
# PID 캡처를 위해 grandchild 가 자기 PID 를 파일에 기록.
|
||||
marker = ROOT / ".orchestrator" / "tmp" / "test_c3_gc_pid.txt"
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
if marker.exists(): marker.unlink()
|
||||
|
||||
# grandchild 의 stdin/stdout/stderr 를 DEVNULL 로 분리 — production 의 claude.exe→python.exe -
|
||||
# 케이스와 동일 (grandchild 가 wrapper 의 pipe 핸들 안 상속). 안 그러면 pipe inheritance 로
|
||||
# communicate() 가 hang.
|
||||
spawn_code = (
|
||||
f"import subprocess, time, sys, os; "
|
||||
f"gc = subprocess.Popen("
|
||||
f" [sys.executable, '-c', 'import time; time.sleep(60)'], "
|
||||
f" stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); "
|
||||
f"open(r'{marker}', 'w').write(str(gc.pid)); "
|
||||
f"time.sleep(60)"
|
||||
)
|
||||
|
||||
with pytest.raises(subprocess.TimeoutExpired):
|
||||
_run_with_tree_kill(
|
||||
[_py(), "-c", spawn_code],
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
# marker 파일에서 grandchild PID 읽기.
|
||||
assert marker.exists(), "grandchild marker not written — parent died too early"
|
||||
gc_pid = int(marker.read_text().strip())
|
||||
|
||||
# 잠시 대기 (cleanup 비동기 가능성) 후 grandchild 죽었는지 확인.
|
||||
deadline = time.time() + 5
|
||||
while time.time() < deadline and _alive(gc_pid):
|
||||
time.sleep(0.2)
|
||||
assert not _alive(gc_pid), f"grandchild PID {gc_pid} still alive after parent timeout"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# C4: 외부 프로세스 보호
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestC4_ExternalProcessProtection:
|
||||
def test_outsider_not_killed(self):
|
||||
# 사용자가 직접 띄운 척하는 외부 프로세스 (orchestrator 가 spawn 안 함).
|
||||
outsider = subprocess.Popen([_py(), "-c", "import time; time.sleep(10)"])
|
||||
try:
|
||||
# _kill_tracked 에 외부 PID 의 (잘못된) signature 넘기면 무시돼야 함.
|
||||
# signature 일치 안 하면 _is_same_process False → kill 안 됨.
|
||||
wrong_sig = [(outsider.pid, 0.0)] # create_time 안 맞음
|
||||
cleaned = _kill_tracked(wrong_sig)
|
||||
assert cleaned == 0
|
||||
assert _alive(outsider.pid), "outsider killed despite wrong create_time"
|
||||
finally:
|
||||
outsider.kill()
|
||||
outsider.wait(timeout=5)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# C5: orchestrator 자살 방지
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestC5_SelfKillProtection:
|
||||
def test_kill_process_tree_self_pid_noop(self):
|
||||
"""orchestrator(=pytest) PID 로 _kill_process_tree 호출해도 죽으면 안 됨."""
|
||||
result = _kill_process_tree(os.getpid())
|
||||
assert result == 0 # ORCH_PID 검사로 즉시 0 반환
|
||||
|
||||
def test_kill_tracked_with_orch_pid_noop(self):
|
||||
# 일부러 self signature 를 tracked 에 넣어도 _is_same_process False → skip.
|
||||
self_p = psutil.Process(os.getpid())
|
||||
self_sig = _proc_signature(self_p)
|
||||
cleaned = _kill_tracked([self_sig])
|
||||
assert cleaned == 0 # 자기 자신 보호
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# C6 (CORE): parent 정상 종료 후 grandchild orphan 정리
|
||||
# — PID 2780 regression test
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestC6_OrphanGrandchildAfterNormalExit:
|
||||
"""PID 2780 path: parent 가 정상 exit 했는데 grandchild 만 살아남는 케이스.
|
||||
monitor thread 가 parent 살아있을 때 grandchild 를 미리 추적해서 finally 에서 정리해야 함."""
|
||||
|
||||
def test_grandchild_killed_after_parent_normal_exit(self):
|
||||
marker = ROOT / ".orchestrator" / "tmp" / "test_c6_gc_pid.txt"
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
if marker.exists(): marker.unlink()
|
||||
|
||||
# parent 가:
|
||||
# 1. grandchild 띄움 (DEVNULL 격리 — production claude.exe→python.exe - 과 동등).
|
||||
# 2. PID 마커에 기록.
|
||||
# 3. monitor 가 1초 polling 으로 catch 할 시간 확보 (2.5초 sleep).
|
||||
# 4. 정상 종료.
|
||||
spawn_code = (
|
||||
f"import subprocess, time, sys, os; "
|
||||
f"gc = subprocess.Popen("
|
||||
f" [sys.executable, '-c', 'import time; time.sleep(60)'], "
|
||||
f" stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); "
|
||||
f"open(r'{marker}', 'w').write(str(gc.pid)); "
|
||||
f"time.sleep(2.5); "
|
||||
f"sys.exit(0)"
|
||||
)
|
||||
|
||||
# 정상 종료 (timeout 안 걸림) — wrapper 의 finally cleanup 만으로 grandchild 잡혀야 함.
|
||||
r = _run_with_tree_kill(
|
||||
[_py(), "-c", spawn_code],
|
||||
timeout=15,
|
||||
)
|
||||
assert r.returncode == 0, "parent did not exit normally"
|
||||
|
||||
# marker 에서 grandchild PID.
|
||||
assert marker.exists(), "grandchild marker missing"
|
||||
gc_pid = int(marker.read_text().strip())
|
||||
|
||||
# 정리 비동기 가능성 → 짧게 대기 후 확인.
|
||||
deadline = time.time() + 5
|
||||
while time.time() < deadline and _alive(gc_pid):
|
||||
time.sleep(0.2)
|
||||
assert not _alive(gc_pid), (
|
||||
f"REGRESSION: grandchild PID {gc_pid} survived parent normal exit "
|
||||
f"(PID 2780 path not fixed)"
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# C7: input + encoding path — run_claude 가 실제 사용하는 호출 모드.
|
||||
# 2026-05-18 production bug: str input + encoding="utf-8" 일 때
|
||||
# wrapper 가 input 을 강제로 bytes 인코딩 → Popen text mode pipe 에
|
||||
# bytes 쓰려다 TypeError: write() argument must be str, not bytes.
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestC7_InputEncodingPath:
|
||||
def test_str_input_with_encoding_utf8(self):
|
||||
"""run_claude 와 동일한 호출 모드 — input=str + encoding='utf-8'."""
|
||||
# stdin 에서 읽은 그대로 stdout 으로 echo. 한글 포함해서 encoding 검증.
|
||||
r = _run_with_tree_kill(
|
||||
[_py(), "-c", "import sys; sys.stdout.write(sys.stdin.read())"],
|
||||
input="hello 안녕\n",
|
||||
encoding="utf-8",
|
||||
timeout=10,
|
||||
)
|
||||
assert r.returncode == 0
|
||||
# encoding= 모드면 stdout 는 str 이어야 함.
|
||||
assert isinstance(r.stdout, str)
|
||||
assert "hello" in r.stdout
|
||||
assert "안녕" in r.stdout
|
||||
|
||||
def test_bytes_input_without_encoding(self):
|
||||
"""encoding 없으면 binary mode — input=bytes 그대로 통과."""
|
||||
r = _run_with_tree_kill(
|
||||
[_py(), "-c", "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())"],
|
||||
input=b"raw bytes",
|
||||
timeout=10,
|
||||
)
|
||||
assert r.returncode == 0
|
||||
assert isinstance(r.stdout, bytes)
|
||||
assert r.stdout == b"raw bytes"
|
||||
|
||||
def test_str_input_without_encoding_auto_encoded(self):
|
||||
"""input=str 인데 encoding 없으면 wrapper 가 자동 utf-8 인코딩."""
|
||||
r = _run_with_tree_kill(
|
||||
[_py(), "-c", "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())"],
|
||||
input="auto encode 한글",
|
||||
timeout=10,
|
||||
)
|
||||
assert r.returncode == 0
|
||||
assert isinstance(r.stdout, bytes)
|
||||
assert r.stdout.decode("utf-8") == "auto encode 한글"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
# Bonus: _SPAWNED discipline — 다중 호출 후 누적 안 됨
|
||||
# ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestSpawnedDiscipline:
|
||||
def test_spawned_drained_between_calls(self):
|
||||
for _ in range(3):
|
||||
_run_with_tree_kill([_py(), "-c", "pass"], timeout=10)
|
||||
# 3 회 호출 후에도 우리 잔재 없음 (wrapper finally 가 discard).
|
||||
assert len(_SPAWNED) == 0
|
||||
Reference in New Issue
Block a user