fix(orchestrator): P4a baseline-diff guard + Stage 5 commit scope

P4 had two production issues blocking #50 integration audit deployment: 1. Stage 3 guard had no baseline awareness — flagged ALL forbidden-path changes including pre-existing dirty WIP. Empirical: 328 such files already in current working tree (tests/matching/ artifacts etc). #50 would have hit reject loops immediately without Claude doing anything wrong. 2. Stage 5 had no commit-scope guard — if Claude ran `git add -A` and committed user's existing WIP, audit commit would be polluted with unrelated production changes. P4a additions: - _audit_baseline_path / _ensure_audit_baseline / _load_audit_baseline: snapshot working-tree dirty paths at run_issue entry for audit issues. Resumed runs preserve existing baseline (no overwrite). - _check_audit_only_violations(baseline=None): accept baseline set, subtract from violations — only flags NEW forbidden changes introduced after audit start. - _check_audit_commit_scope: verify HEAD commit's file list matches AUDIT_ALLOWED_COMMIT_GLOBS (INTEGRATION-AUDIT-*.md, BACKLOG.md). - run_issue: save baseline on audit-mode entry only — no impact on normal issues. - Stage 5 (commit-push) YES gate: new guard rejects on out-of-scope files with remediation prompt (git reset --soft + force-with-lease). 19 new tests: - baseline subtraction (5): pre-existing removed, None=keep-all, empty-set=catch-all, full-coverage filter, Windows path normalize. - baseline persist (5): roundtrip, no-overwrite on resume, missing fallback, corrupt JSON fallback, non-list fallback. - commit scope detection (7): report-only allowed, backlog allowed, src/ rejected, unrelated docs rejected, git error fail-open, Windows backslash, empty commit pass. - allowed globs sanity (2): every glob has audit marker, all under docs/architecture/. Total: 94/94 pytest pass (75 prior + 19 new). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-19 10:29:15 +09:00
parent 4289a500b6
commit e32f632464
2 changed files with 297 additions and 14 deletions
--- a/tests/orchestrator_unit/test_audit_mode.py
+++ b/tests/orchestrator_unit/test_audit_mode.py
@@ -22,8 +22,13 @@ from orchestrator import (  # noqa: E402
    _is_audit_issue,
    _audit_mode,
    _check_audit_only_violations,
+    _check_audit_commit_scope,
+    _ensure_audit_baseline,
+    _load_audit_baseline,
+    _audit_baseline_path,
    AUDIT_ONLY_FORBIDDEN_PREFIXES,
    AUDIT_ONLY_NOTE,
+    AUDIT_ALLOWED_COMMIT_GLOBS,
 )


@@ -199,6 +204,181 @@ class TestCheckAuditOnlyViolations:
 # AUDIT_ONLY_NOTE constants — sanity
 # ─────────────────────────────────────────────────────────────────

+# ─────────────────────────────────────────────────────────────────
+# P4a: baseline-aware violations
+# ─────────────────────────────────────────────────────────────────
+
+class TestBaselineAwareViolations:
+    def test_baseline_subtraction_removes_preexisting(self, monkeypatch):
+        """pre-existing forbidden path 는 baseline 에 있으면 violation 에서 제외."""
+        stdout = (
+            " M src/already_dirty.py\n"   # baseline 안에 있음 — 제외돼야 함
+            " M src/new_violation.py\n"   # baseline 밖 — violation
+        )
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        baseline = {"src/already_dirty.py"}
+        v = _check_audit_only_violations(baseline=baseline)
+        assert v == ["src/new_violation.py"]
+
+    def test_baseline_none_keeps_all(self, monkeypatch):
+        """baseline=None 이면 기존 동작 — 모든 forbidden 잡음."""
+        stdout = " M src/a.py\n M src/b.py\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        v = _check_audit_only_violations(baseline=None)
+        assert set(v) == {"src/a.py", "src/b.py"}
+
+    def test_baseline_empty_set_keeps_all(self, monkeypatch):
+        """baseline=set() 이면 모두 새 violation 으로 잡음."""
+        stdout = " M src/a.py\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        v = _check_audit_only_violations(baseline=set())
+        assert v == ["src/a.py"]
+
+    def test_baseline_filters_all_violations(self, monkeypatch):
+        """모든 violation 이 baseline 에 있으면 빈 list 반환 — clean 으로 판정."""
+        stdout = " M src/a.py\n M templates/b.html\n M tests/c.py\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        baseline = {"src/a.py", "templates/b.html", "tests/c.py"}
+        v = _check_audit_only_violations(baseline=baseline)
+        assert v == []
+
+    def test_baseline_path_normalized_match(self, monkeypatch):
+        """baseline 의 path 는 forward-slash 정규화 형태. Windows backslash 도 매치."""
+        stdout = " M src\\windows_path.py\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        baseline = {"src/windows_path.py"}  # baseline 도 forward-slash 형태로 저장
+        v = _check_audit_only_violations(baseline=baseline)
+        assert v == []
+
+
+# ─────────────────────────────────────────────────────────────────
+# P4a: _ensure_audit_baseline / _load_audit_baseline
+# ─────────────────────────────────────────────────────────────────
+
+class TestAuditBaselinePersist:
+    def test_save_and_load_roundtrip(self, monkeypatch, tmp_path):
+        """baseline 저장 → 로드 → 동일 path set 반환."""
+        # Redirect ORCH_DIR to tmp_path for isolation.
+        monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
+        # Mock git status output.
+        stdout = " M src/a.py\n?? src/b.py\n M docs/c.md\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        _ensure_audit_baseline(999)
+        loaded = _load_audit_baseline(999)
+        assert loaded == {"src/a.py", "src/b.py", "docs/c.md"}
+
+    def test_ensure_does_not_overwrite_existing(self, monkeypatch, tmp_path):
+        """이미 baseline 파일 있으면 덮어쓰지 않음 — resumed run 의 가드 일관성."""
+        monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
+        # First save with one set.
+        stdout1 = " M src/original.py\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout1))
+        _ensure_audit_baseline(999)
+        # Second call with DIFFERENT git status — should NOT overwrite.
+        stdout2 = " M src/different.py\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout2))
+        _ensure_audit_baseline(999)
+        loaded = _load_audit_baseline(999)
+        # Original baseline preserved.
+        assert loaded == {"src/original.py"}
+
+    def test_load_missing_returns_empty_set(self, monkeypatch, tmp_path):
+        monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
+        assert _load_audit_baseline(8888) == set()
+
+    def test_load_corrupt_returns_empty_set(self, monkeypatch, tmp_path):
+        monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
+        # Manually write corrupt JSON.
+        p = tmp_path / "audit_baseline_7777.json"
+        p.write_text("not valid json {{{", encoding="utf-8")
+        assert _load_audit_baseline(7777) == set()
+
+    def test_load_non_list_returns_empty_set(self, monkeypatch, tmp_path):
+        """baseline 파일이 list 가 아닌 다른 JSON (예: dict) 이면 empty set."""
+        monkeypatch.setattr(orchestrator, "ORCH_DIR", tmp_path)
+        p = tmp_path / "audit_baseline_6666.json"
+        p.write_text('{"unexpected": "shape"}', encoding="utf-8")
+        assert _load_audit_baseline(6666) == set()
+
+
+# ─────────────────────────────────────────────────────────────────
+# P4a: _check_audit_commit_scope — Stage 5 guard
+# ─────────────────────────────────────────────────────────────────
+
+class TestAuditCommitScope:
+    def test_clean_commit_audit_report_only(self, monkeypatch):
+        """audit report 파일만 commit 되면 통과."""
+        stdout = (
+            "docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
+            "docs/architecture/INTEGRATION-AUDIT-01-MATRIX.md\n"
+        )
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        assert _check_audit_commit_scope() == []
+
+    def test_backlog_update_allowed(self, monkeypatch):
+        stdout = (
+            "docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
+            "docs/architecture/PHASE-Z-IMPLEMENTATION-ISSUE-BACKLOG.md\n"
+        )
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        assert _check_audit_commit_scope() == []
+
+    def test_src_file_in_commit_detected(self, monkeypatch):
+        """audit commit 에 src/ 파일이 끼면 violation."""
+        stdout = (
+            "docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
+            "src/phase_z2_pipeline.py\n"
+        )
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        v = _check_audit_commit_scope()
+        assert v == ["src/phase_z2_pipeline.py"]
+
+    def test_unrelated_doc_detected(self, monkeypatch):
+        """docs/ 라도 audit 관련 아닌 doc 은 violation."""
+        stdout = (
+            "docs/architecture/INTEGRATION-AUDIT-01-REPORT.md\n"
+            "docs/some_other_doc.md\n"          # 다른 doc
+            "docs/architecture/PHASE-Z-PIPELINE-OVERVIEW.md\n"  # audit 와 무관한 doc
+        )
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        v = _check_audit_commit_scope()
+        assert set(v) == {"docs/some_other_doc.md",
+                          "docs/architecture/PHASE-Z-PIPELINE-OVERVIEW.md"}
+
+    def test_git_error_fails_open(self, monkeypatch):
+        """git show 자체 실패 → 빈 list (가드가 false positive 만들지 않음)."""
+        monkeypatch.setattr(subprocess, "run",
+                            lambda *a, **kw: _FakeCompleted(stdout="", returncode=128))
+        assert _check_audit_commit_scope() == []
+
+    def test_windows_backslash_normalized(self, monkeypatch):
+        """Windows backslash path 도 forward-slash 정규화 후 glob 매치."""
+        stdout = "docs\\architecture\\INTEGRATION-AUDIT-01-REPORT.md\n"
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=stdout))
+        assert _check_audit_commit_scope() == []
+
+    def test_empty_commit_passes(self, monkeypatch):
+        """commit 에 파일 변경 없음 (보통 안 일어나지만) — 위반 없음."""
+        monkeypatch.setattr(subprocess, "run", lambda *a, **kw: _FakeCompleted(stdout=""))
+        assert _check_audit_commit_scope() == []
+
+
+# ─────────────────────────────────────────────────────────────────
+# P4a: allowed-glob shape sanity
+# ─────────────────────────────────────────────────────────────────
+
+class TestAuditCommitAllowedGlobs:
+    def test_globs_have_audit_marker(self):
+        """모든 allowed glob 에 INTEGRATION-AUDIT 또는 BACKLOG 마커 존재."""
+        for g in AUDIT_ALLOWED_COMMIT_GLOBS:
+            assert ("INTEGRATION-AUDIT" in g) or ("BACKLOG" in g)
+
+    def test_globs_under_docs_architecture(self):
+        """모든 allowed path 가 docs/architecture/ 산하 — src/ 등 우발적 허용 차단."""
+        for g in AUDIT_ALLOWED_COMMIT_GLOBS:
+            assert g.startswith("docs/architecture/"), f"glob escapes docs/architecture/: {g}"
+
+
 class TestAuditOnlyConstants:
    def test_note_mentions_forbidden_prefixes(self):
        for p in AUDIT_ONLY_FORBIDDEN_PREFIXES: