221 lines
9.3 KiB
Python
221 lines
9.3 KiB
Python
"""IMP-#85 u7 — subprocess smoke for mdx03 / mdx04 / mdx05 pipeline runs.
|
|
|
|
These smokes exercise the IMP-#85 catalog ↔ contract ↔ builder
|
|
invariant + runtime VP gate end-to-end against real MDX inputs:
|
|
|
|
* mdx03 — non-VP rank-1 path stays clean (exit 0).
|
|
* mdx04 — the original IMP-#85 hard-crash signature
|
|
(``BuilderMissingError ... PAYLOAD_BUILDERS has no such entry``)
|
|
is GONE. u1 converted the uncaught ``ValueError`` into a
|
|
``BuilderMissingError(FitError)`` subclass; the pipeline's
|
|
existing ``except FitError`` at ``src/phase_z2_pipeline.py:4436``
|
|
catches it and the zone is routed to
|
|
``adapter_needed (skip render)``. Anything that crashes
|
|
*downstream* of that routing (e.g. layout_css zone aggregation
|
|
when all live zones are adapter_needed) is a separate axis and
|
|
out of scope for this issue (see follow_up_issue_candidates).
|
|
* mdx05 — IMP-#87 u5 inversion. mdx05 has ZERO V4 evidence for any
|
|
section (``judgments_full32 = 0``, Case B per IMP-#87 Stage 1),
|
|
so the composition planner emits an IMP-#30 u4 EMPTY-SHELL
|
|
placeholder for the whole slide. Before IMP-#87 the pipeline
|
|
reported ``overall=PASS`` + ``full_mdx_coverage=True`` for this
|
|
state — the honesty defect this issue fixes. After IMP-#87 u2/u3
|
|
the same run elevates ``overall`` to
|
|
``EMPTY_SHELL_NO_CONTENT`` and the CLI exits 1 (BLOCKED). The old
|
|
exit-0 mdx05 smoke is therefore stale; this module now (a) keeps
|
|
mdx03 in the exit-0 non-VP parametrization, (b) adds a dedicated
|
|
mdx05 blocked-exit assertion that verifies the new
|
|
``EMPTY_SHELL_NO_CONTENT`` status surface, and (c) preserves the
|
|
IMP-#85 crash-marker guard on the mdx05 path so future
|
|
regressions cannot re-introduce the original uncaught
|
|
``BuilderMissingError`` propagation under cover of the blocked
|
|
exit.
|
|
|
|
Each subprocess gets a unique run_id so the runs do not collide on
|
|
disk when pytest is invoked concurrently or with -x retry.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
SAMPLES_DIR = REPO_ROOT / "samples" / "mdx_batch"
|
|
RUNS_DIR = REPO_ROOT / "data" / "runs"
|
|
|
|
# Original IMP-#85 crash signature (issue body verbatim). u1 converted
|
|
# the uncaught ``ValueError`` raised from the mapper's missing-builder
|
|
# branch into a ``BuilderMissingError(FitError)`` subclass that the
|
|
# pipeline catches. The string below was the marker of the uncaught
|
|
# propagation; it must no longer appear in stdout/stderr of a mdx04
|
|
# subprocess run.
|
|
IMP85_OLD_CRASH_MARKER = "PAYLOAD_BUILDERS has no such entry"
|
|
|
|
|
|
def _run_pipeline(mdx_name: str, run_id: str, timeout: int = 240) -> subprocess.CompletedProcess:
|
|
"""Spawn ``python -m src.phase_z2_pipeline <mdx> <run_id>`` and capture I/O."""
|
|
return subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"src.phase_z2_pipeline",
|
|
str(SAMPLES_DIR / mdx_name),
|
|
run_id,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
cwd=str(REPO_ROOT),
|
|
)
|
|
|
|
|
|
def _unique_run_id(prefix: str) -> str:
|
|
return f"{prefix}_imp85_smoke_{uuid.uuid4().hex[:8]}"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mdx_name,prefix",
|
|
[
|
|
("01.mdx", "mdx01"),
|
|
("02.mdx", "mdx02"),
|
|
("03.mdx", "mdx03"),
|
|
],
|
|
)
|
|
def test_non_vp_smoke_runs_clean(mdx_name: str, prefix: str) -> None:
|
|
"""mdx01/02/03 hit non-VP rank-1 frames; the pipeline runs to exit 0.
|
|
|
|
Non-VP rank-1 selection is the normal Phase Z path and the
|
|
primary regression guard that IMP-#85 u1-u6 do not perturb
|
|
mapper / pipeline behaviour for non-VP routes. IMP-#91 u1 extends
|
|
the parametrization from mdx03-only to the mdx01/02/03 acceptance
|
|
triple — closing the subprocess-axis coverage gap for the two
|
|
non-VP mdx that had only in-process B4 SHA parity coverage
|
|
(tests/regression/test_b4_mapper_source_sha_parity.py).
|
|
|
|
IMP-#87 u5 — mdx05 was removed from this parametrization because
|
|
its V4 evidence is empty for every aligned section (Case B,
|
|
Stage 1 lock). The IMP-#87 u2 ``EMPTY_SHELL_NO_CONTENT`` enum
|
|
+ u3 BLOCKED CLI exit make the post-IMP-#87 mdx05 run exit 1,
|
|
not 0, so an exit-0 parametrization would now be stale. The
|
|
dedicated mdx05 blocked-exit coverage lives in
|
|
``test_mdx05_blocked_exit_empty_shell_no_content`` below.
|
|
"""
|
|
cp = _run_pipeline(mdx_name, _unique_run_id(prefix))
|
|
assert cp.returncode == 0, (
|
|
f"{mdx_name} pipeline returncode={cp.returncode}\n"
|
|
f"--- stderr tail ---\n{cp.stderr[-1500:]}\n"
|
|
f"--- stdout tail ---\n{cp.stdout[-1500:]}"
|
|
)
|
|
|
|
|
|
def test_mdx05_blocked_exit_empty_shell_no_content() -> None:
|
|
"""mdx05 must exit 1 (BLOCKED) with ``overall=EMPTY_SHELL_NO_CONTENT``.
|
|
|
|
IMP-#87 u5 — mdx05 is the canonical Case B fixture (zero V4
|
|
evidence for any aligned section per Stage 1; ``judgments_full32 = 0``
|
|
in step05). The pre-IMP-#87 pipeline mislabelled this state as
|
|
``overall=PASS`` + ``full_mdx_coverage=True`` because the only
|
|
rendered unit was an IMP-#30 u4 EMPTY-SHELL placeholder
|
|
(``frame_template_id="__empty__"``) which trivially passes the
|
|
Selenium overflow check. IMP-#87 u1 splits content-rendered
|
|
coverage from legacy ``covered_section_ids``, u2 elevates the
|
|
overall enum to ``EMPTY_SHELL_NO_CONTENT`` before the legacy
|
|
ladder, and u3 routes that enum to a BLOCKED CLI exit (1).
|
|
|
|
This smoke pins the post-IMP-#87 contract on the real mdx05
|
|
pipeline run:
|
|
|
|
* subprocess returncode == 1 (BLOCKED, u3 axis A4).
|
|
* ``step20_slide_status.json`` ``overall`` ==
|
|
``"EMPTY_SHELL_NO_CONTENT"`` (u2 axis A3 precedence over the
|
|
legacy 4-way ladder).
|
|
* ``step20_slide_status.json`` ``full_mdx_coverage`` is False
|
|
(u1 axis A2 content-rendered coverage split).
|
|
* The IMP-#85 original crash marker
|
|
(``PAYLOAD_BUILDERS has no such entry``) is absent from both
|
|
stdout and stderr — the IMP-#85 crash-marker guard is
|
|
preserved on the mdx05 path even though mdx05 itself no
|
|
longer exits 0.
|
|
"""
|
|
run_id = _unique_run_id("mdx05")
|
|
cp = _run_pipeline("05.mdx", run_id)
|
|
|
|
assert cp.returncode == 1, (
|
|
f"mdx05 expected BLOCKED exit 1, got {cp.returncode}\n"
|
|
f"--- stderr tail ---\n{cp.stderr[-1500:]}\n"
|
|
f"--- stdout tail ---\n{cp.stdout[-1500:]}"
|
|
)
|
|
|
|
combined = cp.stdout + cp.stderr
|
|
assert IMP85_OLD_CRASH_MARKER not in combined, (
|
|
"IMP-#85 original crash signature regressed on mdx05 path:\n"
|
|
f"--- stderr tail ---\n{cp.stderr[-1500:]}\n"
|
|
f"--- stdout tail ---\n{cp.stdout[-1500:]}"
|
|
)
|
|
|
|
status_path = RUNS_DIR / run_id / "phase_z2" / "steps" / "step20_slide_status.json"
|
|
assert status_path.is_file(), (
|
|
f"mdx05 step20_slide_status.json not found at {status_path}\n"
|
|
f"--- stderr tail ---\n{cp.stderr[-1500:]}\n"
|
|
f"--- stdout tail ---\n{cp.stdout[-1500:]}"
|
|
)
|
|
status_payload = json.loads(status_path.read_text(encoding="utf-8"))
|
|
status_data = status_payload.get("data") or {}
|
|
assert status_data.get("overall") == "EMPTY_SHELL_NO_CONTENT", (
|
|
f"mdx05 overall expected EMPTY_SHELL_NO_CONTENT, got "
|
|
f"{status_data.get('overall')!r}"
|
|
)
|
|
assert status_data.get("full_mdx_coverage") is False, (
|
|
f"mdx05 full_mdx_coverage expected False, got "
|
|
f"{status_data.get('full_mdx_coverage')!r}"
|
|
)
|
|
|
|
|
|
def test_mdx04_no_longer_emits_imp85_crash_signature() -> None:
|
|
"""mdx04 must no longer surface the IMP-#85 uncaught crash marker.
|
|
|
|
Before u1: missing-builder ``ValueError``
|
|
(``'PAYLOAD_BUILDERS has no such entry'``) propagated uncaught and
|
|
killed the pipeline at the mapper call site
|
|
(``src/phase_z2_pipeline.py:4411-4413``, ``except FitError``
|
|
only). After u1: the mapper raises
|
|
``BuilderMissingError(FitError)``, the pipeline catches it at the
|
|
same ``except FitError`` block, and the zone is recorded under
|
|
``adapter_needed (skip render)``.
|
|
|
|
This smoke asserts only that the original IMP-#85 marker is gone
|
|
from both stdout and stderr — downstream crashes (e.g.
|
|
``build_layout_css`` zone aggregation when all live zones are
|
|
adapter_needed) belong to a separate axis and are tracked as a
|
|
follow-up issue candidate.
|
|
"""
|
|
cp = _run_pipeline("04.mdx", _unique_run_id("mdx04"))
|
|
combined = cp.stdout + cp.stderr
|
|
assert IMP85_OLD_CRASH_MARKER not in combined, (
|
|
"IMP-#85 original crash signature still present in pipeline output:\n"
|
|
f"--- stderr tail ---\n{cp.stderr[-1500:]}\n"
|
|
f"--- stdout tail ---\n{cp.stdout[-1500:]}"
|
|
)
|
|
|
|
|
|
def test_conftest_env_isolation_active_for_ai_fallback_defaults() -> None:
|
|
"""Direct assertion that ``tests/conftest.py`` isolated the AI
|
|
fallback env vars BEFORE ``src.config`` was first imported.
|
|
|
|
With ``AI_FALLBACK_ENABLED=true`` in the live ``.env``, the
|
|
Settings default-OFF contract would otherwise be violated whenever
|
|
a developer runs ``pytest -q tests`` against a checkout that has a
|
|
live operator ``.env``. This test pins the contract to the source
|
|
of truth (``src/config.py`` defaults).
|
|
"""
|
|
from src.config import Settings
|
|
|
|
s = Settings()
|
|
assert s.ai_fallback_enabled is False
|
|
assert s.ai_fallback_auto_cache is False
|