using System.Text.Json; using Recordingtest.Runner; using Xunit; namespace Recordingtest.Runner.Tests; public class TestRunnerTests : IDisposable { private readonly string _root; public TestRunnerTests() { _root = Path.Combine(Path.GetTempPath(), "rt-runner-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(_root); } public void Dispose() { try { Directory.Delete(_root, true); } catch { } } private (string scenariosDir, string baselinesDir, string outDir) MakeDirs() { var s = Path.Combine(_root, "scenarios"); var b = Path.Combine(_root, "baselines"); var o = Path.Combine(_root, "out"); Directory.CreateDirectory(s); Directory.CreateDirectory(b); Directory.CreateDirectory(o); return (s, b, o); } private static string ScenarioYaml(string name) => $@"name: {name} description: test sut: exe: dummy.exe steps: - kind: save value: ctrl+s "; private static void WriteScenario(string dir, string name) => File.WriteAllText(Path.Combine(dir, name + ".yaml"), ScenarioYaml(name)); [Fact] public void TwoScenarios_BothIdentical_ExitZero_AllPass() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); WriteScenario(sDir, "beta"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); File.WriteAllText(Path.Combine(bDir, "beta.json"), content); var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true)); Assert.Equal(2, report.Total); Assert.Equal(2, report.Passed); Assert.Equal(0, report.Failed); Assert.Equal(0, TestRunner.ToExitCode(report)); } [Fact] public void OneScenarioDiffers_ExitOne_HunkCount() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: false, hunkCount: 1)); Assert.Equal(1, TestRunner.ToExitCode(report)); Assert.Equal("fail", report.Scenarios[0].Status); Assert.Equal(1, report.Scenarios[0].Hunks); } [Fact] public void PlayerThrows_ExitTwo_ErrorStatus() { var (sDir, bDir, oDir) = MakeDirs(); // scenario with a click step so the throw triggers var name = "boom"; var yaml = @"name: boom sut: exe: dummy.exe steps: - kind: click target: uia_path: /Window offset: [0.5, 0.5] "; File.WriteAllText(Path.Combine(sDir, name + ".yaml"), yaml); var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}", throwOnClick: true), new SpyNormalizer(), new StubDiffer(identical: true)); Assert.True(report.Errored >= 1); Assert.Equal(2, TestRunner.ToExitCode(report)); } [Fact] public void EmptyScenariosDir_ExitZero_TotalZero() { var (sDir, bDir, oDir) = MakeDirs(); var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}"), new SpyNormalizer(), new StubDiffer(identical: true)); Assert.Equal(0, report.Total); Assert.Equal(0, TestRunner.ToExitCode(report)); } [Fact] public void ProfileOverride_IsPassedToNormalizer() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var spy = new SpyNormalizer(); var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir, Profile = "strict" }; new TestRunner().RunAll(opts, new FakeHostFactory(content), spy, new StubDiffer(identical: true)); Assert.Contains("strict", spy.Profiles); Assert.DoesNotContain("default", spy.Profiles); } [Fact] public void ReportJson_HasExpectedSchema_And_ReportMd_Exists() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true)); var jsonPath = Path.Combine(oDir, "report.json"); var mdPath = Path.Combine(oDir, "report.md"); Assert.True(File.Exists(jsonPath)); Assert.True(File.Exists(mdPath)); using var doc = JsonDocument.Parse(File.ReadAllText(jsonPath)); var root = doc.RootElement; Assert.True(root.TryGetProperty("runAt", out _)); Assert.True(root.TryGetProperty("total", out _)); Assert.True(root.TryGetProperty("passed", out _)); Assert.True(root.TryGetProperty("failed", out _)); Assert.True(root.TryGetProperty("errored", out _)); Assert.True(root.TryGetProperty("scenarios", out var scenarios)); var first = scenarios[0]; Assert.True(first.TryGetProperty("name", out _)); Assert.True(first.TryGetProperty("status", out _)); Assert.True(first.TryGetProperty("hunks", out _)); Assert.True(first.TryGetProperty("checkpointCount", out _)); Assert.True(first.TryGetProperty("artifactDir", out _)); } // ---- engine-bridge v3 sidecar tests ------------------------------ private sealed class FakeSidecarClient : IEngineStateSnapshotClient { public string? Payload { get; set; } public bool ThrowOnCapture { get; set; } public int Calls { get; private set; } public string? TryCapture() { Calls++; if (ThrowOnCapture) throw new InvalidOperationException("boom"); return Payload; } } [Fact] public void Sidecar_NullClient_SkippedStatus() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll( opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true), sidecarClient: null); var s = report.Scenarios[0]; Assert.False(s.SidecarCaptured); Assert.Equal("skipped", s.SidecarStatus); Assert.Equal("pass", s.Status); } [Fact] public void Sidecar_ClientReturnsNull_UnavailableStatus() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var fake = new FakeSidecarClient { Payload = null }; var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll( opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true), sidecarClient: fake); Assert.Equal(1, fake.Calls); var s = report.Scenarios[0]; Assert.False(s.SidecarCaptured); Assert.Equal("unavailable", s.SidecarStatus); Assert.Equal("pass", s.Status); // main result still wins } [Fact] public void Sidecar_Throws_UnavailableStatus_MainStillPasses() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var fake = new FakeSidecarClient { ThrowOnCapture = true }; var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll( opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true), sidecarClient: fake); var s = report.Scenarios[0]; Assert.False(s.SidecarCaptured); Assert.Equal("unavailable", s.SidecarStatus); Assert.Equal("pass", s.Status); } [Fact] public void Sidecar_Captured_NoBaseline_MissingBaseline_And_WritesReceivedFile() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var fake = new FakeSidecarClient { Payload = "{\"scene\":{\"object_count\":4}}" }; var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll( opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true), sidecarClient: fake); var s = report.Scenarios[0]; Assert.True(s.SidecarCaptured); Assert.Equal("missing_baseline", s.SidecarStatus); Assert.Equal("pass", s.Status); var receivedPath = Path.Combine(s.ArtifactDir, "engine-state.received.json"); Assert.True(File.Exists(receivedPath)); Assert.Contains("object_count", File.ReadAllText(receivedPath)); } [Fact] public void Sidecar_Captured_BaselineIdentical_PassPass() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); var sidecarPayload = "{\"scene\":{\"object_count\":4}}"; File.WriteAllText( Path.Combine(bDir, "alpha.engine-state.approved.json"), sidecarPayload); var fake = new FakeSidecarClient { Payload = sidecarPayload }; var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll( opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true), sidecarClient: fake); var s = report.Scenarios[0]; Assert.True(s.SidecarCaptured); Assert.Equal("pass", s.SidecarStatus); Assert.Equal(0, s.SidecarHunks); Assert.Equal("pass", s.Status); } [Fact] public void Sidecar_Captured_BaselineDivergent_PromotesScenarioToFail() { var (sDir, bDir, oDir) = MakeDirs(); WriteScenario(sDir, "alpha"); var content = "{\"x\":1}"; // main result matches baseline (identical differ below) File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); // sidecar baseline exists too File.WriteAllText( Path.Combine(bDir, "alpha.engine-state.approved.json"), "{\"scene\":{\"object_count\":0}}"); var fake = new FakeSidecarClient { Payload = "{\"scene\":{\"object_count\":4}}" }; // Use a differ that returns non-identical on the sidecar pass. // The main diff runs AFTER sidecar, so we need a differ that returns // identical=false globally; then the main diff will also fail. // For now we assert that sidecar hunks > 0 and status is "fail". var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; var report = new TestRunner().RunAll( opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: false, hunkCount: 2), sidecarClient: fake); var s = report.Scenarios[0]; Assert.True(s.SidecarCaptured); Assert.Equal("fail", s.SidecarStatus); Assert.Equal(2, s.SidecarHunks); Assert.Equal("fail", s.Status); } }