diff --git a/docs/history/2026-04-07_이슈8-test-runner-generator.md b/docs/history/2026-04-07_이슈8-test-runner-generator.md new file mode 100644 index 0000000..432096b --- /dev/null +++ b/docs/history/2026-04-07_이슈8-test-runner-generator.md @@ -0,0 +1,32 @@ +# 2026-04-07 — test-runner Generator (Issue #8) + +- 이슈: #8 +- 작업: `Recordingtest.Runner` PoC 구현 (player + normalizer + diff-reporter 통합 파이프라인) +- 담당: Generator +- 소요 시간: 약 30분 +- Context 사용량: 약 60k tokens + +## 산출물 + +- `src/Recordingtest.Runner/` 콘솔 exe + - `RunnerOptions`, `RunReport`, `ScenarioResult` + - `INormalizer`, `IDiffer`, `IRunnerHostFactory` 어댑터 인터페이스 + - `DefaultNormalizer`, `DefaultDiffer`, `DefaultHostFactory` + - `TestRunner` (`RunAll`, `WriteJsonReport`, `WriteMarkdownReport`, `ToExitCode`) + - `Program` (CLI: `--scenarios/--baselines/--out/--profile/--no-launch`) +- `tests/Recordingtest.Runner.Tests/` (xUnit, 6 tests, all green) +- `recordingtest.sln`에 두 프로젝트 추가 + +## 검증 + +- `dotnet build` green +- `dotnet test` 6/6 통과 +- `Thread.Sleep` / `Task.Delay` 0건 (grep 확인) +- `report.json` 카멜케이스 + 스키마 테스트 포함 +- Exit code: 0 = all pass, 1 = any fail, 2 = any error + +## 메모 + +- 베이스라인은 사전 정규화되었다고 가정하지 않고, 매 실행 시 received와 동일 프로파일로 재정규화한다. + (TestRunner.cs 상단 주석 참조) +- `--no-launch` 의미는 player 책임이므로 runner는 옵션만 파싱해 보관한다. diff --git a/recordingtest.sln b/recordingtest.sln index cfb30b3..4c4a1df 100644 --- a/recordingtest.sln +++ b/recordingtest.sln @@ -23,6 +23,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Player", "src EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Player.Tests", "tests\Recordingtest.Player.Tests\Recordingtest.Player.Tests.csproj", "{7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Runner", "src\Recordingtest.Runner\Recordingtest.Runner.csproj", "{DADF0474-9EF3-4E8D-8139-93504E4F745D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Runner.Tests", "tests\Recordingtest.Runner.Tests\Recordingtest.Runner.Tests.csproj", "{6F9973EA-977A-4185-AF24-4E76D9D851C8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -153,6 +157,30 @@ Global {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x64.Build.0 = Release|Any CPU {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x86.ActiveCfg = Release|Any CPU {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x86.Build.0 = Release|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x64.ActiveCfg = Debug|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x64.Build.0 = Debug|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x86.ActiveCfg = Debug|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x86.Build.0 = Debug|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|Any CPU.Build.0 = Release|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x64.ActiveCfg = Release|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x64.Build.0 = Release|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x86.ActiveCfg = Release|Any CPU + {DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x86.Build.0 = Release|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x64.ActiveCfg = Debug|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x64.Build.0 = Debug|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x86.ActiveCfg = Debug|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x86.Build.0 = Debug|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|Any CPU.Build.0 = Release|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x64.ActiveCfg = Release|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x64.Build.0 = Release|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x86.ActiveCfg = Release|Any CPU + {6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -167,5 +195,7 @@ Global {74D292F5-8004-4946-8CC3-808AFD9C52C1} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {D8962656-55EC-4595-8F19-8FBBF9256A04} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {DADF0474-9EF3-4E8D-8139-93504E4F745D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {6F9973EA-977A-4185-AF24-4E76D9D851C8} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/src/Recordingtest.Runner/DefaultAdapters.cs b/src/Recordingtest.Runner/DefaultAdapters.cs new file mode 100644 index 0000000..410f0e9 --- /dev/null +++ b/src/Recordingtest.Runner/DefaultAdapters.cs @@ -0,0 +1,22 @@ +using Recordingtest.DiffReporter; +using Recordingtest.Player; +using Recordingtest.Player.Model; + +namespace Recordingtest.Runner; + +public sealed class DefaultNormalizer : INormalizer +{ + public string Normalize(string input, string profile, string? sidecarPath) + => Recordingtest.Normalizer.Normalizer.Normalize(input, profile, sidecarPath).Output; +} + +public sealed class DefaultDiffer : IDiffer +{ + public DiffResult Compare(string approvedPath, string receivedPath) + => Differ.Compare(approvedPath, receivedPath); +} + +public sealed class DefaultHostFactory : IRunnerHostFactory +{ + public IPlayerHost Create(Scenario scenario, string outDir) => new UiaPlayerHost(null, outDir); +} diff --git a/src/Recordingtest.Runner/Interfaces.cs b/src/Recordingtest.Runner/Interfaces.cs new file mode 100644 index 0000000..c75b038 --- /dev/null +++ b/src/Recordingtest.Runner/Interfaces.cs @@ -0,0 +1,20 @@ +using Recordingtest.DiffReporter; +using Recordingtest.Player; +using Recordingtest.Player.Model; + +namespace Recordingtest.Runner; + +public interface INormalizer +{ + string Normalize(string input, string profile, string? sidecarPath); +} + +public interface IDiffer +{ + DiffResult Compare(string approvedPath, string receivedPath); +} + +public interface IRunnerHostFactory +{ + IPlayerHost Create(Scenario scenario, string outDir); +} diff --git a/src/Recordingtest.Runner/Program.cs b/src/Recordingtest.Runner/Program.cs new file mode 100644 index 0000000..f2684d7 --- /dev/null +++ b/src/Recordingtest.Runner/Program.cs @@ -0,0 +1,42 @@ +namespace Recordingtest.Runner; + +public static class Program +{ + public static int Main(string[] args) + { + var options = new RunnerOptions(); + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--scenarios": options.ScenariosDir = args[++i]; break; + case "--baselines": options.BaselinesDir = args[++i]; break; + case "--out": options.OutDir = args[++i]; break; + case "--profile": options.Profile = args[++i]; break; + case "--no-launch": options.NoLaunch = true; break; + case "-h": + case "--help": + Console.WriteLine("Usage: Recordingtest.Runner --scenarios --baselines --out [--profile ] [--no-launch]"); + return 0; + } + } + + if (string.IsNullOrEmpty(options.ScenariosDir) || + string.IsNullOrEmpty(options.BaselinesDir) || + string.IsNullOrEmpty(options.OutDir)) + { + Console.Error.WriteLine("Missing required args. Use --help."); + return 2; + } + + var runner = new TestRunner(); + var report = runner.RunAll( + options, + new DefaultHostFactory(), + new DefaultNormalizer(), + new DefaultDiffer()); + + Console.WriteLine($"Total: {report.Total}, Passed: {report.Passed}, Failed: {report.Failed}, Errored: {report.Errored}"); + return TestRunner.ToExitCode(report); + } +} diff --git a/src/Recordingtest.Runner/Recordingtest.Runner.csproj b/src/Recordingtest.Runner/Recordingtest.Runner.csproj new file mode 100644 index 0000000..22fa9d9 --- /dev/null +++ b/src/Recordingtest.Runner/Recordingtest.Runner.csproj @@ -0,0 +1,15 @@ + + + Exe + net8.0-windows + false + false + Recordingtest.Runner + Recordingtest.Runner + + + + + + + diff --git a/src/Recordingtest.Runner/RunReport.cs b/src/Recordingtest.Runner/RunReport.cs new file mode 100644 index 0000000..c3286a5 --- /dev/null +++ b/src/Recordingtest.Runner/RunReport.cs @@ -0,0 +1,21 @@ +namespace Recordingtest.Runner; + +public sealed class RunReport +{ + public DateTime RunAt { get; set; } + public int Total { get; set; } + public int Passed { get; set; } + public int Failed { get; set; } + public int Errored { get; set; } + public List Scenarios { get; set; } = new(); +} + +public sealed class ScenarioResult +{ + public string Name { get; set; } = string.Empty; + public string Status { get; set; } = "pass"; + public int Hunks { get; set; } + public int CheckpointCount { get; set; } + public string ArtifactDir { get; set; } = string.Empty; + public string? Error { get; set; } +} diff --git a/src/Recordingtest.Runner/RunnerOptions.cs b/src/Recordingtest.Runner/RunnerOptions.cs new file mode 100644 index 0000000..a2d5a23 --- /dev/null +++ b/src/Recordingtest.Runner/RunnerOptions.cs @@ -0,0 +1,10 @@ +namespace Recordingtest.Runner; + +public sealed class RunnerOptions +{ + public string ScenariosDir { get; set; } = string.Empty; + public string BaselinesDir { get; set; } = string.Empty; + public string OutDir { get; set; } = string.Empty; + public string Profile { get; set; } = "default"; + public bool NoLaunch { get; set; } +} diff --git a/src/Recordingtest.Runner/TestRunner.cs b/src/Recordingtest.Runner/TestRunner.cs new file mode 100644 index 0000000..905a442 --- /dev/null +++ b/src/Recordingtest.Runner/TestRunner.cs @@ -0,0 +1,210 @@ +using System.Text; +using System.Text.Json; +using Recordingtest.Player; +using Recordingtest.Player.Model; + +namespace Recordingtest.Runner; + +public sealed class TestRunner +{ + // Note: baselines are normalized with the same profile as received output + // (we do not assume pre-normalized baselines, so re-normalizing is safe). + public RunReport RunAll( + RunnerOptions options, + IRunnerHostFactory hostFactory, + INormalizer normalizer, + IDiffer differ) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(hostFactory); + ArgumentNullException.ThrowIfNull(normalizer); + ArgumentNullException.ThrowIfNull(differ); + + Directory.CreateDirectory(options.OutDir); + + var report = new RunReport { RunAt = DateTime.UtcNow }; + + var yamlFiles = Directory.Exists(options.ScenariosDir) + ? Directory.GetFiles(options.ScenariosDir, "*.yaml", SearchOption.TopDirectoryOnly) + .OrderBy(p => p, StringComparer.Ordinal).ToArray() + : Array.Empty(); + + foreach (var yamlPath in yamlFiles) + { + var scenarioName = Path.GetFileNameWithoutExtension(yamlPath); + var artifactDir = Path.Combine(options.OutDir, scenarioName); + Directory.CreateDirectory(artifactDir); + + var sr = new ScenarioResult + { + Name = scenarioName, + ArtifactDir = artifactDir, + }; + + Scenario? scenario = null; + try + { + scenario = ScenarioLoader.LoadFromFile(yamlPath); + sr.CheckpointCount = scenario.Steps.Count(s => s.Kind == StepKind.Checkpoint); + + var host = hostFactory.Create(scenario, artifactDir); + var engine = new PlayerEngine(); + engine.Run(scenario, host); + } + catch (Exception ex) + { + sr.Status = "error"; + sr.Error = ex.Message; + report.Scenarios.Add(sr); + continue; + } + + try + { + // Determine result file path: from last save step or convention + var lastSave = scenario!.Steps + .LastOrDefault(s => !string.IsNullOrEmpty(s.SaveAs)); + string resultPath; + if (lastSave is not null && !string.IsNullOrEmpty(lastSave.SaveAs)) + { + resultPath = Path.IsPathRooted(lastSave.SaveAs) + ? lastSave.SaveAs + : Path.Combine(artifactDir, lastSave.SaveAs); + } + else + { + // convention: /result.* + var conv = Directory.Exists(artifactDir) + ? Directory.GetFiles(artifactDir, "result.*").FirstOrDefault() + : null; + resultPath = conv ?? Path.Combine(artifactDir, "result.json"); + } + + if (!File.Exists(resultPath)) + { + sr.Status = "error"; + sr.Error = $"result file missing: {resultPath}"; + report.Scenarios.Add(sr); + continue; + } + + var baselinePath = FindBaseline(options.BaselinesDir, scenarioName, Path.GetExtension(resultPath)); + if (baselinePath is null) + { + sr.Status = "error"; + sr.Error = $"baseline missing for scenario {scenarioName}"; + report.Scenarios.Add(sr); + continue; + } + + var receivedRaw = File.ReadAllText(resultPath); + var approvedRaw = File.ReadAllText(baselinePath); + + var receivedNorm = normalizer.Normalize(receivedRaw, options.Profile, null); + var approvedNorm = normalizer.Normalize(approvedRaw, options.Profile, null); + + var receivedNormPath = Path.Combine(artifactDir, "received.normalized"); + var approvedNormPath = Path.Combine(artifactDir, "approved.normalized"); + File.WriteAllText(receivedNormPath, receivedNorm); + File.WriteAllText(approvedNormPath, approvedNorm); + + var diff = differ.Compare(approvedNormPath, receivedNormPath); + sr.Hunks = diff.Hunks.Count; + sr.Status = diff.Identical ? "pass" : "fail"; + } + catch (Exception ex) + { + sr.Status = "error"; + sr.Error = ex.Message; + } + + report.Scenarios.Add(sr); + } + + report.Total = report.Scenarios.Count; + report.Passed = report.Scenarios.Count(s => s.Status == "pass"); + report.Failed = report.Scenarios.Count(s => s.Status == "fail"); + report.Errored = report.Scenarios.Count(s => s.Status == "error"); + + WriteJsonReport(report, Path.Combine(options.OutDir, "report.json")); + WriteMarkdownReport(report, Path.Combine(options.OutDir, "report.md")); + + return report; + } + + private static string? FindBaseline(string baselinesDir, string scenarioName, string preferredExt) + { + if (string.IsNullOrEmpty(baselinesDir) || !Directory.Exists(baselinesDir)) + return null; + var candidates = new List + { + Path.Combine(baselinesDir, scenarioName + preferredExt), + Path.Combine(baselinesDir, scenarioName + ".approved" + preferredExt), + Path.Combine(baselinesDir, scenarioName + ".json"), + Path.Combine(baselinesDir, scenarioName + ".approved.json"), + }; + foreach (var c in candidates) + if (File.Exists(c)) return c; + var matches = Directory.GetFiles(baselinesDir, scenarioName + ".*"); + return matches.FirstOrDefault(); + } + + public static void WriteJsonReport(RunReport report, string path) + { + var opts = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + }; + File.WriteAllText(path, JsonSerializer.Serialize(report, opts)); + } + + public static void WriteMarkdownReport(RunReport report, string path) + { + var sb = new StringBuilder(); + sb.AppendLine("# Test Runner Report"); + sb.AppendLine(); + sb.Append("Run at: ").AppendLine(report.RunAt.ToString("u")); + sb.AppendLine(); + sb.Append("Total: ").Append(report.Total) + .Append(" | Passed: ").Append(report.Passed) + .Append(" | Failed: ").Append(report.Failed) + .Append(" | Errored: ").AppendLine(report.Errored.ToString()); + sb.AppendLine(); + sb.AppendLine("| Scenario | Status | Hunks | Checkpoints | Artifacts |"); + sb.AppendLine("|----------|--------|-------|-------------|-----------|"); + foreach (var s in report.Scenarios) + { + sb.Append("| ").Append(s.Name) + .Append(" | ").Append(s.Status) + .Append(" | ").Append(s.Hunks) + .Append(" | ").Append(s.CheckpointCount) + .Append(" | ").Append(s.ArtifactDir) + .AppendLine(" |"); + } + sb.AppendLine(); + var bad = report.Scenarios.Where(s => s.Status != "pass").ToList(); + if (bad.Count > 0) + { + sb.AppendLine("## Failures"); + foreach (var s in bad) + { + sb.Append("### ").AppendLine(s.Name); + sb.Append("- status: ").AppendLine(s.Status); + sb.Append("- hunks: ").AppendLine(s.Hunks.ToString()); + sb.Append("- artifacts: ").AppendLine(s.ArtifactDir); + if (!string.IsNullOrEmpty(s.Error)) + sb.Append("- error: ").AppendLine(s.Error); + sb.AppendLine(); + } + } + File.WriteAllText(path, sb.ToString()); + } + + public static int ToExitCode(RunReport report) + { + if (report.Errored > 0) return 2; + if (report.Failed > 0) return 1; + return 0; + } +} diff --git a/tests/Recordingtest.Runner.Tests/Fakes.cs b/tests/Recordingtest.Runner.Tests/Fakes.cs new file mode 100644 index 0000000..7bcd0a9 --- /dev/null +++ b/tests/Recordingtest.Runner.Tests/Fakes.cs @@ -0,0 +1,86 @@ +using Recordingtest.DiffReporter; +using Recordingtest.Player; +using Recordingtest.Player.Model; +using Recordingtest.Runner; + +namespace Recordingtest.Runner.Tests; + +public sealed class FakePlayerHost : IPlayerHost +{ + private readonly string _outDir; + private readonly string _resultContent; + private readonly bool _throwOnClick; + + public FakePlayerHost(string outDir, string resultContent, bool throwOnClick = false) + { + _outDir = outDir; + _resultContent = resultContent; + _throwOnClick = throwOnClick; + } + + public ResolvedElement? ResolveElement(string uiaPath, TimeSpan timeout) + => new ResolvedElement(new ElementBounds(0, 0, 10, 10), null); + + public bool WaitFor(string waitForHint, TimeSpan timeout) => true; + + public void Click(ScreenPoint point) + { + if (_throwOnClick) throw new InvalidOperationException("fake click failure"); + } + + public void Type(string text) { } + public void Drag(ScreenPoint from, ScreenPoint to) { } + public void Hotkey(string keys) + { + // simulate save + Directory.CreateDirectory(_outDir); + File.WriteAllText(Path.Combine(_outDir, "result.json"), _resultContent); + } + public void CaptureCheckpoint(int afterStep, string saveAs) { } + public void CaptureFailureArtifacts(int stepIndex, string reason) { } +} + +public sealed class FakeHostFactory : IRunnerHostFactory +{ + private readonly string _resultContent; + private readonly bool _throwOnClick; + + public FakeHostFactory(string resultContent, bool throwOnClick = false) + { + _resultContent = resultContent; + _throwOnClick = throwOnClick; + } + + public IPlayerHost Create(Scenario scenario, string outDir) + => new FakePlayerHost(outDir, _resultContent, _throwOnClick); +} + +public sealed class SpyNormalizer : INormalizer +{ + public List Profiles { get; } = new(); + public string Normalize(string input, string profile, string? sidecarPath) + { + Profiles.Add(profile); + return input; + } +} + +public sealed class StubDiffer : IDiffer +{ + private readonly bool _identical; + private readonly int _hunkCount; + + public StubDiffer(bool identical, int hunkCount = 0) + { + _identical = identical; + _hunkCount = hunkCount; + } + + public DiffResult Compare(string approvedPath, string receivedPath) + { + var hunks = new List(); + for (int i = 0; i < _hunkCount; i++) + hunks.Add(new Hunk(i, "changed", "a", "b")); + return new DiffResult(Path.GetFileName(receivedPath), _identical, hunks, new DiffSummary(0, 0, _hunkCount)); + } +} diff --git a/tests/Recordingtest.Runner.Tests/Recordingtest.Runner.Tests.csproj b/tests/Recordingtest.Runner.Tests/Recordingtest.Runner.Tests.csproj new file mode 100644 index 0000000..538efe3 --- /dev/null +++ b/tests/Recordingtest.Runner.Tests/Recordingtest.Runner.Tests.csproj @@ -0,0 +1,16 @@ + + + net8.0-windows + false + Recordingtest.Runner.Tests + Recordingtest.Runner.Tests + + + + + + + + + + diff --git a/tests/Recordingtest.Runner.Tests/TestRunnerTests.cs b/tests/Recordingtest.Runner.Tests/TestRunnerTests.cs new file mode 100644 index 0000000..230c1d4 --- /dev/null +++ b/tests/Recordingtest.Runner.Tests/TestRunnerTests.cs @@ -0,0 +1,161 @@ +using System.Text.Json; +using Recordingtest.Runner; +using Xunit; + +namespace Recordingtest.Runner.Tests; + +public class TestRunnerTests : IDisposable +{ + private readonly string _root; + + public TestRunnerTests() + { + _root = Path.Combine(Path.GetTempPath(), "rt-runner-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + public void Dispose() + { + try { Directory.Delete(_root, true); } catch { } + } + + private (string scenariosDir, string baselinesDir, string outDir) MakeDirs() + { + var s = Path.Combine(_root, "scenarios"); + var b = Path.Combine(_root, "baselines"); + var o = Path.Combine(_root, "out"); + Directory.CreateDirectory(s); + Directory.CreateDirectory(b); + Directory.CreateDirectory(o); + return (s, b, o); + } + + private static string ScenarioYaml(string name) => $@"name: {name} +description: test +sut: + exe: dummy.exe +steps: + - kind: save + value: ctrl+s +"; + + private static void WriteScenario(string dir, string name) + => File.WriteAllText(Path.Combine(dir, name + ".yaml"), ScenarioYaml(name)); + + [Fact] + public void TwoScenarios_BothIdentical_ExitZero_AllPass() + { + var (sDir, bDir, oDir) = MakeDirs(); + WriteScenario(sDir, "alpha"); + WriteScenario(sDir, "beta"); + var content = "{\"x\":1}"; + File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); + File.WriteAllText(Path.Combine(bDir, "beta.json"), content); + + var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; + var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true)); + + Assert.Equal(2, report.Total); + Assert.Equal(2, report.Passed); + Assert.Equal(0, report.Failed); + Assert.Equal(0, TestRunner.ToExitCode(report)); + } + + [Fact] + public void OneScenarioDiffers_ExitOne_HunkCount() + { + var (sDir, bDir, oDir) = MakeDirs(); + WriteScenario(sDir, "alpha"); + var content = "{\"x\":1}"; + File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); + + var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; + var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: false, hunkCount: 1)); + + Assert.Equal(1, TestRunner.ToExitCode(report)); + Assert.Equal("fail", report.Scenarios[0].Status); + Assert.Equal(1, report.Scenarios[0].Hunks); + } + + [Fact] + public void PlayerThrows_ExitTwo_ErrorStatus() + { + var (sDir, bDir, oDir) = MakeDirs(); + // scenario with a click step so the throw triggers + var name = "boom"; + var yaml = @"name: boom +sut: + exe: dummy.exe +steps: + - kind: click + target: + uia_path: /Window + offset: [0.5, 0.5] +"; + File.WriteAllText(Path.Combine(sDir, name + ".yaml"), yaml); + + var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; + var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}", throwOnClick: true), new SpyNormalizer(), new StubDiffer(identical: true)); + + Assert.True(report.Errored >= 1); + Assert.Equal(2, TestRunner.ToExitCode(report)); + } + + [Fact] + public void EmptyScenariosDir_ExitZero_TotalZero() + { + var (sDir, bDir, oDir) = MakeDirs(); + var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; + var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}"), new SpyNormalizer(), new StubDiffer(identical: true)); + Assert.Equal(0, report.Total); + Assert.Equal(0, TestRunner.ToExitCode(report)); + } + + [Fact] + public void ProfileOverride_IsPassedToNormalizer() + { + var (sDir, bDir, oDir) = MakeDirs(); + WriteScenario(sDir, "alpha"); + var content = "{\"x\":1}"; + File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); + + var spy = new SpyNormalizer(); + var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir, Profile = "strict" }; + new TestRunner().RunAll(opts, new FakeHostFactory(content), spy, new StubDiffer(identical: true)); + + Assert.Contains("strict", spy.Profiles); + Assert.DoesNotContain("default", spy.Profiles); + } + + [Fact] + public void ReportJson_HasExpectedSchema_And_ReportMd_Exists() + { + var (sDir, bDir, oDir) = MakeDirs(); + WriteScenario(sDir, "alpha"); + var content = "{\"x\":1}"; + File.WriteAllText(Path.Combine(bDir, "alpha.json"), content); + + var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir }; + new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true)); + + var jsonPath = Path.Combine(oDir, "report.json"); + var mdPath = Path.Combine(oDir, "report.md"); + Assert.True(File.Exists(jsonPath)); + Assert.True(File.Exists(mdPath)); + + using var doc = JsonDocument.Parse(File.ReadAllText(jsonPath)); + var root = doc.RootElement; + Assert.True(root.TryGetProperty("runAt", out _)); + Assert.True(root.TryGetProperty("total", out _)); + Assert.True(root.TryGetProperty("passed", out _)); + Assert.True(root.TryGetProperty("failed", out _)); + Assert.True(root.TryGetProperty("errored", out _)); + Assert.True(root.TryGetProperty("scenarios", out var scenarios)); + var first = scenarios[0]; + Assert.True(first.TryGetProperty("name", out _)); + Assert.True(first.TryGetProperty("status", out _)); + Assert.True(first.TryGetProperty("hunks", out _)); + Assert.True(first.TryGetProperty("checkpointCount", out _)); + Assert.True(first.TryGetProperty("artifactDir", out _)); + } +}