Implement test-runner PoC (#8)

2026-04-07 15:21:03 +09:00
parent 836afea5ee
commit 96df2ef65d
12 changed files with 665 additions and 0 deletions
--- a/docs/history/2026-04-07_이슈8-test-runner-generator.md
+++ b/docs/history/2026-04-07_이슈8-test-runner-generator.md
@@ -0,0 +1,32 @@
+# 2026-04-07 — test-runner Generator (Issue #8)
+
+- 이슈: #8
+- 작업: `Recordingtest.Runner` PoC 구현 (player + normalizer + diff-reporter 통합 파이프라인)
+- 담당: Generator
+- 소요 시간: 약 30분
+- Context 사용량: 약 60k tokens
+
+## 산출물
+
+- `src/Recordingtest.Runner/` 콘솔 exe
+  - `RunnerOptions`, `RunReport`, `ScenarioResult`
+  - `INormalizer`, `IDiffer`, `IRunnerHostFactory` 어댑터 인터페이스
+  - `DefaultNormalizer`, `DefaultDiffer`, `DefaultHostFactory`
+  - `TestRunner` (`RunAll`, `WriteJsonReport`, `WriteMarkdownReport`, `ToExitCode`)
+  - `Program` (CLI: `--scenarios/--baselines/--out/--profile/--no-launch`)
+- `tests/Recordingtest.Runner.Tests/` (xUnit, 6 tests, all green)
+- `recordingtest.sln`에 두 프로젝트 추가
+
+## 검증
+
+- `dotnet build` green
+- `dotnet test` 6/6 통과
+- `Thread.Sleep` / `Task.Delay` 0건 (grep 확인)
+- `report.json` 카멜케이스 + 스키마 테스트 포함
+- Exit code: 0 = all pass, 1 = any fail, 2 = any error
+
+## 메모
+
+- 베이스라인은 사전 정규화되었다고 가정하지 않고, 매 실행 시 received와 동일 프로파일로 재정규화한다.
+  (TestRunner.cs 상단 주석 참조)
+- `--no-launch` 의미는 player 책임이므로 runner는 옵션만 파싱해 보관한다.
--- a/recordingtest.sln
+++ b/recordingtest.sln
@@ -23,6 +23,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Player", "src
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Player.Tests", "tests\Recordingtest.Player.Tests\Recordingtest.Player.Tests.csproj", "{7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Runner", "src\Recordingtest.Runner\Recordingtest.Runner.csproj", "{DADF0474-9EF3-4E8D-8139-93504E4F745D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Runner.Tests", "tests\Recordingtest.Runner.Tests\Recordingtest.Runner.Tests.csproj", "{6F9973EA-977A-4185-AF24-4E76D9D851C8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -153,6 +157,30 @@ Global
 		{7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x64.Build.0 = Release|Any CPU
 		{7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x86.ActiveCfg = Release|Any CPU
 		{7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x86.Build.0 = Release|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x64.Build.0 = Debug|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Debug|x86.Build.0 = Debug|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x64.ActiveCfg = Release|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x64.Build.0 = Release|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x86.ActiveCfg = Release|Any CPU
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D}.Release|x86.Build.0 = Release|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x64.Build.0 = Debug|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Debug|x86.Build.0 = Debug|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x64.ActiveCfg = Release|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x64.Build.0 = Release|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x86.ActiveCfg = Release|Any CPU
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -167,5 +195,7 @@ Global
 		{74D292F5-8004-4946-8CC3-808AFD9C52C1} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{D8962656-55EC-4595-8F19-8FBBF9256A04} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{DADF0474-9EF3-4E8D-8139-93504E4F745D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{6F9973EA-977A-4185-AF24-4E76D9D851C8} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 	EndGlobalSection
 EndGlobal
--- a/src/Recordingtest.Runner/DefaultAdapters.cs
+++ b/src/Recordingtest.Runner/DefaultAdapters.cs
@@ -0,0 +1,22 @@
+using Recordingtest.DiffReporter;
+using Recordingtest.Player;
+using Recordingtest.Player.Model;
+
+namespace Recordingtest.Runner;
+
+public sealed class DefaultNormalizer : INormalizer
+{
+    public string Normalize(string input, string profile, string? sidecarPath)
+        => Recordingtest.Normalizer.Normalizer.Normalize(input, profile, sidecarPath).Output;
+}
+
+public sealed class DefaultDiffer : IDiffer
+{
+    public DiffResult Compare(string approvedPath, string receivedPath)
+        => Differ.Compare(approvedPath, receivedPath);
+}
+
+public sealed class DefaultHostFactory : IRunnerHostFactory
+{
+    public IPlayerHost Create(Scenario scenario, string outDir) => new UiaPlayerHost(null, outDir);
+}
--- a/src/Recordingtest.Runner/Interfaces.cs
+++ b/src/Recordingtest.Runner/Interfaces.cs
@@ -0,0 +1,20 @@
+using Recordingtest.DiffReporter;
+using Recordingtest.Player;
+using Recordingtest.Player.Model;
+
+namespace Recordingtest.Runner;
+
+public interface INormalizer
+{
+    string Normalize(string input, string profile, string? sidecarPath);
+}
+
+public interface IDiffer
+{
+    DiffResult Compare(string approvedPath, string receivedPath);
+}
+
+public interface IRunnerHostFactory
+{
+    IPlayerHost Create(Scenario scenario, string outDir);
+}
--- a/src/Recordingtest.Runner/Program.cs
+++ b/src/Recordingtest.Runner/Program.cs
@@ -0,0 +1,42 @@
+namespace Recordingtest.Runner;
+
+public static class Program
+{
+    public static int Main(string[] args)
+    {
+        var options = new RunnerOptions();
+        for (int i = 0; i < args.Length; i++)
+        {
+            switch (args[i])
+            {
+                case "--scenarios": options.ScenariosDir = args[++i]; break;
+                case "--baselines": options.BaselinesDir = args[++i]; break;
+                case "--out": options.OutDir = args[++i]; break;
+                case "--profile": options.Profile = args[++i]; break;
+                case "--no-launch": options.NoLaunch = true; break;
+                case "-h":
+                case "--help":
+                    Console.WriteLine("Usage: Recordingtest.Runner --scenarios <dir> --baselines <dir> --out <dir> [--profile <name>] [--no-launch]");
+                    return 0;
+            }
+        }
+
+        if (string.IsNullOrEmpty(options.ScenariosDir) ||
+            string.IsNullOrEmpty(options.BaselinesDir) ||
+            string.IsNullOrEmpty(options.OutDir))
+        {
+            Console.Error.WriteLine("Missing required args. Use --help.");
+            return 2;
+        }
+
+        var runner = new TestRunner();
+        var report = runner.RunAll(
+            options,
+            new DefaultHostFactory(),
+            new DefaultNormalizer(),
+            new DefaultDiffer());
+
+        Console.WriteLine($"Total: {report.Total}, Passed: {report.Passed}, Failed: {report.Failed}, Errored: {report.Errored}");
+        return TestRunner.ToExitCode(report);
+    }
+}
--- a/src/Recordingtest.Runner/Recordingtest.Runner.csproj
+++ b/src/Recordingtest.Runner/Recordingtest.Runner.csproj
@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0-windows</TargetFramework>
+    <UseWPF>false</UseWPF>
+    <UseWindowsForms>false</UseWindowsForms>
+    <AssemblyName>Recordingtest.Runner</AssemblyName>
+    <RootNamespace>Recordingtest.Runner</RootNamespace>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Recordingtest.Player\Recordingtest.Player.csproj" />
+    <ProjectReference Include="..\Recordingtest.Normalizer\Recordingtest.Normalizer.csproj" />
+    <ProjectReference Include="..\Recordingtest.DiffReporter\Recordingtest.DiffReporter.csproj" />
+  </ItemGroup>
+</Project>
--- a/src/Recordingtest.Runner/RunReport.cs
+++ b/src/Recordingtest.Runner/RunReport.cs
@@ -0,0 +1,21 @@
+namespace Recordingtest.Runner;
+
+public sealed class RunReport
+{
+    public DateTime RunAt { get; set; }
+    public int Total { get; set; }
+    public int Passed { get; set; }
+    public int Failed { get; set; }
+    public int Errored { get; set; }
+    public List<ScenarioResult> Scenarios { get; set; } = new();
+}
+
+public sealed class ScenarioResult
+{
+    public string Name { get; set; } = string.Empty;
+    public string Status { get; set; } = "pass";
+    public int Hunks { get; set; }
+    public int CheckpointCount { get; set; }
+    public string ArtifactDir { get; set; } = string.Empty;
+    public string? Error { get; set; }
+}
--- a/src/Recordingtest.Runner/RunnerOptions.cs
+++ b/src/Recordingtest.Runner/RunnerOptions.cs
@@ -0,0 +1,10 @@
+namespace Recordingtest.Runner;
+
+public sealed class RunnerOptions
+{
+    public string ScenariosDir { get; set; } = string.Empty;
+    public string BaselinesDir { get; set; } = string.Empty;
+    public string OutDir { get; set; } = string.Empty;
+    public string Profile { get; set; } = "default";
+    public bool NoLaunch { get; set; }
+}
--- a/src/Recordingtest.Runner/TestRunner.cs
+++ b/src/Recordingtest.Runner/TestRunner.cs
@@ -0,0 +1,210 @@
+using System.Text;
+using System.Text.Json;
+using Recordingtest.Player;
+using Recordingtest.Player.Model;
+
+namespace Recordingtest.Runner;
+
+public sealed class TestRunner
+{
+    // Note: baselines are normalized with the same profile as received output
+    // (we do not assume pre-normalized baselines, so re-normalizing is safe).
+    public RunReport RunAll(
+        RunnerOptions options,
+        IRunnerHostFactory hostFactory,
+        INormalizer normalizer,
+        IDiffer differ)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        ArgumentNullException.ThrowIfNull(hostFactory);
+        ArgumentNullException.ThrowIfNull(normalizer);
+        ArgumentNullException.ThrowIfNull(differ);
+
+        Directory.CreateDirectory(options.OutDir);
+
+        var report = new RunReport { RunAt = DateTime.UtcNow };
+
+        var yamlFiles = Directory.Exists(options.ScenariosDir)
+            ? Directory.GetFiles(options.ScenariosDir, "*.yaml", SearchOption.TopDirectoryOnly)
+                .OrderBy(p => p, StringComparer.Ordinal).ToArray()
+            : Array.Empty<string>();
+
+        foreach (var yamlPath in yamlFiles)
+        {
+            var scenarioName = Path.GetFileNameWithoutExtension(yamlPath);
+            var artifactDir = Path.Combine(options.OutDir, scenarioName);
+            Directory.CreateDirectory(artifactDir);
+
+            var sr = new ScenarioResult
+            {
+                Name = scenarioName,
+                ArtifactDir = artifactDir,
+            };
+
+            Scenario? scenario = null;
+            try
+            {
+                scenario = ScenarioLoader.LoadFromFile(yamlPath);
+                sr.CheckpointCount = scenario.Steps.Count(s => s.Kind == StepKind.Checkpoint);
+
+                var host = hostFactory.Create(scenario, artifactDir);
+                var engine = new PlayerEngine();
+                engine.Run(scenario, host);
+            }
+            catch (Exception ex)
+            {
+                sr.Status = "error";
+                sr.Error = ex.Message;
+                report.Scenarios.Add(sr);
+                continue;
+            }
+
+            try
+            {
+                // Determine result file path: <scenario.save_as> from last save step or convention
+                var lastSave = scenario!.Steps
+                    .LastOrDefault(s => !string.IsNullOrEmpty(s.SaveAs));
+                string resultPath;
+                if (lastSave is not null && !string.IsNullOrEmpty(lastSave.SaveAs))
+                {
+                    resultPath = Path.IsPathRooted(lastSave.SaveAs)
+                        ? lastSave.SaveAs
+                        : Path.Combine(artifactDir, lastSave.SaveAs);
+                }
+                else
+                {
+                    // convention: <artifactDir>/result.*
+                    var conv = Directory.Exists(artifactDir)
+                        ? Directory.GetFiles(artifactDir, "result.*").FirstOrDefault()
+                        : null;
+                    resultPath = conv ?? Path.Combine(artifactDir, "result.json");
+                }
+
+                if (!File.Exists(resultPath))
+                {
+                    sr.Status = "error";
+                    sr.Error = $"result file missing: {resultPath}";
+                    report.Scenarios.Add(sr);
+                    continue;
+                }
+
+                var baselinePath = FindBaseline(options.BaselinesDir, scenarioName, Path.GetExtension(resultPath));
+                if (baselinePath is null)
+                {
+                    sr.Status = "error";
+                    sr.Error = $"baseline missing for scenario {scenarioName}";
+                    report.Scenarios.Add(sr);
+                    continue;
+                }
+
+                var receivedRaw = File.ReadAllText(resultPath);
+                var approvedRaw = File.ReadAllText(baselinePath);
+
+                var receivedNorm = normalizer.Normalize(receivedRaw, options.Profile, null);
+                var approvedNorm = normalizer.Normalize(approvedRaw, options.Profile, null);
+
+                var receivedNormPath = Path.Combine(artifactDir, "received.normalized");
+                var approvedNormPath = Path.Combine(artifactDir, "approved.normalized");
+                File.WriteAllText(receivedNormPath, receivedNorm);
+                File.WriteAllText(approvedNormPath, approvedNorm);
+
+                var diff = differ.Compare(approvedNormPath, receivedNormPath);
+                sr.Hunks = diff.Hunks.Count;
+                sr.Status = diff.Identical ? "pass" : "fail";
+            }
+            catch (Exception ex)
+            {
+                sr.Status = "error";
+                sr.Error = ex.Message;
+            }
+
+            report.Scenarios.Add(sr);
+        }
+
+        report.Total = report.Scenarios.Count;
+        report.Passed = report.Scenarios.Count(s => s.Status == "pass");
+        report.Failed = report.Scenarios.Count(s => s.Status == "fail");
+        report.Errored = report.Scenarios.Count(s => s.Status == "error");
+
+        WriteJsonReport(report, Path.Combine(options.OutDir, "report.json"));
+        WriteMarkdownReport(report, Path.Combine(options.OutDir, "report.md"));
+
+        return report;
+    }
+
+    private static string? FindBaseline(string baselinesDir, string scenarioName, string preferredExt)
+    {
+        if (string.IsNullOrEmpty(baselinesDir) || !Directory.Exists(baselinesDir))
+            return null;
+        var candidates = new List<string>
+        {
+            Path.Combine(baselinesDir, scenarioName + preferredExt),
+            Path.Combine(baselinesDir, scenarioName + ".approved" + preferredExt),
+            Path.Combine(baselinesDir, scenarioName + ".json"),
+            Path.Combine(baselinesDir, scenarioName + ".approved.json"),
+        };
+        foreach (var c in candidates)
+            if (File.Exists(c)) return c;
+        var matches = Directory.GetFiles(baselinesDir, scenarioName + ".*");
+        return matches.FirstOrDefault();
+    }
+
+    public static void WriteJsonReport(RunReport report, string path)
+    {
+        var opts = new JsonSerializerOptions
+        {
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+            WriteIndented = true,
+        };
+        File.WriteAllText(path, JsonSerializer.Serialize(report, opts));
+    }
+
+    public static void WriteMarkdownReport(RunReport report, string path)
+    {
+        var sb = new StringBuilder();
+        sb.AppendLine("# Test Runner Report");
+        sb.AppendLine();
+        sb.Append("Run at: ").AppendLine(report.RunAt.ToString("u"));
+        sb.AppendLine();
+        sb.Append("Total: ").Append(report.Total)
+          .Append(" | Passed: ").Append(report.Passed)
+          .Append(" | Failed: ").Append(report.Failed)
+          .Append(" | Errored: ").AppendLine(report.Errored.ToString());
+        sb.AppendLine();
+        sb.AppendLine("| Scenario | Status | Hunks | Checkpoints | Artifacts |");
+        sb.AppendLine("|----------|--------|-------|-------------|-----------|");
+        foreach (var s in report.Scenarios)
+        {
+            sb.Append("| ").Append(s.Name)
+              .Append(" | ").Append(s.Status)
+              .Append(" | ").Append(s.Hunks)
+              .Append(" | ").Append(s.CheckpointCount)
+              .Append(" | ").Append(s.ArtifactDir)
+              .AppendLine(" |");
+        }
+        sb.AppendLine();
+        var bad = report.Scenarios.Where(s => s.Status != "pass").ToList();
+        if (bad.Count > 0)
+        {
+            sb.AppendLine("## Failures");
+            foreach (var s in bad)
+            {
+                sb.Append("### ").AppendLine(s.Name);
+                sb.Append("- status: ").AppendLine(s.Status);
+                sb.Append("- hunks: ").AppendLine(s.Hunks.ToString());
+                sb.Append("- artifacts: ").AppendLine(s.ArtifactDir);
+                if (!string.IsNullOrEmpty(s.Error))
+                    sb.Append("- error: ").AppendLine(s.Error);
+                sb.AppendLine();
+            }
+        }
+        File.WriteAllText(path, sb.ToString());
+    }
+
+    public static int ToExitCode(RunReport report)
+    {
+        if (report.Errored > 0) return 2;
+        if (report.Failed > 0) return 1;
+        return 0;
+    }
+}
--- a/tests/Recordingtest.Runner.Tests/Fakes.cs
+++ b/tests/Recordingtest.Runner.Tests/Fakes.cs
@@ -0,0 +1,86 @@
+using Recordingtest.DiffReporter;
+using Recordingtest.Player;
+using Recordingtest.Player.Model;
+using Recordingtest.Runner;
+
+namespace Recordingtest.Runner.Tests;
+
+public sealed class FakePlayerHost : IPlayerHost
+{
+    private readonly string _outDir;
+    private readonly string _resultContent;
+    private readonly bool _throwOnClick;
+
+    public FakePlayerHost(string outDir, string resultContent, bool throwOnClick = false)
+    {
+        _outDir = outDir;
+        _resultContent = resultContent;
+        _throwOnClick = throwOnClick;
+    }
+
+    public ResolvedElement? ResolveElement(string uiaPath, TimeSpan timeout)
+        => new ResolvedElement(new ElementBounds(0, 0, 10, 10), null);
+
+    public bool WaitFor(string waitForHint, TimeSpan timeout) => true;
+
+    public void Click(ScreenPoint point)
+    {
+        if (_throwOnClick) throw new InvalidOperationException("fake click failure");
+    }
+
+    public void Type(string text) { }
+    public void Drag(ScreenPoint from, ScreenPoint to) { }
+    public void Hotkey(string keys)
+    {
+        // simulate save
+        Directory.CreateDirectory(_outDir);
+        File.WriteAllText(Path.Combine(_outDir, "result.json"), _resultContent);
+    }
+    public void CaptureCheckpoint(int afterStep, string saveAs) { }
+    public void CaptureFailureArtifacts(int stepIndex, string reason) { }
+}
+
+public sealed class FakeHostFactory : IRunnerHostFactory
+{
+    private readonly string _resultContent;
+    private readonly bool _throwOnClick;
+
+    public FakeHostFactory(string resultContent, bool throwOnClick = false)
+    {
+        _resultContent = resultContent;
+        _throwOnClick = throwOnClick;
+    }
+
+    public IPlayerHost Create(Scenario scenario, string outDir)
+        => new FakePlayerHost(outDir, _resultContent, _throwOnClick);
+}
+
+public sealed class SpyNormalizer : INormalizer
+{
+    public List<string> Profiles { get; } = new();
+    public string Normalize(string input, string profile, string? sidecarPath)
+    {
+        Profiles.Add(profile);
+        return input;
+    }
+}
+
+public sealed class StubDiffer : IDiffer
+{
+    private readonly bool _identical;
+    private readonly int _hunkCount;
+
+    public StubDiffer(bool identical, int hunkCount = 0)
+    {
+        _identical = identical;
+        _hunkCount = hunkCount;
+    }
+
+    public DiffResult Compare(string approvedPath, string receivedPath)
+    {
+        var hunks = new List<Hunk>();
+        for (int i = 0; i < _hunkCount; i++)
+            hunks.Add(new Hunk(i, "changed", "a", "b"));
+        return new DiffResult(Path.GetFileName(receivedPath), _identical, hunks, new DiffSummary(0, 0, _hunkCount));
+    }
+}
--- a/tests/Recordingtest.Runner.Tests/Recordingtest.Runner.Tests.csproj
+++ b/tests/Recordingtest.Runner.Tests/Recordingtest.Runner.Tests.csproj
@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0-windows</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <RootNamespace>Recordingtest.Runner.Tests</RootNamespace>
+    <AssemblyName>Recordingtest.Runner.Tests</AssemblyName>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Recordingtest.Runner\Recordingtest.Runner.csproj" />
+  </ItemGroup>
+</Project>
--- a/tests/Recordingtest.Runner.Tests/TestRunnerTests.cs
+++ b/tests/Recordingtest.Runner.Tests/TestRunnerTests.cs
@@ -0,0 +1,161 @@
+using System.Text.Json;
+using Recordingtest.Runner;
+using Xunit;
+
+namespace Recordingtest.Runner.Tests;
+
+public class TestRunnerTests : IDisposable
+{
+    private readonly string _root;
+
+    public TestRunnerTests()
+    {
+        _root = Path.Combine(Path.GetTempPath(), "rt-runner-" + Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(_root);
+    }
+
+    public void Dispose()
+    {
+        try { Directory.Delete(_root, true); } catch { }
+    }
+
+    private (string scenariosDir, string baselinesDir, string outDir) MakeDirs()
+    {
+        var s = Path.Combine(_root, "scenarios");
+        var b = Path.Combine(_root, "baselines");
+        var o = Path.Combine(_root, "out");
+        Directory.CreateDirectory(s);
+        Directory.CreateDirectory(b);
+        Directory.CreateDirectory(o);
+        return (s, b, o);
+    }
+
+    private static string ScenarioYaml(string name) => $@"name: {name}
+description: test
+sut:
+  exe: dummy.exe
+steps:
+  - kind: save
+    value: ctrl+s
+";
+
+    private static void WriteScenario(string dir, string name)
+        => File.WriteAllText(Path.Combine(dir, name + ".yaml"), ScenarioYaml(name));
+
+    [Fact]
+    public void TwoScenarios_BothIdentical_ExitZero_AllPass()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        WriteScenario(sDir, "beta");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+        File.WriteAllText(Path.Combine(bDir, "beta.json"), content);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true));
+
+        Assert.Equal(2, report.Total);
+        Assert.Equal(2, report.Passed);
+        Assert.Equal(0, report.Failed);
+        Assert.Equal(0, TestRunner.ToExitCode(report));
+    }
+
+    [Fact]
+    public void OneScenarioDiffers_ExitOne_HunkCount()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: false, hunkCount: 1));
+
+        Assert.Equal(1, TestRunner.ToExitCode(report));
+        Assert.Equal("fail", report.Scenarios[0].Status);
+        Assert.Equal(1, report.Scenarios[0].Hunks);
+    }
+
+    [Fact]
+    public void PlayerThrows_ExitTwo_ErrorStatus()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        // scenario with a click step so the throw triggers
+        var name = "boom";
+        var yaml = @"name: boom
+sut:
+  exe: dummy.exe
+steps:
+  - kind: click
+    target:
+      uia_path: /Window
+      offset: [0.5, 0.5]
+";
+        File.WriteAllText(Path.Combine(sDir, name + ".yaml"), yaml);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}", throwOnClick: true), new SpyNormalizer(), new StubDiffer(identical: true));
+
+        Assert.True(report.Errored >= 1);
+        Assert.Equal(2, TestRunner.ToExitCode(report));
+    }
+
+    [Fact]
+    public void EmptyScenariosDir_ExitZero_TotalZero()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}"), new SpyNormalizer(), new StubDiffer(identical: true));
+        Assert.Equal(0, report.Total);
+        Assert.Equal(0, TestRunner.ToExitCode(report));
+    }
+
+    [Fact]
+    public void ProfileOverride_IsPassedToNormalizer()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+
+        var spy = new SpyNormalizer();
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir, Profile = "strict" };
+        new TestRunner().RunAll(opts, new FakeHostFactory(content), spy, new StubDiffer(identical: true));
+
+        Assert.Contains("strict", spy.Profiles);
+        Assert.DoesNotContain("default", spy.Profiles);
+    }
+
+    [Fact]
+    public void ReportJson_HasExpectedSchema_And_ReportMd_Exists()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true));
+
+        var jsonPath = Path.Combine(oDir, "report.json");
+        var mdPath = Path.Combine(oDir, "report.md");
+        Assert.True(File.Exists(jsonPath));
+        Assert.True(File.Exists(mdPath));
+
+        using var doc = JsonDocument.Parse(File.ReadAllText(jsonPath));
+        var root = doc.RootElement;
+        Assert.True(root.TryGetProperty("runAt", out _));
+        Assert.True(root.TryGetProperty("total", out _));
+        Assert.True(root.TryGetProperty("passed", out _));
+        Assert.True(root.TryGetProperty("failed", out _));
+        Assert.True(root.TryGetProperty("errored", out _));
+        Assert.True(root.TryGetProperty("scenarios", out var scenarios));
+        var first = scenarios[0];
+        Assert.True(first.TryGetProperty("name", out _));
+        Assert.True(first.TryGetProperty("status", out _));
+        Assert.True(first.TryGetProperty("hunks", out _));
+        Assert.True(first.TryGetProperty("checkpointCount", out _));
+        Assert.True(first.TryGetProperty("artifactDir", out _));
+    }
+}