Implement test-runner PoC (#8)

tests/Recordingtest.Runner.Tests/Fakes.cs

@@ -0,0 +1,86 @@
+using Recordingtest.DiffReporter;
+using Recordingtest.Player;
+using Recordingtest.Player.Model;
+using Recordingtest.Runner;
+
+namespace Recordingtest.Runner.Tests;
+
+public sealed class FakePlayerHost : IPlayerHost
+{
+    private readonly string _outDir;
+    private readonly string _resultContent;
+    private readonly bool _throwOnClick;
+
+    public FakePlayerHost(string outDir, string resultContent, bool throwOnClick = false)
+    {
+        _outDir = outDir;
+        _resultContent = resultContent;
+        _throwOnClick = throwOnClick;
+    }
+
+    public ResolvedElement? ResolveElement(string uiaPath, TimeSpan timeout)
+        => new ResolvedElement(new ElementBounds(0, 0, 10, 10), null);
+
+    public bool WaitFor(string waitForHint, TimeSpan timeout) => true;
+
+    public void Click(ScreenPoint point)
+    {
+        if (_throwOnClick) throw new InvalidOperationException("fake click failure");
+    }
+
+    public void Type(string text) { }
+    public void Drag(ScreenPoint from, ScreenPoint to) { }
+    public void Hotkey(string keys)
+    {
+        // simulate save
+        Directory.CreateDirectory(_outDir);
+        File.WriteAllText(Path.Combine(_outDir, "result.json"), _resultContent);
+    }
+    public void CaptureCheckpoint(int afterStep, string saveAs) { }
+    public void CaptureFailureArtifacts(int stepIndex, string reason) { }
+}
+
+public sealed class FakeHostFactory : IRunnerHostFactory
+{
+    private readonly string _resultContent;
+    private readonly bool _throwOnClick;
+
+    public FakeHostFactory(string resultContent, bool throwOnClick = false)
+    {
+        _resultContent = resultContent;
+        _throwOnClick = throwOnClick;
+    }
+
+    public IPlayerHost Create(Scenario scenario, string outDir)
+        => new FakePlayerHost(outDir, _resultContent, _throwOnClick);
+}
+
+public sealed class SpyNormalizer : INormalizer
+{
+    public List<string> Profiles { get; } = new();
+    public string Normalize(string input, string profile, string? sidecarPath)
+    {
+        Profiles.Add(profile);
+        return input;
+    }
+}
+
+public sealed class StubDiffer : IDiffer
+{
+    private readonly bool _identical;
+    private readonly int _hunkCount;
+
+    public StubDiffer(bool identical, int hunkCount = 0)
+    {
+        _identical = identical;
+        _hunkCount = hunkCount;
+    }
+
+    public DiffResult Compare(string approvedPath, string receivedPath)
+    {
+        var hunks = new List<Hunk>();
+        for (int i = 0; i < _hunkCount; i++)
+            hunks.Add(new Hunk(i, "changed", "a", "b"));
+        return new DiffResult(Path.GetFileName(receivedPath), _identical, hunks, new DiffSummary(0, 0, _hunkCount));
+    }
+}

tests/Recordingtest.Runner.Tests/Recordingtest.Runner.Tests.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0-windows</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <RootNamespace>Recordingtest.Runner.Tests</RootNamespace>
+    <AssemblyName>Recordingtest.Runner.Tests</AssemblyName>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Recordingtest.Runner\Recordingtest.Runner.csproj" />
+  </ItemGroup>
+</Project>

tests/Recordingtest.Runner.Tests/TestRunnerTests.cs

@@ -0,0 +1,161 @@
+using System.Text.Json;
+using Recordingtest.Runner;
+using Xunit;
+
+namespace Recordingtest.Runner.Tests;
+
+public class TestRunnerTests : IDisposable
+{
+    private readonly string _root;
+
+    public TestRunnerTests()
+    {
+        _root = Path.Combine(Path.GetTempPath(), "rt-runner-" + Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(_root);
+    }
+
+    public void Dispose()
+    {
+        try { Directory.Delete(_root, true); } catch { }
+    }
+
+    private (string scenariosDir, string baselinesDir, string outDir) MakeDirs()
+    {
+        var s = Path.Combine(_root, "scenarios");
+        var b = Path.Combine(_root, "baselines");
+        var o = Path.Combine(_root, "out");
+        Directory.CreateDirectory(s);
+        Directory.CreateDirectory(b);
+        Directory.CreateDirectory(o);
+        return (s, b, o);
+    }
+
+    private static string ScenarioYaml(string name) => $@"name: {name}
+description: test
+sut:
+  exe: dummy.exe
+steps:
+  - kind: save
+    value: ctrl+s
+";
+
+    private static void WriteScenario(string dir, string name)
+        => File.WriteAllText(Path.Combine(dir, name + ".yaml"), ScenarioYaml(name));
+
+    [Fact]
+    public void TwoScenarios_BothIdentical_ExitZero_AllPass()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        WriteScenario(sDir, "beta");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+        File.WriteAllText(Path.Combine(bDir, "beta.json"), content);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true));
+
+        Assert.Equal(2, report.Total);
+        Assert.Equal(2, report.Passed);
+        Assert.Equal(0, report.Failed);
+        Assert.Equal(0, TestRunner.ToExitCode(report));
+    }
+
+    [Fact]
+    public void OneScenarioDiffers_ExitOne_HunkCount()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: false, hunkCount: 1));
+
+        Assert.Equal(1, TestRunner.ToExitCode(report));
+        Assert.Equal("fail", report.Scenarios[0].Status);
+        Assert.Equal(1, report.Scenarios[0].Hunks);
+    }
+
+    [Fact]
+    public void PlayerThrows_ExitTwo_ErrorStatus()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        // scenario with a click step so the throw triggers
+        var name = "boom";
+        var yaml = @"name: boom
+sut:
+  exe: dummy.exe
+steps:
+  - kind: click
+    target:
+      uia_path: /Window
+      offset: [0.5, 0.5]
+";
+        File.WriteAllText(Path.Combine(sDir, name + ".yaml"), yaml);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}", throwOnClick: true), new SpyNormalizer(), new StubDiffer(identical: true));
+
+        Assert.True(report.Errored >= 1);
+        Assert.Equal(2, TestRunner.ToExitCode(report));
+    }
+
+    [Fact]
+    public void EmptyScenariosDir_ExitZero_TotalZero()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        var report = new TestRunner().RunAll(opts, new FakeHostFactory("{}"), new SpyNormalizer(), new StubDiffer(identical: true));
+        Assert.Equal(0, report.Total);
+        Assert.Equal(0, TestRunner.ToExitCode(report));
+    }
+
+    [Fact]
+    public void ProfileOverride_IsPassedToNormalizer()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+
+        var spy = new SpyNormalizer();
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir, Profile = "strict" };
+        new TestRunner().RunAll(opts, new FakeHostFactory(content), spy, new StubDiffer(identical: true));
+
+        Assert.Contains("strict", spy.Profiles);
+        Assert.DoesNotContain("default", spy.Profiles);
+    }
+
+    [Fact]
+    public void ReportJson_HasExpectedSchema_And_ReportMd_Exists()
+    {
+        var (sDir, bDir, oDir) = MakeDirs();
+        WriteScenario(sDir, "alpha");
+        var content = "{\"x\":1}";
+        File.WriteAllText(Path.Combine(bDir, "alpha.json"), content);
+
+        var opts = new RunnerOptions { ScenariosDir = sDir, BaselinesDir = bDir, OutDir = oDir };
+        new TestRunner().RunAll(opts, new FakeHostFactory(content), new SpyNormalizer(), new StubDiffer(identical: true));
+
+        var jsonPath = Path.Combine(oDir, "report.json");
+        var mdPath = Path.Combine(oDir, "report.md");
+        Assert.True(File.Exists(jsonPath));
+        Assert.True(File.Exists(mdPath));
+
+        using var doc = JsonDocument.Parse(File.ReadAllText(jsonPath));
+        var root = doc.RootElement;
+        Assert.True(root.TryGetProperty("runAt", out _));
+        Assert.True(root.TryGetProperty("total", out _));
+        Assert.True(root.TryGetProperty("passed", out _));
+        Assert.True(root.TryGetProperty("failed", out _));
+        Assert.True(root.TryGetProperty("errored", out _));
+        Assert.True(root.TryGetProperty("scenarios", out var scenarios));
+        var first = scenarios[0];
+        Assert.True(first.TryGetProperty("name", out _));
+        Assert.True(first.TryGetProperty("status", out _));
+        Assert.True(first.TryGetProperty("hunks", out _));
+        Assert.True(first.TryGetProperty("checkpointCount", out _));
+        Assert.True(first.TryGetProperty("artifactDir", out _));
+    }
+}