kimminsung/recordingtest
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Implement test-runner PoC (#8)

Browse Source
This commit is contained in:
minsung
2026-04-07 15:21:03 +09:00
parent 836afea5ee
commit 96df2ef65d
12 changed files with 665 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/Recordingtest.Runner/DefaultAdapters.cs Normal file
View File

@@ -0,0 +1,22 @@
using Recordingtest.DiffReporter;
using Recordingtest.Player;
using Recordingtest.Player.Model;
namespace Recordingtest.Runner;
public sealed class DefaultNormalizer : INormalizer
{
public string Normalize(string input, string profile, string? sidecarPath)
=> Recordingtest.Normalizer.Normalizer.Normalize(input, profile, sidecarPath).Output;
}
public sealed class DefaultDiffer : IDiffer
{
public DiffResult Compare(string approvedPath, string receivedPath)
=> Differ.Compare(approvedPath, receivedPath);
}
public sealed class DefaultHostFactory : IRunnerHostFactory
{
public IPlayerHost Create(Scenario scenario, string outDir) => new UiaPlayerHost(null, outDir);
}

20
src/Recordingtest.Runner/Interfaces.cs Normal file
View File

@@ -0,0 +1,20 @@
using Recordingtest.DiffReporter;
using Recordingtest.Player;
using Recordingtest.Player.Model;
namespace Recordingtest.Runner;
public interface INormalizer
{
string Normalize(string input, string profile, string? sidecarPath);
}
public interface IDiffer
{
DiffResult Compare(string approvedPath, string receivedPath);
}
public interface IRunnerHostFactory
{
IPlayerHost Create(Scenario scenario, string outDir);
}

42
src/Recordingtest.Runner/Program.cs Normal file
View File

@@ -0,0 +1,42 @@
namespace Recordingtest.Runner;
public static class Program
{
public static int Main(string[] args)
{
var options = new RunnerOptions();
for (int i = 0; i < args.Length; i++)
{
switch (args[i])
{
case "--scenarios": options.ScenariosDir = args[++i]; break;
case "--baselines": options.BaselinesDir = args[++i]; break;
case "--out": options.OutDir = args[++i]; break;
case "--profile": options.Profile = args[++i]; break;
case "--no-launch": options.NoLaunch = true; break;
case "-h":
case "--help":
Console.WriteLine("Usage: Recordingtest.Runner --scenarios <dir> --baselines <dir> --out <dir> [--profile <name>] [--no-launch]");
return 0;
}
}
if (string.IsNullOrEmpty(options.ScenariosDir) ||
string.IsNullOrEmpty(options.BaselinesDir) ||
string.IsNullOrEmpty(options.OutDir))
{
Console.Error.WriteLine("Missing required args. Use --help.");
return 2;
}
var runner = new TestRunner();
var report = runner.RunAll(
options,
new DefaultHostFactory(),
new DefaultNormalizer(),
new DefaultDiffer());
Console.WriteLine($"Total: {report.Total}, Passed: {report.Passed}, Failed: {report.Failed}, Errored: {report.Errored}");
return TestRunner.ToExitCode(report);
}
}

15
src/Recordingtest.Runner/Recordingtest.Runner.csproj Normal file
View File

@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0-windows</TargetFramework>
<UseWPF>false</UseWPF>
<UseWindowsForms>false</UseWindowsForms>
<AssemblyName>Recordingtest.Runner</AssemblyName>
<RootNamespace>Recordingtest.Runner</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Recordingtest.Player\Recordingtest.Player.csproj" />
<ProjectReference Include="..\Recordingtest.Normalizer\Recordingtest.Normalizer.csproj" />
<ProjectReference Include="..\Recordingtest.DiffReporter\Recordingtest.DiffReporter.csproj" />
</ItemGroup>
</Project>

21
src/Recordingtest.Runner/RunReport.cs Normal file
View File

@@ -0,0 +1,21 @@
namespace Recordingtest.Runner;
public sealed class RunReport
{
public DateTime RunAt { get; set; }
public int Total { get; set; }
public int Passed { get; set; }
public int Failed { get; set; }
public int Errored { get; set; }
public List<ScenarioResult> Scenarios { get; set; } = new();
}
public sealed class ScenarioResult
{
public string Name { get; set; } = string.Empty;
public string Status { get; set; } = "pass";
public int Hunks { get; set; }
public int CheckpointCount { get; set; }
public string ArtifactDir { get; set; } = string.Empty;
public string? Error { get; set; }
}

10
src/Recordingtest.Runner/RunnerOptions.cs Normal file
View File

@@ -0,0 +1,10 @@
namespace Recordingtest.Runner;
public sealed class RunnerOptions
{
public string ScenariosDir { get; set; } = string.Empty;
public string BaselinesDir { get; set; } = string.Empty;
public string OutDir { get; set; } = string.Empty;
public string Profile { get; set; } = "default";
public bool NoLaunch { get; set; }
}

210
src/Recordingtest.Runner/TestRunner.cs Normal file
View File

@@ -0,0 +1,210 @@
using System.Text;
using System.Text.Json;
using Recordingtest.Player;
using Recordingtest.Player.Model;
namespace Recordingtest.Runner;
public sealed class TestRunner
{
// Note: baselines are normalized with the same profile as received output
// (we do not assume pre-normalized baselines, so re-normalizing is safe).
public RunReport RunAll(
RunnerOptions options,
IRunnerHostFactory hostFactory,
INormalizer normalizer,
IDiffer differ)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(hostFactory);
ArgumentNullException.ThrowIfNull(normalizer);
ArgumentNullException.ThrowIfNull(differ);
Directory.CreateDirectory(options.OutDir);
var report = new RunReport { RunAt = DateTime.UtcNow };
var yamlFiles = Directory.Exists(options.ScenariosDir)
? Directory.GetFiles(options.ScenariosDir, "*.yaml", SearchOption.TopDirectoryOnly)
.OrderBy(p => p, StringComparer.Ordinal).ToArray()
: Array.Empty<string>();
foreach (var yamlPath in yamlFiles)
{
var scenarioName = Path.GetFileNameWithoutExtension(yamlPath);
var artifactDir = Path.Combine(options.OutDir, scenarioName);
Directory.CreateDirectory(artifactDir);
var sr = new ScenarioResult
{
Name = scenarioName,
ArtifactDir = artifactDir,
};
Scenario? scenario = null;
try
{
scenario = ScenarioLoader.LoadFromFile(yamlPath);
sr.CheckpointCount = scenario.Steps.Count(s => s.Kind == StepKind.Checkpoint);
var host = hostFactory.Create(scenario, artifactDir);
var engine = new PlayerEngine();
engine.Run(scenario, host);
}
catch (Exception ex)
{
sr.Status = "error";
sr.Error = ex.Message;
report.Scenarios.Add(sr);
continue;
}
try
{
// Determine result file path: <scenario.save_as> from last save step or convention
var lastSave = scenario!.Steps
.LastOrDefault(s => !string.IsNullOrEmpty(s.SaveAs));
string resultPath;
if (lastSave is not null && !string.IsNullOrEmpty(lastSave.SaveAs))
{
resultPath = Path.IsPathRooted(lastSave.SaveAs)
? lastSave.SaveAs
: Path.Combine(artifactDir, lastSave.SaveAs);
}
else
{
// convention: <artifactDir>/result.*
var conv = Directory.Exists(artifactDir)
? Directory.GetFiles(artifactDir, "result.*").FirstOrDefault()
: null;
resultPath = conv ?? Path.Combine(artifactDir, "result.json");
}
if (!File.Exists(resultPath))
{
sr.Status = "error";
sr.Error = $"result file missing: {resultPath}";
report.Scenarios.Add(sr);
continue;
}
var baselinePath = FindBaseline(options.BaselinesDir, scenarioName, Path.GetExtension(resultPath));
if (baselinePath is null)
{
sr.Status = "error";
sr.Error = $"baseline missing for scenario {scenarioName}";
report.Scenarios.Add(sr);
continue;
}
var receivedRaw = File.ReadAllText(resultPath);
var approvedRaw = File.ReadAllText(baselinePath);
var receivedNorm = normalizer.Normalize(receivedRaw, options.Profile, null);
var approvedNorm = normalizer.Normalize(approvedRaw, options.Profile, null);
var receivedNormPath = Path.Combine(artifactDir, "received.normalized");
var approvedNormPath = Path.Combine(artifactDir, "approved.normalized");
File.WriteAllText(receivedNormPath, receivedNorm);
File.WriteAllText(approvedNormPath, approvedNorm);
var diff = differ.Compare(approvedNormPath, receivedNormPath);
sr.Hunks = diff.Hunks.Count;
sr.Status = diff.Identical ? "pass" : "fail";
}
catch (Exception ex)
{
sr.Status = "error";
sr.Error = ex.Message;
}
report.Scenarios.Add(sr);
}
report.Total = report.Scenarios.Count;
report.Passed = report.Scenarios.Count(s => s.Status == "pass");
report.Failed = report.Scenarios.Count(s => s.Status == "fail");
report.Errored = report.Scenarios.Count(s => s.Status == "error");
WriteJsonReport(report, Path.Combine(options.OutDir, "report.json"));
WriteMarkdownReport(report, Path.Combine(options.OutDir, "report.md"));
return report;
}
private static string? FindBaseline(string baselinesDir, string scenarioName, string preferredExt)
{
if (string.IsNullOrEmpty(baselinesDir) || !Directory.Exists(baselinesDir))
return null;
var candidates = new List<string>
{
Path.Combine(baselinesDir, scenarioName + preferredExt),
Path.Combine(baselinesDir, scenarioName + ".approved" + preferredExt),
Path.Combine(baselinesDir, scenarioName + ".json"),
Path.Combine(baselinesDir, scenarioName + ".approved.json"),
};
foreach (var c in candidates)
if (File.Exists(c)) return c;
var matches = Directory.GetFiles(baselinesDir, scenarioName + ".*");
return matches.FirstOrDefault();
}
public static void WriteJsonReport(RunReport report, string path)
{
var opts = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true,
};
File.WriteAllText(path, JsonSerializer.Serialize(report, opts));
}
public static void WriteMarkdownReport(RunReport report, string path)
{
var sb = new StringBuilder();
sb.AppendLine("# Test Runner Report");
sb.AppendLine();
sb.Append("Run at: ").AppendLine(report.RunAt.ToString("u"));
sb.AppendLine();
sb.Append("Total: ").Append(report.Total)
.Append(" | Passed: ").Append(report.Passed)
.Append(" | Failed: ").Append(report.Failed)
.Append(" | Errored: ").AppendLine(report.Errored.ToString());
sb.AppendLine();
sb.AppendLine("| Scenario | Status | Hunks | Checkpoints | Artifacts |");
sb.AppendLine("|----------|--------|-------|-------------|-----------|");
foreach (var s in report.Scenarios)
{
sb.Append("| ").Append(s.Name)
.Append(" | ").Append(s.Status)
.Append(" | ").Append(s.Hunks)
.Append(" | ").Append(s.CheckpointCount)
.Append(" | ").Append(s.ArtifactDir)
.AppendLine(" |");
}
sb.AppendLine();
var bad = report.Scenarios.Where(s => s.Status != "pass").ToList();
if (bad.Count > 0)
{
sb.AppendLine("## Failures");
foreach (var s in bad)
{
sb.Append("### ").AppendLine(s.Name);
sb.Append("- status: ").AppendLine(s.Status);
sb.Append("- hunks: ").AppendLine(s.Hunks.ToString());
sb.Append("- artifacts: ").AppendLine(s.ArtifactDir);
if (!string.IsNullOrEmpty(s.Error))
sb.Append("- error: ").AppendLine(s.Error);
sb.AppendLine();
}
}
File.WriteAllText(path, sb.ToString());
}
public static int ToExitCode(RunReport report)
{
if (report.Errored > 0) return 2;
if (report.Failed > 0) return 1;
return 0;
}
}