kimminsung/recordingtest
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

player: raw scenario replay without manual cleanup (#14)

Browse Source 
First time box-v6.yaml (raw recorder output, 676 lines) replays end-to-end
and actually creates a Box in the SUT — no AI post-editing of target paths
or offsets required. This is the counterpart to #13's recorder-side fixes:
the player now absorbs the remaining record→replay gaps instead of demanding
a hand-cleaned scenario.

Changes (all in Recordingtest.Player):

- PlayerEngine: null-target fallbacks
  - Type with null target → host.Type() against current focus
  - Click with null target + raw_coord → click at screen-absolute raw_coord
  - Other null targets still skipped
- PlayerEngine: strip leading alt+tab hotkey steps (recording-startup noise
  that fights the player's own foreground switch)
- PlayerEngine: preserve recorded inter-step timing, clamped 150ms–3s,
  routed through new IPlayerHost.Delay so the engine itself stays Sleep-free
  (keeps the existing "no fixed sleep" DoD test passing)
- PlayerEngine: per-step console log for live debugging
- UiaPlayerHost: BringSutToForeground() — SetForeground + Focus + 600ms
  settle, called from Program.cs before engine.Run
- Step model: add RawCoord (int[]) and Ts (long?) fields, auto-mapped from
  YAML raw_coord / ts keys

Tests updated:
- PlayerEngine_NullTarget_SkipsWithoutCalling → _Fallback_Issue14
  (verifies the new Click-with-raw_coord and Type behavior)
- FakePlayerHost (both player.tests and runner.tests) implement Delay

Live smoke: box-v6.yaml raw replay produced the expected Box geometry on
the 2nd attempt; 1st attempt dropped the initial "BOX" keystrokes, tracked
as a follow-up (foreground settle is still threshold-sensitive at 600ms).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
minsung
2026-04-08 19:26:41 +09:00
parent 4ba5b3d74b
commit 70bf5703b3
12 changed files with 237 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/Recordingtest.Player/IPlayerHost.cs
View File

@@ -25,4 +25,9 @@ public interface IPlayerHost
void CaptureCheckpoint(int afterStep, string saveAs);
void CaptureFailureArtifacts(int stepIndex, string reason);
// Issue #14: delay between steps. Kept on the host (not in the engine)
// because PlayerEngine contract forbids fixed sleeps; the host is free
// to implement real time or a virtual clock for tests.
void Delay(TimeSpan duration);
}

6
src/Recordingtest.Player/Model/Step.cs
View File

@@ -22,6 +22,12 @@ public sealed class Step
public string? WaitFor { get; set; }
public int? AfterStep { get; set; }
public string? SaveAs { get; set; }
// Issue #14: recorder-captured screen-absolute coordinates used as
// fallback when Target is null (Click). Optional; null for non-mouse steps.
public int[]? RawCoord { get; set; }
// Issue #14: recorder-captured absolute timestamp (ms). Used by the
// engine to preserve inter-step pacing during playback.
public long? Ts { get; set; }
}
public sealed class Target

79
src/Recordingtest.Player/PlayerEngine.cs
View File

@@ -6,6 +6,12 @@ public sealed class PlayerEngineOptions
{
public TimeSpan ResolveTimeout { get; set; } = TimeSpan.FromSeconds(10);
public TimeSpan WaitForTimeout { get; set; } = TimeSpan.FromSeconds(15);
// Issue #14: preserve recorded inter-step delays (clamped). When true the
// engine sleeps step.Ts - prevStep.Ts between steps, bounded by Min/Max.
public bool PreserveTiming { get; set; } = true;
public TimeSpan MinStepDelay { get; set; } = TimeSpan.FromMilliseconds(150);
public TimeSpan MaxStepDelay { get; set; } = TimeSpan.FromSeconds(3);
}
public sealed class PlayerEngine
@@ -22,9 +28,51 @@ public sealed class PlayerEngine
ArgumentNullException.ThrowIfNull(scenario);
ArgumentNullException.ThrowIfNull(host);
for (int i = 0; i < scenario.Steps.Count; i++)
// Issue #14: strip leading alt+tab hotkey steps. These are recording
// startup noise (user tabbing from their editor into the SUT at the
// start of the session). At replay time the player already puts the
// SUT in the foreground, so re-running alt+tab here just switches
// focus AWAY from the SUT and breaks subsequent Type steps.
int start = 0;
while (start < scenario.Steps.Count)
{
var s = scenario.Steps[start];
if (s.Kind == StepKind.Hotkey &&
string.Equals(s.Value, "alt+tab", StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine(
$"[player] info: skipping leading alt+tab step {start} (issue #14)");
start++;
continue;
}
break;
}
// Seed prevTs so the FIRST executed step also gets a pre-delay
// (MinStepDelay). Without this, step 2's Type can fire before the
// SUT has fully settled after foreground switch.
long? prevTs = start < scenario.Steps.Count && scenario.Steps[start].Ts is long firstTs
? firstTs - (long)_options.MinStepDelay.TotalMilliseconds
: null;
for (int i = start; i < scenario.Steps.Count; i++)
{
var step = scenario.Steps[i];
if (_options.PreserveTiming && step.Ts is long ts)
{
if (prevTs is long p)
{
var delta = ts - p;
if (delta < _options.MinStepDelay.TotalMilliseconds)
delta = (long)_options.MinStepDelay.TotalMilliseconds;
if (delta > _options.MaxStepDelay.TotalMilliseconds)
delta = (long)_options.MaxStepDelay.TotalMilliseconds;
host.Delay(TimeSpan.FromMilliseconds(delta));
}
prevTs = ts;
}
Console.WriteLine($"[player] step {i} kind={step.Kind} value={step.Value ?? ""}");
try
{
ExecuteStep(i, step, host);
@@ -70,12 +118,29 @@ public sealed class PlayerEngine
}
else if (StepRequiresTarget(step.Kind))
{
// Issue #11: recorder may emit Click/Drag/Type/Focus steps with
// null target. Never click/drag/type at (0,0) on the desktop —
// skip with a warning instead.
Console.WriteLine(
$"[player] warn: skipping step {index} kind={step.Kind} — target is null (issue #11)");
return;
// Issue #14: recorder emits Type/Click with null target when the
// focused element / UIA path at record time could not be resolved
// (e.g. typing into a CommandBox before any mouse click, clicks on
// canvas children that don't expose AutomationId). Fall back to:
// - Type → send keystrokes to whatever currently has focus
// - Click → use recorded raw_coord (screen-absolute) directly
// This mirrors the manual cleanup that produced box-v5-clean.yaml.
if (step.Kind == StepKind.Type)
{
}
else if (step.Kind == StepKind.Click
&& step.RawCoord is { Length: >= 2 })
{
point = new ScreenPoint(step.RawCoord[0], step.RawCoord[1]);
Console.WriteLine(
$"[player] info: step {index} kind=Click null target — using raw_coord ({point.X},{point.Y}) (issue #14)");
}
else
{
Console.WriteLine(
$"[player] warn: skipping step {index} kind={step.Kind} — target is null and no fallback (issue #14)");
return;
}
}
switch (step.Kind)

7
src/Recordingtest.Player/Program.cs
View File

@@ -59,6 +59,13 @@ else
}
using var host = new UiaPlayerHost(app, artifactDir);
// Issue #14: ensure SUT is the foreground window before playback so that
// keystrokes (Type/Hotkey) land on the SUT instead of whatever shell the
// player was launched from (PowerShell, VS Code, etc.). Without this, the
// very first "BOX" type step gets typed into the launching terminal.
host.BringSutToForeground();
var engine = new PlayerEngine();
try
{

35
src/Recordingtest.Player/UiaPlayerHost.cs
View File

@@ -206,6 +206,41 @@ public sealed class UiaPlayerHost : IPlayerHost, IDisposable
}
}
/// <summary>
/// Issue #14 — force the SUT main window to foreground and give it keyboard
/// focus before playback starts. Handles the common case where the player
/// was launched from a shell that still has focus when the first Type/Hotkey
/// step fires.
/// </summary>
public void BringSutToForeground()
{
try
{
var w = _app?.GetMainWindow(_automation, TimeSpan.FromSeconds(5));
if (w is null) return;
try { w.SetForeground(); } catch { /* best-effort */ }
try { w.Focus(); } catch { /* best-effort */ }
// Small settle so the OS-level focus switch takes effect before
// the first SendInput. 150ms is enough in practice on Win10.
// Increased from 150→600ms because FlaUI Keyboard.Type drops the
// first couple of characters if SendInput fires before the OS
// finishes the focus transition (observed: "BOX" lost on first
// step, "10" succeeded later once the app had settled).
System.Threading.Thread.Sleep(600);
}
catch
{
// best-effort; if this fails the user will see the BOX text land
// in the wrong window and can re-run with the SUT focused manually.
}
}
public void Delay(TimeSpan duration)
{
if (duration > TimeSpan.Zero)
System.Threading.Thread.Sleep(duration);
}
public void Dispose()
{
_automation.Dispose();