kimminsung/recordingtest
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix smoke gaps: recorder target + VK translation, player enum + null guard (#11)

Browse Source
This commit is contained in:
minsung
2026-04-07 17:30:53 +09:00
parent a0609f8f0e
commit 139fbbc0bc
10 changed files with 515 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/Recordingtest.Player/Model/Step.cs
View File

@@ -9,6 +9,9 @@ public enum StepKind
Wait,
Checkpoint,
Save,
// Added for issue #11 — recorder emits these kinds from the smoke test.
Wheel,
Focus,
}
public sealed class Step

30
src/Recordingtest.Player/PlayerEngine.cs
View File

@@ -68,6 +68,15 @@ public sealed class PlayerEngine
}
point = ComputeScreenPoint(element.Value.Bounds, step.Target.Offset);
}
else if (StepRequiresTarget(step.Kind))
{
// Issue #11: recorder may emit Click/Drag/Type/Focus steps with
// null target. Never click/drag/type at (0,0) on the desktop —
// skip with a warning instead.
Console.WriteLine(
$"[player] warn: skipping step {index} kind={step.Kind} — target is null (issue #11)");
return;
}
switch (step.Kind)
{
@@ -101,9 +110,30 @@ public sealed class PlayerEngine
case StepKind.Save:
host.Hotkey(step.Value ?? "ctrl+s");
break;
case StepKind.Wheel:
// Issue #11: wheel replay not yet implemented — log & no-op so
// scenarios that contain wheel events don't crash the engine.
Console.WriteLine(
$"[player] info: wheel step {index} value={step.Value} — no-op (issue #11)");
break;
case StepKind.Focus:
// Issue #11: focus replay deferred. IPlayerHost has no Focus()
// method yet; we log and continue.
Console.WriteLine(
$"[player] info: focus step {index} — no-op (issue #11)");
break;
}
}
private static bool StepRequiresTarget(StepKind kind) => kind switch
{
StepKind.Click => true,
StepKind.Drag => true,
StepKind.Type => true,
StepKind.Focus => true,
_ => false,
};
public static ScreenPoint ComputeScreenPoint(ElementBounds bounds, double[] offset)
{
var ox = offset.Length > 0 ? offset[0] : 0.5;

119
src/Recordingtest.Recorder/DragCollapser.cs
View File

@@ -31,8 +31,47 @@ public sealed class DragCollapser
int lastX = 0, lastY = 0;
int maxDistSq = 0;
// Accumulator for consecutive printable key_down events → single type step (issue #11).
var typeBuf = new System.Text.StringBuilder();
RawEvent? typeFirst = null;
UiaResolution? typeRes = null;
// Active modifiers (ctrl/shift/alt/win) held down.
var modsDown = new HashSet<string>(StringComparer.Ordinal);
void FlushType()
{
if (typeBuf.Length == 0 || typeFirst is null) return;
var step = new ScenarioStep
{
Kind = "type",
Ts = typeFirst.TimestampMs,
Value = typeBuf.ToString(),
};
if (typeRes is not null)
{
step.Target = new ScenarioTarget
{
UiaPath = typeRes.UiaPath,
Offset = new[] { 0.5, 0.5 },
};
if (MaskPolicy.IsMasked(typeRes.Snapshot))
{
step.Value = MaskPolicy.MaskedValue;
}
}
steps.Add(step);
typeBuf.Clear();
typeFirst = null;
typeRes = null;
}
foreach (var ev in events)
{
// Any non-key-down event flushes the pending type buffer.
if (ev.Kind != "key_down" && ev.Kind != "key_up")
{
FlushType();
}
switch (ev.Kind)
{
case "mouse_down_l":
@@ -143,28 +182,87 @@ public sealed class DragCollapser
break;
}
case "key_up":
{
var tr = KeyTranslator.Translate(ev.Code);
if (tr.Category == KeyTranslator.KeyCategory.Modifier)
{
modsDown.Remove(tr.Text);
}
break;
}
case "key_down":
{
var res = resolver(ev);
var step = new ScenarioStep
var tr = KeyTranslator.Translate(ev.Code);
if (tr.Category == KeyTranslator.KeyCategory.Modifier)
{
Kind = "type",
// Modifier held — don't emit standalone step; flush any in-progress
// type buffer so upcoming printable keys start a fresh step.
FlushType();
modsDown.Add(tr.Text);
break;
}
var res = resolver(ev);
if (modsDown.Count > 0)
{
// Combined hotkey (e.g., ctrl+c).
FlushType();
var parts = new List<string>();
if (modsDown.Contains("ctrl")) parts.Add("ctrl");
if (modsDown.Contains("shift")) parts.Add("shift");
if (modsDown.Contains("alt")) parts.Add("alt");
if (modsDown.Contains("win")) parts.Add("win");
parts.Add(tr.Text.ToLowerInvariant());
var hk = new ScenarioStep
{
Kind = "hotkey",
Ts = ev.TimestampMs,
Value = string.Join("+", parts),
RawVk = ev.Code,
};
if (res is not null)
{
hk.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { 0.5, 0.5 },
};
}
steps.Add(hk);
break;
}
if (tr.Category == KeyTranslator.KeyCategory.Printable)
{
if (typeFirst is null)
{
typeFirst = ev;
typeRes = res;
}
typeBuf.Append(tr.Text);
break;
}
// Named non-printable key → flush buffer and emit hotkey step.
FlushType();
var named = new ScenarioStep
{
Kind = "hotkey",
Ts = ev.TimestampMs,
Value = ev.Code.ToString(System.Globalization.CultureInfo.InvariantCulture),
Value = tr.Text,
RawVk = ev.Code,
};
if (res is not null)
{
step.Target = new ScenarioTarget
named.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { 0.5, 0.5 },
};
if (MaskPolicy.IsMasked(res.Snapshot))
{
step.Value = MaskPolicy.MaskedValue;
}
}
steps.Add(step);
steps.Add(named);
break;
}
@@ -213,6 +311,7 @@ public sealed class DragCollapser
}
}
FlushType();
return steps;
}
}

91
src/Recordingtest.Recorder/KeyTranslator.cs Normal file
View File

@@ -0,0 +1,91 @@
using System.Globalization;
namespace Recordingtest.Recorder;
/// <summary>
/// Translates Win32 virtual-key codes into human-readable strings for
/// scenario yaml. See issue #11 — prior recorder emitted raw VK ints which
/// broke replay. Rules:
/// - modifiers (Ctrl/Shift/Alt/Win) are tracked separately and not emitted as standalone steps
/// - printable keys (letters/digits/space) become single-character strings
/// - other named keys become lowercase names (enter, tab, esc, f1..f12, arrows, ...)
/// </summary>
public static class KeyTranslator
{
public enum KeyCategory
{
Modifier,
Printable,
Named,
}
public readonly record struct Translated(KeyCategory Category, string Text, uint RawVk);
// Convention chosen for issue #11: printable letters are emitted UPPERCASE
// matching VK semantics (VK for 'A' is 0x41). Case handling is the player's
// concern (Shift state is tracked separately as a modifier).
public static Translated Translate(uint vk)
{
switch (vk)
{
case 0x10: // VK_SHIFT
case 0xA0: // VK_LSHIFT
case 0xA1: // VK_RSHIFT
return new Translated(KeyCategory.Modifier, "shift", vk);
case 0x11: // VK_CONTROL
case 0xA2: // VK_LCONTROL
case 0xA3: // VK_RCONTROL
return new Translated(KeyCategory.Modifier, "ctrl", vk);
case 0x12: // VK_MENU (Alt)
case 0xA4: // VK_LMENU
case 0xA5: // VK_RMENU
return new Translated(KeyCategory.Modifier, "alt", vk);
case 0x5B: // VK_LWIN
case 0x5C: // VK_RWIN
return new Translated(KeyCategory.Modifier, "win", vk);
case 0x08: return new Translated(KeyCategory.Named, "backspace", vk);
case 0x09: return new Translated(KeyCategory.Named, "tab", vk);
case 0x0D: return new Translated(KeyCategory.Named, "enter", vk);
case 0x1B: return new Translated(KeyCategory.Named, "escape", vk);
case 0x20: return new Translated(KeyCategory.Printable, " ", vk);
case 0x21: return new Translated(KeyCategory.Named, "pageup", vk);
case 0x22: return new Translated(KeyCategory.Named, "pagedown", vk);
case 0x23: return new Translated(KeyCategory.Named, "end", vk);
case 0x24: return new Translated(KeyCategory.Named, "home", vk);
case 0x25: return new Translated(KeyCategory.Named, "left", vk);
case 0x26: return new Translated(KeyCategory.Named, "up", vk);
case 0x27: return new Translated(KeyCategory.Named, "right", vk);
case 0x28: return new Translated(KeyCategory.Named, "down", vk);
case 0x2D: return new Translated(KeyCategory.Named, "insert", vk);
case 0x2E: return new Translated(KeyCategory.Named, "delete", vk);
}
// Letters A-Z (VK 0x41..0x5A)
if (vk >= 0x41 && vk <= 0x5A)
{
var c = (char)vk;
return new Translated(KeyCategory.Printable, c.ToString(CultureInfo.InvariantCulture), vk);
}
// Top-row digits 0-9 (VK 0x30..0x39)
if (vk >= 0x30 && vk <= 0x39)
{
var c = (char)vk;
return new Translated(KeyCategory.Printable, c.ToString(CultureInfo.InvariantCulture), vk);
}
// Numpad digits (VK 0x60..0x69)
if (vk >= 0x60 && vk <= 0x69)
{
var c = (char)('0' + (vk - 0x60));
return new Translated(KeyCategory.Printable, c.ToString(CultureInfo.InvariantCulture), vk);
}
// F1..F24 (VK 0x70..0x87)
if (vk >= 0x70 && vk <= 0x87)
{
var n = (int)(vk - 0x70) + 1;
return new Translated(KeyCategory.Named, "f" + n.ToString(CultureInfo.InvariantCulture), vk);
}
return new Translated(KeyCategory.Named, "vk" + vk.ToString(CultureInfo.InvariantCulture), vk);
}
}

34
src/Recordingtest.Recorder/Program.cs
View File

@@ -11,6 +11,7 @@ namespace Recordingtest.Recorder;
public static class Program
{
[STAThread]
public static int Main(string[] args)
{
var parsed = ParseArgs(args);
@@ -126,7 +127,8 @@ public static class Program
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
int eventCount = 0;
int unresolved = 0;
int unresolvedPaths = 0; // resolver ran but returned null
int noResolverAttempt = 0; // resolver skipped entirely (e.g. automation null, key event)
var sw = Stopwatch.StartNew();
var rawBuffer = new System.Collections.Generic.List<RawEvent>();
@@ -146,13 +148,24 @@ public static class Program
var collapser = new DragCollapser();
UiaResolution? Resolve(RawEvent ev)
{
if (automation is null) return null;
// Key events have no meaningful coordinate — resolver cannot attempt
// a point-based lookup. Count them separately from genuine misses.
if (ev.Kind == "key_down" || ev.Kind == "key_up")
{
noResolverAttempt++;
return null;
}
if (automation is null)
{
noResolverAttempt++;
return null;
}
try
{
var snap = ResolveAt(automation, ev.X, ev.Y);
if (snap is null)
{
unresolved++;
unresolvedPaths++;
return null;
}
var path = ElementPathBuilder.Build(snap);
@@ -160,7 +173,7 @@ public static class Program
}
catch
{
unresolved++;
unresolvedPaths++;
return null;
}
}
@@ -168,8 +181,19 @@ public static class Program
{
scenario.Steps.Add(step);
}
int nullTargetSteps = 0;
foreach (var s in scenario.Steps)
{
if (s.Target is null && s.Kind != "wait" && s.Kind != "checkpoint")
{
nullTargetSteps++;
}
}
ScenarioWriter.WriteToFile(scenario, args.OutputPath);
Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}");
Console.WriteLine(
$"[recorder] done. events={eventCount} elapsed={sw.Elapsed} " +
$"unresolved_paths={unresolvedPaths} no_resolver_attempt={noResolverAttempt} " +
$"null_target_steps={nullTargetSteps}");
automation?.Dispose();
return 0;

2
src/Recordingtest.Recorder/Scenario.cs
View File

@@ -31,6 +31,8 @@ public sealed class ScenarioStep
public double[]? EndOffset { get; set; }
/// <summary>For drag steps: end raw coordinate [x, y].</summary>
public int[]? EndRawCoord { get; set; }
/// <summary>Raw Win32 virtual-key code for diagnostics (issue #11).</summary>
public uint? RawVk { get; set; }
}
public sealed class ScenarioTarget