kimminsung/recordingtest
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix recorder drag collapse, focus events, ts/raw_coord (#6)

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
minsung
2026-04-07 14:33:46 +09:00
parent f17e764678
commit 56b7233500
6 changed files with 408 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docs/history/2026-04-07_이슈6-recorder-iteration2.md Normal file
View File

@@ -0,0 +1,16 @@
# 이슈 #6 — Recorder Iteration 2
- 소요 시간: 약 20분
- Context 사용량: 약 35K tokens
- 이슈: #6 (recorder)
## 변경 요약
1. `DragCollapser.cs` 신규: down/move/up 상태 머신으로 click 또는 drag 스텝 생성. focus_change/wheel/key_down/right-click도 처리.
2. `Scenario.ScenarioStep`에 `Ts`, `RawCoord`, `EndOffset`, `EndRawCoord` 필드 추가 (snake_case 직렬화).
3. `RawEvent`에 `FocusedElementPath` 필드 추가.
4. `Program.cs`: ConsumeAsync는 raw 이벤트 버퍼링만, 종료 시 DragCollapser로 일괄 변환. `UIA3Automation.RegisterFocusChangedEvent` 호출 추가 (callback에서는 element 캡처 + path build만 수행하고 큐로 push).
5. 테스트 4개 추가 (drag/click/focus/yaml roundtrip), 총 9개.
## 결과
- `dotnet build recordingtest.sln`: 0 warning / 0 error
- `dotnet test tests/Recordingtest.Recorder.Tests`: 9 passed

218
src/Recordingtest.Recorder/DragCollapser.cs Normal file
View File

@@ -0,0 +1,218 @@
using System;
using System.Collections.Generic;
namespace Recordingtest.Recorder;
/// <summary>
/// Resolution of a raw event into an UIA element snapshot + path.
/// Provided by the caller (usually backed by FlaUI from-point lookup).
/// </summary>
public sealed record UiaResolution(IElementSnapshot Snapshot, string UiaPath);
/// <summary>
/// Pure state machine that collapses a raw event stream into ScenarioSteps.
/// Recognizes click vs drag based on movement between mouse_down/mouse_up.
/// </summary>
public sealed class DragCollapser
{
public int DragThresholdPx { get; }
public DragCollapser(int dragThresholdPx = 4)
{
DragThresholdPx = dragThresholdPx;
}
public IReadOnlyList<ScenarioStep> Collapse(
IEnumerable<RawEvent> events,
Func<RawEvent, UiaResolution?> resolver)
{
var steps = new List<ScenarioStep>();
RawEvent? down = null;
int lastX = 0, lastY = 0;
int maxDistSq = 0;
foreach (var ev in events)
{
switch (ev.Kind)
{
case "mouse_down_l":
down = ev;
lastX = ev.X;
lastY = ev.Y;
maxDistSq = 0;
break;
case "move":
if (down is not null)
{
int dx = ev.X - down.X;
int dy = ev.Y - down.Y;
int d2 = dx * dx + dy * dy;
if (d2 > maxDistSq) maxDistSq = d2;
lastX = ev.X;
lastY = ev.Y;
}
break;
case "mouse_up_l":
if (down is not null)
{
int fdx = ev.X - down.X;
int fdy = ev.Y - down.Y;
int finalDistSq = fdx * fdx + fdy * fdy;
int useSq = Math.Max(maxDistSq, finalDistSq);
var threshSq = DragThresholdPx * DragThresholdPx;
var downRes = resolver(down);
if (useSq >= threshSq)
{
// drag step
var step = new ScenarioStep
{
Kind = "drag",
Ts = down.TimestampMs,
RawCoord = new[] { down.X, down.Y },
EndRawCoord = new[] { ev.X, ev.Y },
};
if (downRes is not null)
{
var (sx, sy) = OffsetNormalizer.Normalize(
downRes.Snapshot.BoundingRectangle, down.X, down.Y);
var (ex, ey) = OffsetNormalizer.Normalize(
downRes.Snapshot.BoundingRectangle, ev.X, ev.Y);
step.Target = new ScenarioTarget
{
UiaPath = downRes.UiaPath,
Offset = new[] { sx, sy },
};
step.EndOffset = new[] { ex, ey };
}
steps.Add(step);
}
else
{
// click step at down point
var step = new ScenarioStep
{
Kind = "click",
Ts = down.TimestampMs,
RawCoord = new[] { down.X, down.Y },
};
if (downRes is not null)
{
var (ox, oy) = OffsetNormalizer.Normalize(
downRes.Snapshot.BoundingRectangle, down.X, down.Y);
step.Target = new ScenarioTarget
{
UiaPath = downRes.UiaPath,
Offset = new[] { ox, oy },
};
if (MaskPolicy.IsMasked(downRes.Snapshot))
{
step.Value = MaskPolicy.MaskedValue;
}
}
steps.Add(step);
}
down = null;
maxDistSq = 0;
}
break;
case "mouse_down_r":
{
var res = resolver(ev);
var step = new ScenarioStep
{
Kind = "click",
Ts = ev.TimestampMs,
RawCoord = new[] { ev.X, ev.Y },
Value = "right",
};
if (res is not null)
{
var (ox, oy) = OffsetNormalizer.Normalize(
res.Snapshot.BoundingRectangle, ev.X, ev.Y);
step.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { ox, oy },
};
}
steps.Add(step);
break;
}
case "key_down":
{
var res = resolver(ev);
var step = new ScenarioStep
{
Kind = "type",
Ts = ev.TimestampMs,
Value = ev.Code.ToString(System.Globalization.CultureInfo.InvariantCulture),
};
if (res is not null)
{
step.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { 0.5, 0.5 },
};
if (MaskPolicy.IsMasked(res.Snapshot))
{
step.Value = MaskPolicy.MaskedValue;
}
}
steps.Add(step);
break;
}
case "wheel":
{
var res = resolver(ev);
var step = new ScenarioStep
{
Kind = "wheel",
Ts = ev.TimestampMs,
RawCoord = new[] { ev.X, ev.Y },
Value = ev.WheelDelta.ToString(System.Globalization.CultureInfo.InvariantCulture),
};
if (res is not null)
{
var (ox, oy) = OffsetNormalizer.Normalize(
res.Snapshot.BoundingRectangle, ev.X, ev.Y);
step.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { ox, oy },
};
}
steps.Add(step);
break;
}
case "focus_change":
{
var step = new ScenarioStep
{
Kind = "focus",
Ts = ev.TimestampMs,
};
if (!string.IsNullOrEmpty(ev.FocusedElementPath))
{
step.Target = new ScenarioTarget
{
UiaPath = ev.FocusedElementPath!,
Offset = new[] { 0.5, 0.5 },
};
}
steps.Add(step);
break;
}
}
}
return steps;
}
}

2
src/Recordingtest.Recorder/LowLevelHook.cs
View File

@@ -4,7 +4,7 @@ using System.Threading.Channels;
namespace Recordingtest.Recorder;
public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta);
public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta, string? FocusedElementPath = null);
/// <summary>
/// Installs WH_KEYBOARD_LL and WH_MOUSE_LL hooks on a dedicated thread with its own message loop.

108
src/Recordingtest.Recorder/Program.cs
View File

@@ -94,16 +94,46 @@ public static class Program
cts.Cancel();
};
// Register UIA focus changed event. The callback only captures the
// element path and pushes a synthetic RawEvent into the same queue;
// it does NOT compute anything else inside the UIA callback.
try
{
if (automation is not null)
{
automation.RegisterFocusChangedEvent(el =>
{
try
{
if (el is null) return;
var snap = new FlaUiSnapshot(el);
var path = ElementPathBuilder.Build(snap);
channel.Writer.TryWrite(new RawEvent(
DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
"focus_change", 0, 0, 0, 0, path));
}
catch
{
// never throw from UIA callback
}
});
}
}
catch (Exception ex)
{
Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}");
}
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
int eventCount = 0;
int unresolved = 0;
var sw = Stopwatch.StartNew();
var rawBuffer = new System.Collections.Generic.List<RawEvent>();
try
{
ConsumeAsync(channel.Reader, scenario, mainWindow, automation, cts.Token,
onEvent: () => eventCount++,
onUnresolved: () => unresolved++).GetAwaiter().GetResult();
ConsumeAsync(channel.Reader, rawBuffer, cts.Token,
onEvent: () => eventCount++).GetAwaiter().GetResult();
}
catch (OperationCanceledException)
{
@@ -111,6 +141,33 @@ public static class Program
}
sw.Stop();
// Collapse buffered raw events into scenario steps via DragCollapser.
var collapser = new DragCollapser();
UiaResolution? Resolve(RawEvent ev)
{
if (automation is null) return null;
try
{
var snap = ResolveAt(automation, ev.X, ev.Y);
if (snap is null)
{
unresolved++;
return null;
}
var path = ElementPathBuilder.Build(snap);
return new UiaResolution(snap, path);
}
catch
{
unresolved++;
return null;
}
}
foreach (var step in collapser.Collapse(rawBuffer, Resolve))
{
scenario.Steps.Add(step);
}
ScenarioWriter.WriteToFile(scenario, args.OutputPath);
Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}");
@@ -155,59 +212,20 @@ public static class Program
private static async Task ConsumeAsync(
ChannelReader<RawEvent> reader,
Scenario scenario,
AutomationElement? mainWindow,
UIA3Automation? automation,
System.Collections.Generic.List<RawEvent> buffer,
CancellationToken ct,
Action onEvent,
Action onUnresolved)
Action onEvent)
{
while (await reader.WaitToReadAsync(ct).ConfigureAwait(false))
{
while (reader.TryRead(out var ev))
{
onEvent();
if (!IsInterestingForStep(ev.Kind)) continue;
IElementSnapshot? snap = null;
if (automation is not null)
{
try
{
snap = ResolveAt(automation, ev.X, ev.Y);
}
catch
{
snap = null;
}
}
if (snap is null)
{
onUnresolved();
continue;
}
var path = ElementPathBuilder.Build(snap);
var (dx, dy) = OffsetNormalizer.Normalize(snap.BoundingRectangle, ev.X, ev.Y);
var step = new ScenarioStep
{
Kind = ev.Kind.StartsWith("key", StringComparison.Ordinal) ? "type" : "click",
Target = new ScenarioTarget
{
UiaPath = path,
Offset = new[] { dx, dy },
},
Value = MaskPolicy.IsMasked(snap) ? MaskPolicy.MaskedValue : null,
};
scenario.Steps.Add(step);
buffer.Add(ev);
}
}
}
private static bool IsInterestingForStep(string kind) =>
kind == "mouse_down_l" || kind == "mouse_down_r" || kind == "key_down";
private static IElementSnapshot? ResolveAt(UIA3Automation automation, int x, int y)
{
var raw = automation.FromPoint(new System.Drawing.Point(x, y));

10
src/Recordingtest.Recorder/Scenario.cs
View File

@@ -18,11 +18,19 @@ public sealed class ScenarioSut
public sealed class ScenarioStep
{
/// <summary>click | type | drag | hotkey | wait</summary>
/// <summary>click | type | drag | hotkey | wait | focus</summary>
public string Kind { get; set; } = "click";
public ScenarioTarget? Target { get; set; }
public string? Value { get; set; }
public string? WaitFor { get; set; }
/// <summary>ms since recording start (or epoch ms from RawEvent).</summary>
public long Ts { get; set; }
/// <summary>Raw screen coordinate [x, y] when applicable.</summary>
public int[]? RawCoord { get; set; }
/// <summary>For drag steps: end offset within target element.</summary>
public double[]? EndOffset { get; set; }
/// <summary>For drag steps: end raw coordinate [x, y].</summary>
public int[]? EndRawCoord { get; set; }
}
public sealed class ScenarioTarget

101
tests/Recordingtest.Recorder.Tests/RecorderTests.cs
View File

@@ -123,6 +123,107 @@ public class RecorderTests
Assert.Equal(s.Steps[1].Value, parsed.Steps[1].Value);
}
private static FakeElement MakeRectElement(string path, double l, double t, double w, double h) =>
new FakeElement
{
ClassName = "Window",
Name = path,
BoundingRectangle = (l, t, w, h),
};
[Fact]
public void DragCollapser_DownMoveUp_BeyondThreshold_EmitsDrag()
{
var el = MakeRectElement("Canvas", 0, 0, 1000, 1000);
var path = "Window[@Name='Canvas']";
UiaResolution? Resolver(RawEvent _) => new UiaResolution(el, path);
var events = new[]
{
new RawEvent(100, "mouse_down_l", 100, 100, 0, 0),
new RawEvent(110, "move", 150, 120, 0, 0),
new RawEvent(120, "move", 300, 400, 0, 0),
new RawEvent(130, "mouse_up_l", 300, 400, 0, 0),
};
var steps = new DragCollapser().Collapse(events, Resolver);
Assert.Single(steps);
Assert.Equal("drag", steps[0].Kind);
Assert.Equal(path, steps[0].Target!.UiaPath);
Assert.Equal(new[] { 100, 100 }, steps[0].RawCoord);
Assert.Equal(new[] { 300, 400 }, steps[0].EndRawCoord);
Assert.NotNull(steps[0].EndOffset);
Assert.Equal(100L, steps[0].Ts);
}
[Fact]
public void DragCollapser_DownUp_BelowThreshold_EmitsClick()
{
var el = MakeRectElement("Btn", 0, 0, 100, 100);
UiaResolution? Resolver(RawEvent _) => new UiaResolution(el, "Window[@Name='Btn']");
var events = new[]
{
new RawEvent(50, "mouse_down_l", 10, 10, 0, 0),
new RawEvent(55, "mouse_up_l", 11, 11, 0, 0),
};
var steps = new DragCollapser().Collapse(events, Resolver);
Assert.Single(steps);
Assert.Equal("click", steps[0].Kind);
Assert.Equal(new[] { 10, 10 }, steps[0].RawCoord);
Assert.Equal(50L, steps[0].Ts);
}
[Fact]
public void DragCollapser_FocusChangeEvent_EmitsFocusStep()
{
var events = new[]
{
new RawEvent(200, "focus_change", 0, 0, 0, 0, "Window[@Name='Main']/Edit[@AutomationId='Pwd']"),
};
var steps = new DragCollapser().Collapse(events, _ => null);
Assert.Single(steps);
Assert.Equal("focus", steps[0].Kind);
Assert.Equal("Window[@Name='Main']/Edit[@AutomationId='Pwd']", steps[0].Target!.UiaPath);
Assert.Equal(200L, steps[0].Ts);
}
[Fact]
public void ScenarioStep_YamlRoundtrip_PreservesTsAndRawCoord()
{
var s = new Scenario
{
Name = "ts-test",
Steps =
{
new ScenarioStep
{
Kind = "click",
Ts = 12345,
RawCoord = new[] { 640, 480 },
Target = new ScenarioTarget
{
UiaPath = "Window[@Name='X']",
Offset = new[] { 0.5, 0.5 },
},
},
},
};
var yaml = ScenarioWriter.Serialize(s);
Assert.Contains("ts:", yaml);
Assert.Contains("raw_coord", yaml);
var parsed = ScenarioWriter.Deserialize(yaml);
Assert.Equal(12345L, parsed.Steps[0].Ts);
Assert.Equal(new[] { 640, 480 }, parsed.Steps[0].RawCoord);
}
[Fact]
public void Cli_MissingAttach_ExitTwo()
{