Fix recorder drag collapse, focus events, ts/raw_coord (#6)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

docs/history/2026-04-07_이슈6-recorder-iteration2.md

@@ -0,0 +1,16 @@
+# 이슈 #6 — Recorder Iteration 2
+
+- 소요 시간: 약 20분
+- Context 사용량: 약 35K tokens
+- 이슈: #6 (recorder)
+
+## 변경 요약
+1. `DragCollapser.cs` 신규: down/move/up 상태 머신으로 click 또는 drag 스텝 생성. focus_change/wheel/key_down/right-click도 처리.
+2. `Scenario.ScenarioStep`에 `Ts`, `RawCoord`, `EndOffset`, `EndRawCoord` 필드 추가 (snake_case 직렬화).
+3. `RawEvent`에 `FocusedElementPath` 필드 추가.
+4. `Program.cs`: ConsumeAsync는 raw 이벤트 버퍼링만, 종료 시 DragCollapser로 일괄 변환. `UIA3Automation.RegisterFocusChangedEvent` 호출 추가 (callback에서는 element 캡처 + path build만 수행하고 큐로 push).
+5. 테스트 4개 추가 (drag/click/focus/yaml roundtrip), 총 9개.
+
+## 결과
+- `dotnet build recordingtest.sln`: 0 warning / 0 error
+- `dotnet test tests/Recordingtest.Recorder.Tests`: 9 passed

src/Recordingtest.Recorder/DragCollapser.cs

@@ -0,0 +1,218 @@
+using System;
+using System.Collections.Generic;
+
+namespace Recordingtest.Recorder;
+
+/// <summary>
+/// Resolution of a raw event into an UIA element snapshot + path.
+/// Provided by the caller (usually backed by FlaUI from-point lookup).
+/// </summary>
+public sealed record UiaResolution(IElementSnapshot Snapshot, string UiaPath);
+
+/// <summary>
+/// Pure state machine that collapses a raw event stream into ScenarioSteps.
+/// Recognizes click vs drag based on movement between mouse_down/mouse_up.
+/// </summary>
+public sealed class DragCollapser
+{
+    public int DragThresholdPx { get; }
+
+    public DragCollapser(int dragThresholdPx = 4)
+    {
+        DragThresholdPx = dragThresholdPx;
+    }
+
+    public IReadOnlyList<ScenarioStep> Collapse(
+        IEnumerable<RawEvent> events,
+        Func<RawEvent, UiaResolution?> resolver)
+    {
+        var steps = new List<ScenarioStep>();
+        RawEvent? down = null;
+        int lastX = 0, lastY = 0;
+        int maxDistSq = 0;
+
+        foreach (var ev in events)
+        {
+            switch (ev.Kind)
+            {
+                case "mouse_down_l":
+                    down = ev;
+                    lastX = ev.X;
+                    lastY = ev.Y;
+                    maxDistSq = 0;
+                    break;
+
+                case "move":
+                    if (down is not null)
+                    {
+                        int dx = ev.X - down.X;
+                        int dy = ev.Y - down.Y;
+                        int d2 = dx * dx + dy * dy;
+                        if (d2 > maxDistSq) maxDistSq = d2;
+                        lastX = ev.X;
+                        lastY = ev.Y;
+                    }
+                    break;
+
+                case "mouse_up_l":
+                    if (down is not null)
+                    {
+                        int fdx = ev.X - down.X;
+                        int fdy = ev.Y - down.Y;
+                        int finalDistSq = fdx * fdx + fdy * fdy;
+                        int useSq = Math.Max(maxDistSq, finalDistSq);
+                        var threshSq = DragThresholdPx * DragThresholdPx;
+
+                        var downRes = resolver(down);
+                        if (useSq >= threshSq)
+                        {
+                            // drag step
+                            var step = new ScenarioStep
+                            {
+                                Kind = "drag",
+                                Ts = down.TimestampMs,
+                                RawCoord = new[] { down.X, down.Y },
+                                EndRawCoord = new[] { ev.X, ev.Y },
+                            };
+                            if (downRes is not null)
+                            {
+                                var (sx, sy) = OffsetNormalizer.Normalize(
+                                    downRes.Snapshot.BoundingRectangle, down.X, down.Y);
+                                var (ex, ey) = OffsetNormalizer.Normalize(
+                                    downRes.Snapshot.BoundingRectangle, ev.X, ev.Y);
+                                step.Target = new ScenarioTarget
+                                {
+                                    UiaPath = downRes.UiaPath,
+                                    Offset = new[] { sx, sy },
+                                };
+                                step.EndOffset = new[] { ex, ey };
+                            }
+                            steps.Add(step);
+                        }
+                        else
+                        {
+                            // click step at down point
+                            var step = new ScenarioStep
+                            {
+                                Kind = "click",
+                                Ts = down.TimestampMs,
+                                RawCoord = new[] { down.X, down.Y },
+                            };
+                            if (downRes is not null)
+                            {
+                                var (ox, oy) = OffsetNormalizer.Normalize(
+                                    downRes.Snapshot.BoundingRectangle, down.X, down.Y);
+                                step.Target = new ScenarioTarget
+                                {
+                                    UiaPath = downRes.UiaPath,
+                                    Offset = new[] { ox, oy },
+                                };
+                                if (MaskPolicy.IsMasked(downRes.Snapshot))
+                                {
+                                    step.Value = MaskPolicy.MaskedValue;
+                                }
+                            }
+                            steps.Add(step);
+                        }
+                        down = null;
+                        maxDistSq = 0;
+                    }
+                    break;
+
+                case "mouse_down_r":
+                {
+                    var res = resolver(ev);
+                    var step = new ScenarioStep
+                    {
+                        Kind = "click",
+                        Ts = ev.TimestampMs,
+                        RawCoord = new[] { ev.X, ev.Y },
+                        Value = "right",
+                    };
+                    if (res is not null)
+                    {
+                        var (ox, oy) = OffsetNormalizer.Normalize(
+                            res.Snapshot.BoundingRectangle, ev.X, ev.Y);
+                        step.Target = new ScenarioTarget
+                        {
+                            UiaPath = res.UiaPath,
+                            Offset = new[] { ox, oy },
+                        };
+                    }
+                    steps.Add(step);
+                    break;
+                }
+
+                case "key_down":
+                {
+                    var res = resolver(ev);
+                    var step = new ScenarioStep
+                    {
+                        Kind = "type",
+                        Ts = ev.TimestampMs,
+                        Value = ev.Code.ToString(System.Globalization.CultureInfo.InvariantCulture),
+                    };
+                    if (res is not null)
+                    {
+                        step.Target = new ScenarioTarget
+                        {
+                            UiaPath = res.UiaPath,
+                            Offset = new[] { 0.5, 0.5 },
+                        };
+                        if (MaskPolicy.IsMasked(res.Snapshot))
+                        {
+                            step.Value = MaskPolicy.MaskedValue;
+                        }
+                    }
+                    steps.Add(step);
+                    break;
+                }
+
+                case "wheel":
+                {
+                    var res = resolver(ev);
+                    var step = new ScenarioStep
+                    {
+                        Kind = "wheel",
+                        Ts = ev.TimestampMs,
+                        RawCoord = new[] { ev.X, ev.Y },
+                        Value = ev.WheelDelta.ToString(System.Globalization.CultureInfo.InvariantCulture),
+                    };
+                    if (res is not null)
+                    {
+                        var (ox, oy) = OffsetNormalizer.Normalize(
+                            res.Snapshot.BoundingRectangle, ev.X, ev.Y);
+                        step.Target = new ScenarioTarget
+                        {
+                            UiaPath = res.UiaPath,
+                            Offset = new[] { ox, oy },
+                        };
+                    }
+                    steps.Add(step);
+                    break;
+                }
+
+                case "focus_change":
+                {
+                    var step = new ScenarioStep
+                    {
+                        Kind = "focus",
+                        Ts = ev.TimestampMs,
+                    };
+                    if (!string.IsNullOrEmpty(ev.FocusedElementPath))
+                    {
+                        step.Target = new ScenarioTarget
+                        {
+                            UiaPath = ev.FocusedElementPath!,
+                            Offset = new[] { 0.5, 0.5 },
+                        };
+                    }
+                    steps.Add(step);
+                    break;
+                }
+            }
+        }
+
+        return steps;
+    }
+}

src/Recordingtest.Recorder/LowLevelHook.cs

@@ -4,7 +4,7 @@ using System.Threading.Channels;
 
 namespace Recordingtest.Recorder;
 
-public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta);
+public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta, string? FocusedElementPath = null);
 
 /// <summary>
 /// Installs WH_KEYBOARD_LL and WH_MOUSE_LL hooks on a dedicated thread with its own message loop.

src/Recordingtest.Recorder/Program.cs

@@ -94,16 +94,46 @@ public static class Program
             cts.Cancel();
         };
 
+        // Register UIA focus changed event. The callback only captures the
+        // element path and pushes a synthetic RawEvent into the same queue;
+        // it does NOT compute anything else inside the UIA callback.
+        try
+        {
+            if (automation is not null)
+            {
+                automation.RegisterFocusChangedEvent(el =>
+                {
+                    try
+                    {
+                        if (el is null) return;
+                        var snap = new FlaUiSnapshot(el);
+                        var path = ElementPathBuilder.Build(snap);
+                        channel.Writer.TryWrite(new RawEvent(
+                            DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                            "focus_change", 0, 0, 0, 0, path));
+                    }
+                    catch
+                    {
+                        // never throw from UIA callback
+                    }
+                });
+            }
+        }
+        catch (Exception ex)
+        {
+            Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}");
+        }
+
         Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
         int eventCount = 0;
         int unresolved = 0;
         var sw = Stopwatch.StartNew();
+        var rawBuffer = new System.Collections.Generic.List<RawEvent>();
 
         try
         {
-            ConsumeAsync(channel.Reader, scenario, mainWindow, automation, cts.Token,
+            ConsumeAsync(channel.Reader, rawBuffer, cts.Token,
-                onEvent: () => eventCount++,
+                onEvent: () => eventCount++).GetAwaiter().GetResult();
-                onUnresolved: () => unresolved++).GetAwaiter().GetResult();
         }
         catch (OperationCanceledException)
         {
@@ -111,6 +141,33 @@ public static class Program
         }
 
         sw.Stop();
+
+        // Collapse buffered raw events into scenario steps via DragCollapser.
+        var collapser = new DragCollapser();
+        UiaResolution? Resolve(RawEvent ev)
+        {
+            if (automation is null) return null;
+            try
+            {
+                var snap = ResolveAt(automation, ev.X, ev.Y);
+                if (snap is null)
+                {
+                    unresolved++;
+                    return null;
+                }
+                var path = ElementPathBuilder.Build(snap);
+                return new UiaResolution(snap, path);
+            }
+            catch
+            {
+                unresolved++;
+                return null;
+            }
+        }
+        foreach (var step in collapser.Collapse(rawBuffer, Resolve))
+        {
+            scenario.Steps.Add(step);
+        }
         ScenarioWriter.WriteToFile(scenario, args.OutputPath);
         Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}");
 
@@ -155,59 +212,20 @@ public static class Program
 
     private static async Task ConsumeAsync(
         ChannelReader<RawEvent> reader,
-        Scenario scenario,
+        System.Collections.Generic.List<RawEvent> buffer,
-        AutomationElement? mainWindow,
-        UIA3Automation? automation,
         CancellationToken ct,
-        Action onEvent,
+        Action onEvent)
-        Action onUnresolved)
     {
         while (await reader.WaitToReadAsync(ct).ConfigureAwait(false))
         {
             while (reader.TryRead(out var ev))
             {
                 onEvent();
-                if (!IsInterestingForStep(ev.Kind)) continue;
+                buffer.Add(ev);
-
-                IElementSnapshot? snap = null;
-                if (automation is not null)
-                {
-                    try
-                    {
-                        snap = ResolveAt(automation, ev.X, ev.Y);
-                    }
-                    catch
-                    {
-                        snap = null;
-                    }
-                }
-
-                if (snap is null)
-                {
-                    onUnresolved();
-                    continue;
-                }
-
-                var path = ElementPathBuilder.Build(snap);
-                var (dx, dy) = OffsetNormalizer.Normalize(snap.BoundingRectangle, ev.X, ev.Y);
-                var step = new ScenarioStep
-                {
-                    Kind = ev.Kind.StartsWith("key", StringComparison.Ordinal) ? "type" : "click",
-                    Target = new ScenarioTarget
-                    {
-                        UiaPath = path,
-                        Offset = new[] { dx, dy },
-                    },
-                    Value = MaskPolicy.IsMasked(snap) ? MaskPolicy.MaskedValue : null,
-                };
-                scenario.Steps.Add(step);
             }
         }
     }
 
-    private static bool IsInterestingForStep(string kind) =>
-        kind == "mouse_down_l" || kind == "mouse_down_r" || kind == "key_down";
-
     private static IElementSnapshot? ResolveAt(UIA3Automation automation, int x, int y)
     {
         var raw = automation.FromPoint(new System.Drawing.Point(x, y));

src/Recordingtest.Recorder/Scenario.cs

@@ -18,11 +18,19 @@ public sealed class ScenarioSut
 
 public sealed class ScenarioStep
 {
-    /// <summary>click | type | drag | hotkey | wait</summary>
+    /// <summary>click | type | drag | hotkey | wait | focus</summary>
     public string Kind { get; set; } = "click";
     public ScenarioTarget? Target { get; set; }
     public string? Value { get; set; }
     public string? WaitFor { get; set; }
+    /// <summary>ms since recording start (or epoch ms from RawEvent).</summary>
+    public long Ts { get; set; }
+    /// <summary>Raw screen coordinate [x, y] when applicable.</summary>
+    public int[]? RawCoord { get; set; }
+    /// <summary>For drag steps: end offset within target element.</summary>
+    public double[]? EndOffset { get; set; }
+    /// <summary>For drag steps: end raw coordinate [x, y].</summary>
+    public int[]? EndRawCoord { get; set; }
 }
 
 public sealed class ScenarioTarget

tests/Recordingtest.Recorder.Tests/RecorderTests.cs

@@ -123,6 +123,107 @@ public class RecorderTests
         Assert.Equal(s.Steps[1].Value, parsed.Steps[1].Value);
     }
 
+    private static FakeElement MakeRectElement(string path, double l, double t, double w, double h) =>
+        new FakeElement
+        {
+            ClassName = "Window",
+            Name = path,
+            BoundingRectangle = (l, t, w, h),
+        };
+
+    [Fact]
+    public void DragCollapser_DownMoveUp_BeyondThreshold_EmitsDrag()
+    {
+        var el = MakeRectElement("Canvas", 0, 0, 1000, 1000);
+        var path = "Window[@Name='Canvas']";
+        UiaResolution? Resolver(RawEvent _) => new UiaResolution(el, path);
+
+        var events = new[]
+        {
+            new RawEvent(100, "mouse_down_l", 100, 100, 0, 0),
+            new RawEvent(110, "move", 150, 120, 0, 0),
+            new RawEvent(120, "move", 300, 400, 0, 0),
+            new RawEvent(130, "mouse_up_l", 300, 400, 0, 0),
+        };
+
+        var steps = new DragCollapser().Collapse(events, Resolver);
+
+        Assert.Single(steps);
+        Assert.Equal("drag", steps[0].Kind);
+        Assert.Equal(path, steps[0].Target!.UiaPath);
+        Assert.Equal(new[] { 100, 100 }, steps[0].RawCoord);
+        Assert.Equal(new[] { 300, 400 }, steps[0].EndRawCoord);
+        Assert.NotNull(steps[0].EndOffset);
+        Assert.Equal(100L, steps[0].Ts);
+    }
+
+    [Fact]
+    public void DragCollapser_DownUp_BelowThreshold_EmitsClick()
+    {
+        var el = MakeRectElement("Btn", 0, 0, 100, 100);
+        UiaResolution? Resolver(RawEvent _) => new UiaResolution(el, "Window[@Name='Btn']");
+
+        var events = new[]
+        {
+            new RawEvent(50, "mouse_down_l", 10, 10, 0, 0),
+            new RawEvent(55, "mouse_up_l", 11, 11, 0, 0),
+        };
+
+        var steps = new DragCollapser().Collapse(events, Resolver);
+
+        Assert.Single(steps);
+        Assert.Equal("click", steps[0].Kind);
+        Assert.Equal(new[] { 10, 10 }, steps[0].RawCoord);
+        Assert.Equal(50L, steps[0].Ts);
+    }
+
+    [Fact]
+    public void DragCollapser_FocusChangeEvent_EmitsFocusStep()
+    {
+        var events = new[]
+        {
+            new RawEvent(200, "focus_change", 0, 0, 0, 0, "Window[@Name='Main']/Edit[@AutomationId='Pwd']"),
+        };
+
+        var steps = new DragCollapser().Collapse(events, _ => null);
+
+        Assert.Single(steps);
+        Assert.Equal("focus", steps[0].Kind);
+        Assert.Equal("Window[@Name='Main']/Edit[@AutomationId='Pwd']", steps[0].Target!.UiaPath);
+        Assert.Equal(200L, steps[0].Ts);
+    }
+
+    [Fact]
+    public void ScenarioStep_YamlRoundtrip_PreservesTsAndRawCoord()
+    {
+        var s = new Scenario
+        {
+            Name = "ts-test",
+            Steps =
+            {
+                new ScenarioStep
+                {
+                    Kind = "click",
+                    Ts = 12345,
+                    RawCoord = new[] { 640, 480 },
+                    Target = new ScenarioTarget
+                    {
+                        UiaPath = "Window[@Name='X']",
+                        Offset = new[] { 0.5, 0.5 },
+                    },
+                },
+            },
+        };
+
+        var yaml = ScenarioWriter.Serialize(s);
+        Assert.Contains("ts:", yaml);
+        Assert.Contains("raw_coord", yaml);
+
+        var parsed = ScenarioWriter.Deserialize(yaml);
+        Assert.Equal(12345L, parsed.Steps[0].Ts);
+        Assert.Equal(new[] { 640, 480 }, parsed.Steps[0].RawCoord);
+    }
+
     [Fact]
     public void Cli_MissingAttach_ExitTwo()
     {