kimminsung/recordingtest
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix recorder drag collapse, focus events, ts/raw_coord (#6)

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
minsung
2026-04-07 14:33:46 +09:00
parent f17e764678
commit 56b7233500
6 changed files with 408 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
src/Recordingtest.Recorder/DragCollapser.cs Normal file
View File

@@ -0,0 +1,218 @@
using System;
using System.Collections.Generic;
namespace Recordingtest.Recorder;
/// <summary>
/// Resolution of a raw event into an UIA element snapshot + path.
/// Provided by the caller (usually backed by FlaUI from-point lookup).
/// </summary>
public sealed record UiaResolution(IElementSnapshot Snapshot, string UiaPath);
/// <summary>
/// Pure state machine that collapses a raw event stream into ScenarioSteps.
/// Recognizes click vs drag based on movement between mouse_down/mouse_up.
/// </summary>
public sealed class DragCollapser
{
public int DragThresholdPx { get; }
public DragCollapser(int dragThresholdPx = 4)
{
DragThresholdPx = dragThresholdPx;
}
public IReadOnlyList<ScenarioStep> Collapse(
IEnumerable<RawEvent> events,
Func<RawEvent, UiaResolution?> resolver)
{
var steps = new List<ScenarioStep>();
RawEvent? down = null;
int lastX = 0, lastY = 0;
int maxDistSq = 0;
foreach (var ev in events)
{
switch (ev.Kind)
{
case "mouse_down_l":
down = ev;
lastX = ev.X;
lastY = ev.Y;
maxDistSq = 0;
break;
case "move":
if (down is not null)
{
int dx = ev.X - down.X;
int dy = ev.Y - down.Y;
int d2 = dx * dx + dy * dy;
if (d2 > maxDistSq) maxDistSq = d2;
lastX = ev.X;
lastY = ev.Y;
}
break;
case "mouse_up_l":
if (down is not null)
{
int fdx = ev.X - down.X;
int fdy = ev.Y - down.Y;
int finalDistSq = fdx * fdx + fdy * fdy;
int useSq = Math.Max(maxDistSq, finalDistSq);
var threshSq = DragThresholdPx * DragThresholdPx;
var downRes = resolver(down);
if (useSq >= threshSq)
{
// drag step
var step = new ScenarioStep
{
Kind = "drag",
Ts = down.TimestampMs,
RawCoord = new[] { down.X, down.Y },
EndRawCoord = new[] { ev.X, ev.Y },
};
if (downRes is not null)
{
var (sx, sy) = OffsetNormalizer.Normalize(
downRes.Snapshot.BoundingRectangle, down.X, down.Y);
var (ex, ey) = OffsetNormalizer.Normalize(
downRes.Snapshot.BoundingRectangle, ev.X, ev.Y);
step.Target = new ScenarioTarget
{
UiaPath = downRes.UiaPath,
Offset = new[] { sx, sy },
};
step.EndOffset = new[] { ex, ey };
}
steps.Add(step);
}
else
{
// click step at down point
var step = new ScenarioStep
{
Kind = "click",
Ts = down.TimestampMs,
RawCoord = new[] { down.X, down.Y },
};
if (downRes is not null)
{
var (ox, oy) = OffsetNormalizer.Normalize(
downRes.Snapshot.BoundingRectangle, down.X, down.Y);
step.Target = new ScenarioTarget
{
UiaPath = downRes.UiaPath,
Offset = new[] { ox, oy },
};
if (MaskPolicy.IsMasked(downRes.Snapshot))
{
step.Value = MaskPolicy.MaskedValue;
}
}
steps.Add(step);
}
down = null;
maxDistSq = 0;
}
break;
case "mouse_down_r":
{
var res = resolver(ev);
var step = new ScenarioStep
{
Kind = "click",
Ts = ev.TimestampMs,
RawCoord = new[] { ev.X, ev.Y },
Value = "right",
};
if (res is not null)
{
var (ox, oy) = OffsetNormalizer.Normalize(
res.Snapshot.BoundingRectangle, ev.X, ev.Y);
step.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { ox, oy },
};
}
steps.Add(step);
break;
}
case "key_down":
{
var res = resolver(ev);
var step = new ScenarioStep
{
Kind = "type",
Ts = ev.TimestampMs,
Value = ev.Code.ToString(System.Globalization.CultureInfo.InvariantCulture),
};
if (res is not null)
{
step.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { 0.5, 0.5 },
};
if (MaskPolicy.IsMasked(res.Snapshot))
{
step.Value = MaskPolicy.MaskedValue;
}
}
steps.Add(step);
break;
}
case "wheel":
{
var res = resolver(ev);
var step = new ScenarioStep
{
Kind = "wheel",
Ts = ev.TimestampMs,
RawCoord = new[] { ev.X, ev.Y },
Value = ev.WheelDelta.ToString(System.Globalization.CultureInfo.InvariantCulture),
};
if (res is not null)
{
var (ox, oy) = OffsetNormalizer.Normalize(
res.Snapshot.BoundingRectangle, ev.X, ev.Y);
step.Target = new ScenarioTarget
{
UiaPath = res.UiaPath,
Offset = new[] { ox, oy },
};
}
steps.Add(step);
break;
}
case "focus_change":
{
var step = new ScenarioStep
{
Kind = "focus",
Ts = ev.TimestampMs,
};
if (!string.IsNullOrEmpty(ev.FocusedElementPath))
{
step.Target = new ScenarioTarget
{
UiaPath = ev.FocusedElementPath!,
Offset = new[] { 0.5, 0.5 },
};
}
steps.Add(step);
break;
}
}
}
return steps;
}
}

2
src/Recordingtest.Recorder/LowLevelHook.cs
View File

@@ -4,7 +4,7 @@ using System.Threading.Channels;
namespace Recordingtest.Recorder;
public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta);
public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta, string? FocusedElementPath = null);
/// <summary>
/// Installs WH_KEYBOARD_LL and WH_MOUSE_LL hooks on a dedicated thread with its own message loop.

108
src/Recordingtest.Recorder/Program.cs
View File

@@ -94,16 +94,46 @@ public static class Program
cts.Cancel();
};
// Register UIA focus changed event. The callback only captures the
// element path and pushes a synthetic RawEvent into the same queue;
// it does NOT compute anything else inside the UIA callback.
try
{
if (automation is not null)
{
automation.RegisterFocusChangedEvent(el =>
{
try
{
if (el is null) return;
var snap = new FlaUiSnapshot(el);
var path = ElementPathBuilder.Build(snap);
channel.Writer.TryWrite(new RawEvent(
DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
"focus_change", 0, 0, 0, 0, path));
}
catch
{
// never throw from UIA callback
}
});
}
}
catch (Exception ex)
{
Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}");
}
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
int eventCount = 0;
int unresolved = 0;
var sw = Stopwatch.StartNew();
var rawBuffer = new System.Collections.Generic.List<RawEvent>();
try
{
ConsumeAsync(channel.Reader, scenario, mainWindow, automation, cts.Token,
onEvent: () => eventCount++,
onUnresolved: () => unresolved++).GetAwaiter().GetResult();
ConsumeAsync(channel.Reader, rawBuffer, cts.Token,
onEvent: () => eventCount++).GetAwaiter().GetResult();
}
catch (OperationCanceledException)
{
@@ -111,6 +141,33 @@ public static class Program
}
sw.Stop();
// Collapse buffered raw events into scenario steps via DragCollapser.
var collapser = new DragCollapser();
UiaResolution? Resolve(RawEvent ev)
{
if (automation is null) return null;
try
{
var snap = ResolveAt(automation, ev.X, ev.Y);
if (snap is null)
{
unresolved++;
return null;
}
var path = ElementPathBuilder.Build(snap);
return new UiaResolution(snap, path);
}
catch
{
unresolved++;
return null;
}
}
foreach (var step in collapser.Collapse(rawBuffer, Resolve))
{
scenario.Steps.Add(step);
}
ScenarioWriter.WriteToFile(scenario, args.OutputPath);
Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}");
@@ -155,59 +212,20 @@ public static class Program
private static async Task ConsumeAsync(
ChannelReader<RawEvent> reader,
Scenario scenario,
AutomationElement? mainWindow,
UIA3Automation? automation,
System.Collections.Generic.List<RawEvent> buffer,
CancellationToken ct,
Action onEvent,
Action onUnresolved)
Action onEvent)
{
while (await reader.WaitToReadAsync(ct).ConfigureAwait(false))
{
while (reader.TryRead(out var ev))
{
onEvent();
if (!IsInterestingForStep(ev.Kind)) continue;
IElementSnapshot? snap = null;
if (automation is not null)
{
try
{
snap = ResolveAt(automation, ev.X, ev.Y);
}
catch
{
snap = null;
}
}
if (snap is null)
{
onUnresolved();
continue;
}
var path = ElementPathBuilder.Build(snap);
var (dx, dy) = OffsetNormalizer.Normalize(snap.BoundingRectangle, ev.X, ev.Y);
var step = new ScenarioStep
{
Kind = ev.Kind.StartsWith("key", StringComparison.Ordinal) ? "type" : "click",
Target = new ScenarioTarget
{
UiaPath = path,
Offset = new[] { dx, dy },
},
Value = MaskPolicy.IsMasked(snap) ? MaskPolicy.MaskedValue : null,
};
scenario.Steps.Add(step);
buffer.Add(ev);
}
}
}
private static bool IsInterestingForStep(string kind) =>
kind == "mouse_down_l" || kind == "mouse_down_r" || kind == "key_down";
private static IElementSnapshot? ResolveAt(UIA3Automation automation, int x, int y)
{
var raw = automation.FromPoint(new System.Drawing.Point(x, y));

10
src/Recordingtest.Recorder/Scenario.cs
View File

@@ -18,11 +18,19 @@ public sealed class ScenarioSut
public sealed class ScenarioStep
{
/// <summary>click | type | drag | hotkey | wait</summary>
/// <summary>click | type | drag | hotkey | wait | focus</summary>
public string Kind { get; set; } = "click";
public ScenarioTarget? Target { get; set; }
public string? Value { get; set; }
public string? WaitFor { get; set; }
/// <summary>ms since recording start (or epoch ms from RawEvent).</summary>
public long Ts { get; set; }
/// <summary>Raw screen coordinate [x, y] when applicable.</summary>
public int[]? RawCoord { get; set; }
/// <summary>For drag steps: end offset within target element.</summary>
public double[]? EndOffset { get; set; }
/// <summary>For drag steps: end raw coordinate [x, y].</summary>
public int[]? EndRawCoord { get; set; }
}
public sealed class ScenarioTarget