Fix recorder drag collapse, focus events, ts/raw_coord (#6)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
minsung
2026-04-07 14:33:46 +09:00
parent f17e764678
commit 56b7233500
6 changed files with 408 additions and 47 deletions

View File

@@ -94,16 +94,46 @@ public static class Program
cts.Cancel();
};
// Register UIA focus changed event. The callback only captures the
// element path and pushes a synthetic RawEvent into the same queue;
// it does NOT compute anything else inside the UIA callback.
try
{
if (automation is not null)
{
automation.RegisterFocusChangedEvent(el =>
{
try
{
if (el is null) return;
var snap = new FlaUiSnapshot(el);
var path = ElementPathBuilder.Build(snap);
channel.Writer.TryWrite(new RawEvent(
DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
"focus_change", 0, 0, 0, 0, path));
}
catch
{
// never throw from UIA callback
}
});
}
}
catch (Exception ex)
{
Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}");
}
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
int eventCount = 0;
int unresolved = 0;
var sw = Stopwatch.StartNew();
var rawBuffer = new System.Collections.Generic.List<RawEvent>();
try
{
ConsumeAsync(channel.Reader, scenario, mainWindow, automation, cts.Token,
onEvent: () => eventCount++,
onUnresolved: () => unresolved++).GetAwaiter().GetResult();
ConsumeAsync(channel.Reader, rawBuffer, cts.Token,
onEvent: () => eventCount++).GetAwaiter().GetResult();
}
catch (OperationCanceledException)
{
@@ -111,6 +141,33 @@ public static class Program
}
sw.Stop();
// Collapse buffered raw events into scenario steps via DragCollapser.
var collapser = new DragCollapser();
UiaResolution? Resolve(RawEvent ev)
{
if (automation is null) return null;
try
{
var snap = ResolveAt(automation, ev.X, ev.Y);
if (snap is null)
{
unresolved++;
return null;
}
var path = ElementPathBuilder.Build(snap);
return new UiaResolution(snap, path);
}
catch
{
unresolved++;
return null;
}
}
foreach (var step in collapser.Collapse(rawBuffer, Resolve))
{
scenario.Steps.Add(step);
}
ScenarioWriter.WriteToFile(scenario, args.OutputPath);
Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}");
@@ -155,59 +212,20 @@ public static class Program
private static async Task ConsumeAsync(
ChannelReader<RawEvent> reader,
Scenario scenario,
AutomationElement? mainWindow,
UIA3Automation? automation,
System.Collections.Generic.List<RawEvent> buffer,
CancellationToken ct,
Action onEvent,
Action onUnresolved)
Action onEvent)
{
while (await reader.WaitToReadAsync(ct).ConfigureAwait(false))
{
while (reader.TryRead(out var ev))
{
onEvent();
if (!IsInterestingForStep(ev.Kind)) continue;
IElementSnapshot? snap = null;
if (automation is not null)
{
try
{
snap = ResolveAt(automation, ev.X, ev.Y);
}
catch
{
snap = null;
}
}
if (snap is null)
{
onUnresolved();
continue;
}
var path = ElementPathBuilder.Build(snap);
var (dx, dy) = OffsetNormalizer.Normalize(snap.BoundingRectangle, ev.X, ev.Y);
var step = new ScenarioStep
{
Kind = ev.Kind.StartsWith("key", StringComparison.Ordinal) ? "type" : "click",
Target = new ScenarioTarget
{
UiaPath = path,
Offset = new[] { dx, dy },
},
Value = MaskPolicy.IsMasked(snap) ? MaskPolicy.MaskedValue : null,
};
scenario.Steps.Add(step);
buffer.Add(ev);
}
}
}
private static bool IsInterestingForStep(string kind) =>
kind == "mouse_down_l" || kind == "mouse_down_r" || kind == "key_down";
private static IElementSnapshot? ResolveAt(UIA3Automation automation, int x, int y)
{
var raw = automation.FromPoint(new System.Drawing.Point(x, y));