Fix recorder drag collapse, focus events, ts/raw_coord (#6)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -94,16 +94,46 @@ public static class Program
|
||||
cts.Cancel();
|
||||
};
|
||||
|
||||
// Register UIA focus changed event. The callback only captures the
|
||||
// element path and pushes a synthetic RawEvent into the same queue;
|
||||
// it does NOT compute anything else inside the UIA callback.
|
||||
try
|
||||
{
|
||||
if (automation is not null)
|
||||
{
|
||||
automation.RegisterFocusChangedEvent(el =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (el is null) return;
|
||||
var snap = new FlaUiSnapshot(el);
|
||||
var path = ElementPathBuilder.Build(snap);
|
||||
channel.Writer.TryWrite(new RawEvent(
|
||||
DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
||||
"focus_change", 0, 0, 0, 0, path));
|
||||
}
|
||||
catch
|
||||
{
|
||||
// never throw from UIA callback
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}");
|
||||
}
|
||||
|
||||
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
|
||||
int eventCount = 0;
|
||||
int unresolved = 0;
|
||||
var sw = Stopwatch.StartNew();
|
||||
var rawBuffer = new System.Collections.Generic.List<RawEvent>();
|
||||
|
||||
try
|
||||
{
|
||||
ConsumeAsync(channel.Reader, scenario, mainWindow, automation, cts.Token,
|
||||
onEvent: () => eventCount++,
|
||||
onUnresolved: () => unresolved++).GetAwaiter().GetResult();
|
||||
ConsumeAsync(channel.Reader, rawBuffer, cts.Token,
|
||||
onEvent: () => eventCount++).GetAwaiter().GetResult();
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -111,6 +141,33 @@ public static class Program
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
|
||||
// Collapse buffered raw events into scenario steps via DragCollapser.
|
||||
var collapser = new DragCollapser();
|
||||
UiaResolution? Resolve(RawEvent ev)
|
||||
{
|
||||
if (automation is null) return null;
|
||||
try
|
||||
{
|
||||
var snap = ResolveAt(automation, ev.X, ev.Y);
|
||||
if (snap is null)
|
||||
{
|
||||
unresolved++;
|
||||
return null;
|
||||
}
|
||||
var path = ElementPathBuilder.Build(snap);
|
||||
return new UiaResolution(snap, path);
|
||||
}
|
||||
catch
|
||||
{
|
||||
unresolved++;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
foreach (var step in collapser.Collapse(rawBuffer, Resolve))
|
||||
{
|
||||
scenario.Steps.Add(step);
|
||||
}
|
||||
ScenarioWriter.WriteToFile(scenario, args.OutputPath);
|
||||
Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}");
|
||||
|
||||
@@ -155,59 +212,20 @@ public static class Program
|
||||
|
||||
private static async Task ConsumeAsync(
|
||||
ChannelReader<RawEvent> reader,
|
||||
Scenario scenario,
|
||||
AutomationElement? mainWindow,
|
||||
UIA3Automation? automation,
|
||||
System.Collections.Generic.List<RawEvent> buffer,
|
||||
CancellationToken ct,
|
||||
Action onEvent,
|
||||
Action onUnresolved)
|
||||
Action onEvent)
|
||||
{
|
||||
while (await reader.WaitToReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
while (reader.TryRead(out var ev))
|
||||
{
|
||||
onEvent();
|
||||
if (!IsInterestingForStep(ev.Kind)) continue;
|
||||
|
||||
IElementSnapshot? snap = null;
|
||||
if (automation is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
snap = ResolveAt(automation, ev.X, ev.Y);
|
||||
}
|
||||
catch
|
||||
{
|
||||
snap = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (snap is null)
|
||||
{
|
||||
onUnresolved();
|
||||
continue;
|
||||
}
|
||||
|
||||
var path = ElementPathBuilder.Build(snap);
|
||||
var (dx, dy) = OffsetNormalizer.Normalize(snap.BoundingRectangle, ev.X, ev.Y);
|
||||
var step = new ScenarioStep
|
||||
{
|
||||
Kind = ev.Kind.StartsWith("key", StringComparison.Ordinal) ? "type" : "click",
|
||||
Target = new ScenarioTarget
|
||||
{
|
||||
UiaPath = path,
|
||||
Offset = new[] { dx, dy },
|
||||
},
|
||||
Value = MaskPolicy.IsMasked(snap) ? MaskPolicy.MaskedValue : null,
|
||||
};
|
||||
scenario.Steps.Add(step);
|
||||
buffer.Add(ev);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsInterestingForStep(string kind) =>
|
||||
kind == "mouse_down_l" || kind == "mouse_down_r" || kind == "key_down";
|
||||
|
||||
private static IElementSnapshot? ResolveAt(UIA3Automation automation, int x, int y)
|
||||
{
|
||||
var raw = automation.FromPoint(new System.Drawing.Point(x, y));
|
||||
|
||||
Reference in New Issue
Block a user