recorder: focus poller PoC for Gap I-1 (deferred, #14)
Adds a background focus poller that periodically calls
Automation.FocusedElement() and stamps the path onto key_down RawEvents,
so DragCollapser can fill type-step targets without relying on the stale
post-hoc Resolve() pass. Plumbing:
Program.cs — focus poller Task + diagnostic counters
LowLevelHook — volatile CurrentFocusedPath, stamped on key_down
RawEvent — FocusedElementPath already existed (focus_change)
DragCollapser— typeFocusPath captured at first printable key_down,
takes precedence over lastFocusPath/lastMousePath
Result on box-v7.yaml live recording: null_target_steps unchanged (13).
Root cause: EG-BIM Modeler's CommandBox and similar input controls lack
AutomationPeer, so UIA-based focus tracking — from any external process —
cannot see them. The WPF-internal Keyboard.FocusedElement is in-process
only and unreachable from the recorder.
Deferred. The plumbing stays in place because the same stamping path can
be reused by a future generic WPF DLL-injection probe. Player's existing
null-target fallback (Type→OS focus, Click→raw_coord) remains the official
strategy and successfully replays box-v7 end-to-end.
See docs/history/2026-04-08_gap-i1-deferred.md for analysis and future
options (generic WPF injection / AutomationPeer AI attachment).
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -155,6 +155,63 @@ public static class Program
|
||||
Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}");
|
||||
}
|
||||
|
||||
// Issue #14 Gap I-1 — background focus poller. Periodically queries
|
||||
// Automation.FocusedElement() and publishes the element path into
|
||||
// LowLevelHook.CurrentFocusedPath, so key_down events captured on the
|
||||
// hook thread can be stamped with the live focused-element path at
|
||||
// the exact instant the user started typing. This catches custom WPF
|
||||
// controls (e.g. CommandBox) that do NOT raise UIA focus_changed
|
||||
// events reliably.
|
||||
var pollerCts = new CancellationTokenSource();
|
||||
Task? pollerTask = null;
|
||||
int pollerSuccess = 0;
|
||||
int pollerNullFocus = 0;
|
||||
int pollerWrongPid = 0;
|
||||
int pollerErrors = 0;
|
||||
string? pollerLastError = null;
|
||||
if (automation is not null)
|
||||
{
|
||||
var auto = automation;
|
||||
var pid = sutPid;
|
||||
pollerTask = Task.Run(async () =>
|
||||
{
|
||||
while (!pollerCts.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var focused = auto.FocusedElement();
|
||||
if (focused is null)
|
||||
{
|
||||
System.Threading.Interlocked.Increment(ref pollerNullFocus);
|
||||
}
|
||||
else
|
||||
{
|
||||
int elPid = 0;
|
||||
try { elPid = focused.Properties.ProcessId.ValueOrDefault; }
|
||||
catch { elPid = 0; }
|
||||
if (pid == 0 || elPid == pid)
|
||||
{
|
||||
var snap = new FlaUiSnapshot(focused);
|
||||
hook.CurrentFocusedPath = ElementPathBuilder.Build(snap);
|
||||
System.Threading.Interlocked.Increment(ref pollerSuccess);
|
||||
}
|
||||
else
|
||||
{
|
||||
System.Threading.Interlocked.Increment(ref pollerWrongPid);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
System.Threading.Interlocked.Increment(ref pollerErrors);
|
||||
pollerLastError = ex.GetType().Name + ": " + ex.Message;
|
||||
}
|
||||
try { await Task.Delay(100, pollerCts.Token); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
}, pollerCts.Token);
|
||||
}
|
||||
|
||||
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
|
||||
int eventCount = 0;
|
||||
int unresolvedPaths = 0; // resolver ran but returned null
|
||||
@@ -174,12 +231,20 @@ public static class Program
|
||||
|
||||
sw.Stop();
|
||||
|
||||
// Stop the focus poller before UIA teardown.
|
||||
pollerCts.Cancel();
|
||||
try { pollerTask?.Wait(500); } catch { /* ignore */ }
|
||||
|
||||
// Collapse buffered raw events into scenario steps via DragCollapser.
|
||||
var collapser = new DragCollapser();
|
||||
UiaResolution? Resolve(RawEvent ev)
|
||||
{
|
||||
// Key events have no meaningful coordinate — resolver cannot attempt
|
||||
// a point-based lookup. Count them separately from genuine misses.
|
||||
// Issue #14 Gap I-1 — key events: Resolve() runs at collapse time
|
||||
// (after the recording ended), so querying FocusedElement() HERE
|
||||
// would be stale (focus has already left the SUT). The focused
|
||||
// element path is captured at key_down time by the FocusPoller
|
||||
// and baked into RawEvent.FocusedElementPath. DragCollapser reads
|
||||
// that directly; Resolve() simply returns null for key events.
|
||||
if (ev.Kind == "key_down" || ev.Kind == "key_up")
|
||||
{
|
||||
noResolverAttempt++;
|
||||
@@ -225,6 +290,11 @@ public static class Program
|
||||
$"[recorder] done. events={eventCount} elapsed={sw.Elapsed} " +
|
||||
$"unresolved_paths={unresolvedPaths} no_resolver_attempt={noResolverAttempt} " +
|
||||
$"null_target_steps={nullTargetSteps}");
|
||||
Console.WriteLine(
|
||||
$"[recorder] focus_poller success={pollerSuccess} null_focus={pollerNullFocus} " +
|
||||
$"wrong_pid={pollerWrongPid} errors={pollerErrors} last_path={hook.CurrentFocusedPath ?? "<null>"}");
|
||||
if (pollerLastError is not null)
|
||||
Console.WriteLine($"[recorder] focus_poller last_error: {pollerLastError}");
|
||||
|
||||
automation?.Dispose();
|
||||
return 0;
|
||||
|
||||
Reference in New Issue
Block a user