kimminsung/recordingtest
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

recorder: focus poller PoC for Gap I-1 (deferred, #14)

Browse Source 
Adds a background focus poller that periodically calls
Automation.FocusedElement() and stamps the path onto key_down RawEvents,
so DragCollapser can fill type-step targets without relying on the stale
post-hoc Resolve() pass. Plumbing:

  Program.cs   — focus poller Task + diagnostic counters
  LowLevelHook — volatile CurrentFocusedPath, stamped on key_down
  RawEvent     — FocusedElementPath already existed (focus_change)
  DragCollapser— typeFocusPath captured at first printable key_down,
                 takes precedence over lastFocusPath/lastMousePath

Result on box-v7.yaml live recording: null_target_steps unchanged (13).
Root cause: EG-BIM Modeler's CommandBox and similar input controls lack
AutomationPeer, so UIA-based focus tracking — from any external process —
cannot see them. The WPF-internal Keyboard.FocusedElement is in-process
only and unreachable from the recorder.

Deferred. The plumbing stays in place because the same stamping path can
be reused by a future generic WPF DLL-injection probe. Player's existing
null-target fallback (Type→OS focus, Click→raw_coord) remains the official
strategy and successfully replays box-v7 end-to-end.

See docs/history/2026-04-08_gap-i1-deferred.md for analysis and future
options (generic WPF injection / AutomationPeer AI attachment).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
minsung
2026-04-08 20:49:08 +09:00
parent 98d801442b
commit a771352bcb
6 changed files with 139 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/Recordingtest.Recorder/DragCollapser.cs
View File

@@ -35,6 +35,11 @@ public sealed class DragCollapser
var typeBuf = new System.Text.StringBuilder();
RawEvent? typeFirst = null;
UiaResolution? typeRes = null;
// Issue #14 Gap I-1: path captured directly from the focus poller at
// the first key_down of a type burst. Takes precedence over the older
// lastFocusPath / lastMousePath fallbacks because it's pinned to the
// instant the user actually started typing.
string? typeFocusPath = null;
// Active modifiers (ctrl/shift/alt/win) held down.
var modsDown = new HashSet<string>(StringComparer.Ordinal);
@@ -69,7 +74,7 @@ public sealed class DragCollapser
}
else
{
var fallback = lastFocusPath ?? lastMousePath;
var fallback = typeFocusPath ?? lastFocusPath ?? lastMousePath;
if (!string.IsNullOrEmpty(fallback))
{
step.Target = new ScenarioTarget
@@ -83,6 +88,7 @@ public sealed class DragCollapser
typeBuf.Clear();
typeFirst = null;
typeRes = null;
typeFocusPath = null;
}
foreach (var ev in events)
@@ -265,6 +271,9 @@ public sealed class DragCollapser
{
typeFirst = ev;
typeRes = res;
// Issue #14 Gap I-1: capture focused-element path
// snapshotted by the poller at key_down time.
typeFocusPath = ev.FocusedElementPath;
}
typeBuf.Append(tr.Text);
break;

11
src/Recordingtest.Recorder/LowLevelHook.cs
View File

@@ -27,6 +27,14 @@ public sealed class LowLevelHook : IDisposable
/// </summary>
public IWindowFilter Filter { get; set; } = new PassThroughWindowFilter();
/// <summary>
/// Issue #14 Gap I-1 — latest UIA focused-element path observed by a
/// background poller. Stamped onto key_down RawEvents so the collapser
/// can assign a target to the resulting Type step without relying on
/// the stale post-hoc Resolve() pass. Null until the first poll.
/// </summary>
public volatile string? CurrentFocusedPath;
public LowLevelHook(Channel<RawEvent> channel)
{
_channel = channel;
@@ -83,7 +91,8 @@ public sealed class LowLevelHook : IDisposable
NativeMethods.WM_KEYUP or NativeMethods.WM_SYSKEYUP => "key_up",
_ => "key",
};
var ev = new RawEvent(NowMs(), kind, 0, 0, data.vkCode, 0);
var ev = new RawEvent(
NowMs(), kind, 0, 0, data.vkCode, 0, CurrentFocusedPath);
if (Filter.ShouldKeep(ev))
{
_channel.Writer.TryWrite(ev);

74
src/Recordingtest.Recorder/Program.cs
View File

@@ -155,6 +155,63 @@ public static class Program
Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}");
}
// Issue #14 Gap I-1 — background focus poller. Periodically queries
// Automation.FocusedElement() and publishes the element path into
// LowLevelHook.CurrentFocusedPath, so key_down events captured on the
// hook thread can be stamped with the live focused-element path at
// the exact instant the user started typing. This catches custom WPF
// controls (e.g. CommandBox) that do NOT raise UIA focus_changed
// events reliably.
var pollerCts = new CancellationTokenSource();
Task? pollerTask = null;
int pollerSuccess = 0;
int pollerNullFocus = 0;
int pollerWrongPid = 0;
int pollerErrors = 0;
string? pollerLastError = null;
if (automation is not null)
{
var auto = automation;
var pid = sutPid;
pollerTask = Task.Run(async () =>
{
while (!pollerCts.IsCancellationRequested)
{
try
{
var focused = auto.FocusedElement();
if (focused is null)
{
System.Threading.Interlocked.Increment(ref pollerNullFocus);
}
else
{
int elPid = 0;
try { elPid = focused.Properties.ProcessId.ValueOrDefault; }
catch { elPid = 0; }
if (pid == 0 || elPid == pid)
{
var snap = new FlaUiSnapshot(focused);
hook.CurrentFocusedPath = ElementPathBuilder.Build(snap);
System.Threading.Interlocked.Increment(ref pollerSuccess);
}
else
{
System.Threading.Interlocked.Increment(ref pollerWrongPid);
}
}
}
catch (Exception ex)
{
System.Threading.Interlocked.Increment(ref pollerErrors);
pollerLastError = ex.GetType().Name + ": " + ex.Message;
}
try { await Task.Delay(100, pollerCts.Token); }
catch (OperationCanceledException) { break; }
}
}, pollerCts.Token);
}
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
int eventCount = 0;
int unresolvedPaths = 0; // resolver ran but returned null
@@ -174,12 +231,20 @@ public static class Program
sw.Stop();
// Stop the focus poller before UIA teardown.
pollerCts.Cancel();
try { pollerTask?.Wait(500); } catch { /* ignore */ }
// Collapse buffered raw events into scenario steps via DragCollapser.
var collapser = new DragCollapser();
UiaResolution? Resolve(RawEvent ev)
{
// Key events have no meaningful coordinate — resolver cannot attempt
// a point-based lookup. Count them separately from genuine misses.
// Issue #14 Gap I-1 — key events: Resolve() runs at collapse time
// (after the recording ended), so querying FocusedElement() HERE
// would be stale (focus has already left the SUT). The focused
// element path is captured at key_down time by the FocusPoller
// and baked into RawEvent.FocusedElementPath. DragCollapser reads
// that directly; Resolve() simply returns null for key events.
if (ev.Kind == "key_down" || ev.Kind == "key_up")
{
noResolverAttempt++;
@@ -225,6 +290,11 @@ public static class Program
$"[recorder] done. events={eventCount} elapsed={sw.Elapsed} " +
$"unresolved_paths={unresolvedPaths} no_resolver_attempt={noResolverAttempt} " +
$"null_target_steps={nullTargetSteps}");
Console.WriteLine(
$"[recorder] focus_poller success={pollerSuccess} null_focus={pollerNullFocus} " +
$"wrong_pid={pollerWrongPid} errors={pollerErrors} last_path={hook.CurrentFocusedPath ?? "<null>"}");
if (pollerLastError is not null)
Console.WriteLine($"[recorder] focus_poller last_error: {pollerLastError}");
automation?.Dispose();
return 0;