From a771352bcbf6ec254a8f5859abf75edc1fc35310 Mon Sep 17 00:00:00 2001 From: minsung Date: Wed, 8 Apr 2026 20:49:08 +0900 Subject: [PATCH] recorder: focus poller PoC for Gap I-1 (deferred, #14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a background focus poller that periodically calls Automation.FocusedElement() and stamps the path onto key_down RawEvents, so DragCollapser can fill type-step targets without relying on the stale post-hoc Resolve() pass. Plumbing: Program.cs — focus poller Task + diagnostic counters LowLevelHook — volatile CurrentFocusedPath, stamped on key_down RawEvent — FocusedElementPath already existed (focus_change) DragCollapser— typeFocusPath captured at first printable key_down, takes precedence over lastFocusPath/lastMousePath Result on box-v7.yaml live recording: null_target_steps unchanged (13). Root cause: EG-BIM Modeler's CommandBox and similar input controls lack AutomationPeer, so UIA-based focus tracking — from any external process — cannot see them. The WPF-internal Keyboard.FocusedElement is in-process only and unreachable from the recorder. Deferred. The plumbing stays in place because the same stamping path can be reused by a future generic WPF DLL-injection probe. Player's existing null-target fallback (Type→OS focus, Click→raw_coord) remains the official strategy and successfully replays box-v7 end-to-end. See docs/history/2026-04-08_gap-i1-deferred.md for analysis and future options (generic WPF injection / AutomationPeer AI attachment). Co-Authored-By: Claude Opus 4.6 (1M context) --- PLAN.md | 4 +- PROGRESS.md | 2 +- docs/history/2026-04-08_gap-i1-deferred.md | 44 ++++++++++++ src/Recordingtest.Recorder/DragCollapser.cs | 11 ++- src/Recordingtest.Recorder/LowLevelHook.cs | 11 ++- src/Recordingtest.Recorder/Program.cs | 74 ++++++++++++++++++++- 6 files changed, 139 insertions(+), 7 deletions(-) create mode 100644 docs/history/2026-04-08_gap-i1-deferred.md diff --git a/PLAN.md b/PLAN.md index 4c40d5e..6622e24 100644 --- a/PLAN.md +++ b/PLAN.md @@ -10,8 +10,8 @@ ## P1 — 라이브 검증 (사용자 환경 필요) -4. **recorder Gap I-1** — type 스텝 target을 `Automation.FocusedElement`로 직접 채워 null_target_steps=0 달성. Player fallback 의존도 줄이기. -5. **engine-bridge v3** — ReflectionEngineStateProvider 실매핑 (smoke test 이후) +4. **engine-bridge v3** — `ReflectionEngineStateProvider` 실매핑. v2의 plugin masquerade + HttpListener 위에 카메라/선택/씬그래프 상태를 reflection으로 노출. 골든파일 sidecar JSON. `/contract engine-bridge-v3`. +5. ~~recorder Gap I-1~~ — **deferred**. UIA poller PoC 결과 본질적 한계 확인 (AutomationPeer 부재 컨트롤은 못 봄). generic WPF DLL injection 또는 AutomationPeer AI 부착 PoC가 선결. ## Follow-ups (non-blocking) diff --git a/PROGRESS.md b/PROGRESS.md index 6146b4a..979a289 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -60,7 +60,7 @@ _(없음)_ - [ ] player: `UiaPlayerHost` uia_path resolver가 마지막 `@AutomationId`만 사용 — 전체 ancestor chain 지원 필요. - [ ] recorder: IME 조합 키 처리 (contract risks). - [x] ~~player: foreground settle 안정화~~ — 능동 대기(`GetForegroundWindow` polling 2s + 100ms settle)로 전환, 1차 재생 성공 확인 -- [ ] recorder: null_target 이벤트 자체를 줄이기 — `Automation.FocusedElement` 직접 조회해 type 스텝 target 채우기 (issue #14 Gap I-1). 현재는 player fallback으로 우회. +- [~] recorder Gap I-1 — UIA `Automation.FocusedElement` 폴링 PoC 시도(commit pending). 결과: SUT의 CommandBox 등 AutomationPeer 미부착 컨트롤은 UIA 외부에서 본질적으로 못 봄. **deferred** — generic WPF DLL injection 또는 SUT-side AutomationPeer 부착 PoC가 필요. 현재는 Player fallback(null target → OS 키 입력 / raw_coord 클릭)이 공식 전략. ## Blocked diff --git a/docs/history/2026-04-08_gap-i1-deferred.md b/docs/history/2026-04-08_gap-i1-deferred.md new file mode 100644 index 0000000..379171b --- /dev/null +++ b/docs/history/2026-04-08_gap-i1-deferred.md @@ -0,0 +1,44 @@ +# 2026-04-08 — Gap I-1 (recorder focus poller) deferred + +**소요 시간**: ~45분 +**Context 사용량**: ~25k tokens (Opus 4.6, 동일 세션 누적) + +## 시도 + +issue #14의 recorder Gap I-1: type 스텝 target이 항상 null로 남는 문제를 root에서 풀기 위해 시도. + +접근: +1. 백그라운드 Task가 100ms 주기로 `automation.FocusedElement()` 폴링 +2. 결과 path를 `LowLevelHook.CurrentFocusedPath` volatile 필드에 저장 +3. `KeyboardProc`가 key_down RawEvent 생성 시 이 필드를 `FocusedElementPath`에 stamp +4. `DragCollapser`가 type burst 시작 시 `typeFocusPath` 로컬에 캡처, FlushType fallback에서 우선 사용 +5. 진단 카운터 추가 (success/null_focus/wrong_pid/errors/last_path) + +## 결과 + +box-v7.yaml 녹화 → `null_target_steps=13` (이전 12와 사실상 동일). type 스텝의 `target:` 여전히 비어있음. + +## 원인 + +UIA `FocusedElement()`는 **AutomationPeer가 부착된 컨트롤만** 보고할 수 있다. EG-BIM Modeler의 CommandBox 등 핵심 입력 컨트롤은 AutomationPeer가 없어서 UIA 트리에 의미 있게 노출되지 않음. 외부 프로세스에서 어떤 UIA API를 호출해도 동일한 한계. + +WPF의 진짜 포커스(`Keyboard.FocusedElement`)는 **in-process API**라 외부 recorder에서는 직접 호출 불가. + +## 결정 + +**Gap I-1 deferred.** 현재는 Player fallback이 공식 전략: +- Type with null target → OS 레벨 키보드 입력 (SUT의 WPF가 자기 포커스 컨트롤로 라우팅) +- Click with null target + raw_coord → 화면 절대좌표 클릭 + +이 fallback으로 box-v6/v7 모두 E2E 재생 성공 확인됨. 결정성/진단성은 떨어지지만 실행 자체엔 충분. + +## 향후 옵션 (선결 PoC 필요) + +1. **Generic WPF DLL injection** — CreateRemoteThread + LoadLibrary로 임의 WPF 프로세스에 probe DLL 주입, Dispatcher 위에서 `Keyboard.FocusedElement` 읽어 named pipe로 노출. 권한 이슈 있음. +2. **AutomationPeer AI 부착 PoC** (메모리 `project_recordingtest_automationpeer_ai.md`) — SUT fork에 AI로 AutomationPeer 자동 부착하는 별도 PR 트랙. SUT 협조 필요. + +둘 다 본 이슈 범위를 벗어나므로 별도 트랙. PLAN.md에서 제거됨. + +## 남긴 코드 + +진단 가능한 형태로 commit (revert하지 않음). 향후 generic injection PoC가 들어올 때 같은 stamping 메커니즘(`LowLevelHook.CurrentFocusedPath` → `RawEvent.FocusedElementPath` → `DragCollapser.typeFocusPath`) 그대로 재사용 가능. diff --git a/src/Recordingtest.Recorder/DragCollapser.cs b/src/Recordingtest.Recorder/DragCollapser.cs index ddc6dee..ad4586f 100644 --- a/src/Recordingtest.Recorder/DragCollapser.cs +++ b/src/Recordingtest.Recorder/DragCollapser.cs @@ -35,6 +35,11 @@ public sealed class DragCollapser var typeBuf = new System.Text.StringBuilder(); RawEvent? typeFirst = null; UiaResolution? typeRes = null; + // Issue #14 Gap I-1: path captured directly from the focus poller at + // the first key_down of a type burst. Takes precedence over the older + // lastFocusPath / lastMousePath fallbacks because it's pinned to the + // instant the user actually started typing. + string? typeFocusPath = null; // Active modifiers (ctrl/shift/alt/win) held down. var modsDown = new HashSet(StringComparer.Ordinal); @@ -69,7 +74,7 @@ public sealed class DragCollapser } else { - var fallback = lastFocusPath ?? lastMousePath; + var fallback = typeFocusPath ?? lastFocusPath ?? lastMousePath; if (!string.IsNullOrEmpty(fallback)) { step.Target = new ScenarioTarget @@ -83,6 +88,7 @@ public sealed class DragCollapser typeBuf.Clear(); typeFirst = null; typeRes = null; + typeFocusPath = null; } foreach (var ev in events) @@ -265,6 +271,9 @@ public sealed class DragCollapser { typeFirst = ev; typeRes = res; + // Issue #14 Gap I-1: capture focused-element path + // snapshotted by the poller at key_down time. + typeFocusPath = ev.FocusedElementPath; } typeBuf.Append(tr.Text); break; diff --git a/src/Recordingtest.Recorder/LowLevelHook.cs b/src/Recordingtest.Recorder/LowLevelHook.cs index 3e2b3c1..eb9effa 100644 --- a/src/Recordingtest.Recorder/LowLevelHook.cs +++ b/src/Recordingtest.Recorder/LowLevelHook.cs @@ -27,6 +27,14 @@ public sealed class LowLevelHook : IDisposable /// public IWindowFilter Filter { get; set; } = new PassThroughWindowFilter(); + /// + /// Issue #14 Gap I-1 — latest UIA focused-element path observed by a + /// background poller. Stamped onto key_down RawEvents so the collapser + /// can assign a target to the resulting Type step without relying on + /// the stale post-hoc Resolve() pass. Null until the first poll. + /// + public volatile string? CurrentFocusedPath; + public LowLevelHook(Channel channel) { _channel = channel; @@ -83,7 +91,8 @@ public sealed class LowLevelHook : IDisposable NativeMethods.WM_KEYUP or NativeMethods.WM_SYSKEYUP => "key_up", _ => "key", }; - var ev = new RawEvent(NowMs(), kind, 0, 0, data.vkCode, 0); + var ev = new RawEvent( + NowMs(), kind, 0, 0, data.vkCode, 0, CurrentFocusedPath); if (Filter.ShouldKeep(ev)) { _channel.Writer.TryWrite(ev); diff --git a/src/Recordingtest.Recorder/Program.cs b/src/Recordingtest.Recorder/Program.cs index 350be71..ef54de9 100644 --- a/src/Recordingtest.Recorder/Program.cs +++ b/src/Recordingtest.Recorder/Program.cs @@ -155,6 +155,63 @@ public static class Program Console.Error.WriteLine($"[recorder] focus subscribe failed: {ex.Message}"); } + // Issue #14 Gap I-1 — background focus poller. Periodically queries + // Automation.FocusedElement() and publishes the element path into + // LowLevelHook.CurrentFocusedPath, so key_down events captured on the + // hook thread can be stamped with the live focused-element path at + // the exact instant the user started typing. This catches custom WPF + // controls (e.g. CommandBox) that do NOT raise UIA focus_changed + // events reliably. + var pollerCts = new CancellationTokenSource(); + Task? pollerTask = null; + int pollerSuccess = 0; + int pollerNullFocus = 0; + int pollerWrongPid = 0; + int pollerErrors = 0; + string? pollerLastError = null; + if (automation is not null) + { + var auto = automation; + var pid = sutPid; + pollerTask = Task.Run(async () => + { + while (!pollerCts.IsCancellationRequested) + { + try + { + var focused = auto.FocusedElement(); + if (focused is null) + { + System.Threading.Interlocked.Increment(ref pollerNullFocus); + } + else + { + int elPid = 0; + try { elPid = focused.Properties.ProcessId.ValueOrDefault; } + catch { elPid = 0; } + if (pid == 0 || elPid == pid) + { + var snap = new FlaUiSnapshot(focused); + hook.CurrentFocusedPath = ElementPathBuilder.Build(snap); + System.Threading.Interlocked.Increment(ref pollerSuccess); + } + else + { + System.Threading.Interlocked.Increment(ref pollerWrongPid); + } + } + } + catch (Exception ex) + { + System.Threading.Interlocked.Increment(ref pollerErrors); + pollerLastError = ex.GetType().Name + ": " + ex.Message; + } + try { await Task.Delay(100, pollerCts.Token); } + catch (OperationCanceledException) { break; } + } + }, pollerCts.Token); + } + Console.WriteLine("[recorder] capturing... press Ctrl+C to stop."); int eventCount = 0; int unresolvedPaths = 0; // resolver ran but returned null @@ -174,12 +231,20 @@ public static class Program sw.Stop(); + // Stop the focus poller before UIA teardown. + pollerCts.Cancel(); + try { pollerTask?.Wait(500); } catch { /* ignore */ } + // Collapse buffered raw events into scenario steps via DragCollapser. var collapser = new DragCollapser(); UiaResolution? Resolve(RawEvent ev) { - // Key events have no meaningful coordinate — resolver cannot attempt - // a point-based lookup. Count them separately from genuine misses. + // Issue #14 Gap I-1 — key events: Resolve() runs at collapse time + // (after the recording ended), so querying FocusedElement() HERE + // would be stale (focus has already left the SUT). The focused + // element path is captured at key_down time by the FocusPoller + // and baked into RawEvent.FocusedElementPath. DragCollapser reads + // that directly; Resolve() simply returns null for key events. if (ev.Kind == "key_down" || ev.Kind == "key_up") { noResolverAttempt++; @@ -225,6 +290,11 @@ public static class Program $"[recorder] done. events={eventCount} elapsed={sw.Elapsed} " + $"unresolved_paths={unresolvedPaths} no_resolver_attempt={noResolverAttempt} " + $"null_target_steps={nullTargetSteps}"); + Console.WriteLine( + $"[recorder] focus_poller success={pollerSuccess} null_focus={pollerNullFocus} " + + $"wrong_pid={pollerWrongPid} errors={pollerErrors} last_path={hook.CurrentFocusedPath ?? ""}"); + if (pollerLastError is not null) + Console.WriteLine($"[recorder] focus_poller last_error: {pollerLastError}"); automation?.Dispose(); return 0;