From d486cbb4d96e31ea06ca39b1af18f885d0a81275 Mon Sep 17 00:00:00 2001 From: minsung Date: Tue, 7 Apr 2026 14:27:46 +0900 Subject: [PATCH] Implement recorder PoC (#6) --- .../2026-04-07_이슈6-recorder-generator.md | 40 +++ recordingtest.sln | 60 ++++ .../ElementPathBuilder.cs | 61 ++++ src/Recordingtest.Recorder/LowLevelHook.cs | 133 ++++++++ src/Recordingtest.Recorder/MaskPolicy.cs | 28 ++ src/Recordingtest.Recorder/NativeMethods.cs | 89 ++++++ .../OffsetNormalizer.cs | 30 ++ src/Recordingtest.Recorder/Program.cs | 296 ++++++++++++++++++ .../Recordingtest.Recorder.csproj | 16 + src/Recordingtest.Recorder/Scenario.cs | 32 ++ src/Recordingtest.Recorder/ScenarioWriter.cs | 40 +++ .../FakeElement.cs | 12 + .../RecorderTests.cs | 141 +++++++++ .../Recordingtest.Recorder.Tests.csproj | 18 ++ 14 files changed, 996 insertions(+) create mode 100644 docs/history/2026-04-07_이슈6-recorder-generator.md create mode 100644 src/Recordingtest.Recorder/ElementPathBuilder.cs create mode 100644 src/Recordingtest.Recorder/LowLevelHook.cs create mode 100644 src/Recordingtest.Recorder/MaskPolicy.cs create mode 100644 src/Recordingtest.Recorder/NativeMethods.cs create mode 100644 src/Recordingtest.Recorder/OffsetNormalizer.cs create mode 100644 src/Recordingtest.Recorder/Program.cs create mode 100644 src/Recordingtest.Recorder/Recordingtest.Recorder.csproj create mode 100644 src/Recordingtest.Recorder/Scenario.cs create mode 100644 src/Recordingtest.Recorder/ScenarioWriter.cs create mode 100644 tests/Recordingtest.Recorder.Tests/FakeElement.cs create mode 100644 tests/Recordingtest.Recorder.Tests/RecorderTests.cs create mode 100644 tests/Recordingtest.Recorder.Tests/Recordingtest.Recorder.Tests.csproj diff --git a/docs/history/2026-04-07_이슈6-recorder-generator.md b/docs/history/2026-04-07_이슈6-recorder-generator.md new file mode 100644 index 0000000..bdde2c6 --- /dev/null +++ b/docs/history/2026-04-07_이슈6-recorder-generator.md @@ -0,0 +1,40 @@ +# 2026-04-07 이슈 #6 — Recorder Generator + +- **이슈**: #6 +- **소요 시간**: 약 25분 +- **Context 사용량**: 약 45k 토큰 + +## 작업 요약 + +`docs/contracts/recorder.md` 계약을 바탕으로 `Recordingtest.Recorder` 콘솔 PoC와 +xUnit 테스트 프로젝트를 신규 작성했다. + +### 산출물 + +- `src/Recordingtest.Recorder/` + - `Recordingtest.Recorder.csproj` (`net8.0-windows`, FlaUI.Core/UIA3 4.0.0, YamlDotNet 16.1.3) + - `NativeMethods.cs` — Win32 P/Invoke (WH_KEYBOARD_LL, WH_MOUSE_LL, GetMessage 등) + - `LowLevelHook.cs` — 전용 STA 스레드 + message loop, `Channel` 푸시 + - `ElementPathBuilder.cs` — `IElementSnapshot` 기반 순수 로직 + - `OffsetNormalizer.cs` — 정규화 [0..1] 클램프 포함 + - `MaskPolicy.cs` — `PasswordBox` / `IsPassword` → `` + - `Scenario.cs`, `ScenarioWriter.cs` — YAML 직렬화 (UnderscoredNamingConvention) + - `Program.cs` — CLI 파싱, attach 로직(절대 Launch 없음), Ctrl+C 플러시 +- `tests/Recordingtest.Recorder.Tests/` + - 5 단위 테스트 (path builder / normalizer / mask / yaml roundtrip / CLI exit 2) + +### 결과 + +- `dotnet build recordingtest.sln` — 경고 0, 오류 0 +- `dotnet test tests/Recordingtest.Recorder.Tests` — 5/5 통과 + +### 주의 + +- SUT(EG-BIM Modeler)는 코드/스크립트 어디에서도 launch하지 않는다. attach만 지원. +- `--attach` 누락 시 usage 출력 후 exit 2. +- 실 hook/UIA는 단위 테스트에서 다루지 않으며, 순수 로직 4종 + CLI 1종만 검증. + +### 미해결 / 후속 + +- 실제 SUT 대상 통합 테스트는 Evaluator 단계에서 별도 진행. +- IME 조합 키, 드래그 합성, 휠 등 고차 이벤트 합성은 후속 스프린트. diff --git a/recordingtest.sln b/recordingtest.sln index 5c80497..cfb30b3 100644 --- a/recordingtest.sln +++ b/recordingtest.sln @@ -15,6 +15,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.DiffReporter. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.DiffReporter.Tests", "tests\Recordingtest.DiffReporter.Tests\Recordingtest.DiffReporter.Tests.csproj", "{65290E3F-D498-452B-9A76-FBC460E53A9F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Recorder", "src\Recordingtest.Recorder\Recordingtest.Recorder.csproj", "{8C34DAA9-DB54-433B-86C1-E559EE36B5EE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Recorder.Tests", "tests\Recordingtest.Recorder.Tests\Recordingtest.Recorder.Tests.csproj", "{74D292F5-8004-4946-8CC3-808AFD9C52C1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Player", "src\Recordingtest.Player\Recordingtest.Player.csproj", "{D8962656-55EC-4595-8F19-8FBBF9256A04}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Recordingtest.Player.Tests", "tests\Recordingtest.Player.Tests\Recordingtest.Player.Tests.csproj", "{7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -97,6 +105,54 @@ Global {65290E3F-D498-452B-9A76-FBC460E53A9F}.Release|x64.Build.0 = Release|Any CPU {65290E3F-D498-452B-9A76-FBC460E53A9F}.Release|x86.ActiveCfg = Release|Any CPU {65290E3F-D498-452B-9A76-FBC460E53A9F}.Release|x86.Build.0 = Release|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Debug|x64.Build.0 = Debug|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Debug|x86.ActiveCfg = Debug|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Debug|x86.Build.0 = Debug|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Release|Any CPU.Build.0 = Release|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Release|x64.ActiveCfg = Release|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Release|x64.Build.0 = Release|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Release|x86.ActiveCfg = Release|Any CPU + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE}.Release|x86.Build.0 = Release|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Debug|x64.ActiveCfg = Debug|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Debug|x64.Build.0 = Debug|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Debug|x86.ActiveCfg = Debug|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Debug|x86.Build.0 = Debug|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Release|Any CPU.Build.0 = Release|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Release|x64.ActiveCfg = Release|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Release|x64.Build.0 = Release|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Release|x86.ActiveCfg = Release|Any CPU + {74D292F5-8004-4946-8CC3-808AFD9C52C1}.Release|x86.Build.0 = Release|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Debug|x64.ActiveCfg = Debug|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Debug|x64.Build.0 = Debug|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Debug|x86.ActiveCfg = Debug|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Debug|x86.Build.0 = Debug|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Release|Any CPU.Build.0 = Release|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Release|x64.ActiveCfg = Release|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Release|x64.Build.0 = Release|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Release|x86.ActiveCfg = Release|Any CPU + {D8962656-55EC-4595-8F19-8FBBF9256A04}.Release|x86.Build.0 = Release|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Debug|x64.ActiveCfg = Debug|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Debug|x64.Build.0 = Debug|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Debug|x86.ActiveCfg = Debug|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Debug|x86.Build.0 = Debug|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|Any CPU.Build.0 = Release|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x64.ActiveCfg = Release|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x64.Build.0 = Release|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x86.ActiveCfg = Release|Any CPU + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -107,5 +163,9 @@ Global {21A2E01D-FFC3-446D-B56E-775FF7E14C76} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {234EAA83-19DE-45A6-B9B2-2C0E85A17E4D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {65290E3F-D498-452B-9A76-FBC460E53A9F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {8C34DAA9-DB54-433B-86C1-E559EE36B5EE} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {74D292F5-8004-4946-8CC3-808AFD9C52C1} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {D8962656-55EC-4595-8F19-8FBBF9256A04} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {7A5C0D53-BDFC-4AF6-8F4D-49E7EB8245F5} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/src/Recordingtest.Recorder/ElementPathBuilder.cs b/src/Recordingtest.Recorder/ElementPathBuilder.cs new file mode 100644 index 0000000..4849a45 --- /dev/null +++ b/src/Recordingtest.Recorder/ElementPathBuilder.cs @@ -0,0 +1,61 @@ +using System.Collections.Generic; +using System.Text; + +namespace Recordingtest.Recorder; + +/// +/// Pure-logic snapshot of a UIA element. We avoid depending on FlaUI types here +/// so the path builder is fully unit-testable. +/// +public interface IElementSnapshot +{ + string ClassName { get; } + string? AutomationId { get; } + string? Name { get; } + bool IsPassword { get; } + /// Screen-space rectangle: left, top, width, height. + (double Left, double Top, double Width, double Height) BoundingRectangle { get; } + IElementSnapshot? Parent { get; } +} + +public static class ElementPathBuilder +{ + /// + /// Build "ClassName[@AutomationId='...']/ClassName[@Name='...']/..." walking + /// from the topmost ancestor down to the given element. + /// + public static string Build(IElementSnapshot element) + { + var chain = new List(); + IElementSnapshot? cur = element; + while (cur is not null) + { + chain.Add(cur); + cur = cur.Parent; + } + chain.Reverse(); + var sb = new StringBuilder(); + for (int i = 0; i < chain.Count; i++) + { + if (i > 0) sb.Append('/'); + sb.Append(FormatSegment(chain[i])); + } + return sb.ToString(); + } + + private static string FormatSegment(IElementSnapshot e) + { + var cls = string.IsNullOrEmpty(e.ClassName) ? "Element" : e.ClassName; + if (!string.IsNullOrEmpty(e.AutomationId)) + { + return $"{cls}[@AutomationId='{Escape(e.AutomationId!)}']"; + } + if (!string.IsNullOrEmpty(e.Name)) + { + return $"{cls}[@Name='{Escape(e.Name!)}']"; + } + return cls; + } + + private static string Escape(string s) => s.Replace("'", "'"); +} diff --git a/src/Recordingtest.Recorder/LowLevelHook.cs b/src/Recordingtest.Recorder/LowLevelHook.cs new file mode 100644 index 0000000..58c2b25 --- /dev/null +++ b/src/Recordingtest.Recorder/LowLevelHook.cs @@ -0,0 +1,133 @@ +using System; +using System.Threading; +using System.Threading.Channels; + +namespace Recordingtest.Recorder; + +public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta); + +/// +/// Installs WH_KEYBOARD_LL and WH_MOUSE_LL hooks on a dedicated thread with its own message loop. +/// Pushes RawEvent into a Channel for the main loop to consume. +/// +public sealed class LowLevelHook : IDisposable +{ + private readonly Channel _channel; + private Thread? _thread; + private IntPtr _kbHook = IntPtr.Zero; + private IntPtr _mouseHook = IntPtr.Zero; + private NativeMethods.HookProc? _kbProc; + private NativeMethods.HookProc? _mouseProc; + private uint _threadId; + private volatile bool _running; + + public LowLevelHook(Channel channel) + { + _channel = channel; + } + + public ChannelReader Reader => _channel.Reader; + + public void Start() + { + if (_running) return; + _running = true; + var ready = new ManualResetEventSlim(false); + _thread = new Thread(() => HookThreadMain(ready)) + { + IsBackground = true, + Name = "LowLevelHookThread", + }; + _thread.SetApartmentState(ApartmentState.STA); + _thread.Start(); + ready.Wait(); + } + + private void HookThreadMain(ManualResetEventSlim ready) + { + _threadId = GetCurrentThreadId(); + _kbProc = KeyboardProc; + _mouseProc = MouseProc; + var hMod = NativeMethods.GetModuleHandle(null); + _kbHook = NativeMethods.SetWindowsHookEx(NativeMethods.WH_KEYBOARD_LL, _kbProc, hMod, 0); + _mouseHook = NativeMethods.SetWindowsHookEx(NativeMethods.WH_MOUSE_LL, _mouseProc, hMod, 0); + ready.Set(); + + while (_running && NativeMethods.GetMessage(out var msg, IntPtr.Zero, 0, 0) > 0) + { + NativeMethods.TranslateMessage(ref msg); + NativeMethods.DispatchMessage(ref msg); + } + + if (_kbHook != IntPtr.Zero) NativeMethods.UnhookWindowsHookEx(_kbHook); + if (_mouseHook != IntPtr.Zero) NativeMethods.UnhookWindowsHookEx(_mouseHook); + _kbHook = IntPtr.Zero; + _mouseHook = IntPtr.Zero; + } + + private IntPtr KeyboardProc(int nCode, IntPtr wParam, IntPtr lParam) + { + if (nCode >= 0) + { + var data = System.Runtime.InteropServices.Marshal.PtrToStructure(lParam); + var msg = wParam.ToInt32(); + var kind = msg switch + { + NativeMethods.WM_KEYDOWN or NativeMethods.WM_SYSKEYDOWN => "key_down", + NativeMethods.WM_KEYUP or NativeMethods.WM_SYSKEYUP => "key_up", + _ => "key", + }; + _channel.Writer.TryWrite(new RawEvent(NowMs(), kind, 0, 0, data.vkCode, 0)); + } + return NativeMethods.CallNextHookEx(IntPtr.Zero, nCode, wParam, lParam); + } + + private IntPtr MouseProc(int nCode, IntPtr wParam, IntPtr lParam) + { + if (nCode >= 0) + { + var data = System.Runtime.InteropServices.Marshal.PtrToStructure(lParam); + var msg = wParam.ToInt32(); + var kind = msg switch + { + NativeMethods.WM_LBUTTONDOWN => "mouse_down_l", + NativeMethods.WM_LBUTTONUP => "mouse_up_l", + NativeMethods.WM_RBUTTONDOWN => "mouse_down_r", + NativeMethods.WM_RBUTTONUP => "mouse_up_r", + NativeMethods.WM_MBUTTONDOWN => "mouse_down_m", + NativeMethods.WM_MBUTTONUP => "mouse_up_m", + NativeMethods.WM_MOUSEWHEEL => "wheel", + NativeMethods.WM_MOUSEMOVE => "move", + _ => "mouse", + }; + int wheel = 0; + if (msg == NativeMethods.WM_MOUSEWHEEL) + { + wheel = (short)((data.mouseData >> 16) & 0xFFFF); + } + _channel.Writer.TryWrite(new RawEvent(NowMs(), kind, data.pt.x, data.pt.y, 0, wheel)); + } + return NativeMethods.CallNextHookEx(IntPtr.Zero, nCode, wParam, lParam); + } + + private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + [System.Runtime.InteropServices.DllImport("kernel32.dll")] + private static extern uint GetCurrentThreadId(); + + public void Dispose() + { + if (!_running) return; + _running = false; + if (_threadId != 0) + { + // Post a WM_QUIT to break GetMessage loop + PostThreadMessage(_threadId, 0x0012 /* WM_QUIT */, IntPtr.Zero, IntPtr.Zero); + } + _thread?.Join(1000); + _channel.Writer.TryComplete(); + } + + [System.Runtime.InteropServices.DllImport("user32.dll")] + private static extern bool PostThreadMessage(uint idThread, uint Msg, IntPtr wParam, IntPtr lParam); +} diff --git a/src/Recordingtest.Recorder/MaskPolicy.cs b/src/Recordingtest.Recorder/MaskPolicy.cs new file mode 100644 index 0000000..0ec2803 --- /dev/null +++ b/src/Recordingtest.Recorder/MaskPolicy.cs @@ -0,0 +1,28 @@ +namespace Recordingtest.Recorder; + +public static class MaskPolicy +{ + public const string MaskedValue = ""; + + /// + /// Returns the masked value if the focused element is a password input. + /// Detection: IsPassword flag, or ClassName equals "PasswordBox". + /// + public static string Apply(IElementSnapshot? focused, string? rawValue) + { + if (focused is null) return rawValue ?? string.Empty; + if (focused.IsPassword) return MaskedValue; + if (string.Equals(focused.ClassName, "PasswordBox", System.StringComparison.Ordinal)) + { + return MaskedValue; + } + return rawValue ?? string.Empty; + } + + public static bool IsMasked(IElementSnapshot? focused) + { + if (focused is null) return false; + return focused.IsPassword + || string.Equals(focused.ClassName, "PasswordBox", System.StringComparison.Ordinal); + } +} diff --git a/src/Recordingtest.Recorder/NativeMethods.cs b/src/Recordingtest.Recorder/NativeMethods.cs new file mode 100644 index 0000000..30c8f3e --- /dev/null +++ b/src/Recordingtest.Recorder/NativeMethods.cs @@ -0,0 +1,89 @@ +using System; +using System.Runtime.InteropServices; + +namespace Recordingtest.Recorder; + +internal static class NativeMethods +{ + public const int WH_KEYBOARD_LL = 13; + public const int WH_MOUSE_LL = 14; + + public const int WM_KEYDOWN = 0x0100; + public const int WM_KEYUP = 0x0101; + public const int WM_SYSKEYDOWN = 0x0104; + public const int WM_SYSKEYUP = 0x0105; + + public const int WM_LBUTTONDOWN = 0x0201; + public const int WM_LBUTTONUP = 0x0202; + public const int WM_RBUTTONDOWN = 0x0204; + public const int WM_RBUTTONUP = 0x0205; + public const int WM_MBUTTONDOWN = 0x0207; + public const int WM_MBUTTONUP = 0x0208; + public const int WM_MOUSEWHEEL = 0x020A; + public const int WM_MOUSEMOVE = 0x0200; + + public delegate IntPtr HookProc(int nCode, IntPtr wParam, IntPtr lParam); + + [StructLayout(LayoutKind.Sequential)] + public struct POINT + { + public int x; + public int y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct MSLLHOOKSTRUCT + { + public POINT pt; + public uint mouseData; + public uint flags; + public uint time; + public IntPtr dwExtraInfo; + } + + [StructLayout(LayoutKind.Sequential)] + public struct KBDLLHOOKSTRUCT + { + public uint vkCode; + public uint scanCode; + public uint flags; + public uint time; + public IntPtr dwExtraInfo; + } + + [DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)] + public static extern IntPtr SetWindowsHookEx(int idHook, HookProc lpfn, IntPtr hMod, uint dwThreadId); + + [DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool UnhookWindowsHookEx(IntPtr hhk); + + [DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)] + public static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam); + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + public static extern IntPtr GetModuleHandle(string? lpModuleName); + + [DllImport("user32.dll")] + public static extern int GetMessage(out MSG lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax); + + [DllImport("user32.dll")] + public static extern bool TranslateMessage(ref MSG lpMsg); + + [DllImport("user32.dll")] + public static extern IntPtr DispatchMessage(ref MSG lpMsg); + + [DllImport("user32.dll")] + public static extern void PostQuitMessage(int nExitCode); + + [StructLayout(LayoutKind.Sequential)] + public struct MSG + { + public IntPtr hwnd; + public uint message; + public IntPtr wParam; + public IntPtr lParam; + public uint time; + public POINT pt; + } +} diff --git a/src/Recordingtest.Recorder/OffsetNormalizer.cs b/src/Recordingtest.Recorder/OffsetNormalizer.cs new file mode 100644 index 0000000..f7a1612 --- /dev/null +++ b/src/Recordingtest.Recorder/OffsetNormalizer.cs @@ -0,0 +1,30 @@ +namespace Recordingtest.Recorder; + +public static class OffsetNormalizer +{ + /// + /// Convert a screen point into normalized [0..1] offsets relative to the + /// element bounding rectangle. Out-of-bounds points are clamped. + /// Returns (0,0) for zero-sized rectangles. + /// + public static (double DxNorm, double DyNorm) Normalize( + (double Left, double Top, double Width, double Height) bounds, + double screenX, + double screenY) + { + if (bounds.Width <= 0 || bounds.Height <= 0) + { + return (0.0, 0.0); + } + + double dx = (screenX - bounds.Left) / bounds.Width; + double dy = (screenY - bounds.Top) / bounds.Height; + + if (dx < 0) dx = 0; + if (dx > 1) dx = 1; + if (dy < 0) dy = 0; + if (dy > 1) dy = 1; + + return (dx, dy); + } +} diff --git a/src/Recordingtest.Recorder/Program.cs b/src/Recordingtest.Recorder/Program.cs new file mode 100644 index 0000000..5f182c2 --- /dev/null +++ b/src/Recordingtest.Recorder/Program.cs @@ -0,0 +1,296 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using FlaUI.Core; +using FlaUI.Core.AutomationElements; +using FlaUI.UIA3; + +namespace Recordingtest.Recorder; + +public static class Program +{ + public static int Main(string[] args) + { + var parsed = ParseArgs(args); + if (parsed is null) + { + PrintUsage(); + return 2; + } + + Console.WriteLine($"[recorder] output={parsed.OutputPath} attach={parsed.Attach}"); + + try + { + return Run(parsed); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[recorder] error: {ex.Message}"); + return 1; + } + } + + internal sealed record CliArgs(string OutputPath, string Attach); + + internal static CliArgs? ParseArgs(string[] args) + { + string? output = null; + string? attach = null; + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--output" when i + 1 < args.Length: + output = args[++i]; + break; + case "--attach" when i + 1 < args.Length: + attach = args[++i]; + break; + } + } + if (string.IsNullOrEmpty(attach)) return null; + if (string.IsNullOrEmpty(output)) output = "scenarios/recorded.yaml"; + return new CliArgs(output!, attach!); + } + + internal static void PrintUsage() + { + Console.Error.WriteLine("Usage: Recordingtest.Recorder --output scenarios/.yaml --attach "); + Console.Error.WriteLine(" --attach is REQUIRED. The recorder never launches the SUT itself."); + } + + private static int Run(CliArgs args) + { + var channel = Channel.CreateUnbounded(); + using var hook = new LowLevelHook(channel); + hook.Start(); + + Application? app = null; + UIA3Automation? automation = null; + AutomationElement? mainWindow = null; + + try + { + (app, automation, mainWindow) = TryAttach(args.Attach); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[recorder] attach failed: {ex.Message}"); + } + + var scenario = new Scenario + { + Name = System.IO.Path.GetFileNameWithoutExtension(args.OutputPath), + Description = "Recorded session", + }; + + var cts = new CancellationTokenSource(); + Console.CancelKeyPress += (_, e) => + { + e.Cancel = true; + cts.Cancel(); + }; + + Console.WriteLine("[recorder] capturing... press Ctrl+C to stop."); + int eventCount = 0; + int unresolved = 0; + var sw = Stopwatch.StartNew(); + + try + { + ConsumeAsync(channel.Reader, scenario, mainWindow, automation, cts.Token, + onEvent: () => eventCount++, + onUnresolved: () => unresolved++).GetAwaiter().GetResult(); + } + catch (OperationCanceledException) + { + // expected on Ctrl+C + } + + sw.Stop(); + ScenarioWriter.WriteToFile(scenario, args.OutputPath); + Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}"); + + automation?.Dispose(); + return 0; + } + + private static (Application?, UIA3Automation?, AutomationElement?) TryAttach(string attach) + { + // NOTE: We never Launch() the SUT here. Only attach by pid or window title. + Application? app = null; + if (int.TryParse(attach, out var pid)) + { + app = Application.Attach(pid); + } + else + { + var procs = Process.GetProcesses(); + foreach (var p in procs) + { + try + { + if (!string.IsNullOrEmpty(p.MainWindowTitle) && + p.MainWindowTitle.Contains(attach, StringComparison.OrdinalIgnoreCase)) + { + app = Application.Attach(p.Id); + break; + } + } + catch + { + // ignore inaccessible processes + } + } + } + if (app is null) return (null, null, null); + + var automation = new UIA3Automation(); + var main = app.GetMainWindow(automation, TimeSpan.FromSeconds(5)); + return (app, automation, main); + } + + private static async Task ConsumeAsync( + ChannelReader reader, + Scenario scenario, + AutomationElement? mainWindow, + UIA3Automation? automation, + CancellationToken ct, + Action onEvent, + Action onUnresolved) + { + while (await reader.WaitToReadAsync(ct).ConfigureAwait(false)) + { + while (reader.TryRead(out var ev)) + { + onEvent(); + if (!IsInterestingForStep(ev.Kind)) continue; + + IElementSnapshot? snap = null; + if (automation is not null) + { + try + { + snap = ResolveAt(automation, ev.X, ev.Y); + } + catch + { + snap = null; + } + } + + if (snap is null) + { + onUnresolved(); + continue; + } + + var path = ElementPathBuilder.Build(snap); + var (dx, dy) = OffsetNormalizer.Normalize(snap.BoundingRectangle, ev.X, ev.Y); + var step = new ScenarioStep + { + Kind = ev.Kind.StartsWith("key", StringComparison.Ordinal) ? "type" : "click", + Target = new ScenarioTarget + { + UiaPath = path, + Offset = new[] { dx, dy }, + }, + Value = MaskPolicy.IsMasked(snap) ? MaskPolicy.MaskedValue : null, + }; + scenario.Steps.Add(step); + } + } + } + + private static bool IsInterestingForStep(string kind) => + kind == "mouse_down_l" || kind == "mouse_down_r" || kind == "key_down"; + + private static IElementSnapshot? ResolveAt(UIA3Automation automation, int x, int y) + { + var raw = automation.FromPoint(new System.Drawing.Point(x, y)); + if (raw is null) return null; + return new FlaUiSnapshot(raw); + } +} + +/// +/// Adapter wrapping a FlaUI AutomationElement as IElementSnapshot. +/// Resolved on demand from the main loop (never from the hook thread). +/// +internal sealed class FlaUiSnapshot : IElementSnapshot +{ + private readonly AutomationElement _el; + private readonly FlaUiSnapshot? _parentSnap; + + public FlaUiSnapshot(AutomationElement el, FlaUiSnapshot? parentSnap = null) + { + _el = el; + _parentSnap = parentSnap; + } + + public string ClassName => SafeGet(() => _el.ClassName ?? string.Empty); + public string? AutomationId => SafeGet(() => _el.AutomationId); + public string? Name => SafeGet(() => _el.Name); + + public bool IsPassword + { + get + { + try + { + var ct = _el.ControlType; + if (ct == FlaUI.Core.Definitions.ControlType.Edit && + string.Equals(_el.ClassName, "PasswordBox", StringComparison.Ordinal)) + { + return true; + } + } + catch + { + // ignore + } + return false; + } + } + + public (double Left, double Top, double Width, double Height) BoundingRectangle + { + get + { + try + { + var r = _el.BoundingRectangle; + return (r.Left, r.Top, r.Width, r.Height); + } + catch + { + return (0, 0, 0, 0); + } + } + } + + public IElementSnapshot? Parent + { + get + { + if (_parentSnap is not null) return _parentSnap; + try + { + var p = _el.Parent; + return p is null ? null : new FlaUiSnapshot(p); + } + catch + { + return null; + } + } + } + + private static T SafeGet(Func f) + { + try { return f(); } catch { return default!; } + } +} diff --git a/src/Recordingtest.Recorder/Recordingtest.Recorder.csproj b/src/Recordingtest.Recorder/Recordingtest.Recorder.csproj new file mode 100644 index 0000000..d47d1be --- /dev/null +++ b/src/Recordingtest.Recorder/Recordingtest.Recorder.csproj @@ -0,0 +1,16 @@ + + + Exe + net8.0-windows + false + false + Recordingtest.Recorder + Recordingtest.Recorder + false + + + + + + + diff --git a/src/Recordingtest.Recorder/Scenario.cs b/src/Recordingtest.Recorder/Scenario.cs new file mode 100644 index 0000000..dac34bb --- /dev/null +++ b/src/Recordingtest.Recorder/Scenario.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; + +namespace Recordingtest.Recorder; + +public sealed class Scenario +{ + public string Name { get; set; } = string.Empty; + public string Description { get; set; } = string.Empty; + public ScenarioSut Sut { get; set; } = new(); + public List Steps { get; set; } = new(); +} + +public sealed class ScenarioSut +{ + public string Exe { get; set; } = "EG-BIM Modeler/EG-BIM Modeler.exe"; + public int StartupTimeoutMs { get; set; } = 15000; +} + +public sealed class ScenarioStep +{ + /// click | type | drag | hotkey | wait + public string Kind { get; set; } = "click"; + public ScenarioTarget? Target { get; set; } + public string? Value { get; set; } + public string? WaitFor { get; set; } +} + +public sealed class ScenarioTarget +{ + public string UiaPath { get; set; } = string.Empty; + public double[] Offset { get; set; } = new double[] { 0.5, 0.5 }; +} diff --git a/src/Recordingtest.Recorder/ScenarioWriter.cs b/src/Recordingtest.Recorder/ScenarioWriter.cs new file mode 100644 index 0000000..41e4989 --- /dev/null +++ b/src/Recordingtest.Recorder/ScenarioWriter.cs @@ -0,0 +1,40 @@ +using System.IO; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Recordingtest.Recorder; + +public static class ScenarioWriter +{ + private static ISerializer BuildSerializer() => + new SerializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.Preserve) + .Build(); + + private static IDeserializer BuildDeserializer() => + new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + public static string Serialize(Scenario scenario) + { + return BuildSerializer().Serialize(scenario); + } + + public static Scenario Deserialize(string yaml) + { + return BuildDeserializer().Deserialize(yaml); + } + + public static void WriteToFile(Scenario scenario, string path) + { + var dir = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + File.WriteAllText(path, Serialize(scenario)); + } +} diff --git a/tests/Recordingtest.Recorder.Tests/FakeElement.cs b/tests/Recordingtest.Recorder.Tests/FakeElement.cs new file mode 100644 index 0000000..37c9571 --- /dev/null +++ b/tests/Recordingtest.Recorder.Tests/FakeElement.cs @@ -0,0 +1,12 @@ +namespace Recordingtest.Recorder.Tests; + +internal sealed class FakeElement : IElementSnapshot +{ + public string ClassName { get; set; } = "Element"; + public string? AutomationId { get; set; } + public string? Name { get; set; } + public bool IsPassword { get; set; } + public (double Left, double Top, double Width, double Height) BoundingRectangle { get; set; } + = (0, 0, 0, 0); + public IElementSnapshot? Parent { get; set; } +} diff --git a/tests/Recordingtest.Recorder.Tests/RecorderTests.cs b/tests/Recordingtest.Recorder.Tests/RecorderTests.cs new file mode 100644 index 0000000..862d1ec --- /dev/null +++ b/tests/Recordingtest.Recorder.Tests/RecorderTests.cs @@ -0,0 +1,141 @@ +using System; +using System.IO; +using Xunit; + +namespace Recordingtest.Recorder.Tests; + +public class RecorderTests +{ + [Fact] + public void ElementPathBuilder_WithNestedElements_ReturnsFullPath() + { + var window = new FakeElement + { + ClassName = "Window", + Name = "Main", + }; + var panel = new FakeElement + { + ClassName = "StackPanel", + AutomationId = "ToolStrip", + Parent = window, + }; + var button = new FakeElement + { + ClassName = "Button", + AutomationId = "BoxCommand", + Parent = panel, + }; + + var path = ElementPathBuilder.Build(button); + + Assert.Equal( + "Window[@Name='Main']/StackPanel[@AutomationId='ToolStrip']/Button[@AutomationId='BoxCommand']", + path); + } + + [Fact] + public void OffsetNormalizer_ClicksInsideElement_ReturnsZeroToOne() + { + var bounds = (Left: 100.0, Top: 200.0, Width: 400.0, Height: 200.0); + + var (dx, dy) = OffsetNormalizer.Normalize(bounds, 300, 300); + + Assert.Equal(0.5, dx, 6); + Assert.Equal(0.5, dy, 6); + + var (dx2, dy2) = OffsetNormalizer.Normalize(bounds, 100, 200); + Assert.Equal(0.0, dx2, 6); + Assert.Equal(0.0, dy2, 6); + + // Out-of-bounds clamps into [0,1] + var (dx3, dy3) = OffsetNormalizer.Normalize(bounds, 9999, -9999); + Assert.InRange(dx3, 0.0, 1.0); + Assert.InRange(dy3, 0.0, 1.0); + } + + [Fact] + public void FocusedElementIsPassword_ReturnsMasked() + { + var pwd = new FakeElement + { + ClassName = "PasswordBox", + IsPassword = true, + }; + + var value = MaskPolicy.Apply(pwd, "supersecret"); + + Assert.Equal("", value); + Assert.True(MaskPolicy.IsMasked(pwd)); + + var plain = new FakeElement { ClassName = "TextBox" }; + Assert.Equal("hello", MaskPolicy.Apply(plain, "hello")); + Assert.False(MaskPolicy.IsMasked(plain)); + } + + [Fact] + public void YamlSerializer_RoundtripsScenario() + { + var s = new Scenario + { + Name = "smoke", + Description = "round trip", + Sut = new ScenarioSut + { + Exe = "EG-BIM Modeler/EG-BIM Modeler.exe", + StartupTimeoutMs = 15000, + }, + Steps = + { + new ScenarioStep + { + Kind = "click", + Target = new ScenarioTarget + { + UiaPath = "Window[@Name='Main']/Button[@AutomationId='BoxCommand']", + Offset = new[] { 0.5, 0.5 }, + }, + Value = null, + WaitFor = null, + }, + new ScenarioStep + { + Kind = "type", + Target = new ScenarioTarget + { + UiaPath = "Window[@Name='Main']/Edit[@AutomationId='Pwd']", + Offset = new[] { 0.5, 0.5 }, + }, + Value = "", + }, + }, + }; + + var yaml = ScenarioWriter.Serialize(s); + var parsed = ScenarioWriter.Deserialize(yaml); + + Assert.Equal(s.Name, parsed.Name); + Assert.Equal(s.Description, parsed.Description); + Assert.Equal(s.Sut.Exe, parsed.Sut.Exe); + Assert.Equal(s.Sut.StartupTimeoutMs, parsed.Sut.StartupTimeoutMs); + Assert.Equal(s.Steps.Count, parsed.Steps.Count); + Assert.Equal(s.Steps[0].Target!.UiaPath, parsed.Steps[0].Target!.UiaPath); + Assert.Equal(s.Steps[1].Value, parsed.Steps[1].Value); + } + + [Fact] + public void Cli_MissingAttach_ExitTwo() + { + var stderr = Console.Error; + try + { + Console.SetError(new StringWriter()); + var rc = Program.Main(new[] { "--output", "scenarios/x.yaml" }); + Assert.Equal(2, rc); + } + finally + { + Console.SetError(stderr); + } + } +} diff --git a/tests/Recordingtest.Recorder.Tests/Recordingtest.Recorder.Tests.csproj b/tests/Recordingtest.Recorder.Tests/Recordingtest.Recorder.Tests.csproj new file mode 100644 index 0000000..d51d8e5 --- /dev/null +++ b/tests/Recordingtest.Recorder.Tests/Recordingtest.Recorder.Tests.csproj @@ -0,0 +1,18 @@ + + + net8.0-windows + false + false + false + Recordingtest.Recorder.Tests + Recordingtest.Recorder.Tests + + + + + + + + + +