Implement recorder PoC (#6)

This commit is contained in:
minsung
2026-04-07 14:27:46 +09:00
parent e3d2ff6c77
commit d486cbb4d9
14 changed files with 996 additions and 0 deletions

View File

@@ -0,0 +1,61 @@
using System.Collections.Generic;
using System.Text;
namespace Recordingtest.Recorder;
/// <summary>
/// Pure-logic snapshot of a UIA element. We avoid depending on FlaUI types here
/// so the path builder is fully unit-testable.
/// </summary>
public interface IElementSnapshot
{
string ClassName { get; }
string? AutomationId { get; }
string? Name { get; }
bool IsPassword { get; }
/// <summary>Screen-space rectangle: left, top, width, height.</summary>
(double Left, double Top, double Width, double Height) BoundingRectangle { get; }
IElementSnapshot? Parent { get; }
}
public static class ElementPathBuilder
{
/// <summary>
/// Build "ClassName[@AutomationId='...']/ClassName[@Name='...']/..." walking
/// from the topmost ancestor down to the given element.
/// </summary>
public static string Build(IElementSnapshot element)
{
var chain = new List<IElementSnapshot>();
IElementSnapshot? cur = element;
while (cur is not null)
{
chain.Add(cur);
cur = cur.Parent;
}
chain.Reverse();
var sb = new StringBuilder();
for (int i = 0; i < chain.Count; i++)
{
if (i > 0) sb.Append('/');
sb.Append(FormatSegment(chain[i]));
}
return sb.ToString();
}
private static string FormatSegment(IElementSnapshot e)
{
var cls = string.IsNullOrEmpty(e.ClassName) ? "Element" : e.ClassName;
if (!string.IsNullOrEmpty(e.AutomationId))
{
return $"{cls}[@AutomationId='{Escape(e.AutomationId!)}']";
}
if (!string.IsNullOrEmpty(e.Name))
{
return $"{cls}[@Name='{Escape(e.Name!)}']";
}
return cls;
}
private static string Escape(string s) => s.Replace("'", "&apos;");
}

View File

@@ -0,0 +1,133 @@
using System;
using System.Threading;
using System.Threading.Channels;
namespace Recordingtest.Recorder;
public sealed record RawEvent(long TimestampMs, string Kind, int X, int Y, uint Code, int WheelDelta);
/// <summary>
/// Installs WH_KEYBOARD_LL and WH_MOUSE_LL hooks on a dedicated thread with its own message loop.
/// Pushes RawEvent into a Channel for the main loop to consume.
/// </summary>
public sealed class LowLevelHook : IDisposable
{
private readonly Channel<RawEvent> _channel;
private Thread? _thread;
private IntPtr _kbHook = IntPtr.Zero;
private IntPtr _mouseHook = IntPtr.Zero;
private NativeMethods.HookProc? _kbProc;
private NativeMethods.HookProc? _mouseProc;
private uint _threadId;
private volatile bool _running;
public LowLevelHook(Channel<RawEvent> channel)
{
_channel = channel;
}
public ChannelReader<RawEvent> Reader => _channel.Reader;
public void Start()
{
if (_running) return;
_running = true;
var ready = new ManualResetEventSlim(false);
_thread = new Thread(() => HookThreadMain(ready))
{
IsBackground = true,
Name = "LowLevelHookThread",
};
_thread.SetApartmentState(ApartmentState.STA);
_thread.Start();
ready.Wait();
}
private void HookThreadMain(ManualResetEventSlim ready)
{
_threadId = GetCurrentThreadId();
_kbProc = KeyboardProc;
_mouseProc = MouseProc;
var hMod = NativeMethods.GetModuleHandle(null);
_kbHook = NativeMethods.SetWindowsHookEx(NativeMethods.WH_KEYBOARD_LL, _kbProc, hMod, 0);
_mouseHook = NativeMethods.SetWindowsHookEx(NativeMethods.WH_MOUSE_LL, _mouseProc, hMod, 0);
ready.Set();
while (_running && NativeMethods.GetMessage(out var msg, IntPtr.Zero, 0, 0) > 0)
{
NativeMethods.TranslateMessage(ref msg);
NativeMethods.DispatchMessage(ref msg);
}
if (_kbHook != IntPtr.Zero) NativeMethods.UnhookWindowsHookEx(_kbHook);
if (_mouseHook != IntPtr.Zero) NativeMethods.UnhookWindowsHookEx(_mouseHook);
_kbHook = IntPtr.Zero;
_mouseHook = IntPtr.Zero;
}
private IntPtr KeyboardProc(int nCode, IntPtr wParam, IntPtr lParam)
{
if (nCode >= 0)
{
var data = System.Runtime.InteropServices.Marshal.PtrToStructure<NativeMethods.KBDLLHOOKSTRUCT>(lParam);
var msg = wParam.ToInt32();
var kind = msg switch
{
NativeMethods.WM_KEYDOWN or NativeMethods.WM_SYSKEYDOWN => "key_down",
NativeMethods.WM_KEYUP or NativeMethods.WM_SYSKEYUP => "key_up",
_ => "key",
};
_channel.Writer.TryWrite(new RawEvent(NowMs(), kind, 0, 0, data.vkCode, 0));
}
return NativeMethods.CallNextHookEx(IntPtr.Zero, nCode, wParam, lParam);
}
private IntPtr MouseProc(int nCode, IntPtr wParam, IntPtr lParam)
{
if (nCode >= 0)
{
var data = System.Runtime.InteropServices.Marshal.PtrToStructure<NativeMethods.MSLLHOOKSTRUCT>(lParam);
var msg = wParam.ToInt32();
var kind = msg switch
{
NativeMethods.WM_LBUTTONDOWN => "mouse_down_l",
NativeMethods.WM_LBUTTONUP => "mouse_up_l",
NativeMethods.WM_RBUTTONDOWN => "mouse_down_r",
NativeMethods.WM_RBUTTONUP => "mouse_up_r",
NativeMethods.WM_MBUTTONDOWN => "mouse_down_m",
NativeMethods.WM_MBUTTONUP => "mouse_up_m",
NativeMethods.WM_MOUSEWHEEL => "wheel",
NativeMethods.WM_MOUSEMOVE => "move",
_ => "mouse",
};
int wheel = 0;
if (msg == NativeMethods.WM_MOUSEWHEEL)
{
wheel = (short)((data.mouseData >> 16) & 0xFFFF);
}
_channel.Writer.TryWrite(new RawEvent(NowMs(), kind, data.pt.x, data.pt.y, 0, wheel));
}
return NativeMethods.CallNextHookEx(IntPtr.Zero, nCode, wParam, lParam);
}
private static long NowMs() => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
[System.Runtime.InteropServices.DllImport("kernel32.dll")]
private static extern uint GetCurrentThreadId();
public void Dispose()
{
if (!_running) return;
_running = false;
if (_threadId != 0)
{
// Post a WM_QUIT to break GetMessage loop
PostThreadMessage(_threadId, 0x0012 /* WM_QUIT */, IntPtr.Zero, IntPtr.Zero);
}
_thread?.Join(1000);
_channel.Writer.TryComplete();
}
[System.Runtime.InteropServices.DllImport("user32.dll")]
private static extern bool PostThreadMessage(uint idThread, uint Msg, IntPtr wParam, IntPtr lParam);
}

View File

@@ -0,0 +1,28 @@
namespace Recordingtest.Recorder;
public static class MaskPolicy
{
public const string MaskedValue = "<MASKED>";
/// <summary>
/// Returns the masked value if the focused element is a password input.
/// Detection: IsPassword flag, or ClassName equals "PasswordBox".
/// </summary>
public static string Apply(IElementSnapshot? focused, string? rawValue)
{
if (focused is null) return rawValue ?? string.Empty;
if (focused.IsPassword) return MaskedValue;
if (string.Equals(focused.ClassName, "PasswordBox", System.StringComparison.Ordinal))
{
return MaskedValue;
}
return rawValue ?? string.Empty;
}
public static bool IsMasked(IElementSnapshot? focused)
{
if (focused is null) return false;
return focused.IsPassword
|| string.Equals(focused.ClassName, "PasswordBox", System.StringComparison.Ordinal);
}
}

View File

@@ -0,0 +1,89 @@
using System;
using System.Runtime.InteropServices;
namespace Recordingtest.Recorder;
internal static class NativeMethods
{
public const int WH_KEYBOARD_LL = 13;
public const int WH_MOUSE_LL = 14;
public const int WM_KEYDOWN = 0x0100;
public const int WM_KEYUP = 0x0101;
public const int WM_SYSKEYDOWN = 0x0104;
public const int WM_SYSKEYUP = 0x0105;
public const int WM_LBUTTONDOWN = 0x0201;
public const int WM_LBUTTONUP = 0x0202;
public const int WM_RBUTTONDOWN = 0x0204;
public const int WM_RBUTTONUP = 0x0205;
public const int WM_MBUTTONDOWN = 0x0207;
public const int WM_MBUTTONUP = 0x0208;
public const int WM_MOUSEWHEEL = 0x020A;
public const int WM_MOUSEMOVE = 0x0200;
public delegate IntPtr HookProc(int nCode, IntPtr wParam, IntPtr lParam);
[StructLayout(LayoutKind.Sequential)]
public struct POINT
{
public int x;
public int y;
}
[StructLayout(LayoutKind.Sequential)]
public struct MSLLHOOKSTRUCT
{
public POINT pt;
public uint mouseData;
public uint flags;
public uint time;
public IntPtr dwExtraInfo;
}
[StructLayout(LayoutKind.Sequential)]
public struct KBDLLHOOKSTRUCT
{
public uint vkCode;
public uint scanCode;
public uint flags;
public uint time;
public IntPtr dwExtraInfo;
}
[DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)]
public static extern IntPtr SetWindowsHookEx(int idHook, HookProc lpfn, IntPtr hMod, uint dwThreadId);
[DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
public static extern bool UnhookWindowsHookEx(IntPtr hhk);
[DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)]
public static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode, IntPtr wParam, IntPtr lParam);
[DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
public static extern IntPtr GetModuleHandle(string? lpModuleName);
[DllImport("user32.dll")]
public static extern int GetMessage(out MSG lpMsg, IntPtr hWnd, uint wMsgFilterMin, uint wMsgFilterMax);
[DllImport("user32.dll")]
public static extern bool TranslateMessage(ref MSG lpMsg);
[DllImport("user32.dll")]
public static extern IntPtr DispatchMessage(ref MSG lpMsg);
[DllImport("user32.dll")]
public static extern void PostQuitMessage(int nExitCode);
[StructLayout(LayoutKind.Sequential)]
public struct MSG
{
public IntPtr hwnd;
public uint message;
public IntPtr wParam;
public IntPtr lParam;
public uint time;
public POINT pt;
}
}

View File

@@ -0,0 +1,30 @@
namespace Recordingtest.Recorder;
public static class OffsetNormalizer
{
/// <summary>
/// Convert a screen point into normalized [0..1] offsets relative to the
/// element bounding rectangle. Out-of-bounds points are clamped.
/// Returns (0,0) for zero-sized rectangles.
/// </summary>
public static (double DxNorm, double DyNorm) Normalize(
(double Left, double Top, double Width, double Height) bounds,
double screenX,
double screenY)
{
if (bounds.Width <= 0 || bounds.Height <= 0)
{
return (0.0, 0.0);
}
double dx = (screenX - bounds.Left) / bounds.Width;
double dy = (screenY - bounds.Top) / bounds.Height;
if (dx < 0) dx = 0;
if (dx > 1) dx = 1;
if (dy < 0) dy = 0;
if (dy > 1) dy = 1;
return (dx, dy);
}
}

View File

@@ -0,0 +1,296 @@
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using FlaUI.Core;
using FlaUI.Core.AutomationElements;
using FlaUI.UIA3;
namespace Recordingtest.Recorder;
public static class Program
{
public static int Main(string[] args)
{
var parsed = ParseArgs(args);
if (parsed is null)
{
PrintUsage();
return 2;
}
Console.WriteLine($"[recorder] output={parsed.OutputPath} attach={parsed.Attach}");
try
{
return Run(parsed);
}
catch (Exception ex)
{
Console.Error.WriteLine($"[recorder] error: {ex.Message}");
return 1;
}
}
internal sealed record CliArgs(string OutputPath, string Attach);
internal static CliArgs? ParseArgs(string[] args)
{
string? output = null;
string? attach = null;
for (int i = 0; i < args.Length; i++)
{
switch (args[i])
{
case "--output" when i + 1 < args.Length:
output = args[++i];
break;
case "--attach" when i + 1 < args.Length:
attach = args[++i];
break;
}
}
if (string.IsNullOrEmpty(attach)) return null;
if (string.IsNullOrEmpty(output)) output = "scenarios/recorded.yaml";
return new CliArgs(output!, attach!);
}
internal static void PrintUsage()
{
Console.Error.WriteLine("Usage: Recordingtest.Recorder --output scenarios/<name>.yaml --attach <pid|title>");
Console.Error.WriteLine(" --attach is REQUIRED. The recorder never launches the SUT itself.");
}
private static int Run(CliArgs args)
{
var channel = Channel.CreateUnbounded<RawEvent>();
using var hook = new LowLevelHook(channel);
hook.Start();
Application? app = null;
UIA3Automation? automation = null;
AutomationElement? mainWindow = null;
try
{
(app, automation, mainWindow) = TryAttach(args.Attach);
}
catch (Exception ex)
{
Console.Error.WriteLine($"[recorder] attach failed: {ex.Message}");
}
var scenario = new Scenario
{
Name = System.IO.Path.GetFileNameWithoutExtension(args.OutputPath),
Description = "Recorded session",
};
var cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
cts.Cancel();
};
Console.WriteLine("[recorder] capturing... press Ctrl+C to stop.");
int eventCount = 0;
int unresolved = 0;
var sw = Stopwatch.StartNew();
try
{
ConsumeAsync(channel.Reader, scenario, mainWindow, automation, cts.Token,
onEvent: () => eventCount++,
onUnresolved: () => unresolved++).GetAwaiter().GetResult();
}
catch (OperationCanceledException)
{
// expected on Ctrl+C
}
sw.Stop();
ScenarioWriter.WriteToFile(scenario, args.OutputPath);
Console.WriteLine($"[recorder] done. events={eventCount} elapsed={sw.Elapsed} unresolved_paths={unresolved}");
automation?.Dispose();
return 0;
}
private static (Application?, UIA3Automation?, AutomationElement?) TryAttach(string attach)
{
// NOTE: We never Launch() the SUT here. Only attach by pid or window title.
Application? app = null;
if (int.TryParse(attach, out var pid))
{
app = Application.Attach(pid);
}
else
{
var procs = Process.GetProcesses();
foreach (var p in procs)
{
try
{
if (!string.IsNullOrEmpty(p.MainWindowTitle) &&
p.MainWindowTitle.Contains(attach, StringComparison.OrdinalIgnoreCase))
{
app = Application.Attach(p.Id);
break;
}
}
catch
{
// ignore inaccessible processes
}
}
}
if (app is null) return (null, null, null);
var automation = new UIA3Automation();
var main = app.GetMainWindow(automation, TimeSpan.FromSeconds(5));
return (app, automation, main);
}
private static async Task ConsumeAsync(
ChannelReader<RawEvent> reader,
Scenario scenario,
AutomationElement? mainWindow,
UIA3Automation? automation,
CancellationToken ct,
Action onEvent,
Action onUnresolved)
{
while (await reader.WaitToReadAsync(ct).ConfigureAwait(false))
{
while (reader.TryRead(out var ev))
{
onEvent();
if (!IsInterestingForStep(ev.Kind)) continue;
IElementSnapshot? snap = null;
if (automation is not null)
{
try
{
snap = ResolveAt(automation, ev.X, ev.Y);
}
catch
{
snap = null;
}
}
if (snap is null)
{
onUnresolved();
continue;
}
var path = ElementPathBuilder.Build(snap);
var (dx, dy) = OffsetNormalizer.Normalize(snap.BoundingRectangle, ev.X, ev.Y);
var step = new ScenarioStep
{
Kind = ev.Kind.StartsWith("key", StringComparison.Ordinal) ? "type" : "click",
Target = new ScenarioTarget
{
UiaPath = path,
Offset = new[] { dx, dy },
},
Value = MaskPolicy.IsMasked(snap) ? MaskPolicy.MaskedValue : null,
};
scenario.Steps.Add(step);
}
}
}
private static bool IsInterestingForStep(string kind) =>
kind == "mouse_down_l" || kind == "mouse_down_r" || kind == "key_down";
private static IElementSnapshot? ResolveAt(UIA3Automation automation, int x, int y)
{
var raw = automation.FromPoint(new System.Drawing.Point(x, y));
if (raw is null) return null;
return new FlaUiSnapshot(raw);
}
}
/// <summary>
/// Adapter wrapping a FlaUI AutomationElement as IElementSnapshot.
/// Resolved on demand from the main loop (never from the hook thread).
/// </summary>
internal sealed class FlaUiSnapshot : IElementSnapshot
{
private readonly AutomationElement _el;
private readonly FlaUiSnapshot? _parentSnap;
public FlaUiSnapshot(AutomationElement el, FlaUiSnapshot? parentSnap = null)
{
_el = el;
_parentSnap = parentSnap;
}
public string ClassName => SafeGet(() => _el.ClassName ?? string.Empty);
public string? AutomationId => SafeGet(() => _el.AutomationId);
public string? Name => SafeGet(() => _el.Name);
public bool IsPassword
{
get
{
try
{
var ct = _el.ControlType;
if (ct == FlaUI.Core.Definitions.ControlType.Edit &&
string.Equals(_el.ClassName, "PasswordBox", StringComparison.Ordinal))
{
return true;
}
}
catch
{
// ignore
}
return false;
}
}
public (double Left, double Top, double Width, double Height) BoundingRectangle
{
get
{
try
{
var r = _el.BoundingRectangle;
return (r.Left, r.Top, r.Width, r.Height);
}
catch
{
return (0, 0, 0, 0);
}
}
}
public IElementSnapshot? Parent
{
get
{
if (_parentSnap is not null) return _parentSnap;
try
{
var p = _el.Parent;
return p is null ? null : new FlaUiSnapshot(p);
}
catch
{
return null;
}
}
}
private static T SafeGet<T>(Func<T> f)
{
try { return f(); } catch { return default!; }
}
}

View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0-windows</TargetFramework>
<UseWPF>false</UseWPF>
<UseWindowsForms>false</UseWindowsForms>
<AssemblyName>Recordingtest.Recorder</AssemblyName>
<RootNamespace>Recordingtest.Recorder</RootNamespace>
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FlaUI.Core" Version="4.0.0" />
<PackageReference Include="FlaUI.UIA3" Version="4.0.0" />
<PackageReference Include="YamlDotNet" Version="16.1.3" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,32 @@
using System.Collections.Generic;
namespace Recordingtest.Recorder;
public sealed class Scenario
{
public string Name { get; set; } = string.Empty;
public string Description { get; set; } = string.Empty;
public ScenarioSut Sut { get; set; } = new();
public List<ScenarioStep> Steps { get; set; } = new();
}
public sealed class ScenarioSut
{
public string Exe { get; set; } = "EG-BIM Modeler/EG-BIM Modeler.exe";
public int StartupTimeoutMs { get; set; } = 15000;
}
public sealed class ScenarioStep
{
/// <summary>click | type | drag | hotkey | wait</summary>
public string Kind { get; set; } = "click";
public ScenarioTarget? Target { get; set; }
public string? Value { get; set; }
public string? WaitFor { get; set; }
}
public sealed class ScenarioTarget
{
public string UiaPath { get; set; } = string.Empty;
public double[] Offset { get; set; } = new double[] { 0.5, 0.5 };
}

View File

@@ -0,0 +1,40 @@
using System.IO;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace Recordingtest.Recorder;
public static class ScenarioWriter
{
private static ISerializer BuildSerializer() =>
new SerializerBuilder()
.WithNamingConvention(UnderscoredNamingConvention.Instance)
.ConfigureDefaultValuesHandling(DefaultValuesHandling.Preserve)
.Build();
private static IDeserializer BuildDeserializer() =>
new DeserializerBuilder()
.WithNamingConvention(UnderscoredNamingConvention.Instance)
.IgnoreUnmatchedProperties()
.Build();
public static string Serialize(Scenario scenario)
{
return BuildSerializer().Serialize(scenario);
}
public static Scenario Deserialize(string yaml)
{
return BuildDeserializer().Deserialize<Scenario>(yaml);
}
public static void WriteToFile(Scenario scenario, string path)
{
var dir = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
File.WriteAllText(path, Serialize(scenario));
}
}