빌드 경고 해결 및 데이터 검증 진행중.

This commit is contained in:
Lectom C Han
2026-01-08 18:09:21 +09:00
parent 2d84a26053
commit 7b93467c6e
6 changed files with 206 additions and 81 deletions

View File

@@ -9,6 +9,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="MiniExcel" Version="1.42.0" /> <PackageReference Include="MiniExcel" Version="1.42.0" />
<PackageReference Include="StackExchange.Redis" Version="2.10.1" /> <PackageReference Include="StackExchange.Redis" Version="2.10.1" />
<PackageReference Include="DocumentFormat.OpenXml" Version="3.0.2" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@@ -3,6 +3,8 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using MiniExcelLibs; using MiniExcelLibs;
using DocumentFormat.OpenXml.Packaging;
using DocumentFormat.OpenXml.Spreadsheet;
namespace ExcelKv.Core; namespace ExcelKv.Core;
@@ -72,38 +74,46 @@ public class ExcelLoader
Console.WriteLine($"[Core.Loader] Processing {filePath} Sheet: {sheetName}..."); Console.WriteLine($"[Core.Loader] Processing {filePath} Sheet: {sheetName}...");
var rows = MiniExcel.Query(filePath, sheetName: sheetName, useHeaderRow: false).ToList(); var rows = MiniExcel.Query(filePath, sheetName: sheetName, useHeaderRow: false).ToList();
var mergedRanges = GetMergedRanges(filePath, sheetName);
// Validation // Validation
int dataStartRow = config.TopHeaderStartRow + config.TopHeaderDepth; int dataStartRow = config.TopHeaderStartRow + config.TopHeaderDepth;
if (rows.Count <= dataStartRow) return 0; if (rows.Count <= dataStartRow) return 0;
// 1. Analyze Top Headers (Data Columns) int limitRow = config.DataEndRow.HasValue ? Math.Min(rows.Count, config.DataEndRow.Value + 1) : rows.Count;
var topHeaderRows = new List<string[]>();
for (int i = config.TopHeaderStartRow; i < dataStartRow; i++) // Flatten all rows up to limit for consistent indexing
var flattenedRows = new List<string[]>();
int globalMaxCol = 0;
for (int i = 0; i < limitRow; i++)
{ {
topHeaderRows.Add(FlattenDictionaryRow((IDictionary<string, object>)rows[i])); var flat = FlattenDictionaryRow((IDictionary<string, object>)rows[i]);
flattenedRows.Add(flat);
if (flat.Length > globalMaxCol) globalMaxCol = flat.Length;
} }
// Calculate Global Max Col from Data Rows to ensure we don't truncate data int maxMergeCol = mergedRanges.Any() ? mergedRanges.Max(m => m.EndCol + 1) : 0;
// Optimization: check a sample or assume logical limit. For correctness, check valid rows. int normalizedMaxCol = Math.Max(globalMaxCol, maxMergeCol);
int globalMaxCol = 0;
int limitRow = config.DataEndRow.HasValue ? Math.Min(rows.Count, config.DataEndRow.Value + 1) : rows.Count; ApplyMergedValues(flattenedRows, mergedRanges, normalizedMaxCol);
for(int i = dataStartRow; i < limitRow; i++)
{ // 1. Analyze Top Headers (Data Columns)
var d = (IDictionary<string, object>)rows[i]; var topHeaderRows = flattenedRows
if(d.Count > globalMaxCol) globalMaxCol = d.Count; // Approximate .Skip(config.TopHeaderStartRow)
// FlattenDictionaryRow is cleaner but expensive to call just for count. .Take(config.TopHeaderDepth)
// d.Keys.Count is effectively the column count for that row. .ToList();
}
int headerMaxCol = topHeaderRows.Any() ? Math.Max(normalizedMaxCol, topHeaderRows.Max(r => r.Length)) : normalizedMaxCol;
var filledHeaders = topHeaderRows;
// Flatten Data Headers (Right Side) // Flatten Data Headers (Right Side)
var topAxisKeys = FlattenTopHeaders(topHeaderRows, config.LeftHeaderStartCol + config.LeftHeaderWidth, globalMaxCol); var topAxisKeys = FlattenTopHeaders(filledHeaders, config.LeftHeaderStartCol + config.LeftHeaderWidth, headerMaxCol);
// ... (Left Axis Header logic omitted for brevity, unchanged) ... // ... (Left Axis Header logic omitted for brevity, unchanged) ...
// ** New: Extract Left Axis Headers (Corner Region) ** // ** New: Extract Left Axis Headers (Corner Region) **
// These are the headers *above* the Left Key columns. // These are the headers *above* the Left Key columns.
var leftAxisHeaders = new List<string>(); var leftAxisHeaders = new List<string>();
var bottomHeaderRow = FlattenDictionaryRow((IDictionary<string, object>)rows[dataStartRow - 1]); var bottomHeaderRow = flattenedRows[dataStartRow - 1];
for (int c = 0; c < config.LeftHeaderWidth; c++) for (int c = 0; c < config.LeftHeaderWidth; c++)
{ {
@@ -122,8 +132,7 @@ public class ExcelLoader
for (int i = dataStartRow; i < limitRow; i++) for (int i = dataStartRow; i < limitRow; i++)
{ {
var rowDict = (IDictionary<string, object>)rows[i]; var rowVals = flattenedRows[i];
var rowVals = FlattenDictionaryRow(rowDict);
// ... (Left Key Logic Unchanged) ... // ... (Left Key Logic Unchanged) ...
var currentLeftParts = new List<string>(); var currentLeftParts = new List<string>();
@@ -140,7 +149,7 @@ public class ExcelLoader
} }
} }
if (!rowHasContent) continue; if (!rowHasContent) continue;
string leftKey = string.Join("----", currentLeftParts); string leftKey = string.Join("__", currentLeftParts);
// B. Map Values // B. Map Values
int limitCol = config.DataEndCol.HasValue ? Math.Min(rowVals.Length, config.DataEndCol.Value + 1) : rowVals.Length; int limitCol = config.DataEndCol.HasValue ? Math.Min(rowVals.Length, config.DataEndCol.Value + 1) : rowVals.Length;
@@ -168,7 +177,7 @@ public class ExcelLoader
// Update: User said "소수점 6자리까지 표현하는걸로 통일". Could mean truncate or round. Safe bet is Round. // Update: User said "소수점 6자리까지 표현하는걸로 통일". Could mean truncate or round. Safe bet is Round.
} }
string fullKey = $"{sheetName}:{leftKey}*{topKey}"; string fullKey = $"{sheetName}:{leftKey}----{topKey}";
await storage.SetAsync(fullKey, val, i, c); await storage.SetAsync(fullKey, val, i, c);
@@ -193,11 +202,43 @@ public class ExcelLoader
for (int i = 0; i < sortedKeys.Count; i++) for (int i = 0; i < sortedKeys.Count; i++)
{ {
var val = rowDict[sortedKeys[i]]; var val = rowDict[sortedKeys[i]];
result[i] = val?.ToString() ?? ""; result[i] = ExtractCellValue(val);
} }
return result; return result;
} }
// Normalize cell value, preferring cached result for formulas.
private static string ExtractCellValue(object val)
{
if (val == null) return "";
string ToStr(object? o) => o?.ToString() ?? "";
var type = val.GetType();
var typeName = type.FullName ?? "";
// MiniExcel may return an internal ExcelFormula type; try to read cached value.
if (typeName.Contains("Formula", StringComparison.OrdinalIgnoreCase))
{
var cached = type.GetProperty("Value")?.GetValue(val)
?? type.GetProperty("CachedValue")?.GetValue(val)
?? type.GetProperty("Result")?.GetValue(val);
if (cached != null) return ToStr(cached);
}
var s = ToStr(val);
if (s.StartsWith("="))
{
var cached = type.GetProperty("CachedValue")?.GetValue(val)
?? type.GetProperty("Value")?.GetValue(val)
?? type.GetProperty("Result")?.GetValue(val);
if (cached != null) return ToStr(cached);
// If no cached value, avoid persisting the formula string.
return "";
}
return s;
}
private static List<string> FlattenTopHeaders(List<string[]> headerRows, int startCol, int globalMaxCol) private static List<string> FlattenTopHeaders(List<string[]> headerRows, int startCol, int globalMaxCol)
{ {
if (headerRows.Count == 0) return new List<string>(); if (headerRows.Count == 0) return new List<string>();
@@ -206,8 +247,6 @@ public class ExcelLoader
var flatHeaders = new List<string>(); var flatHeaders = new List<string>();
// Iterate Columns
var lastValues = new string[headerRows.Count];
for (int c = startCol; c < maxCol; c++) for (int c = startCol; c < maxCol; c++)
{ {
var parts = new List<string>(); var parts = new List<string>();
@@ -215,19 +254,102 @@ public class ExcelLoader
for (int r = 0; r < headerRows.Count; r++) for (int r = 0; r < headerRows.Count; r++)
{ {
string val = (c < headerRows[r].Length) ? headerRows[r][c] : ""; string val = (c < headerRows[r].Length) ? headerRows[r][c] : "";
// Horizontal Forward Fill (Re-enabled for Merged Headers)
if (string.IsNullOrWhiteSpace(val)) val = lastValues[r];
else lastValues[r] = val;
if (!string.IsNullOrWhiteSpace(val)) parts.Add(val); if (!string.IsNullOrWhiteSpace(val)) parts.Add(val);
} }
// If empty, use "Col_Index" fallback or keep empty? // If empty, use "Col_Index" fallback or keep empty?
// User schema usually requires keys. If empty, it's skipped in mapping. // User schema usually requires keys. If empty, it's skipped in mapping.
// Let's keep it empty, but if data exists, it won't map unless we have a key. // Let's keep it empty, but if data exists, it won't map unless we have a key.
// If parts is empty, let's leave valid empty string so mapping can decide. // If parts is empty, let's leave valid empty string so mapping can decide.
flatHeaders.Add(string.Join("----", parts)); flatHeaders.Add(string.Join("__", parts));
} }
return flatHeaders; return flatHeaders;
} }
private record MergeRange(int StartRow, int EndRow, int StartCol, int EndCol);
private static List<MergeRange> GetMergedRanges(string filePath, string sheetName)
{
var result = new List<MergeRange>();
try
{
using var doc = SpreadsheetDocument.Open(filePath, false);
var sheet = doc.WorkbookPart?.Workbook.Descendants<Sheet>().FirstOrDefault(s => s.Name == sheetName);
if (sheet == null) return result;
var wsPart = doc.WorkbookPart?.GetPartById(sheet.Id!) as WorksheetPart;
var mergeCells = wsPart?.Worksheet.Elements<MergeCells>().FirstOrDefault();
if (mergeCells == null) return result;
foreach (var mc in mergeCells.Elements<MergeCell>())
{
var (sr, er, sc, ec) = ParseRange(mc.Reference?.Value ?? "");
result.Add(new MergeRange(sr, er, sc, ec));
}
}
catch
{
// If merge metadata cannot be read, fall back to no-op.
}
return result;
}
private static (int startRow, int endRow, int startCol, int endCol) ParseRange(string reference)
{
if (string.IsNullOrWhiteSpace(reference)) return (0, 0, 0, 0);
if (!reference.Contains(":"))
{
var (r, c) = ParseCell(reference);
return (r, r, c, c);
}
var parts = reference.Split(':');
var (r1, c1) = ParseCell(parts[0]);
var (r2, c2) = ParseCell(parts[1]);
return (Math.Min(r1, r2), Math.Max(r1, r2), Math.Min(c1, c2), Math.Max(c1, c2));
}
private static (int row, int col) ParseCell(string cellRef)
{
int row = 0, col = 0;
int i = 0;
while (i < cellRef.Length && char.IsLetter(cellRef[i]))
{
col = col * 26 + (char.ToUpperInvariant(cellRef[i]) - 'A' + 1);
i++;
}
string rowStr = cellRef[i..];
int.TryParse(rowStr, out row);
// Convert to zero-based indices
return (Math.Max(0, row - 1), Math.Max(0, col - 1));
}
private static void ApplyMergedValues(List<string[]> rows, List<MergeRange> merges, int maxCols)
{
NormalizeRows(rows, maxCols);
foreach (var merge in merges)
{
if (merge.StartRow >= rows.Count) continue;
string anchor = (merge.StartCol < rows[merge.StartRow].Length) ? rows[merge.StartRow][merge.StartCol] : "";
for (int r = merge.StartRow; r <= merge.EndRow && r < rows.Count; r++)
{
var row = rows[r];
for (int c = merge.StartCol; c <= merge.EndCol && c < row.Length; c++)
{
row[c] = anchor;
}
}
}
}
private static void NormalizeRows(List<string[]> rows, int maxCols)
{
for (int i = 0; i < rows.Count; i++)
{
if (rows[i].Length < maxCols)
{
var padded = new string[maxCols];
Array.Copy(rows[i], padded, rows[i].Length);
rows[i] = padded;
}
}
}
} }

View File

@@ -6,15 +6,6 @@ using System.Text.RegularExpressions;
using System.Threading.Tasks; using System.Threading.Tasks;
using Garnet; using Garnet;
using MiniExcelLibs; using MiniExcelLibs;
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Garnet;
using MiniExcelLibs;
using ExcelKv.Core; // Use Shared Library using ExcelKv.Core; // Use Shared Library
namespace ExcelKvPoC; namespace ExcelKvPoC;
@@ -41,6 +32,13 @@ public class GarnetClientAdapter : IStorageWrapper, IDisposable
await Task.CompletedTask; // Fire and forget in batch context mainly await Task.CompletedTask; // Fire and forget in batch context mainly
} }
public async Task SetAsync(string key, string value, int row, int col)
{
// For this adapter we store only key/value; row/col metadata can be added later if needed.
_tasks.Add(_batch.StringSetAsync(key, value));
await Task.CompletedTask;
}
public async Task IncrementAsync(string key, double value) public async Task IncrementAsync(string key, double value)
{ {
_tasks.Add(_batch.StringIncrementAsync(key, value)); _tasks.Add(_batch.StringIncrementAsync(key, value));

View File

@@ -9,7 +9,7 @@
- **Embedded Garnet**: Redis 호환 고성능 인메모리 스토어 내장. - **Embedded Garnet**: Redis 호환 고성능 인메모리 스토어 내장.
- **2D Flattening**: - **2D Flattening**:
- 병합된 Cross-Tab 엑셀 헤더를 논리적 Key로 자동 평탄화. - 병합된 Cross-Tab 엑셀 헤더를 논리적 Key로 자동 평탄화.
- Key Format: `{Sheet}:{LeftAxis}*{TopAxis}` (예: `Station.No1*Concrete.TypeA`) - Key Format: `{Sheet}:{LeftAxis}----{TopAxis}` (예: `Station__No1----Concrete__.__TypeA`)
### 2. Smart Schema Management ### 2. Smart Schema Management
- **Interactive Region**: 헤더와 데이터 영역을 동적으로 지정 가능 (`RegionConfig`). - **Interactive Region**: 헤더와 데이터 영역을 동적으로 지정 가능 (`RegionConfig`).

View File

@@ -9,10 +9,10 @@ namespace SchemaEditor.Services;
public class GarnetClientService : IStorageWrapper, IDisposable public class GarnetClientService : IStorageWrapper, IDisposable
{ {
private ConnectionMultiplexer _redis; private ConnectionMultiplexer? _redis;
private IDatabase _db; private IDatabase? _db;
public bool IsConnected => _redis != null && _redis.IsConnected; public bool IsConnected => _redis?.IsConnected == true;
public void Connect(string connectionString = "localhost:3187") public void Connect(string connectionString = "localhost:3187")
{ {
@@ -26,7 +26,7 @@ public class GarnetClientService : IStorageWrapper, IDisposable
public async Task SetAsync(string key, string value) public async Task SetAsync(string key, string value)
{ {
if (_db == null) Connect(); if (_db == null) Connect();
await _db.StringSetAsync(key, value); if (_db != null) await _db.StringSetAsync(key, value);
} }
public async Task SetAsync(string key, string value, int row, int col) public async Task SetAsync(string key, string value, int row, int col)
@@ -35,19 +35,21 @@ public class GarnetClientService : IStorageWrapper, IDisposable
// Traceability metadata (row, col) could be stored in a hash or side key if needed. // Traceability metadata (row, col) could be stored in a hash or side key if needed.
// For now, we just proceed with standard storage. // For now, we just proceed with standard storage.
if (_db == null) Connect(); if (_db == null) Connect();
await _db.StringSetAsync(key, value); if (_db != null) await _db.StringSetAsync(key, value);
} }
public async Task IncrementAsync(string key, double value) public async Task IncrementAsync(string key, double value)
{ {
if (_db == null) Connect(); if (_db == null) Connect();
await _db.StringIncrementAsync(key, value); if (_db != null) await _db.StringIncrementAsync(key, value);
} }
// For Data Explorer // For Data Explorer
public async Task<List<string>> SearchKeysAsync(string pattern) public async Task<List<string>> SearchKeysAsync(string pattern)
{ {
if (_db == null) Connect(); if (_db == null) Connect();
if (_redis == null || _db == null) return await Task.FromResult(new List<string>());
var server = _redis.GetServer(_redis.GetEndPoints().First()); var server = _redis.GetServer(_redis.GetEndPoints().First());
// Use Keys for simplicity in Schema Editor (low traffic) // Use Keys for simplicity in Schema Editor (low traffic)
// In high production, use SCAN // In high production, use SCAN
@@ -58,7 +60,9 @@ public class GarnetClientService : IStorageWrapper, IDisposable
public async Task<string> GetValueAsync(string key) public async Task<string> GetValueAsync(string key)
{ {
if (_db == null) Connect(); if (_db == null) Connect();
return await _db.StringGetAsync(key); if (_db == null) return string.Empty;
var val = await _db.StringGetAsync(key);
return val.HasValue ? val.ToString() : string.Empty;
} }
public void Dispose() public void Dispose()

View File

@@ -17,7 +17,7 @@ public class GarnetHost : IHostedService, IDisposable
var serverArgs = new string[] { "--port", "3187" }; // Changed to 3187 var serverArgs = new string[] { "--port", "3187" }; // Changed to 3187
_server = new GarnetServer(serverArgs); _server = new GarnetServer(serverArgs);
_server.Start(); _server.Start();
Console.WriteLine("[GarnetHost] Server started on port 3278"); Console.WriteLine("[GarnetHost] Server started on port 3187");
} }
catch (Exception ex) catch (Exception ex)
{ {