kimminsung/sum-parts-test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record results so far and fix the memory blowup in the palette decode

Browse Source 
Adds STATUS.md as the handoff document: benchmark numbers, the bare-earth
metrics that actually matter for this project, the Korean-data domain gap that
retraining will not fix, and what to do on the 24 GB machine.

The memory problem was in how a prediction's colours were turned back into
class indices. Every consumer built an (N, 13, 3) float64 temporary:

    d = ((rgb[:, None, :] - COLOR_MAP[None, :, :]) ** 2).sum(axis=2)

That is ~250 MB of intermediates per 800k-point tile, several live at once, and
a full 4.7M-point block pushes it into gigabytes. main.py writes exact palette
entries, so an exact hash lookup resolves nearly every point with no large
temporary; only leftovers fall back to a chunked distance search. Peak RSS on a
470k-point tile drops to 61 MB. Extracted to sumparts_palette.py and shared by
coarse_eval.py and split_by_class.py.

Also from this round:

- patch_cm_mutation.sh: ConfusionMatrix.update() rewrote the caller's pred
  tensor in place, folding every ignore_index point into class num_classes-1.
  test() saves its visualization from that same tensor afterwards, so an
  unlabelled tile came out 100% wall and the model looked degenerate when it
  was not.
- patch_class_mask.sh: SUMPARTS_MASK_CLASSES drops known-absent classes from
  the argmax. Measured on Seosan and it does not help - the runner-up for
  "water" is "wall", not "terrain" - but the experiment is worth keeping.
- split_by_class.py now writes .ply alongside .obj. A vertex-only OBJ has zero
  faces and most viewers render nothing, which is why the first export looked
  broken.
- verify_outputs.sh reads exported files back with a parser, so "here are your
  files" can be checked rather than asserted.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
nbright
2026-08-24 09:07:31 +09:00
co-authored by Claude Opus 5
parent 3fdd7ab3f0
commit f39b093106
19 changed files with 1144 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/coarse_eval.py
+64 -29
View File
@@ -37,7 +37,9 @@ import sys
from pathlib import Path
import numpy as np
from plyfile import PlyData
sys.path.insert(0, str(Path(__file__).resolve().parent))
from sumparts_palette import read_point_classes # noqa: E402
FINE = ['unclassified', 'terrain', 'high_vegetation', 'facade_surface',
'water', 'car', 'boat', 'roof_surface', 'chimney', 'dormer',
@@ -73,28 +75,14 @@ COLOR_MAP = np.array([
def load_labels(path: Path) -> np.ndarray:
"""Fine class per point: from a `label` field, else decoded from colour."""
v = PlyData.read(str(path))["vertex"]
props = [p.name for p in v.properties]
"""Fine class per point.
if "label" in props:
lab = np.asarray(v["label"]).astype(np.int64).ravel()
# a prediction ply may carry a placeholder label; fall through if so
if lab.max() >= 0 and not (lab == lab[0]).all():
return lab
if lab.max() >= 0 and lab[0] >= 0:
return lab
rgb_set = next((s for s in (("red", "green", "blue"), ("r", "g", "b"))
if all(c in props for c in s)), None)
if rgb_set is None:
raise SystemExit(f"{path}: no label field and no colour to decode")
rgb = np.stack([np.asarray(v[c], dtype=np.float64) for c in rgb_set], axis=1)
if rgb.max() <= 1.0:
rgb *= 255.0
d = ((rgb[:, None, :] - COLOR_MAP[None, :, :]) ** 2).sum(axis=2)
return d.argmin(axis=1).astype(np.int64)
Delegates to sumparts_palette, whose colour decode avoids building an
(N, 13, 3) temporary -- that pattern needs ~250 MB of intermediates per
800k-point tile and takes the machine down on a full block.
"""
_, cls = read_point_classes(path)
return cls.astype(np.int64)
def confusion(pred: np.ndarray, true: np.ndarray, n: int) -> np.ndarray:
@@ -104,19 +92,24 @@ def confusion(pred: np.ndarray, true: np.ndarray, n: int) -> np.ndarray:
def report(cm: np.ndarray, names: list[str], skip: set[int]) -> None:
tp = np.diag(cm).astype(np.float64)
actual = cm.sum(axis=1).astype(np.float64)
predicted = cm.sum(axis=0).astype(np.float64)
actual = cm.sum(axis=1).astype(np.float64) # ground truth per class
predicted = cm.sum(axis=0).astype(np.float64) # predictions per class
union = actual + predicted - tp
print(f" {'class':<12} {'IoU':>7} {'recall':>8} {'points':>12}")
# precision matters as much as recall here, and for some jobs more.
# Stripping structures off a terrain model is precision-first: a hole where
# ground was missed can be interpolated, but a wall left sitting in the
# surface is a fake landform.
print(f" {'class':<12} {'IoU':>7} {'prec':>8} {'recall':>8} {'points':>12}")
ious = []
for i, name in enumerate(names):
if i in skip:
continue
iou = 100.0 * tp[i] / union[i] if union[i] > 0 else 0.0
rec = 100.0 * tp[i] / actual[i] if actual[i] > 0 else 0.0
iou = 100.0 * tp[i] / union[i] if union[i] > 0 else 0.0
prec = 100.0 * tp[i] / predicted[i] if predicted[i] > 0 else 0.0
rec = 100.0 * tp[i] / actual[i] if actual[i] > 0 else 0.0
ious.append(iou)
print(f" {name:<12} {iou:>6.2f}% {rec:>7.2f}% {int(actual[i]):>12,}")
print(f" {name:<12} {iou:>6.2f}% {prec:>7.2f}% {rec:>7.2f}% {int(actual[i]):>12,}")
scored = [i for i in range(len(names)) if i not in skip]
oa_tp = tp[scored].sum()
@@ -126,6 +119,44 @@ def report(cm: np.ndarray, names: list[str], skip: set[int]) -> None:
print(f" OA : {100.0 * oa_tp / oa_n if oa_n else 0:.2f}%")
def report_bare_earth(cm_fine: np.ndarray) -> None:
"""Ground vs everything else, the way a terrain model is actually judged.
Reported separately from the 4-class view because the failure modes are not
symmetric. Ground missed -> a hole, which interpolation fills. Non-ground
kept -> a retaining wall or a roof baked into the terrain, which nothing
downstream can tell from a real landform.
"""
ground = {1} # terrain
n = cm_fine.shape[0]
rest = [i for i in range(n) if i not in ground and i != 0]
tp = cm_fine[1, 1]
fp = cm_fine[np.ix_(rest, [1])].sum() # non-ground predicted as ground
fn = cm_fine[np.ix_([1], rest)].sum() # ground predicted as something else
prec = 100.0 * tp / (tp + fp) if tp + fp else 0.0
rec = 100.0 * tp / (tp + fn) if tp + fn else 0.0
iou = 100.0 * tp / (tp + fp + fn) if tp + fp + fn else 0.0
f1 = 2 * prec * rec / (prec + rec) if prec + rec else 0.0
print(f" ground kept correctly {int(tp):>12,}")
print(f" non-ground leaked in {int(fp):>12,} <- contaminates the surface")
print(f" ground missed {int(fn):>12,} <- holes, interpolable")
print()
print(f" precision : {prec:6.2f}% (of what we call ground, how much is)")
print(f" recall : {rec:6.2f}% (of real ground, how much we caught)")
print(f" IoU : {iou:6.2f}%")
print(f" F1 : {f1:6.2f}%")
print()
if fp:
print(" where the contamination comes from:")
leaks = [(int(cm_fine[i, 1]), FINE[i]) for i in rest if cm_fine[i, 1] > 0]
for cnt, name in sorted(leaks, reverse=True)[:8]:
print(f" {name:<20} {cnt:>10,} {100.0*cnt/fp:>6.2f}% of leak")
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -171,9 +202,13 @@ def main() -> None:
report(cm_fine, FINE, skip={0})
print()
print("=== coarse (what this project needs) ===")
print("=== coarse (building / vegetation / vehicle / ground) ===")
report(cm_coarse, COARSE, skip={0})
print()
print("=== bare earth (ground vs everything else) ===")
report_bare_earth(cm_fine)
if __name__ == "__main__":
main()