nbrightandClaude Opus 5
609d9a6972
Reproduces the SUM Parts (CVPR 2025) face-labeling benchmark on a single consumer GPU, then applies it to drone-photogrammetry road survey meshes. Verified on RTX 3060 12GB / WSL2 Ubuntu 22.04 / CUDA 11.8 / torch 2.0.1: - CUDA extensions build (pointnet2_batch, pointops, chamfer_dist, emd, subsampling) - PointNet 100 epochs reaches mIoU 17.19, matching the paper's reported 15.1 - OBJ -> PLY conversion round-trips through the model and yields per-point predictions Four upstream source patches, all idempotent, originals preserved: - numpy aliases removed in 1.24 (np.long etc.) and collections ABCs moved in python 3.10 - the blind test split ships label = -1, which crashed ConfusionMatrix - mode=val referenced `epoch` before assignment Documents the traps that cost the most time, including VRAM overflow silently falling back to host RAM on WSL2 (25-100x slowdown, no OOM) and the colour scale mismatch between r/g/b float32 and red/green/blue uint8. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
56 lines
1.8 KiB
Bash
56 lines
1.8 KiB
Bash
#!/usr/bin/env bash
|
|
# SUM Parts - did the run come back to healthy speed?
|
|
#
|
|
# Healthy for pointnet at voxel_max 64000 on this box is ~0.29 s/it (benchmark)
|
|
# and ~103 s/epoch. The degraded state was ~3.1 s/it and ~1100 s/epoch, caused
|
|
# by the caching allocator pool overflowing VRAM into host RAM.
|
|
set -uo pipefail
|
|
|
|
RUNROOT="$HOME/sum-parts/runs"
|
|
D="${1:-$(cat "$RUNROOT/.latest" 2>/dev/null)}"
|
|
LOG="$D/train.log"
|
|
|
|
echo "run: $D"
|
|
[ -f "$LOG" ] || { echo " no train.log yet"; exit 1; }
|
|
|
|
echo
|
|
echo "=== resume confirmation ==="
|
|
grep -aE 'Resume|Successful Loading|start_epoch' "$LOG" | tail -3 || echo " (none found)"
|
|
|
|
echo
|
|
echo "=== current epoch ==="
|
|
grep -aoE 'Train Epoch \[[0-9]+/[0-9]+\]' "$LOG" | tail -1 || echo " not started"
|
|
|
|
echo
|
|
echo "=== recent iteration rates ==="
|
|
grep -aoE '[0-9.]+(s/it|it/s)' "$LOG" | tail -12 | tr '\n' ' '
|
|
echo
|
|
|
|
echo
|
|
echo "=== GPU ==="
|
|
nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu,power.draw --format=csv,noheader
|
|
|
|
echo
|
|
echo "=== verdict ==="
|
|
# tqdm prints either "N.NNs/it" or "N.NNit/s" depending on which side of 1 the
|
|
# rate falls, so take whichever form appeared LAST and normalise to s/it.
|
|
# (Grepping only for 's/it' picks up a stale warm-up reading and misjudges a
|
|
# run that has since recovered.)
|
|
last=$(grep -aoE '[0-9.]+(s/it|it/s)' "$LOG" | tail -1)
|
|
if [ -z "$last" ]; then
|
|
echo " no rate readings yet"
|
|
exit 0
|
|
fi
|
|
|
|
sec=$(awk -v r="$last" 'BEGIN {
|
|
if (r ~ /it\/s$/) { sub(/it\/s$/, "", r); printf "%.4f", (r > 0 ? 1.0 / r : 0) }
|
|
else { sub(/s\/it$/, "", r); printf "%.4f", r }
|
|
}')
|
|
|
|
echo " last reading: $last -> ${sec} s/it"
|
|
awk -v v="$sec" 'BEGIN {
|
|
if (v < 0.8) print " HEALTHY (benchmark is 0.291 s/it)";
|
|
else if (v < 1.5) print " MARGINAL";
|
|
else print " DEGRADED -- allocator likely spilling to host RAM";
|
|
}'
|