nbrightandClaude Opus 5
609d9a6972
Reproduces the SUM Parts (CVPR 2025) face-labeling benchmark on a single consumer GPU, then applies it to drone-photogrammetry road survey meshes. Verified on RTX 3060 12GB / WSL2 Ubuntu 22.04 / CUDA 11.8 / torch 2.0.1: - CUDA extensions build (pointnet2_batch, pointops, chamfer_dist, emd, subsampling) - PointNet 100 epochs reaches mIoU 17.19, matching the paper's reported 15.1 - OBJ -> PLY conversion round-trips through the model and yields per-point predictions Four upstream source patches, all idempotent, originals preserved: - numpy aliases removed in 1.24 (np.long etc.) and collections ABCs moved in python 3.10 - the blind test split ships label = -1, which crashed ConfusionMatrix - mode=val referenced `epoch` before assignment Documents the traps that cost the most time, including VRAM overflow silently falling back to host RAM on WSL2 (25-100x slowdown, no OOM) and the colour scale mismatch between r/g/b float32 and red/green/blue uint8. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
59 lines
1.8 KiB
Bash
59 lines
1.8 KiB
Bash
#!/usr/bin/env bash
|
|
# SUM Parts - status of the unattended training run
|
|
set -uo pipefail
|
|
|
|
RUNROOT="$HOME/sum-parts/runs"
|
|
|
|
WORKDIR="${1:-}"
|
|
if [ -z "$WORKDIR" ]; then
|
|
if [ -f "$RUNROOT/.latest" ]; then
|
|
WORKDIR=$(cat "$RUNROOT/.latest")
|
|
else
|
|
WORKDIR=$(find "$RUNROOT" -maxdepth 1 -mindepth 1 -type d -printf '%T@ %p\n' 2>/dev/null \
|
|
| sort -rn | head -1 | cut -d' ' -f2-)
|
|
fi
|
|
fi
|
|
|
|
if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
|
|
echo "no run directory found under $RUNROOT"
|
|
exit 1
|
|
fi
|
|
|
|
echo "=== run: $(basename "$WORKDIR") ==="
|
|
[ -f "$WORKDIR/status.txt" ] && cat "$WORKDIR/status.txt"
|
|
|
|
echo
|
|
echo "=== process ==="
|
|
if pgrep -f train_watchdog.sh > /dev/null; then
|
|
pgrep -af "train_watchdog.sh"
|
|
pgrep -af "main.py" | head -3
|
|
else
|
|
echo "watchdog NOT running"
|
|
fi
|
|
|
|
echo
|
|
echo "=== GPU ==="
|
|
nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu \
|
|
--format=csv,noheader 2>/dev/null || echo "nvidia-smi unavailable"
|
|
|
|
echo
|
|
echo "=== epochs completed ==="
|
|
if [ -f "$WORKDIR/train.log" ]; then
|
|
grep -oE 'Epoch [0-9]+ LR [0-9.]+ train_miou [0-9.]+, val_miou [0-9.]+, best val miou [0-9.]+' \
|
|
"$WORKDIR/train.log" 2>/dev/null | tail -8 || echo " (no epoch lines yet)"
|
|
echo
|
|
echo " total epoch lines: $(grep -cE 'Epoch [0-9]+ LR' "$WORKDIR/train.log" 2>/dev/null || echo 0)"
|
|
echo " log size : $(du -h "$WORKDIR/train.log" | cut -f1)"
|
|
echo " last write : $(date -r "$WORKDIR/train.log" '+%F %T')"
|
|
else
|
|
echo " no train.log yet"
|
|
fi
|
|
|
|
echo
|
|
echo "=== watchdog events ==="
|
|
[ -f "$WORKDIR/watchdog.log" ] && tail -12 "$WORKDIR/watchdog.log"
|
|
|
|
echo
|
|
echo "=== best so far ==="
|
|
grep -E 'Find a better ckpt' "$WORKDIR/train.log" 2>/dev/null | tail -3 || echo " none yet"
|