#!/usr/bin/env bash
# SUM Parts - status of the unattended training run
set -uo pipefail

RUNROOT="$HOME/sum-parts/runs"

WORKDIR="${1:-}"
if [ -z "$WORKDIR" ]; then
    if [ -f "$RUNROOT/.latest" ]; then
        WORKDIR=$(cat "$RUNROOT/.latest")
    else
        WORKDIR=$(find "$RUNROOT" -maxdepth 1 -mindepth 1 -type d -printf '%T@ %p\n' 2>/dev/null \
                  | sort -rn | head -1 | cut -d' ' -f2-)
    fi
fi

if [ -z "$WORKDIR" ] || [ ! -d "$WORKDIR" ]; then
    echo "no run directory found under $RUNROOT"
    exit 1
fi

echo "=== run: $(basename "$WORKDIR") ==="
[ -f "$WORKDIR/status.txt" ] && cat "$WORKDIR/status.txt"

echo
echo "=== process ==="
if pgrep -f train_watchdog.sh > /dev/null; then
    pgrep -af "train_watchdog.sh"
    pgrep -af "main.py" | head -3
else
    echo "watchdog NOT running"
fi

echo
echo "=== GPU ==="
nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu \
           --format=csv,noheader 2>/dev/null || echo "nvidia-smi unavailable"

echo
echo "=== epochs completed ==="
if [ -f "$WORKDIR/train.log" ]; then
    grep -oE 'Epoch [0-9]+ LR [0-9.]+ train_miou [0-9.]+, val_miou [0-9.]+, best val miou [0-9.]+' \
        "$WORKDIR/train.log" 2>/dev/null | tail -8 || echo "  (no epoch lines yet)"
    echo
    echo "  total epoch lines: $(grep -cE 'Epoch [0-9]+ LR' "$WORKDIR/train.log" 2>/dev/null || echo 0)"
    echo "  log size         : $(du -h "$WORKDIR/train.log" | cut -f1)"
    echo "  last write       : $(date -r "$WORKDIR/train.log" '+%F %T')"
else
    echo "  no train.log yet"
fi

echo
echo "=== watchdog events ==="
[ -f "$WORKDIR/watchdog.log" ] && tail -12 "$WORKDIR/watchdog.log"

echo
echo "=== best so far ==="
grep -E 'Find a better ckpt' "$WORKDIR/train.log" 2>/dev/null | tail -3 || echo "  none yet"