#!/usr/bin/env bash
# SUM Parts - per-epoch wall time, to locate where a run slowed down
set -uo pipefail

RUNROOT="$HOME/sum-parts/runs"
WORKDIR="${1:-$(cat "$RUNROOT/.latest" 2>/dev/null)}"
LOG="$WORKDIR/train.log"

[ -f "$LOG" ] || { echo "no train.log at $WORKDIR"; exit 1; }

grep -E 'Epoch [0-9]+ LR' "$LOG" | awk '
{
    # [08/20 20:28:44 SUMV2_Triangle]: Epoch 10 LR ...
    #  $1      $2         $3           $4    $5
    split($2, t, ":")
    sec = t[1]*3600 + t[2]*60 + t[3]
    for (i = 1; i <= NF; i++) if ($i == "Epoch") { ep = $(i+1); break }
    if (prev_sec != "") {
        d = sec - prev_sec
        if (d < 0) d += 86400          # crossed midnight
        printf "epoch %3d  %s  +%6.1f s", ep, t[1]":"t[2]":"t[3], d
        if (d > slow_thresh) printf "   <-- SLOW"
        printf "\n"
    } else {
        printf "epoch %3d  %s\n", ep, t[1]":"t[2]":"t[3]
    }
    prev_sec = sec
}
BEGIN { slow_thresh = 300 }
'

echo
echo "--- summary ---"
grep -E 'Epoch [0-9]+ LR' "$LOG" | tail -1
echo "epochs done: $(grep -cE 'Epoch [0-9]+ LR' "$LOG")"