#!/usr/bin/env bash # SUM Parts - compare early vs recent per-iteration speed in a running train log # # Separates "each iteration got slower" from "something between epochs got # slower" (validation, checkpointing, dataloader restart). set -uo pipefail RUNROOT="$HOME/sum-parts/runs" WORKDIR="${1:-$(cat "$RUNROOT/.latest" 2>/dev/null)}" LOG="$WORKDIR/train.log" [ -f "$LOG" ] || { echo "no train.log at ${WORKDIR:-}"; exit 1; } echo "log: $LOG" echo echo "=== earliest 15 rate readings ===" grep -oE '[0-9.]+(s/it|it/s)' "$LOG" | head -15 | tr '\n' ' ' echo echo echo "=== most recent 15 rate readings ===" grep -oE '[0-9.]+(s/it|it/s)' "$LOG" | tail -15 | tr '\n' ' ' echo echo echo "=== Train Epoch progress lines (first / last) ===" grep -oE 'Train Epoch \[[0-9]+/[0-9]+\] Loss [0-9.]+ Acc [0-9.]+' "$LOG" | head -3 echo " ..." grep -oE 'Train Epoch \[[0-9]+/[0-9]+\] Loss [0-9.]+ Acc [0-9.]+' "$LOG" | tail -3 echo echo "=== log growth ===" echo " size : $(du -h "$LOG" | cut -f1)" echo " last write: $(date -r "$LOG" '+%F %T') (now $(date '+%F %T'))"