#!/usr/bin/env bash
# SUM Parts - wait for training to finish, then run every evaluation we care about
#
# Runs unattended so nobody has to sit watching for the last epoch:
#   1. wait for the trainer to exit
#   2. coarse evaluation on SUM val   -> building / vegetation / vehicle / ground
#   3. inference on the Seosan tile   -> does our own data work with this model
set -uo pipefail

SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUT="$HOME/sum-parts/runs/after_train"
LOG="$OUT/after_train.log"

mkdir -p "$OUT"
exec > >(tee -a "$LOG") 2>&1

say() { echo "[$(date '+%F %T')] $*"; }

say "waiting for training to finish"
waited=0
while pgrep -f 'main.py --cfg' > /dev/null; do
    sleep 30
    waited=$((waited + 30))
    [ $((waited % 300)) -eq 0 ] && say "  still running (${waited}s)"
    [ "$waited" -gt 7200 ] && { say "  timed out after 2h"; break; }
done
say "trainer no longer running"

# also wait out the watchdog, so it does not relaunch under us
pkill -f train_watchdog.sh 2>/dev/null && say "stopped the watchdog"
sleep 5

echo
say "=== best checkpoint ==="
CKPT=$(find "$HOME/sum-parts/semantic_segmentation/PointNeXt_bundle/examples/segmentation/log/sumv2_triangle" \
       -name '*pointvector*_ckpt_best.pth' -printf '%T@ %p\n' 2>/dev/null \
       | sort -rn | head -1 | cut -d' ' -f2-)
say "$CKPT"
grep -ahE 'Best ckpt @E' "$HOME/sum-parts/runs/pointvector-xl_"*/train.log 2>/dev/null | tail -2

echo
say "=== 1/2  coarse evaluation on SUM val (4 classes) ==="
TEST_VOXEL_MAX=24000 bash "$SCRIPTS/eval_coarse.sh" || say "eval_coarse returned non-zero"

echo
say "=== 2/2  inference on the Seosan tile ==="
bash "$SCRIPTS/poc_infer.sh" || say "poc_infer returned non-zero"
bash "$SCRIPTS/poc_check_pred.sh" || say "poc_check_pred returned non-zero"

echo
say "AFTER TRAIN DONE -- log: $LOG"