#!/usr/bin/env bash
# SUM Parts - start an unattended training run and return immediately
#
# setsid detaches the watchdog into its own session, so it keeps running after
# the launching shell, the Claude Code session, or the Windows terminal goes
# away. It does NOT survive `wsl --shutdown` or a Windows reboot -- nothing run
# inside WSL does.
#
# Usage:
#   bash launch_overnight.sh                 # pointnet, 100 epochs
#   EPOCHS=100 bash launch_overnight.sh pointnet
#
# Then check on it any time with:
#   bash check_training.sh
set -euo pipefail

SCRIPTS="/mnt/d/MYCLAUDE_PROJECT/sum-parts-test/scripts"
CFG="${1:-pointnet}"
STAMP=$(date +%Y%m%d-%H%M%S)
WORKDIR="$HOME/sum-parts/runs/${CFG}_${STAMP}"

mkdir -p "$WORKDIR"
echo "$WORKDIR" > "$HOME/sum-parts/runs/.latest"

# already running?
if pgrep -f "train_watchdog.sh" > /dev/null; then
    echo "a watchdog is already running:"
    pgrep -af "train_watchdog.sh"
    echo
    echo "stop it first with: bash $SCRIPTS/stop_training.sh"
    exit 1
fi

STAMP="$STAMP" EPOCHS="${EPOCHS:-100}" VAL_FREQ="${VAL_FREQ:-5}" \
MAX_RETRIES="${MAX_RETRIES:-8}" CFG_VOXEL_MAX="${CFG_VOXEL_MAX:-}" \
RESUME_CKPT="${RESUME_CKPT:-}" \
    setsid nohup bash "$SCRIPTS/train_watchdog.sh" "$CFG" \
    > "$WORKDIR/nohup.out" 2>&1 < /dev/null &

sleep 3

echo "launched: $CFG"
echo "workdir : $WORKDIR"
echo
pgrep -af "train_watchdog.sh" || echo "WARNING: watchdog not visible in process list"
echo
echo "check progress:  bash $SCRIPTS/check_training.sh"
echo "stop         :  bash $SCRIPTS/stop_training.sh"