#!/usr/bin/env bash
# SUM Parts - stop the unattended training run
#
# Kills the watchdog first so it does not treat the dying trainer as a crash
# and immediately resume it.
set -uo pipefail

if ! pgrep -f train_watchdog.sh > /dev/null && ! pgrep -f "main.py" > /dev/null; then
    echo "nothing running"
    exit 0
fi

echo "stopping watchdog:"
pgrep -af train_watchdog.sh || true
pkill -f train_watchdog.sh || true
sleep 2

echo "stopping trainer:"
pgrep -af "main.py" || true
pkill -f "examples/segmentation/main.py" || true
sleep 3

if pgrep -f "main.py" > /dev/null; then
    echo "still alive, sending SIGKILL"
    pkill -9 -f "examples/segmentation/main.py" || true
fi

echo
echo "remaining:"
pgrep -af "train_watchdog.sh|main.py" || echo "  clean"
echo
echo "checkpoints are kept -- relaunch resumes from the latest one."