Split the unattended run into a GPU-free phase and a GPU phase
The target machine's card is busy with someone else's job, so a single end-to-end script stalls on work that does not actually need a GPU. Compiling the CUDA extensions needs nvcc, not a device, and downloading 18 GB of data needs neither. Those are the slow parts (~50 min + ~30 min), so phase A now runs entirely without the card: run_setup.sh bootstrap, conda, extensions, patches, data no GPU run_train.sh voxel_max measurement, training, evaluation GPU run_setup reports the GPU but never fails on it, and verify_env.py gained SKIP_CUDA_CHECK so import coverage still runs when no device is visible. TORCH_CUDA_ARCH_LIST is stated rather than probed, since the card may be unavailable at build time. run_train waits for the GPU instead of failing when it is busy: it polls until enough VRAM frees up (12h default), so it can be queued ahead of time. Past the deadline it proceeds anyway and lets the measured voxel_max adapt to whatever is actually free. keepalive.sh now takes the phase to supervise. Replaces run_all.sh and RUN.md with SETUP.md and TRAIN.md. Adds selfcheck.sh, which syntax-checks every script and flags CRLF endings - a shell script with either fails at its first line, which for an unattended weekend run means losing the weekend. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
nbright
+46
-28
@@ -1,31 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# SUM Parts - keep run_all.sh alive across anything that kills it
|
||||
# SUM Parts - keep a phase script alive across anything that kills it
|
||||
#
|
||||
# run_all.sh already retries individual phases, and train_watchdog resumes
|
||||
# training from its checkpoint. This is the layer above both: it restarts
|
||||
# run_all itself if the whole process disappears -- a WSL VM restart, an OOM
|
||||
# kill, a stray pkill.
|
||||
# The phase scripts already retry their own steps, and train_watchdog resumes
|
||||
# training from its checkpoint. This is the layer above both: it relaunches the
|
||||
# phase if the whole process disappears - a WSL VM restart, an OOM kill, a
|
||||
# stray pkill.
|
||||
#
|
||||
# That is safe because every phase is idempotent. A restart re-checks what is
|
||||
# already done (conda env, patches, downloaded archives, training checkpoint)
|
||||
# and continues from there rather than redoing it.
|
||||
# Safe because every phase is idempotent. A restart re-checks what is already
|
||||
# done (conda env, patches, downloaded archives, training checkpoint) and
|
||||
# continues from there rather than redoing it.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/keepalive.sh setup # phase A, no GPU needed
|
||||
# bash scripts/keepalive.sh train # phase B, needs the GPU
|
||||
#
|
||||
# Unattended:
|
||||
# setsid nohup bash scripts/keepalive.sh setup > ~/keepalive.out 2>&1 &
|
||||
#
|
||||
# Stops when:
|
||||
# - STATUS says DONE -> success, exits 0
|
||||
# - STATUS says FAILED -> a hard error like a missing HF token;
|
||||
# retrying cannot fix it, exits 1
|
||||
# - MAX_RESTARTS reached -> exits 1
|
||||
#
|
||||
# Usage (this is the one command to run before leaving):
|
||||
# setsid nohup bash scripts/keepalive.sh > ~/keepalive.out 2>&1 &
|
||||
#
|
||||
# Check on it:
|
||||
# cat ~/sum-parts/runs/run_all/STATUS
|
||||
# tail -f ~/sum-parts/runs/run_all/run.log
|
||||
# STATUS says DONE -> success, exit 0
|
||||
# STATUS says FAILED -> hard error (missing HF token, no GPU for phase B);
|
||||
# retrying cannot fix it, exit 1
|
||||
# MAX_RESTARTS reached -> exit 1
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
OUT="$HOME/sum-parts/runs/run_all"
|
||||
|
||||
TARGET="${1:-}"
|
||||
case "$TARGET" in
|
||||
setup|run_setup) TARGET=setup; SCRIPT="$SCRIPTS/run_setup.sh"; OUT="$HOME/sum-parts/runs/setup" ;;
|
||||
train|run_train) TARGET=train; SCRIPT="$SCRIPTS/run_train.sh"; OUT="$HOME/sum-parts/runs/train" ;;
|
||||
*)
|
||||
echo "usage: bash keepalive.sh {setup|train}"
|
||||
echo
|
||||
echo " setup phase A - bootstrap, conda, CUDA extensions, data. No GPU needed."
|
||||
echo " train phase B - voxel_max measurement, training, evaluation. Needs the GPU."
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
STATUS="$OUT/STATUS"
|
||||
KLOG="$OUT/keepalive.log"
|
||||
|
||||
@@ -41,24 +54,29 @@ state_of() {
|
||||
grep -E '^state' "$STATUS" | head -1 | cut -d: -f2- | tr -d ' '
|
||||
}
|
||||
|
||||
klog "keepalive starting (max $MAX_RESTARTS restarts, ${COOLDOWN}s cooldown)"
|
||||
klog "keepalive starting for phase '$TARGET' (max $MAX_RESTARTS restarts, ${COOLDOWN}s cooldown)"
|
||||
|
||||
restarts=0
|
||||
while :; do
|
||||
s=$(state_of)
|
||||
case "$s" in
|
||||
DONE)
|
||||
klog "run_all reports DONE -- finished"
|
||||
klog "phase '$TARGET' reports DONE"
|
||||
[ "$TARGET" = setup ] && {
|
||||
klog "next, once the GPU is free:"
|
||||
klog " setsid nohup bash $SCRIPTS/keepalive.sh train > ~/keepalive-train.out 2>&1 &"
|
||||
}
|
||||
exit 0
|
||||
;;
|
||||
FAILED)
|
||||
klog "run_all reports FAILED -- a hard error that restarting will not fix:"
|
||||
klog "phase '$TARGET' reports FAILED -- restarting will not fix this:"
|
||||
sed 's/^/ /' "$STATUS" | tee -a "$KLOG"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if pgrep -f "run_all.sh" | grep -qv "$$"; then
|
||||
# already running (started by hand, or by a previous loop)?
|
||||
if pgrep -f "$(basename "$SCRIPT")" | grep -qv "^$$\$"; then
|
||||
sleep 30
|
||||
continue
|
||||
fi
|
||||
@@ -69,14 +87,14 @@ while :; do
|
||||
fi
|
||||
|
||||
if [ "$restarts" -gt 0 ]; then
|
||||
klog "run_all is not running (state='$s') -- restart #$restarts"
|
||||
klog "not running (state='$s') -- restart #$restarts"
|
||||
else
|
||||
klog "launching run_all"
|
||||
klog "launching $(basename "$SCRIPT")"
|
||||
fi
|
||||
|
||||
bash "$SCRIPTS/run_all.sh"
|
||||
bash "$SCRIPT"
|
||||
rc=$?
|
||||
klog "run_all exited rc=$rc, state='$(state_of)'"
|
||||
klog "$(basename "$SCRIPT") exited rc=$rc, state='$(state_of)'"
|
||||
|
||||
restarts=$((restarts + 1))
|
||||
sleep "$COOLDOWN"
|
||||
|
||||
@@ -1,305 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SUM Parts - unattended end-to-end run: bare machine to trained model
|
||||
#
|
||||
# Built for a weekend with nobody at the keyboard. Everything that can stop the
|
||||
# run is checked in PREFLIGHT, before any long step, so a failure surfaces in
|
||||
# the first minute rather than after three hours of setup.
|
||||
#
|
||||
# preflight -> bootstrap -> env -> patches -> verify -> data -> vram -> train -> eval
|
||||
#
|
||||
# The one thing this cannot do for you is accept the HuggingFace dataset gate:
|
||||
# it needs a browser and a logged-in account. It IS per-account though, so if
|
||||
# you already accepted it elsewhere, copying the token to this machine is
|
||||
# enough. Preflight fails immediately if the token is missing or rejected.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_all.sh # blocking, logs to stdout + file
|
||||
# bash scripts/run_all.sh --detach # survives terminal/session close
|
||||
#
|
||||
# Watch it later:
|
||||
# tail -f ~/sum-parts/runs/run_all/run.log
|
||||
# cat ~/sum-parts/runs/run_all/STATUS
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
OUT="$HOME/sum-parts/runs/run_all"
|
||||
LOG="$OUT/run.log"
|
||||
STATUS="$OUT/STATUS"
|
||||
|
||||
CFG="${CFG:-pointvector-xl}"
|
||||
EPOCHS="${EPOCHS:-100}"
|
||||
VAL_FREQ="${VAL_FREQ:-5}"
|
||||
# candidates tried high to low; first one that fits in VRAM wins.
|
||||
# 64000 is the paper setting and needs ~16.5 GB.
|
||||
VOXEL_CANDIDATES="${VOXEL_CANDIDATES:-64000 48000 40000 32000 24000}"
|
||||
|
||||
# ---------------------------------------------------------------- detach
|
||||
|
||||
if [ "${1:-}" = "--detach" ]; then
|
||||
mkdir -p "$OUT"
|
||||
echo "detaching; log: $LOG"
|
||||
setsid nohup bash "${BASH_SOURCE[0]}" > "$OUT/nohup.out" 2>&1 < /dev/null &
|
||||
sleep 2
|
||||
pgrep -af "run_all.sh" | grep -v detach || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
PHASE="starting"
|
||||
STARTED=$(date '+%F %T')
|
||||
|
||||
say() { echo "[$(date '+%F %T')] $*"; }
|
||||
head_() { echo; echo "════ $* ════"; }
|
||||
|
||||
write_status() {
|
||||
{
|
||||
echo "state : $1"
|
||||
echo "phase : $PHASE"
|
||||
echo "cfg : $CFG"
|
||||
echo "voxel_max: ${VOXEL_MAX:-(not chosen yet)}"
|
||||
echo "started : $STARTED"
|
||||
echo "updated : $(date '+%F %T')"
|
||||
[ -n "${EXTRA:-}" ] && echo "note : $EXTRA"
|
||||
echo "log : $LOG"
|
||||
} > "$STATUS"
|
||||
}
|
||||
|
||||
die() {
|
||||
say "GIVING UP in phase '$PHASE': $*"
|
||||
EXTRA="$*" write_status "FAILED"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Every phase is idempotent -- bootstrap skips an existing install, the patches
|
||||
# detect themselves, download_data skips cached archives, training resumes from
|
||||
# its checkpoint. So retrying a phase is always safe, and so is rerunning the
|
||||
# whole script from the top (see keepalive.sh).
|
||||
STEP_RETRIES="${STEP_RETRIES:-4}"
|
||||
STEP_BACKOFF="${STEP_BACKOFF:-60}"
|
||||
|
||||
step() {
|
||||
PHASE="$1"; shift
|
||||
head_ "$PHASE"
|
||||
write_status "running"
|
||||
|
||||
local attempt=1 wait=$STEP_BACKOFF
|
||||
while :; do
|
||||
if "$@"; then
|
||||
[ "$attempt" -gt 1 ] && say "phase '$PHASE' succeeded on attempt $attempt"
|
||||
return 0
|
||||
fi
|
||||
if [ "$attempt" -ge "$STEP_RETRIES" ]; then
|
||||
die "$* (failed $attempt times)"
|
||||
fi
|
||||
say "phase '$PHASE' failed (attempt $attempt/$STEP_RETRIES); retrying in ${wait}s"
|
||||
EXTRA="retrying $PHASE ($attempt/$STEP_RETRIES)" write_status "retrying"
|
||||
sleep "$wait"
|
||||
attempt=$((attempt + 1))
|
||||
wait=$((wait * 2))
|
||||
done
|
||||
}
|
||||
|
||||
# Preflight is the exception: a missing HF token or absent GPU will not fix
|
||||
# itself, so retrying just burns the weekend. Fail loudly and immediately.
|
||||
step_once() {
|
||||
PHASE="$1"; shift
|
||||
head_ "$PHASE"
|
||||
write_status "running"
|
||||
"$@" || die "$*"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- preflight
|
||||
|
||||
preflight() {
|
||||
local fail=0
|
||||
|
||||
say "checking tools"
|
||||
for t in git curl python3 tar; do
|
||||
command -v "$t" > /dev/null || { say " MISSING: $t"; fail=1; }
|
||||
done
|
||||
|
||||
say "checking GPU"
|
||||
if ! command -v nvidia-smi > /dev/null; then
|
||||
say " MISSING: nvidia-smi -- the driver is not exposing a GPU here"
|
||||
fail=1
|
||||
else
|
||||
nvidia-smi --query-gpu=name,memory.total,driver_version \
|
||||
--format=csv,noheader | sed 's/^/ /'
|
||||
VRAM_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1)
|
||||
say " usable VRAM: ${VRAM_MB} MiB"
|
||||
fi
|
||||
|
||||
say "checking disk (need 35 GB free in \$HOME)"
|
||||
local free_gb
|
||||
free_gb=$(df -BG --output=avail "$HOME" | tail -1 | tr -dc '0-9')
|
||||
say " free: ${free_gb} GB"
|
||||
[ "${free_gb:-0}" -lt 35 ] && { say " NOT ENOUGH"; fail=1; }
|
||||
|
||||
say "checking HuggingFace credentials"
|
||||
# the dataset is gated; the token must exist AND the account must already
|
||||
# have accepted the licence in a browser
|
||||
local tok=""
|
||||
[ -n "${HF_TOKEN:-}" ] && tok="$HF_TOKEN"
|
||||
[ -z "$tok" ] && [ -f "$HOME/.cache/huggingface/token" ] \
|
||||
&& tok=$(tr -d '\r\n' < "$HOME/.cache/huggingface/token")
|
||||
if [ -z "$tok" ]; then
|
||||
say " MISSING: no HF token"
|
||||
say " fix: copy the token from a machine that already accepted the gate:"
|
||||
say " mkdir -p ~/.cache/huggingface"
|
||||
say " echo hf_xxxxx > ~/.cache/huggingface/token"
|
||||
say " the gate itself is per-account and needs a browser once:"
|
||||
say " https://huggingface.co/datasets/gwxgrxhyz/SUM-Parts"
|
||||
fail=1
|
||||
else
|
||||
local code
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' -I \
|
||||
-H "Authorization: Bearer $tok" \
|
||||
"https://huggingface.co/datasets/gwxgrxhyz/SUM-Parts/resolve/main/demo.zip")
|
||||
say " gate probe: HTTP $code"
|
||||
case "$code" in
|
||||
200|302) say " gate OK" ;;
|
||||
401|403) say " REJECTED -- token invalid, or this account has not accepted the gate"
|
||||
say " accept it in a browser: https://huggingface.co/datasets/gwxgrxhyz/SUM-Parts"
|
||||
fail=1 ;;
|
||||
*) say " unexpected response; continuing but the download may fail" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
[ "$fail" -eq 0 ] || return 1
|
||||
say "preflight OK"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- vram pick
|
||||
|
||||
pick_voxel_max() {
|
||||
say "measuring which voxel_max fits in VRAM (high to low, first fit wins)"
|
||||
say "NOTE: on WSL2 an oversized value does not OOM -- the driver spills into"
|
||||
say " host RAM and the run completes 25-100x slower. So this measures"
|
||||
say " peak allocation instead of trusting that it 'worked'."
|
||||
|
||||
source "$HOME/miniconda3/etc/profile.d/conda.sh"
|
||||
conda activate sumparts
|
||||
export WANDB_MODE=disabled WANDB_SILENT=true CUDA_HOME="$CONDA_PREFIX"
|
||||
export PYTORCH_CUDA_ALLOC_CONF="garbage_collection_threshold:0.7,max_split_size_mb:128"
|
||||
|
||||
cd "$HOME/sum-parts/semantic_segmentation/PointNeXt_bundle/examples/segmentation" \
|
||||
|| return 1
|
||||
|
||||
local vm log line fits
|
||||
for vm in $VOXEL_CANDIDATES; do
|
||||
log="$OUT/vram_${vm}.log"
|
||||
say " trying voxel_max=$vm"
|
||||
python -u "$SCRIPTS/bench_models.py" --iters 4 --voxel-max "$vm" \
|
||||
--cfgs "$CFG" > "$log" 2>&1
|
||||
line=$(grep -aE "^${CFG} +[0-9]" "$log" | tail -1)
|
||||
if [ -z "$line" ]; then
|
||||
say " no result (probably OOM or an error); see $log"
|
||||
continue
|
||||
fi
|
||||
fits=$(echo "$line" | grep -o 'yes$' || true)
|
||||
say " $(echo "$line" | awk '{print "peak", $5, "s/iter", $6}')"
|
||||
if [ -n "$fits" ]; then
|
||||
VOXEL_MAX="$vm"
|
||||
say " chosen: voxel_max=$VOXEL_MAX"
|
||||
return 0
|
||||
fi
|
||||
say " does not fit -- spilling to host RAM"
|
||||
done
|
||||
|
||||
say " nothing fit; falling back to the smallest candidate"
|
||||
VOXEL_MAX=$(echo "$VOXEL_CANDIDATES" | awk '{print $NF}')
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- phases
|
||||
|
||||
do_verify() {
|
||||
source "$HOME/miniconda3/etc/profile.d/conda.sh"
|
||||
conda activate sumparts
|
||||
WANDB_MODE=disabled python "$SCRIPTS/verify_env.py"
|
||||
}
|
||||
|
||||
do_train() {
|
||||
# Already finished? Don't retrain on a rerun.
|
||||
local done_marker="$OUT/TRAIN_DONE"
|
||||
if [ -f "$done_marker" ]; then
|
||||
say "training already completed (marker: $done_marker)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Pick up an earlier run's progress. train_watchdog only auto-discovers
|
||||
# checkpoints written after IT started, so a fresh invocation would restart
|
||||
# from epoch 1 without this.
|
||||
local ckpt
|
||||
ckpt=$(find "$HOME/sum-parts/semantic_segmentation/PointNeXt_bundle/examples/segmentation/log/sumv2_triangle" \
|
||||
-name '*_ckpt_latest.pth' -printf '%T@ %p\n' 2>/dev/null \
|
||||
| sort -rn | head -1 | cut -d' ' -f2-)
|
||||
|
||||
if [ -n "$ckpt" ]; then
|
||||
say "resuming from $(basename "$ckpt")"
|
||||
else
|
||||
say "no checkpoint found; starting fresh"
|
||||
fi
|
||||
|
||||
say "training $CFG for $EPOCHS epochs at voxel_max=$VOXEL_MAX"
|
||||
CFG_VOXEL_MAX="$VOXEL_MAX" \
|
||||
VAL_VOXEL_MAX="$VOXEL_MAX" \
|
||||
EPOCHS="$EPOCHS" VAL_FREQ="$VAL_FREQ" MAX_RETRIES=8 \
|
||||
RESUME_CKPT="$ckpt" \
|
||||
bash "$SCRIPTS/train_watchdog.sh" "$CFG" || return 1
|
||||
|
||||
touch "$done_marker"
|
||||
return 0
|
||||
}
|
||||
|
||||
do_eval() {
|
||||
bash "$SCRIPTS/final_eval.sh" || say "final_eval reported non-zero (test split is blind; that is expected)"
|
||||
bash "$SCRIPTS/eval_coarse.sh" || say "eval_coarse reported non-zero"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- run
|
||||
|
||||
say "run_all starting -- cfg=$CFG epochs=$EPOCHS"
|
||||
say "log: $LOG"
|
||||
write_status "running"
|
||||
|
||||
step_once "preflight" preflight
|
||||
|
||||
# RUN_ALL_DRYRUN lets you prove the preflight checks pass without starting the
|
||||
# long phases -- worth doing before walking away for the weekend.
|
||||
if [ -n "${RUN_ALL_DRYRUN:-}" ]; then
|
||||
say "DRYRUN set -- preflight passed, stopping before the real work"
|
||||
write_status "dryrun-ok"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
step "bootstrap" bash "$SCRIPTS/bootstrap.sh"
|
||||
step "conda env" bash "$SCRIPTS/setup_env.sh"
|
||||
step "cuda extensions" bash "$SCRIPTS/setup_pointnext.sh"
|
||||
step "patch numpy" bash "$SCRIPTS/patch_numpy_aliases.sh"
|
||||
step "patch test split" bash "$SCRIPTS/patch_unlabeled_test.sh"
|
||||
step "patch val mode" bash "$SCRIPTS/patch_val_mode.sh"
|
||||
step "verify env" do_verify
|
||||
step "download data" bash "$SCRIPTS/download_data.sh" all
|
||||
step "prepare splits" bash "$SCRIPTS/prepare_full_split.sh"
|
||||
step "link data" bash "$SCRIPTS/link_data.sh"
|
||||
step "choose voxel_max" pick_voxel_max
|
||||
step "train" do_train
|
||||
step "evaluate" do_eval
|
||||
|
||||
PHASE="done"
|
||||
write_status "DONE"
|
||||
|
||||
head_ "SUMMARY"
|
||||
say "cfg : $CFG"
|
||||
say "voxel_max : $VOXEL_MAX"
|
||||
grep -aE 'Best ckpt' "$OUT/../"*/train.log 2>/dev/null | tail -2
|
||||
[ -f "$HOME/sum-parts/runs/coarse_eval/coarse.txt" ] && {
|
||||
echo
|
||||
echo "--- coarse (building / vegetation / vehicle / ground) ---"
|
||||
cat "$HOME/sum-parts/runs/coarse_eval/coarse.txt"
|
||||
}
|
||||
say "RUN ALL DONE"
|
||||
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env bash
|
||||
# SUM Parts - PHASE A: everything that does not need the GPU
|
||||
#
|
||||
# Splitting the work here is deliberate. Compiling the CUDA extensions needs
|
||||
# nvcc, not a GPU, and downloading 18 GB of data needs neither. Those are the
|
||||
# slow parts (~50 min + ~30 min), so they can run while the card is busy with
|
||||
# someone else's job.
|
||||
#
|
||||
# preflight -> bootstrap -> conda env -> cuda extensions -> patches
|
||||
# -> verify imports -> download data -> splits -> link
|
||||
#
|
||||
# Ends by writing SETUP_DONE. run_train.sh refuses to start without it.
|
||||
#
|
||||
# The one thing this cannot do for you is accept the HuggingFace dataset gate:
|
||||
# it needs a browser and a logged-in account. The gate IS per-account, so a
|
||||
# token from a machine that already accepted it is enough. Preflight fails
|
||||
# immediately if the token is missing or rejected.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_setup.sh
|
||||
# bash scripts/run_setup.sh --detach
|
||||
# RUN_DRYRUN=1 bash scripts/run_setup.sh # preflight only
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
OUT="$HOME/sum-parts/runs/setup"
|
||||
LOG="$OUT/setup.log"
|
||||
STATUS="$OUT/STATUS"
|
||||
DONE_MARKER="$OUT/SETUP_DONE"
|
||||
|
||||
# The extensions are compiled for this architecture. We may not be able to ask
|
||||
# the GPU which one it is (it can be busy or absent), so it is stated instead.
|
||||
# RTX 3060 / 3070 / 3080 / 3090 = 8.6 (Ampere consumer)
|
||||
# RTX 4090 = 8.9
|
||||
# A100 = 8.0
|
||||
export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-8.6}"
|
||||
|
||||
if [ "${1:-}" = "--detach" ]; then
|
||||
mkdir -p "$OUT"
|
||||
echo "detaching; log: $LOG"
|
||||
setsid nohup bash "${BASH_SOURCE[0]}" > "$OUT/nohup.out" 2>&1 < /dev/null &
|
||||
sleep 2
|
||||
pgrep -af "run_setup.sh" | grep -v detach || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
PHASE="starting"
|
||||
STARTED=$(date '+%F %T')
|
||||
|
||||
say() { echo "[$(date '+%F %T')] $*"; }
|
||||
head_() { echo; echo "════ $* ════"; }
|
||||
|
||||
write_status() {
|
||||
{
|
||||
echo "state : $1"
|
||||
echo "phase : $PHASE"
|
||||
echo "arch : $TORCH_CUDA_ARCH_LIST"
|
||||
echo "started : $STARTED"
|
||||
echo "updated : $(date '+%F %T')"
|
||||
[ -n "${EXTRA:-}" ] && echo "note : $EXTRA"
|
||||
echo "log : $LOG"
|
||||
} > "$STATUS"
|
||||
}
|
||||
|
||||
die() {
|
||||
say "GIVING UP in phase '$PHASE': $*"
|
||||
EXTRA="$*" write_status "FAILED"
|
||||
exit 1
|
||||
}
|
||||
|
||||
STEP_RETRIES="${STEP_RETRIES:-4}"
|
||||
STEP_BACKOFF="${STEP_BACKOFF:-60}"
|
||||
|
||||
# Every phase is idempotent, so retrying one - or rerunning the whole script -
|
||||
# is always safe. Existing installs, applied patches and cached archives are
|
||||
# detected and skipped.
|
||||
step() {
|
||||
PHASE="$1"; shift
|
||||
head_ "$PHASE"
|
||||
write_status "running"
|
||||
local attempt=1 wait=$STEP_BACKOFF
|
||||
while :; do
|
||||
if "$@"; then
|
||||
[ "$attempt" -gt 1 ] && say "phase '$PHASE' succeeded on attempt $attempt"
|
||||
return 0
|
||||
fi
|
||||
[ "$attempt" -ge "$STEP_RETRIES" ] && die "$* (failed $attempt times)"
|
||||
say "phase '$PHASE' failed (attempt $attempt/$STEP_RETRIES); retrying in ${wait}s"
|
||||
EXTRA="retrying $PHASE ($attempt/$STEP_RETRIES)" write_status "retrying"
|
||||
sleep "$wait"
|
||||
attempt=$((attempt + 1)); wait=$((wait * 2))
|
||||
done
|
||||
}
|
||||
|
||||
step_once() {
|
||||
PHASE="$1"; shift
|
||||
head_ "$PHASE"
|
||||
write_status "running"
|
||||
"$@" || die "$*"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- preflight
|
||||
|
||||
preflight() {
|
||||
local fail=0
|
||||
|
||||
say "checking tools"
|
||||
for t in git curl python3 tar; do
|
||||
command -v "$t" > /dev/null || { say " MISSING: $t"; fail=1; }
|
||||
done
|
||||
|
||||
say "checking disk (need 35 GB free in \$HOME)"
|
||||
local free_gb
|
||||
free_gb=$(df -BG --output=avail "$HOME" | tail -1 | tr -dc '0-9')
|
||||
say " free: ${free_gb} GB"
|
||||
[ "${free_gb:-0}" -lt 35 ] && { say " NOT ENOUGH"; fail=1; }
|
||||
|
||||
# A GPU is NOT required for this phase. Report what is there, but never
|
||||
# fail on it -- the whole point of the split is to work while the card is
|
||||
# occupied.
|
||||
say "checking GPU (informational only for this phase)"
|
||||
if command -v nvidia-smi > /dev/null; then
|
||||
nvidia-smi --query-gpu=name,memory.total,memory.used --format=csv,noheader \
|
||||
| sed 's/^/ /' || say " nvidia-smi failed; continuing anyway"
|
||||
else
|
||||
say " nvidia-smi absent -- fine for setup, required before training"
|
||||
fi
|
||||
say " building extensions for arch $TORCH_CUDA_ARCH_LIST"
|
||||
|
||||
say "checking HuggingFace credentials"
|
||||
local tok=""
|
||||
[ -n "${HF_TOKEN:-}" ] && tok="$HF_TOKEN"
|
||||
[ -z "$tok" ] && [ -f "$HOME/.cache/huggingface/token" ] \
|
||||
&& tok=$(tr -d '\r\n' < "$HOME/.cache/huggingface/token")
|
||||
if [ -z "$tok" ]; then
|
||||
say " MISSING: no HF token"
|
||||
say " fix: copy it from a machine that already accepted the gate:"
|
||||
say " mkdir -p ~/.cache/huggingface"
|
||||
say " echo hf_xxxxx > ~/.cache/huggingface/token"
|
||||
say " the gate needs a browser once, per account:"
|
||||
say " https://huggingface.co/datasets/gwxgrxhyz/SUM-Parts"
|
||||
fail=1
|
||||
else
|
||||
local code
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' -I \
|
||||
-H "Authorization: Bearer $tok" \
|
||||
"https://huggingface.co/datasets/gwxgrxhyz/SUM-Parts/resolve/main/demo.zip")
|
||||
say " gate probe: HTTP $code"
|
||||
case "$code" in
|
||||
200|302) say " gate OK" ;;
|
||||
401|403) say " REJECTED -- token invalid, or this account has not accepted the gate"
|
||||
fail=1 ;;
|
||||
*) say " unexpected response; continuing but the download may fail" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
[ "$fail" -eq 0 ] || return 1
|
||||
say "preflight OK -- no GPU needed from here to the end of this phase"
|
||||
}
|
||||
|
||||
do_verify() {
|
||||
source "$HOME/miniconda3/etc/profile.d/conda.sh"
|
||||
conda activate sumparts
|
||||
# SKIP_CUDA_CHECK keeps this to imports only. Querying the device is
|
||||
# harmless even on a busy card, but it fails outright if the driver is not
|
||||
# present yet - and that must not block a GPU-free setup.
|
||||
SKIP_CUDA_CHECK=1 WANDB_MODE=disabled python "$SCRIPTS/verify_env.py"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- run
|
||||
|
||||
say "run_setup starting (phase A - no GPU required)"
|
||||
say "log: $LOG"
|
||||
write_status "running"
|
||||
|
||||
step_once "preflight" preflight
|
||||
|
||||
if [ -n "${RUN_DRYRUN:-}" ]; then
|
||||
say "DRYRUN set -- preflight passed, stopping before the real work"
|
||||
write_status "dryrun-ok"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
step "bootstrap" bash "$SCRIPTS/bootstrap.sh"
|
||||
step "conda env" bash "$SCRIPTS/setup_env.sh"
|
||||
step "cuda extensions" bash "$SCRIPTS/setup_pointnext.sh"
|
||||
step "patch numpy" bash "$SCRIPTS/patch_numpy_aliases.sh"
|
||||
step "patch test split" bash "$SCRIPTS/patch_unlabeled_test.sh"
|
||||
step "patch val mode" bash "$SCRIPTS/patch_val_mode.sh"
|
||||
step "verify imports" do_verify
|
||||
step "download data" bash "$SCRIPTS/download_data.sh" all
|
||||
step "prepare splits" bash "$SCRIPTS/prepare_full_split.sh"
|
||||
step "link data" bash "$SCRIPTS/link_data.sh"
|
||||
|
||||
PHASE="done"
|
||||
touch "$DONE_MARKER"
|
||||
write_status "DONE"
|
||||
|
||||
head_ "PHASE A COMPLETE"
|
||||
say "built for arch : $TORCH_CUDA_ARCH_LIST"
|
||||
say "marker : $DONE_MARKER"
|
||||
echo
|
||||
say "next, once the GPU is free:"
|
||||
say " bash scripts/run_train.sh"
|
||||
say "or, to keep it alive unattended:"
|
||||
say " setsid nohup bash scripts/keepalive.sh run_train > ~/keepalive.out 2>&1 &"
|
||||
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env bash
|
||||
# SUM Parts - PHASE B: the GPU work
|
||||
#
|
||||
# Assumes run_setup.sh already finished (it checks for the marker). Everything
|
||||
# here needs the card to itself:
|
||||
#
|
||||
# preflight -> choose voxel_max (measured) -> train -> evaluate
|
||||
#
|
||||
# Waits for the GPU rather than failing when it is busy: if another job holds
|
||||
# the card, this parks until enough VRAM frees up. So it can be started ahead
|
||||
# of time and left alone.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_train.sh
|
||||
# bash scripts/run_train.sh --detach
|
||||
# RUN_DRYRUN=1 bash scripts/run_train.sh # preflight only
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPTS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SETUP_MARKER="$HOME/sum-parts/runs/setup/SETUP_DONE"
|
||||
OUT="$HOME/sum-parts/runs/train"
|
||||
LOG="$OUT/train_phase.log"
|
||||
STATUS="$OUT/STATUS"
|
||||
DONE_MARKER="$OUT/TRAIN_DONE"
|
||||
|
||||
CFG="${CFG:-pointvector-xl}"
|
||||
EPOCHS="${EPOCHS:-100}"
|
||||
VAL_FREQ="${VAL_FREQ:-5}"
|
||||
# tried high to low; first one that actually fits in VRAM wins.
|
||||
# 64000 is the paper setting and needs ~16.5 GB.
|
||||
VOXEL_CANDIDATES="${VOXEL_CANDIDATES:-64000 48000 40000 32000 24000}"
|
||||
|
||||
# how long to wait for someone else's job to release the card
|
||||
GPU_WAIT_MINUTES="${GPU_WAIT_MINUTES:-720}"
|
||||
GPU_FREE_MB="${GPU_FREE_MB:-16000}"
|
||||
|
||||
if [ "${1:-}" = "--detach" ]; then
|
||||
mkdir -p "$OUT"
|
||||
echo "detaching; log: $LOG"
|
||||
setsid nohup bash "${BASH_SOURCE[0]}" > "$OUT/nohup.out" 2>&1 < /dev/null &
|
||||
sleep 2
|
||||
pgrep -af "run_train.sh" | grep -v detach || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
PHASE="starting"
|
||||
STARTED=$(date '+%F %T')
|
||||
VOXEL_MAX=""
|
||||
|
||||
say() { echo "[$(date '+%F %T')] $*"; }
|
||||
head_() { echo; echo "════ $* ════"; }
|
||||
|
||||
write_status() {
|
||||
{
|
||||
echo "state : $1"
|
||||
echo "phase : $PHASE"
|
||||
echo "cfg : $CFG"
|
||||
echo "voxel_max: ${VOXEL_MAX:-(not chosen yet)}"
|
||||
echo "started : $STARTED"
|
||||
echo "updated : $(date '+%F %T')"
|
||||
[ -n "${EXTRA:-}" ] && echo "note : $EXTRA"
|
||||
echo "log : $LOG"
|
||||
} > "$STATUS"
|
||||
}
|
||||
|
||||
die() {
|
||||
say "GIVING UP in phase '$PHASE': $*"
|
||||
EXTRA="$*" write_status "FAILED"
|
||||
exit 1
|
||||
}
|
||||
|
||||
STEP_RETRIES="${STEP_RETRIES:-4}"
|
||||
STEP_BACKOFF="${STEP_BACKOFF:-60}"
|
||||
|
||||
step() {
|
||||
PHASE="$1"; shift
|
||||
head_ "$PHASE"
|
||||
write_status "running"
|
||||
local attempt=1 wait=$STEP_BACKOFF
|
||||
while :; do
|
||||
if "$@"; then
|
||||
[ "$attempt" -gt 1 ] && say "phase '$PHASE' succeeded on attempt $attempt"
|
||||
return 0
|
||||
fi
|
||||
[ "$attempt" -ge "$STEP_RETRIES" ] && die "$* (failed $attempt times)"
|
||||
say "phase '$PHASE' failed (attempt $attempt/$STEP_RETRIES); retrying in ${wait}s"
|
||||
EXTRA="retrying $PHASE ($attempt/$STEP_RETRIES)" write_status "retrying"
|
||||
sleep "$wait"
|
||||
attempt=$((attempt + 1)); wait=$((wait * 2))
|
||||
done
|
||||
}
|
||||
|
||||
step_once() {
|
||||
PHASE="$1"; shift
|
||||
head_ "$PHASE"
|
||||
write_status "running"
|
||||
"$@" || die "$*"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- preflight
|
||||
|
||||
preflight() {
|
||||
local fail=0
|
||||
|
||||
say "checking phase A completed"
|
||||
if [ -f "$SETUP_MARKER" ]; then
|
||||
say " marker present: $SETUP_MARKER"
|
||||
else
|
||||
say " MISSING: $SETUP_MARKER"
|
||||
say " run phase A first: bash scripts/run_setup.sh"
|
||||
fail=1
|
||||
fi
|
||||
|
||||
say "checking environment"
|
||||
if [ -x "$HOME/miniconda3/bin/conda" ]; then
|
||||
source "$HOME/miniconda3/etc/profile.d/conda.sh"
|
||||
if conda env list | grep -q '^sumparts '; then
|
||||
say " conda env sumparts present"
|
||||
else
|
||||
say " MISSING: conda env 'sumparts'"; fail=1
|
||||
fi
|
||||
else
|
||||
say " MISSING: miniconda"; fail=1
|
||||
fi
|
||||
|
||||
say "checking data"
|
||||
local d="$HOME/sum-parts/data/face_labeling/texsp_pcl"
|
||||
if [ -d "$d/train" ]; then
|
||||
say " train/val/test = $(find -L "$d/train" -name '*.ply' | wc -l)/$(find -L "$d/val" -name '*.ply' 2>/dev/null | wc -l)/$(find -L "$d/test" -name '*.ply' | wc -l)"
|
||||
else
|
||||
say " MISSING: $d/train"; fail=1
|
||||
fi
|
||||
|
||||
say "checking GPU driver"
|
||||
if ! command -v nvidia-smi > /dev/null; then
|
||||
say " MISSING: nvidia-smi -- no GPU visible, and this phase cannot run without one"
|
||||
fail=1
|
||||
else
|
||||
nvidia-smi --query-gpu=name,memory.total,memory.used,utilization.gpu \
|
||||
--format=csv,noheader | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
[ "$fail" -eq 0 ] || return 1
|
||||
say "preflight OK"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ wait for GPU
|
||||
|
||||
wait_for_gpu() {
|
||||
local deadline=$(( $(date +%s) + GPU_WAIT_MINUTES * 60 ))
|
||||
local total used free
|
||||
|
||||
total=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -1)
|
||||
say "card has ${total} MiB; waiting until ${GPU_FREE_MB} MiB is free"
|
||||
say "(will wait up to ${GPU_WAIT_MINUTES} minutes)"
|
||||
|
||||
while :; do
|
||||
used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)
|
||||
free=$(( total - used ))
|
||||
if [ "$free" -ge "$GPU_FREE_MB" ]; then
|
||||
say " ${free} MiB free -- proceeding"
|
||||
return 0
|
||||
fi
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
say " timed out with only ${free} MiB free"
|
||||
say " continuing anyway; voxel_max will be measured against what is actually available"
|
||||
return 0
|
||||
fi
|
||||
EXTRA="waiting for GPU (${free} MiB free, need ${GPU_FREE_MB})" write_status "waiting"
|
||||
say " ${free} MiB free, need ${GPU_FREE_MB} -- checking again in 5 min"
|
||||
sleep 300
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- vram pick
|
||||
|
||||
pick_voxel_max() {
|
||||
say "measuring which voxel_max fits (high to low, first fit wins)"
|
||||
say "NOTE: on WSL2 an oversized value does not OOM -- the driver spills into"
|
||||
say " host RAM and the run completes 25-100x slower. So this checks peak"
|
||||
say " allocation instead of trusting that it 'worked'."
|
||||
|
||||
source "$HOME/miniconda3/etc/profile.d/conda.sh"
|
||||
conda activate sumparts
|
||||
export WANDB_MODE=disabled WANDB_SILENT=true CUDA_HOME="$CONDA_PREFIX"
|
||||
export PYTORCH_CUDA_ALLOC_CONF="garbage_collection_threshold:0.7,max_split_size_mb:128"
|
||||
|
||||
cd "$HOME/sum-parts/semantic_segmentation/PointNeXt_bundle/examples/segmentation" \
|
||||
|| return 1
|
||||
|
||||
local vm log line fits
|
||||
for vm in $VOXEL_CANDIDATES; do
|
||||
log="$OUT/vram_${vm}.log"
|
||||
say " trying voxel_max=$vm"
|
||||
python -u "$SCRIPTS/bench_models.py" --iters 4 --voxel-max "$vm" \
|
||||
--cfgs "$CFG" > "$log" 2>&1
|
||||
line=$(grep -aE "^${CFG} +[0-9]" "$log" | tail -1)
|
||||
if [ -z "$line" ]; then
|
||||
say " no result (OOM or error); see $log"
|
||||
continue
|
||||
fi
|
||||
say " $(echo "$line" | awk '{print "peak", $5, " s/iter", $6}')"
|
||||
fits=$(echo "$line" | grep -o 'yes$' || true)
|
||||
if [ -n "$fits" ]; then
|
||||
VOXEL_MAX="$vm"
|
||||
say " chosen: voxel_max=$VOXEL_MAX"
|
||||
return 0
|
||||
fi
|
||||
say " does not fit -- would spill to host RAM"
|
||||
done
|
||||
|
||||
say " nothing fit; falling back to the smallest candidate"
|
||||
VOXEL_MAX=$(echo "$VOXEL_CANDIDATES" | awk '{print $NF}')
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- train
|
||||
|
||||
do_train() {
|
||||
if [ -f "$DONE_MARKER" ]; then
|
||||
say "training already completed (marker: $DONE_MARKER)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# train_watchdog only auto-discovers checkpoints written after it starts,
|
||||
# so hand it the newest one explicitly or a rerun restarts from epoch 1.
|
||||
local ckpt
|
||||
ckpt=$(find "$HOME/sum-parts/semantic_segmentation/PointNeXt_bundle/examples/segmentation/log/sumv2_triangle" \
|
||||
-name '*_ckpt_latest.pth' -printf '%T@ %p\n' 2>/dev/null \
|
||||
| sort -rn | head -1 | cut -d' ' -f2-)
|
||||
if [ -n "$ckpt" ]; then
|
||||
say "resuming from $(basename "$ckpt")"
|
||||
else
|
||||
say "no checkpoint found; starting fresh"
|
||||
fi
|
||||
|
||||
say "training $CFG for $EPOCHS epochs at voxel_max=$VOXEL_MAX"
|
||||
CFG_VOXEL_MAX="$VOXEL_MAX" VAL_VOXEL_MAX="$VOXEL_MAX" \
|
||||
EPOCHS="$EPOCHS" VAL_FREQ="$VAL_FREQ" MAX_RETRIES=8 \
|
||||
RESUME_CKPT="$ckpt" \
|
||||
bash "$SCRIPTS/train_watchdog.sh" "$CFG" || return 1
|
||||
|
||||
touch "$DONE_MARKER"
|
||||
return 0
|
||||
}
|
||||
|
||||
do_eval() {
|
||||
bash "$SCRIPTS/final_eval.sh" || say "final_eval non-zero (test split is blind; expected)"
|
||||
bash "$SCRIPTS/eval_coarse.sh" || say "eval_coarse non-zero"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- run
|
||||
|
||||
say "run_train starting (phase B - GPU required)"
|
||||
say "cfg=$CFG epochs=$EPOCHS"
|
||||
say "log: $LOG"
|
||||
write_status "running"
|
||||
|
||||
step_once "preflight" preflight
|
||||
|
||||
if [ -n "${RUN_DRYRUN:-}" ]; then
|
||||
say "DRYRUN set -- preflight passed, stopping before the real work"
|
||||
write_status "dryrun-ok"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
step_once "wait for GPU" wait_for_gpu
|
||||
step "choose voxel_max" pick_voxel_max
|
||||
step "train" do_train
|
||||
step "evaluate" do_eval
|
||||
|
||||
PHASE="done"
|
||||
write_status "DONE"
|
||||
|
||||
head_ "PHASE B COMPLETE"
|
||||
say "cfg : $CFG"
|
||||
say "voxel_max : $VOXEL_MAX"
|
||||
grep -ahE 'Best ckpt' "$HOME/sum-parts/runs/"*/train.log 2>/dev/null | tail -2
|
||||
if [ -f "$HOME/sum-parts/runs/coarse_eval/coarse.txt" ]; then
|
||||
echo
|
||||
echo "--- coarse (building / vegetation / vehicle / ground) ---"
|
||||
cat "$HOME/sum-parts/runs/coarse_eval/coarse.txt"
|
||||
fi
|
||||
say "TRAIN PHASE DONE"
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# SUM Parts - syntax-check every script in this directory
|
||||
#
|
||||
# Cheap guard: a shell script with a syntax error fails at the first line it
|
||||
# reaches, which for an unattended weekend run means losing the weekend.
|
||||
set -uo pipefail
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
|
||||
fail=0
|
||||
|
||||
echo "=== bash ==="
|
||||
for f in *.sh; do
|
||||
printf ' %-26s' "$f"
|
||||
if bash -n "$f" 2>/dev/null; then echo "OK"; else echo "SYNTAX ERROR"; fail=1; fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== python ==="
|
||||
for f in *.py; do
|
||||
printf ' %-26s' "$f"
|
||||
if python3 -c "import ast,sys; ast.parse(open(sys.argv[1],encoding='utf-8').read())" "$f" 2>/dev/null
|
||||
then echo "OK"; else echo "SYNTAX ERROR"; fail=1; fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== CRLF check (breaks bash on Linux) ==="
|
||||
if grep -rlU $'\r' ./*.sh ./*.py 2>/dev/null; then
|
||||
echo " ^ these have CRLF line endings and will fail with: bash: \$'\\r': command not found"
|
||||
fail=1
|
||||
else
|
||||
echo " clean"
|
||||
fi
|
||||
|
||||
echo
|
||||
[ "$fail" -eq 0 ] && echo "ALL OK" || echo "PROBLEMS FOUND"
|
||||
exit "$fail"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SUM Parts - run only run_all.sh's preflight, to prove it before leaving
|
||||
#
|
||||
# Sources run_all.sh with RUN_ALL_DRYRUN set so the phase list is skipped and
|
||||
# only the checks execute. Cheap, no side effects, no GPU work.
|
||||
set -uo pipefail
|
||||
RUN_ALL_DRYRUN=1 bash "$(dirname "${BASH_SOURCE[0]}")/run_all.sh"
|
||||
+15
-4
@@ -62,12 +62,23 @@ def main() -> int:
|
||||
if root is None:
|
||||
ok = False
|
||||
|
||||
# SKIP_CUDA_CHECK exists for the GPU-free setup phase: the extensions can be
|
||||
# built and imported without a card present, and querying the device would
|
||||
# fail on a machine whose GPU is absent or still occupied. Import coverage
|
||||
# is unaffected -- only the device query is skipped.
|
||||
skip_cuda = bool(os.environ.get("SKIP_CUDA_CHECK"))
|
||||
|
||||
try:
|
||||
import torch
|
||||
print(f"torch {torch.__version__} | cuda {torch.version.cuda} | "
|
||||
f"available {torch.cuda.is_available()}")
|
||||
if torch.cuda.is_available():
|
||||
print(f"device: {torch.cuda.get_device_name(0)}")
|
||||
line = f"torch {torch.__version__} | cuda {torch.version.cuda}"
|
||||
if skip_cuda:
|
||||
print(line + " | device check skipped (SKIP_CUDA_CHECK)")
|
||||
else:
|
||||
print(line + f" | available {torch.cuda.is_available()}")
|
||||
if torch.cuda.is_available():
|
||||
print(f"device: {torch.cuda.get_device_name(0)}")
|
||||
else:
|
||||
print("device: none visible -- fine for setup, required to train")
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"torch import failed: {e}")
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user