nbrightandClaude Opus 5
3fdd7ab3f0
The target machine's card is busy with someone else's job, so a single end-to-end script stalls on work that does not actually need a GPU. Compiling the CUDA extensions needs nvcc, not a device, and downloading 18 GB of data needs neither. Those are the slow parts (~50 min + ~30 min), so phase A now runs entirely without the card: run_setup.sh bootstrap, conda, extensions, patches, data no GPU run_train.sh voxel_max measurement, training, evaluation GPU run_setup reports the GPU but never fails on it, and verify_env.py gained SKIP_CUDA_CHECK so import coverage still runs when no device is visible. TORCH_CUDA_ARCH_LIST is stated rather than probed, since the card may be unavailable at build time. run_train waits for the GPU instead of failing when it is busy: it polls until enough VRAM frees up (12h default), so it can be queued ahead of time. Past the deadline it proceeds anyway and lets the measured voxel_max adapt to whatever is actually free. keepalive.sh now takes the phase to supervise. Replaces run_all.sh and RUN.md with SETUP.md and TRAIN.md. Adds selfcheck.sh, which syntax-checks every script and flags CRLF endings - a shell script with either fails at its first line, which for an unattended weekend run means losing the weekend. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
112 lines
3.4 KiB
Python
112 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Check that every compiled extension and import the sumv2 training path needs
|
|
is actually present.
|
|
|
|
Extension import names do not match their directory names, which is an easy way
|
|
to waste an hour chasing a build that already succeeded:
|
|
|
|
openpoints/cpp/pointnet2_batch -> pointnet2_batch_cuda
|
|
openpoints/cpp/pointops -> pointops_cuda
|
|
openpoints/cpp/chamfer_dist -> chamfer
|
|
openpoints/cpp/emd -> emd_cuda (package name: emd_ext)
|
|
openpoints/cpp/subsampling -> openpoints.cpp.subsampling.grid_subsampling
|
|
|
|
Run from PointNeXt_bundle/ (or anywhere, if openpoints is importable).
|
|
"""
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
def add_openpoints_to_path() -> Path | None:
|
|
"""Put PointNeXt_bundle on sys.path.
|
|
|
|
main.py does this itself with a hardcoded '../../', but this script lives
|
|
outside the repo, so walk up from cwd (then from the default clone path)
|
|
until a directory containing openpoints/ turns up.
|
|
"""
|
|
candidates = [Path.cwd(), *Path.cwd().parents,
|
|
Path.home() / "sum-parts/semantic_segmentation/PointNeXt_bundle"]
|
|
for c in candidates:
|
|
if (c / "openpoints" / "__init__.py").exists():
|
|
sys.path.insert(0, str(c))
|
|
return c
|
|
return None
|
|
|
|
|
|
MODULES = [
|
|
"torch",
|
|
"numpy",
|
|
"pointnet2_batch_cuda",
|
|
"pointops_cuda",
|
|
"chamfer",
|
|
"emd_cuda",
|
|
"torch_scatter",
|
|
"plyfile",
|
|
"wandb",
|
|
"trimesh",
|
|
]
|
|
|
|
FROM_IMPORTS = [
|
|
("openpoints.cpp.subsampling", "grid_subsampling"),
|
|
("openpoints.models", "build_model_from_cfg"),
|
|
("openpoints.dataset", "build_dataloader_from_cfg"),
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
ok = True
|
|
|
|
root = add_openpoints_to_path()
|
|
print(f"openpoints root: {root or 'NOT FOUND'}")
|
|
if root is None:
|
|
ok = False
|
|
|
|
# SKIP_CUDA_CHECK exists for the GPU-free setup phase: the extensions can be
|
|
# built and imported without a card present, and querying the device would
|
|
# fail on a machine whose GPU is absent or still occupied. Import coverage
|
|
# is unaffected -- only the device query is skipped.
|
|
skip_cuda = bool(os.environ.get("SKIP_CUDA_CHECK"))
|
|
|
|
try:
|
|
import torch
|
|
line = f"torch {torch.__version__} | cuda {torch.version.cuda}"
|
|
if skip_cuda:
|
|
print(line + " | device check skipped (SKIP_CUDA_CHECK)")
|
|
else:
|
|
print(line + f" | available {torch.cuda.is_available()}")
|
|
if torch.cuda.is_available():
|
|
print(f"device: {torch.cuda.get_device_name(0)}")
|
|
else:
|
|
print("device: none visible -- fine for setup, required to train")
|
|
except Exception as e: # noqa: BLE001
|
|
print(f"torch import failed: {e}")
|
|
return 1
|
|
|
|
print()
|
|
for name in MODULES:
|
|
try:
|
|
__import__(name)
|
|
print(f"{name:32s} OK")
|
|
except Exception as e: # noqa: BLE001
|
|
ok = False
|
|
print(f"{name:32s} FAIL {type(e).__name__}: {e}")
|
|
|
|
print()
|
|
for mod, attr in FROM_IMPORTS:
|
|
try:
|
|
m = __import__(mod, fromlist=[attr])
|
|
getattr(m, attr)
|
|
print(f"{mod + '.' + attr:32s} OK")
|
|
except Exception as e: # noqa: BLE001
|
|
ok = False
|
|
print(f"{mod + '.' + attr:32s} FAIL {type(e).__name__}: {e}")
|
|
|
|
print()
|
|
print("ALL OK" if ok else "SOME CHECKS FAILED")
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|