fix #3602

examples/lora_multi_gpu/ds_zero3.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
-# ZeRO-3 enables weight sharding on multiple GPUs
 
-deepspeed --include "localhost:0,1,2,3" \
+NPROC_PER_NODE=4
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 python -m torch.distributed.run \
+    --nproc_per_node $NPROC_PER_NODE \
+    --nnodes 1 \
+    --standalone \
     src/train.py examples/lora_multi_gpu/llama3_lora_sft_ds.yaml