kyy/llm_trainer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix #4005 #4013

Browse Source
This commit is contained in:
hiyouga committed 2024-06-03 19:12:29 +08:00
1 parent 1539c72b94
commit eed33862bc
24 files changed
+52 -52
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/extras/badam/llama3_lora_sft.yaml
+2 -2
View File
@@ -28,10 +28,10 @@ overwrite_output_dir: true
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
learning_rate: 0.0001
learning_rate: 1.0e-4
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_steps: 0.1
warmup_ratio: 0.1
pure_bf16: true
### eval
examples/extras/fsdp_qlora/llama3_lora_sft.yaml
+2 -2
View File
@@ -29,10 +29,10 @@ overwrite_output_dir: true
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
learning_rate: 0.0001
learning_rate: 1.0e-4
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_steps: 0.1
warmup_ratio: 0.1
fp16: true
### eval
examples/extras/galore/llama3_full_sft.yaml
+2 -2
View File
@@ -29,10 +29,10 @@ overwrite_output_dir: true
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 1
learning_rate: 0.0001
learning_rate: 1.0e-4
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_steps: 0.1
warmup_ratio: 0.1
pure_bf16: true
### eval
examples/extras/llama_pro/llama3_freeze_sft.yaml
+2 -2
View File
@@ -27,10 +27,10 @@ overwrite_output_dir: true
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
learning_rate: 0.0001
learning_rate: 1.0e-4
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_steps: 0.1
warmup_ratio: 0.1
fp16: true
### eval
examples/extras/loraplus/llama3_lora_sft.yaml
+2 -2
View File
@@ -26,10 +26,10 @@ overwrite_output_dir: true
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
learning_rate: 0.0001
learning_rate: 1.0e-4
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_steps: 0.1
warmup_ratio: 0.1
fp16: true
### eval
examples/extras/mod/llama3_full_sft.yaml
+2 -2
View File
@@ -26,10 +26,10 @@ overwrite_output_dir: true
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
optim: paged_adamw_8bit
learning_rate: 0.0001
learning_rate: 1.0e-4
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_steps: 0.1
warmup_ratio: 0.1
pure_bf16: true
### eval