support save full model, replace BOS token

2023-06-27 21:40:11 +08:00
parent 1c732e2537
commit 2e01abfda5
3 changed files with 48 additions and 33 deletions
--- a/src/utils/other.py
+++ b/src/utils/other.py
@@ -5,13 +5,11 @@ import torch
 import logging
 from typing import Dict, List, Optional

-from transformers.trainer import TRAINER_STATE_NAME
+from transformers.trainer import TRAINER_STATE_NAME, WEIGHTS_NAME
 from transformers.modeling_utils import PreTrainedModel
 from transformers.generation.utils import LogitsProcessorList
 from transformers.generation.logits_process import LogitsProcessor

-from peft.utils import WEIGHTS_NAME
-

 IGNORE_INDEX = -100
 VALUE_HEAD_FILE_NAME = "value_head.bin"