add codegemma

2024-04-16 00:11:15 +08:00
parent e0dbac2845
commit 6543f3d449
4 changed files with 23 additions and 7 deletions
--- a/src/llmtuner/model/loader.py
+++ b/src/llmtuner/model/loader.py
@@ -48,7 +48,6 @@ def load_tokenizer(model_args: "ModelArguments") -> "PreTrainedTokenizer":
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            use_fast=True,
-            split_special_tokens=model_args.split_special_tokens,
            padding_side="right",
            **init_kwargs,
        )