add codegemma

2024-04-16 00:11:15 +08:00
1 parent e0dbac2845
commit 6543f3d449
@@ -48,7 +48,6 @@ def load_tokenizer(model_args: "ModelArguments") -> "PreTrainedTokenizer":
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            use_fast=True,
-            split_special_tokens=model_args.split_special_tokens,
            padding_side="right",
            **init_kwargs,
        )