kyy/llm_trainer
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix encode

Browse Source
This commit is contained in:
hiyouga committed 2023-08-04 23:27:55 +08:00
1 parent b4852f9406
commit 8172ad1b5e
2 files changed
+19 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/llmtuner/dsets/preprocess.py
+1 -1
View File
@@ -55,7 +55,7 @@ def preprocess_dataset(
for query, response, history, prefix in construct_example(examples):
input_ids, labels = [], []
for source_ids, target_ids in template.get_dialog(tokenizer, query, response, history, prefix):
for source_ids, target_ids in template.get_dialog(tokenizer, query, response, history, prefix): # TODO: fix bos
if len(source_ids) > data_args.max_source_length:
source_ids = source_ids[:data_args.max_source_length]
if len(target_ids) > data_args.max_target_length - 1: # eos token