support val set in streaming mode

This commit is contained in:
hiyouga
2023-08-09 23:00:26 +08:00
parent 572ea3bafb
commit d86ea314a1
10 changed files with 58 additions and 50 deletions

View File

@@ -46,7 +46,7 @@ def run_sft(
data_collator=data_collator,
callbacks=callbacks,
compute_metrics=ComputeMetrics(tokenizer) if training_args.predict_with_generate else None,
**split_dataset(dataset, data_args.dev_ratio, training_args.do_train)
**split_dataset(dataset, data_args, training_args)
)
# Keyword arguments for `model.generate`