36 lines
1.8 KiB
Python
36 lines
1.8 KiB
Python
import os
|
|
import autorag
|
|
import click
|
|
from autorag.evaluator import Evaluator
|
|
from dotenv import load_dotenv
|
|
|
|
from llama_index.llms.ollama import Ollama
|
|
|
|
data_path = './projects/example_01' # 폴더명 확인
|
|
|
|
OLLAMA_BASE_URL = "autorag-ollama:11434"
|
|
autorag.generator_models["ollama"] = autorag.LazyInit(Ollama, base_url=OLLAMA_BASE_URL, model="gemma3:12b", request_timeout=300, num_gpus=1)
|
|
autorag.generator_models["ollama"] = autorag.LazyInit(Ollama, base_url=OLLAMA_BASE_URL, model="phi4", request_timeout=300, num_gpus=1)
|
|
autorag.generator_models["ollama"] = autorag.LazyInit(Ollama, base_url=OLLAMA_BASE_URL, model="deepseek-r1:14b", request_timeout=300, num_gpus=1)
|
|
autorag.generator_models["ollama"] = autorag.LazyInit(Ollama, base_url=OLLAMA_BASE_URL, model="aya-expanse:8b", request_timeout=300, num_gpus=1)
|
|
|
|
data_index=1
|
|
|
|
@click.command()
|
|
@click.option('--config', type=click.Path(exists=True), default=os.path.join(data_path, 'config.yaml'))
|
|
@click.option('--qa_data_path', type=click.Path(exists=True), default=os.path.join(data_path, f'qa_{data_index}.parquet'))
|
|
@click.option('--corpus_data_path', type=click.Path(exists=True), default=os.path.join(data_path, f'corpus_{data_index}.parquet'))
|
|
@click.option('--project_dir', type=click.Path(exists=False), default=os.path.join(data_path, f'benchmark_ck{data_index}'))
|
|
|
|
|
|
def main(config, qa_data_path, corpus_data_path, project_dir):
|
|
load_dotenv()
|
|
if os.getenv('OPENAI_API_KEY') is None:
|
|
raise ValueError('OPENAI_API_KEY environment variable is not set')
|
|
if not os.path.exists(project_dir):
|
|
os.makedirs(project_dir)
|
|
evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir=project_dir)
|
|
evaluator.start_trial(config)
|
|
|
|
if __name__ == '__main__':
|
|
main() |