Initial commit

.dockerignore

@@ -0,0 +1,165 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+venv*/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+pytest.ini
+.DS_Store
+projects/tutorial_1
+!projects/tutorial_1/config.yaml

.env.sample

@@ -0,0 +1,2 @@
+OPENAI_API_KEY=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
+LLAMA_CLOUD_API_KEY=llx-MkHkuDxnSxXEHvsIPAtjEZl4iSB8pHS1mgYDVZQlA690LUub

.gitignore

@@ -0,0 +1,169 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+.DS_Store
+pytest.ini
+projects
+test_projects
+
+# Visual Studio Code
+.vscode/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "autorag-frontend"]
+	path = autorag-frontend
+	url = https://github.com/Auto-RAG/autorag-frontend.git

Dockerfile

@@ -0,0 +1,28 @@
+# Base stage: Install common dependencies
+FROM python:3.10-slim AS base
+
+# Set working directory and environment variables
+WORKDIR /usr/src/app
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1
+
+# Copy only requirements files first to leverage Docker cache
+COPY pyproject.toml ./ 
+
+# Install system and Python dependencies in a single layer
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    gcc \
+    libssl-dev && \
+    pip install --no-cache-dir --upgrade pip setuptools setuptools-scm && \
+    rm -rf /var/lib/apt/lists/* 
+
+# Copy project files
+COPY . .
+
+# Install base project
+RUN pip install --no-cache-dir -e . 
+
+CMD ["bash"]

README.md

@@ -0,0 +1,154 @@
+# AutoRAG Evaluation
+
+이 문서는 AutoRAG을 활용하여 RAG 파이프라인을 설정하고 평가하는 과정에 대한 안내입니다.
+
+---
+
+## 📌 환경 세팅
+
+1. `.env` 파일을 설정합니다. (`.env.sample` 참고)
+2. Docker 이미지를 빌드합니다.
+
+```bash
+   docker build -t autorag-base .
+```
+
+3. Docker Compose를 실행하여 서비스를 시작합니다.
+
+```bash
+   docker compose up -d
+```
+
+4. Hugginface embedding, Ollama LLM을 위한 추가 모듈 설치를 시작합니다.
+
+```bash
+   pip install -r requirements_custom.txt
+```
+
+---
+
+## 📂 데이터 생성
+
+RAG 평가를 위해 **QA 데이터 세트**와 **코퍼스 데이터 세트**가 필요합니다.
+
+### 1️⃣ 프로젝트 폴더 생성
+
+```bash
+cd projects
+mkdir -p "project_name"
+cd "project_name"
+mkdir -p raw_data config
+```
+
+- **`raw_data/`**: 분석할 원본 데이터를 저장 (`.pdf` 등)
+- **`config/`**: 파싱(`parse.yaml`), 청킹(`chunk.yaml`) 설정 파일을 저장
+
+### 2️⃣ 파싱 설정 (`parse.yaml`)
+
+파싱 모듈을 설정합니다.
+
+```yaml
+modules:
+  - module_type: langchain_parse
+    parse_method: pdfminer
+```
+
+여러 개의 파싱 모듈을 동시에 사용할 수도 있습니다.
+
+### 3️⃣ 청킹 설정 (`chunk.yaml`)
+
+청킹 모듈을 설정합니다.
+
+```yaml
+modules:
+  - module_type: llama_index_chunk
+    chunk_method: Token
+    chunk_size: 1024
+    chunk_overlap: 24
+    add_file_name: en
+```
+
+여러 개의 청킹 모듈을 사용할 경우, QA 데이터와 매핑해야 합니다.
+
+### 4️⃣ QA 데이터 생성
+
+`raw_data/`에 저장된 파일을 바탕으로 **파싱 → 청킹 → QA 데이터 생성**을 진행합니다.  
+QA 데이터는 `GPT-4o-mini` 모델을 사용하여 **20건**을 생성합니다.
+
+```bash
+cd autorag-workspace
+sh making.sh
+```
+
+---
+
+## 🔍 RAG Pipeline 평가
+
+### 1️⃣ Ollama 모델 다운로드
+
+WSL(Windows Subsystem for Linux)에서 실행합니다.
+
+```bash
+docker exec -it autorag-ollama bash
+ollama pull phi4
+ollama list
+```
+
+### 2️⃣ AutoRAG 평가 실행
+
+```bash
+cd Autorag-workspace
+python main.py
+```
+
+### 3️⃣ 평가 결과 확인
+
+평가 결과는 프로젝트 폴더 내 `benchmark_*` 경로에 저장됩니다.
+
+#### ✅ 전체 파이프라인 평가 결과
+
+```bash
+cd projects/프로젝트이름/benchmark_{*}/*/
+summary.csv
+```
+
+#### ✅ 세부 평가 결과
+
+- **검색기 노드 라인 평가 결과**
+  ```bash
+  cd ./retrieve_node_line
+  summary.csv
+  ```
+- **검색 노드 평가 결과**
+  ```bash
+  cd ./retrieve_node_line/retrival
+  summary.csv
+  ```
+- **리랭커 노드 평가 결과**
+  ```bash
+  cd ./retrieve_node_line/passage_reranker
+  summary.csv
+  ```
+- **생성기 노드 라인 평가 결과**
+  ```bash
+  cd ./post_retrieve_node_line
+  summary.csv
+  ```
+- **생성 노드 평가 결과**
+  ```bash
+  cd ./post_retrieve_node_line/generator
+  summary.csv
+  ```
+
+> 📌 **참고:** `./projects/example_01` 폴더는 데이터 생성부터 평가까지 진행된 예제입니다.
+
+---
+
+## 📊 평가 대시보드 실행
+
+```bash
+cd Autorag-workspace
+sh dashboard.sh
+```
+
+AutoRAG 평가 결과를 자세히 확인할 수 있습니다.

autorag-workspace/autorag/VERSION

@@ -0,0 +1 @@
+0.3.14

autorag-workspace/autorag/__init__.py

@@ -0,0 +1,113 @@
+import logging
+import logging.config
+import os
+import sys
+from random import random
+from typing import List, Any
+
+from llama_index.core.embeddings.mock_embed_model import MockEmbedding
+from llama_index.core.base.llms.types import CompletionResponse
+from llama_index.core.llms.mock import MockLLM
+from llama_index.llms.bedrock import Bedrock
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.embeddings.openai import OpenAIEmbeddingModelType
+
+from llama_index.llms.openai import OpenAI
+from llama_index.llms.openai_like import OpenAILike
+from langchain_openai.embeddings import OpenAIEmbeddings
+from rich.logging import RichHandler
+
+from llama_index.llms.ollama import Ollama
+
+version_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "VERSION")
+
+with open(version_path, "r") as f:
+	__version__ = f.read().strip()
+
+
+class LazyInit:
+	def __init__(self, factory, *args, **kwargs):
+		self._factory = factory
+		self._args = args
+		self._kwargs = kwargs
+		self._instance = None
+
+	def __call__(self):
+		if self._instance is None:
+			self._instance = self._factory(*self._args, **self._kwargs)
+		return self._instance
+
+	def __getattr__(self, name):
+		if self._instance is None:
+			self._instance = self._factory(*self._args, **self._kwargs)
+		return getattr(self._instance, name)
+
+
+rich_format = "[%(filename)s:%(lineno)s] >> %(message)s"
+logging.basicConfig(
+	level="INFO", format=rich_format, handlers=[RichHandler(rich_tracebacks=True)]
+)
+logger = logging.getLogger("AutoRAG")
+
+
+def handle_exception(exc_type, exc_value, exc_traceback):
+	logger = logging.getLogger("AutoRAG")
+	logger.error("Unexpected exception", exc_info=(exc_type, exc_value, exc_traceback))
+
+
+sys.excepthook = handle_exception
+
+
+class AutoRAGBedrock(Bedrock):
+	async def acomplete(
+		self, prompt: str, formatted: bool = False, **kwargs: Any
+	) -> CompletionResponse:
+		return self.complete(prompt, formatted=formatted, **kwargs)
+
+
+generator_models = {
+	"openai": OpenAI,
+	"openailike": OpenAILike,
+	"mock": MockLLM,
+	"bedrock": AutoRAGBedrock,
+	"ollama": Ollama,
+}
+
+# embedding_models = {
+
+# }
+
+try:
+	from llama_index.llms.huggingface import HuggingFaceLLM
+	from llama_index.llms.ollama import Ollama
+
+	generator_models["huggingfacellm"] = HuggingFaceLLM
+	generator_models["ollama"] = Ollama
+
+except ImportError:
+	logger.info(
+		"You are using API version of AutoRAG. "
+		"To use local version, run pip install 'AutoRAG[gpu]'"
+	)
+
+# try:
+# 	from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+# 	embedding_models["hf_all_mpnet_base_v2"] = HuggingFaceEmbedding # 250312 변경 - 김용연
+# 	embedding_models["hf_KURE-v1"] = HuggingFaceEmbedding # 250312 변경 - 김용연
+# 	embedding_models["hf_snowflake-arctic-embed-l-v2.0-ko"] = HuggingFaceEmbedding # 250313 변경 - 김용연
+
+# except ImportError:
+# 	logger.info(
+# 		"You are using API version of AutoRAG."
+# 		"To use local version, run pip install 'AutoRAG[gpu]'"
+# 	)
+
+try:
+	import transformers
+
+	transformers.logging.set_verbosity_error()
+except ImportError:
+	logger.info(
+		"You are using API version of AutoRAG."
+		"To use local version, run pip install 'AutoRAG[gpu]'"
+	)

autorag-workspace/autorag/chunker.py

@@ -0,0 +1,51 @@
+import logging
+import os
+import shutil
+from typing import Optional
+
+import pandas as pd
+
+from autorag.data.chunk.run import run_chunker
+from autorag.data.utils.util import load_yaml, get_param_combinations
+
+logger = logging.getLogger("AutoRAG")
+
+
+class Chunker:
+	def __init__(self, raw_df: pd.DataFrame, project_dir: Optional[str] = None):
+		self.parsed_raw = raw_df
+		self.project_dir = project_dir if project_dir is not None else os.getcwd()
+
+	@classmethod
+	def from_parquet(
+		cls, parsed_data_path: str, project_dir: Optional[str] = None
+	) -> "Chunker":
+		if not os.path.exists(parsed_data_path):
+			raise ValueError(f"parsed_data_path {parsed_data_path} does not exist.")
+		if not parsed_data_path.endswith("parquet"):
+			raise ValueError(
+				f"parsed_data_path {parsed_data_path} is not a parquet file."
+			)
+		parsed_result = pd.read_parquet(parsed_data_path, engine="pyarrow")
+		return cls(parsed_result, project_dir)
+
+	def start_chunking(self, yaml_path: str):
+		if not os.path.exists(self.project_dir):
+			os.makedirs(self.project_dir)
+
+		# Copy YAML file to the trial directory
+		shutil.copy(yaml_path, os.path.join(self.project_dir, "chunk_config.yaml"))
+
+		# load yaml file
+		modules = load_yaml(yaml_path)
+
+		input_modules, input_params = get_param_combinations(modules)
+
+		logger.info("Chunking Start...")
+		run_chunker(
+			modules=input_modules,
+			module_params=input_params,
+			parsed_result=self.parsed_raw,
+			project_dir=self.project_dir,
+		)
+		logger.info("Chunking Done!")

autorag-workspace/autorag/cli.py

@@ -0,0 +1,209 @@
+import importlib.resources
+import logging
+import os
+import pathlib
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+import click
+import nest_asyncio
+
+from autorag import dashboard
+from autorag.deploy import extract_best_config as original_extract_best_config
+from autorag.deploy.api import ApiRunner
+from autorag.evaluator import Evaluator
+from autorag.validator import Validator
+
+logger = logging.getLogger("AutoRAG")
+
+autorag_dir = os.path.dirname(os.path.realpath(__file__))
+version_file = os.path.join(autorag_dir, "VERSION")
+with open(version_file, "r") as f:
+	__version__ = f.read().strip()
+
+
+@click.group()
+@click.version_option(__version__)
+def cli():
+	pass
+
+
+@click.command()
+@click.option(
+	"--config",
+	"-c",
+	help="Path to config yaml file. Must be yaml or yml file.",
+	type=str,
+)
+@click.option(
+	"--qa_data_path", help="Path to QA dataset. Must be parquet file.", type=str
+)
+@click.option(
+	"--corpus_data_path", help="Path to corpus dataset. Must be parquet file.", type=str
+)
+@click.option(
+	"--project_dir", help="Path to project directory.", type=str, default=None
+)
+@click.option(
+	"--skip_validation",
+	help="Skip validation or not. Default is False.",
+	type=bool,
+	default=False,
+)
+def evaluate(config, qa_data_path, corpus_data_path, project_dir, skip_validation):
+	if not config.endswith(".yaml") and not config.endswith(".yml"):
+		raise ValueError(f"Config file {config} is not a yaml or yml file.")
+	if not os.path.exists(config):
+		raise ValueError(f"Config file {config} does not exist.")
+	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir=project_dir)
+	evaluator.start_trial(config, skip_validation=skip_validation)
+
+
+@click.command()
+@click.option(
+	"--config_path", type=str, help="Path to extracted config yaml file.", default=None
+)
+@click.option("--host", type=str, default="0.0.0.0", help="Host address")
+@click.option("--port", type=int, default=8000, help="Port number")
+@click.option(
+	"--trial_dir",
+	type=click.Path(file_okay=False, dir_okay=True, exists=True),
+	default=None,
+	help="Path to trial directory.",
+)
+@click.option(
+	"--project_dir", help="Path to project directory.", type=str, default=None
+)
+@click.option(
+	"--remote", help="Run the API server in remote mode.", type=bool, default=False
+)
+def run_api(config_path, host, port, trial_dir, project_dir, remote: bool):
+	if trial_dir is None:
+		runner = ApiRunner.from_yaml(config_path, project_dir=project_dir)
+	else:
+		runner = ApiRunner.from_trial_folder(trial_dir)
+	logger.info(f"Running API server at {host}:{port}...")
+	nest_asyncio.apply()
+	runner.run_api_server(host, port, remote=remote)
+
+
+@click.command()
+@click.option(
+	"--yaml_path", type=click.Path(path_type=Path), help="Path to the YAML file."
+)
+@click.option(
+	"--project_dir",
+	type=click.Path(path_type=Path),
+	help="Path to the project directory.",
+)
+@click.option(
+	"--trial_path", type=click.Path(path_type=Path), help="Path to the trial directory."
+)
+def run_web(
+	yaml_path: Optional[str], project_dir: Optional[str], trial_path: Optional[str]
+):
+	try:
+		with importlib.resources.path("autorag", "web.py") as web_path:
+			web_py_path = str(web_path)
+	except ImportError:
+		raise ImportError(
+			"Could not locate the web.py file within the autorag package."
+			" Please ensure that autorag is correctly installed."
+		)
+
+	if not yaml_path and not trial_path:
+		raise ValueError("yaml_path or trial_path must be given.")
+	elif yaml_path and trial_path:
+		raise ValueError("yaml_path and trial_path cannot be given at the same time.")
+	elif yaml_path and not project_dir:
+		subprocess.run(
+			["streamlit", "run", web_py_path, "--", "--yaml_path", yaml_path]
+		)
+	elif yaml_path and project_dir:
+		subprocess.run(
+			[
+				"streamlit",
+				"run",
+				web_py_path,
+				"--",
+				"--yaml_path",
+				yaml_path,
+				"--project_dir",
+				project_dir,
+			]
+		)
+	elif trial_path:
+		subprocess.run(
+			["streamlit", "run", web_py_path, "--", "--trial_path", trial_path]
+		)
+
+
+@click.command()
+@click.option(
+	"--trial_dir",
+	type=click.Path(dir_okay=True, file_okay=False, exists=True),
+	required=True,
+)
+@click.option(
+	"--port", type=int, default=7690, help="Port number. The default is 7690."
+)
+def run_dashboard(trial_dir: str, port: int):
+	dashboard.run(trial_dir, port=port)
+
+
+@click.command()
+@click.option("--trial_path", type=click.Path(), help="Path to the trial directory.")
+@click.option(
+	"--output_path",
+	type=click.Path(),
+	help="Path to the output directory." " Must be .yaml or .yml file.",
+)
+def extract_best_config(trial_path: str, output_path: str):
+	original_extract_best_config(trial_path, output_path)
+
+
+@click.command()
+@click.option("--trial_path", help="Path to trial directory.", type=str)
+def restart_evaluate(trial_path):
+	if not os.path.exists(trial_path):
+		raise ValueError(f"trial_path {trial_path} does not exist.")
+	project_dir = str(pathlib.PurePath(trial_path).parent)
+	qa_data_path = os.path.join(project_dir, "data", "qa.parquet")
+	corpus_data_path = os.path.join(project_dir, "data", "corpus.parquet")
+	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir)
+	evaluator.restart_trial(trial_path)
+
+
+@click.command()
+@click.option(
+	"--config",
+	"-c",
+	help="Path to config yaml file. Must be yaml or yml file.",
+	type=str,
+)
+@click.option(
+	"--qa_data_path", help="Path to QA dataset. Must be parquet file.", type=str
+)
+@click.option(
+	"--corpus_data_path", help="Path to corpus dataset. Must be parquet file.", type=str
+)
+def validate(config, qa_data_path, corpus_data_path):
+	if not config.endswith(".yaml") and not config.endswith(".yml"):
+		raise ValueError(f"Config file {config} is not a parquet file.")
+	if not os.path.exists(config):
+		raise ValueError(f"Config file {config} does not exist.")
+	validator = Validator(qa_data_path=qa_data_path, corpus_data_path=corpus_data_path)
+	validator.validate(config)
+
+
+cli.add_command(evaluate, "evaluate")
+cli.add_command(run_api, "run_api")
+cli.add_command(run_web, "run_web")
+cli.add_command(run_dashboard, "dashboard")
+cli.add_command(extract_best_config, "extract_best_config")
+cli.add_command(restart_evaluate, "restart_evaluate")
+cli.add_command(validate, "validate")
+
+if __name__ == "__main__":
+	cli()

autorag-workspace/autorag/dashboard.py

@@ -0,0 +1,199 @@
+import ast
+import logging
+import os
+from typing import Dict, List
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import panel as pn
+import seaborn as sns
+import yaml
+from bokeh.models import NumberFormatter, BooleanFormatter
+
+from autorag.utils.util import dict_to_markdown, dict_to_markdown_table
+
+pn.extension(
+	"terminal",
+	"tabulator",
+	"mathjax",
+	"ipywidgets",
+	console_output="disable",
+	sizing_mode="stretch_width",
+	css_files=[
+		"https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/all.min.css"
+	],
+)
+logger = logging.getLogger("AutoRAG")
+
+
+def find_node_dir(trial_dir: str) -> List[str]:
+	trial_summary_df = pd.read_csv(os.path.join(trial_dir, "summary.csv"))
+	result_paths = []
+	for idx, row in trial_summary_df.iterrows():
+		node_line_name = row["node_line_name"]
+		node_type = row["node_type"]
+		result_paths.append(os.path.join(trial_dir, node_line_name, node_type))
+	return result_paths
+
+
+def get_metric_values(node_summary_df: pd.DataFrame) -> Dict:
+	non_metric_column_names = [
+		"filename",
+		"module_name",
+		"module_params",
+		"execution_time",
+		"average_output_token",
+		"is_best",
+	]
+	best_row = node_summary_df.loc[node_summary_df["is_best"]].drop(
+		columns=non_metric_column_names, errors="ignore"
+	)
+	assert len(best_row) == 1, "The best module must be only one."
+	return best_row.iloc[0].to_dict()
+
+
+def make_trial_summary_md(trial_dir):
+	markdown_text = f"""# Trial Result Summary
+- Trial Directory : {trial_dir}
+
+"""
+	node_dirs = find_node_dir(trial_dir)
+	for node_dir in node_dirs:
+		node_summary_filepath = os.path.join(node_dir, "summary.csv")
+		node_type = os.path.basename(node_dir)
+		node_summary_df = pd.read_csv(node_summary_filepath)
+		best_row = node_summary_df.loc[node_summary_df["is_best"]].iloc[0]
+		metric_dict = get_metric_values(node_summary_df)
+		markdown_text += f"""---
+
+## {node_type} best module
+
+### Module Name
+
+{best_row['module_name']}
+
+### Module Params
+
+{dict_to_markdown(ast.literal_eval(best_row['module_params']), level=3)}
+
+### Metric Values
+
+{dict_to_markdown_table(metric_dict, key_column_name='metric_name', value_column_name='metric_value')}
+
+"""
+
+	return markdown_text
+
+
+def node_view(node_dir: str):
+	non_metric_column_names = [
+		"filename",
+		"module_name",
+		"module_params",
+		"execution_time",
+		"average_output_token",
+		"is_best",
+	]
+	summary_df = pd.read_csv(os.path.join(node_dir, "summary.csv"))
+	bokeh_formatters = {
+		"float": NumberFormatter(format="0.000"),
+		"bool": BooleanFormatter(),
+	}
+	first_df = pd.read_parquet(os.path.join(node_dir, "0.parquet"), engine="pyarrow")
+
+	each_module_df_widget = pn.widgets.Tabulator(
+		pd.DataFrame(columns=first_df.columns),
+		name="Module DataFrame",
+		formatters=bokeh_formatters,
+		pagination="local",
+		page_size=20,
+		widths=150,
+	)
+
+	def change_module_widget(event):
+		if event.column == "detail":
+			filename = summary_df["filename"].iloc[event.row]
+			filepath = os.path.join(node_dir, filename)
+			each_module_df = pd.read_parquet(filepath, engine="pyarrow")
+			each_module_df_widget.value = each_module_df
+
+	df_widget = pn.widgets.Tabulator(
+		summary_df,
+		name="Summary DataFrame",
+		formatters=bokeh_formatters,
+		buttons={"detail": '<i class="fa fa-eye"></i>'},
+		widths=150,
+	)
+	df_widget.on_click(change_module_widget)
+
+	try:
+		fig, ax = plt.subplots(figsize=(10, 5))
+		metric_df = summary_df.drop(columns=non_metric_column_names, errors="ignore")
+		sns.stripplot(data=metric_df, ax=ax)
+		strip_plot_pane = pn.pane.Matplotlib(fig, tight=True)
+
+		fig2, ax2 = plt.subplots(figsize=(10, 5))
+		sns.boxplot(data=metric_df, ax=ax2)
+		box_plot_pane = pn.pane.Matplotlib(fig2, tight=True)
+		plot_pane = pn.Row(strip_plot_pane, box_plot_pane)
+
+		layout = pn.Column(
+			"## Summary distribution plot",
+			plot_pane,
+			"## Summary DataFrame",
+			df_widget,
+			"## Module Result DataFrame",
+			each_module_df_widget,
+		)
+	except Exception as e:
+		logger.error(f"Skipping make boxplot and stripplot with error {e}")
+		layout = pn.Column("## Summary DataFrame", df_widget)
+	layout.servable()
+	return layout
+
+
+CSS = """
+div.card-margin:nth-child(1) {
+    max-height: 300px;
+}
+div.card-margin:nth-child(2) {
+    max-height: 400px;
+}
+"""
+
+
+def yaml_to_markdown(yaml_filepath):
+	markdown_content = ""
+	with open(yaml_filepath, "r", encoding="utf-8") as file:
+		try:
+			content = yaml.safe_load(file)
+			markdown_content += f"## {os.path.basename(yaml_filepath)}\n```yaml\n{yaml.safe_dump(content, allow_unicode=True)}\n```\n\n"
+		except yaml.YAMLError as exc:
+			print(f"Error in {yaml_filepath}: {exc}")
+	return markdown_content
+
+
+def run(trial_dir: str, port: int = 7690):
+	trial_summary_md = make_trial_summary_md(trial_dir=trial_dir)
+	trial_summary_tab = pn.pane.Markdown(trial_summary_md, sizing_mode="stretch_width")
+
+	node_views = [
+		(str(os.path.basename(node_dir)), node_view(node_dir))
+		for node_dir in find_node_dir(trial_dir)
+	]
+
+	yaml_file_markdown = yaml_to_markdown(os.path.join(trial_dir, "config.yaml"))
+
+	yaml_file = pn.pane.Markdown(yaml_file_markdown, sizing_mode="stretch_width")
+
+	tabs = pn.Tabs(
+		("Summary", trial_summary_tab),
+		*node_views,
+		("Used YAML file", yaml_file),
+		dynamic=True,
+	)
+
+	template = pn.template.FastListTemplate(
+		site="AutoRAG", title="Dashboard", main=[tabs], raw_css=[CSS]
+	).servable()
+	template.show(port=port)

autorag-workspace/autorag/data/__init__.py

@@ -0,0 +1,109 @@
+import logging
+from typing import List, Callable
+
+from langchain_community.document_loaders import (
+	PDFMinerLoader,
+	PDFPlumberLoader,
+	PyPDFium2Loader,
+	PyPDFLoader,
+	PyMuPDFLoader,
+	UnstructuredPDFLoader,
+	CSVLoader,
+	JSONLoader,
+	UnstructuredMarkdownLoader,
+	BSHTMLLoader,
+	UnstructuredXMLLoader,
+	DirectoryLoader,
+)
+from langchain_unstructured import UnstructuredLoader
+from langchain_upstage import UpstageDocumentParseLoader
+
+from llama_index.core.node_parser import (
+	TokenTextSplitter,
+	SentenceSplitter,
+	SentenceWindowNodeParser,
+	SemanticSplitterNodeParser,
+	SemanticDoubleMergingSplitterNodeParser,
+	SimpleFileNodeParser,
+)
+from langchain.text_splitter import (
+	RecursiveCharacterTextSplitter,
+	CharacterTextSplitter,
+	KonlpyTextSplitter,
+	SentenceTransformersTokenTextSplitter,
+)
+
+from autorag import LazyInit
+
+logger = logging.getLogger("AutoRAG")
+
+parse_modules = {
+	# PDF
+	"pdfminer": PDFMinerLoader,
+	"pdfplumber": PDFPlumberLoader,
+	"pypdfium2": PyPDFium2Loader,
+	"pypdf": PyPDFLoader,
+	"pymupdf": PyMuPDFLoader,
+	"unstructuredpdf": UnstructuredPDFLoader,
+	# Common File Types
+	# 1. CSV
+	"csv": CSVLoader,
+	# 2. JSON
+	"json": JSONLoader,
+	# 3. Markdown
+	"unstructuredmarkdown": UnstructuredMarkdownLoader,
+	# 4. HTML
+	"bshtml": BSHTMLLoader,
+	# 5. XML
+	"unstructuredxml": UnstructuredXMLLoader,
+	# 6. All files
+	"directory": DirectoryLoader,
+	"unstructured": UnstructuredLoader,
+	"upstagedocumentparse": UpstageDocumentParseLoader,
+}
+
+chunk_modules = {
+	# Llama Index
+	# Token
+	"token": TokenTextSplitter,
+	# Sentence
+	"sentence": SentenceSplitter,
+	# window
+	"sentencewindow": SentenceWindowNodeParser,
+	# Semantic
+	"semantic_llama_index": SemanticSplitterNodeParser,
+	"semanticdoublemerging": SemanticDoubleMergingSplitterNodeParser,
+	# Simple
+	"simplefile": SimpleFileNodeParser,
+	# LangChain
+	# Token
+	"sentencetransformerstoken": SentenceTransformersTokenTextSplitter,
+	# Character
+	"recursivecharacter": RecursiveCharacterTextSplitter,
+	"character": CharacterTextSplitter,
+	# Sentence
+	"konlpy": KonlpyTextSplitter,
+}
+
+
+def split_by_sentence_kiwi() -> Callable[[str], List[str]]:
+	try:
+		from kiwipiepy import Kiwi
+	except ImportError:
+		raise ImportError(
+			"You need to install kiwipiepy to use 'ko_kiwi' tokenizer. "
+			"Please install kiwipiepy by running 'pip install kiwipiepy'. "
+			"Or install Korean version of AutoRAG by running 'pip install AutoRAG[ko]'."
+		)
+	kiwi = Kiwi()
+
+	def split(text: str) -> List[str]:
+		kiwi_result = kiwi.split_into_sents(text)
+		sentences = list(map(lambda x: x.text, kiwi_result))
+
+		return sentences
+
+	return split
+
+
+sentence_splitter_modules = {"kiwi": LazyInit(split_by_sentence_kiwi)}

autorag-workspace/autorag/data/chunk/__init__.py

@@ -0,0 +1,2 @@
+from .llama_index_chunk import llama_index_chunk
+from .langchain_chunk import langchain_chunk

autorag-workspace/autorag/data/chunk/base.py

@@ -0,0 +1,128 @@
+import functools
+import logging
+from typing import Tuple, List, Dict, Any
+
+import pandas as pd
+
+from autorag.embedding.base import EmbeddingModel
+from autorag.data import chunk_modules, sentence_splitter_modules
+from autorag.utils import result_to_dataframe
+
+logger = logging.getLogger("AutoRAG")
+
+
+def chunker_node(func):
+	@functools.wraps(func)
+	@result_to_dataframe(["doc_id", "contents", "path", "start_end_idx", "metadata"])
+	def wrapper(
+		parsed_result: pd.DataFrame, chunk_method: str, **kwargs
+	) -> Tuple[
+		List[str], List[str], List[str], List[Tuple[int, int]], List[Dict[str, Any]]
+	]:
+		logger.info(f"Running chunker - {func.__name__} module...")
+
+		# get texts from parsed_result
+		texts = parsed_result["texts"].tolist()
+
+		# get filenames from parsed_result when 'add_file_name' is setting
+		file_name_language = kwargs.pop("add_file_name", None)
+		metadata_list = make_metadata_list(parsed_result)
+
+		# run chunk module
+		if func.__name__ in ["llama_index_chunk", "langchain_chunk"]:
+			chunk_instance = __get_chunk_instance(
+				func.__name__, chunk_method.lower(), **kwargs
+			)
+			result = func(
+				texts=texts,
+				chunker=chunk_instance,
+				file_name_language=file_name_language,
+				metadata_list=metadata_list,
+			)
+			del chunk_instance
+			return result
+		else:
+			raise ValueError(f"Unsupported module_type: {func.__name__}")
+
+	return wrapper
+
+
+def make_metadata_list(parsed_result: pd.DataFrame) -> List[Dict[str, str]]:
+	metadata_list = [{} for _ in range(len(parsed_result["texts"]))]
+
+	def _make_metadata_pure(
+		lst: List[str], key: str, metadata_lst: List[Dict[str, str]]
+	):
+		for value, metadata in zip(lst, metadata_lst):
+			metadata[key] = value
+
+	for column in ["page", "last_modified_datetime", "path"]:
+		if column in parsed_result.columns:
+			_make_metadata_pure(parsed_result[column].tolist(), column, metadata_list)
+	return metadata_list
+
+
+def __get_chunk_instance(module_type: str, chunk_method: str, **kwargs):
+	# Add sentence_splitter to kwargs
+	sentence_available_methods = [
+		"semantic_llama_index",
+		"semanticdoublemerging",
+		"sentencewindow",
+	]
+	if chunk_method in sentence_available_methods:
+		# llama index default sentence_splitter is 'nltk -PunktSentenceTokenizer'
+		if "sentence_splitter" in kwargs.keys():
+			sentence_splitter_str = kwargs.pop("sentence_splitter")
+			sentence_splitter_func = sentence_splitter_modules[sentence_splitter_str]()
+			kwargs.update({"sentence_splitter": sentence_splitter_func})
+
+	def get_embedding_model(_embed_model_str: str, _module_type: str):
+		if _embed_model_str == "openai":
+			if _module_type == "langchain_chunk":
+				_embed_model_str = "openai_langchain"
+		return EmbeddingModel.load(_embed_model_str)()
+
+	# Add embed_model to kwargs
+	embedding_available_methods = ["semantic_llama_index", "semantic_langchain"]
+	if chunk_method in embedding_available_methods:
+		# there is no default embed_model, so we have to get it parameter and add it.
+		if "embed_model" not in kwargs.keys():
+			raise ValueError(f"embed_model is required for {chunk_method} method.")
+		embed_model_str = kwargs.pop("embed_model")
+		embed_model = get_embedding_model(embed_model_str, module_type)
+		if chunk_method == "semantic_llama_index":
+			kwargs.update({"embed_model": embed_model})
+		elif chunk_method == "semantic_langchain":
+			kwargs.update({"embeddings": embed_model})
+
+	return chunk_modules[chunk_method](**kwargs)
+
+
+def add_file_name(
+	file_name_language: str, file_names: List[str], chunk_texts: List[str]
+) -> List[str]:
+	if file_name_language == "en":
+		return list(
+			map(
+				lambda x: f"file_name: {x[1]}\n contents: {x[0]}",
+				zip(chunk_texts, file_names),
+			)
+		)
+	elif file_name_language == "ko":
+		return list(
+			map(
+				lambda x: f"파일 제목: {x[1]}\n 내용: {x[0]}",
+				zip(chunk_texts, file_names),
+			)
+		)
+	elif file_name_language == "ja":
+		return list(
+			map(
+				lambda x: f"ファイル名: {x[1]}\n 内容: {x[0]}",
+				zip(chunk_texts, file_names),
+			)
+		)
+	else:
+		raise ValueError(
+			f"Unsupported file_name_language: {file_name_language}. Choose from 'en' ,'ko' or 'ja."
+		)

autorag-workspace/autorag/data/chunk/langchain_chunk.py

@@ -0,0 +1,76 @@
+import os
+from itertools import chain
+import uuid
+from typing import Tuple, List, Dict, Any, Optional
+
+from langchain_text_splitters import TextSplitter
+
+from autorag.data.chunk.base import chunker_node, add_file_name
+from autorag.data.utils.util import add_essential_metadata, get_start_end_idx
+
+
+@chunker_node
+def langchain_chunk(
+	texts: List[str],
+	chunker: TextSplitter,
+	file_name_language: Optional[str] = None,
+	metadata_list: Optional[List[Dict[str, str]]] = None,
+) -> Tuple[
+	List[str], List[str], List[str], List[Tuple[int, int]], List[Dict[str, Any]]
+]:
+	"""
+	Chunk texts from the parsed result to use langchain chunk method
+
+	:param texts: The list of texts to chunk from the parsed result
+	:param chunker: A langchain TextSplitter(Chunker) instance.
+	:param file_name_language: The language to use 'add_file_name' feature.
+	    You need to set one of 'English' and 'Korean'
+	    The 'add_file_name' feature is to add a file_name to chunked_contents.
+	    This is used to prevent hallucination by retrieving contents from the wrong document.
+	    Default form of 'English' is "file_name: {file_name}\n contents: {content}"
+	:param metadata_list: The list of dict of metadata from the parsed result
+	:return: tuple of lists containing the chunked doc_id, contents, path, start_idx, end_idx and metadata
+	"""
+	results = [
+		langchain_chunk_pure(text, chunker, file_name_language, meta)
+		for text, meta in zip(texts, metadata_list)
+	]
+
+	doc_id, contents, path, start_end_idx, metadata = (
+		list(chain.from_iterable(item)) for item in zip(*results)
+	)
+
+	return doc_id, contents, path, start_end_idx, metadata
+
+
+def langchain_chunk_pure(
+	text: str,
+	chunker: TextSplitter,
+	file_name_language: Optional[str] = None,
+	_metadata: Optional[Dict[str, str]] = None,
+):
+	# chunk
+	chunk_results = chunker.create_documents([text], metadatas=[_metadata])
+
+	# make doc_id
+	doc_id = list(str(uuid.uuid4()) for _ in range(len(chunk_results)))
+
+	# make path
+	path_lst = list(map(lambda x: x.metadata.get("path", ""), chunk_results))
+
+	# make contents and start_end_idx
+	if file_name_language:
+		chunked_file_names = list(map(lambda x: os.path.basename(x), path_lst))
+		chunked_texts = list(map(lambda x: x.page_content, chunk_results))
+		start_end_idx = list(map(lambda x: get_start_end_idx(text, x), chunked_texts))
+		contents = add_file_name(file_name_language, chunked_file_names, chunked_texts)
+	else:
+		contents = list(map(lambda node: node.page_content, chunk_results))
+		start_end_idx = list(map(lambda x: get_start_end_idx(text, x), contents))
+
+	# make metadata
+	metadata = list(
+		map(lambda node: add_essential_metadata(node.metadata), chunk_results)
+	)
+
+	return doc_id, contents, path_lst, start_end_idx, metadata

autorag-workspace/autorag/data/chunk/llama_index_chunk.py

@@ -0,0 +1,96 @@
+import os.path
+from itertools import chain
+from typing import Tuple, List, Dict, Any, Optional
+
+from llama_index.core import Document
+from llama_index.core.node_parser.interface import NodeParser
+
+from autorag.utils.util import process_batch, get_event_loop
+from autorag.data.chunk.base import chunker_node, add_file_name
+from autorag.data.utils.util import (
+	add_essential_metadata_llama_text_node,
+	get_start_end_idx,
+)
+
+
+@chunker_node
+def llama_index_chunk(
+	texts: List[str],
+	chunker: NodeParser,
+	file_name_language: Optional[str] = None,
+	metadata_list: Optional[List[Dict[str, str]]] = None,
+	batch: int = 8,
+) -> Tuple[
+	List[str], List[str], List[str], List[Tuple[int, int]], List[Dict[str, Any]]
+]:
+	"""
+	Chunk texts from the parsed result to use llama index chunk method
+
+	:param texts: The list of texts to chunk from the parsed result
+	:param chunker: A llama index NodeParser(Chunker) instance.
+	:param file_name_language: The language to use 'add_file_name' feature.
+	    You need to set one of 'English' and 'Korean'
+	    The 'add_file_name' feature is to add a file_name to chunked_contents.
+	    This is used to prevent hallucination by retrieving contents from the wrong document.
+	    Default form of 'English' is "file_name: {file_name}\n contents: {content}"
+	:param metadata_list: The list of dict of metadata from the parsed result
+	:param batch: The batch size for chunk texts. Default is 8
+	:return: tuple of lists containing the chunked doc_id, contents, path, start_idx, end_idx and metadata
+	"""
+	tasks = [
+		llama_index_chunk_pure(text, chunker, file_name_language, meta)
+		for text, meta in zip(texts, metadata_list)
+	]
+	loop = get_event_loop()
+	results = loop.run_until_complete(process_batch(tasks, batch))
+
+	doc_id, contents, path, start_end_idx, metadata = (
+		list(chain.from_iterable(item)) for item in zip(*results)
+	)
+
+	return list(doc_id), list(contents), list(path), list(start_end_idx), list(metadata)
+
+
+async def llama_index_chunk_pure(
+	text: str,
+	chunker: NodeParser,
+	file_name_language: Optional[str] = None,
+	_metadata: Optional[Dict[str, str]] = None,
+):
+	# set document
+	document = [Document(text=text, metadata=_metadata)]
+
+	# chunk document
+	chunk_results = await chunker.aget_nodes_from_documents(documents=document)
+
+	# make doc_id
+	doc_id = list(map(lambda node: node.node_id, chunk_results))
+
+	# make path
+	path_lst = list(map(lambda x: x.metadata.get("path", ""), chunk_results))
+
+	# make contents and start_end_idx
+	if file_name_language:
+		chunked_file_names = list(map(lambda x: os.path.basename(x), path_lst))
+		chunked_texts = list(map(lambda x: x.text, chunk_results))
+		start_end_idx = list(
+			map(
+				lambda x: get_start_end_idx(text, x),
+				chunked_texts,
+			)
+		)
+		contents = add_file_name(file_name_language, chunked_file_names, chunked_texts)
+	else:
+		contents = list(map(lambda x: x.text, chunk_results))
+		start_end_idx = list(map(lambda x: get_start_end_idx(text, x), contents))
+
+	metadata = list(
+		map(
+			lambda node: add_essential_metadata_llama_text_node(
+				node.metadata, node.relationships
+			),
+			chunk_results,
+		)
+	)
+
+	return doc_id, contents, path_lst, start_end_idx, metadata

autorag-workspace/autorag/data/chunk/run.py

@@ -0,0 +1,38 @@
+import os
+from typing import Callable, List, Dict
+import pandas as pd
+
+from autorag.strategy import measure_speed
+
+
+def run_chunker(
+	modules: List[Callable],
+	module_params: List[Dict],
+	parsed_result: pd.DataFrame,
+	project_dir: str,
+):
+	results, execution_times = zip(
+		*map(
+			lambda x: measure_speed(x[0], parsed_result=parsed_result, **x[1]),
+			zip(modules, module_params),
+		)
+	)
+	average_times = list(map(lambda x: x / len(results[0]), execution_times))
+
+	# save results to parquet files
+	filepaths = list(
+		map(lambda x: os.path.join(project_dir, f"{x}.parquet"), range(len(modules)))
+	)
+	list(map(lambda x: x[0].to_parquet(x[1], index=False), zip(results, filepaths)))
+	filenames = list(map(lambda x: os.path.basename(x), filepaths))
+
+	summary_df = pd.DataFrame(
+		{
+			"filename": filenames,
+			"module_name": list(map(lambda module: module.__name__, modules)),
+			"module_params": module_params,
+			"execution_time": average_times,
+		}
+	)
+	summary_df.to_csv(os.path.join(project_dir, "summary.csv"), index=False)
+	return summary_df

autorag-workspace/autorag/data/legacy/corpus/__init__.py

@@ -0,0 +1,2 @@
+from .langchain import langchain_documents_to_parquet
+from .llama_index import llama_documents_to_parquet, llama_text_node_to_parquet

autorag-workspace/autorag/data/legacy/corpus/langchain.py

@@ -0,0 +1,47 @@
+import uuid
+from typing import List, Optional
+
+import pandas as pd
+from langchain_core.documents import Document
+
+from autorag.data.utils.util import add_essential_metadata
+from autorag.utils.util import save_parquet_safe
+
+
+def langchain_documents_to_parquet(
+	langchain_documents: List[Document],
+	output_filepath: Optional[str] = None,
+	upsert: bool = False,
+) -> pd.DataFrame:
+	"""
+	Langchain documents to corpus dataframe.
+	Corpus dataframe will be saved to filepath(file_dir/filename) if given.
+	Return corpus dataframe whether the filepath is given.
+	You can use this method to create corpus.parquet after load and chunk using Llama Index.
+
+	:param langchain_documents: List of langchain documents.
+	:param output_filepath: Optional filepath to save the parquet file.
+	    If None, the function will return the processed_data as pd.DataFrame, but do not save as parquet.
+	    File directory must exist. File extension must be .parquet
+	:param upsert: If true, the function will overwrite the existing file if it exists.
+	    Default is False.
+	:return: Corpus data as pd.DataFrame
+	"""
+
+	corpus_df = pd.DataFrame(
+		list(
+			map(
+				lambda doc: {
+					"doc_id": str(uuid.uuid4()),
+					"contents": doc.page_content,
+					"metadata": add_essential_metadata(doc.metadata),
+				},
+				langchain_documents,
+			)
+		)
+	)
+
+	if output_filepath is not None:
+		save_parquet_safe(corpus_df, output_filepath, upsert=upsert)
+
+	return corpus_df

autorag-workspace/autorag/data/legacy/corpus/llama_index.py

@@ -0,0 +1,93 @@
+import uuid
+from typing import List, Optional
+
+import pandas as pd
+from llama_index.core import Document
+from llama_index.core.schema import TextNode
+
+from autorag.data.utils.util import (
+	add_essential_metadata,
+	add_essential_metadata_llama_text_node,
+)
+from autorag.utils.util import save_parquet_safe
+
+
+def llama_documents_to_parquet(
+	llama_documents: List[Document],
+	output_filepath: Optional[str] = None,
+	upsert: bool = False,
+) -> pd.DataFrame:
+	"""
+	Llama Index documents to corpus dataframe.
+	Corpus dataframe will be saved to filepath(file_dir/filename) if given.
+	Return corpus dataframe whether the filepath is given.
+	You can use this method to create corpus.parquet after load and chunk using Llama Index.
+
+	:param llama_documents: List[Document]
+	:param output_filepath: Optional filepath to save the parquet file.
+	    If None, the function will return the processed_data as pd.DataFrame, but do not save as parquet.
+	    File directory must exist. File extension must be .parquet
+	:param upsert: If true, the function will overwrite the existing file if it exists.
+	    Default is False.
+	:return: Corpus data as pd.DataFrame
+	"""
+
+	doc_lst = pd.DataFrame(
+		list(
+			map(
+				lambda doc: {
+					"doc_id": str(uuid.uuid4()),
+					"contents": doc.text,
+					"metadata": add_essential_metadata(doc.metadata),
+				},
+				llama_documents,
+			)
+		)
+	)
+
+	processed_df = pd.DataFrame(doc_lst)
+
+	if output_filepath is not None:
+		save_parquet_safe(processed_df, output_filepath, upsert=upsert)
+
+	return processed_df
+
+
+def llama_text_node_to_parquet(
+	text_nodes: List[TextNode],
+	output_filepath: Optional[str] = None,
+	upsert: bool = False,
+) -> pd.DataFrame:
+	"""
+	Llama Index text nodes to corpus dataframe.
+	Corpus dataframe will be saved to filepath(file_dir/filename) if given.
+	Return corpus dataframe whether the filepath is given.
+	You can use this method to create corpus.parquet after load and chunk using Llama Index.
+
+	:param text_nodes: List of llama index text nodes.
+	:param output_filepath: Optional filepath to save the parquet file.
+	    If None, the function will return the processed_data as pd.DataFrame, but do not save as parquet.
+	    File directory must exist. File extension must be .parquet
+	:param upsert: If true, the function will overwrite the existing file if it exists.
+	    Default is False.
+	:return: Corpus data as pd.DataFrame
+	"""
+	corpus_df = pd.DataFrame(
+		list(
+			map(
+				lambda node: {
+					"doc_id": node.node_id,
+					"contents": node.text,
+					"metadata": add_essential_metadata_llama_text_node(
+						node.metadata, node.relationships
+					),
+				},
+				text_nodes,
+			)
+		)
+	)
+
+	if output_filepath is not None:
+		save_parquet_safe(corpus_df, output_filepath, upsert=upsert)
+
+	return corpus_df

autorag-workspace/autorag/data/legacy/qacreation/__init__.py

@@ -0,0 +1,6 @@
+from .base import make_single_content_qa, make_qa_with_existing_qa
+from .llama_index import (
+	generate_qa_llama_index,
+	generate_answers,
+	generate_qa_llama_index_by_ratio,
+)

autorag-workspace/autorag/data/legacy/qacreation/base.py

@@ -0,0 +1,239 @@
+import logging
+import uuid
+from typing import Callable, Optional, List
+
+import chromadb
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+import autorag
+from autorag.nodes.retrieval.vectordb import vectordb_ingest, vectordb_pure
+from autorag.utils.util import (
+	save_parquet_safe,
+	fetch_contents,
+	get_event_loop,
+	process_batch,
+)
+
+logger = logging.getLogger("AutoRAG")
+
+
+def make_single_content_qa(
+	corpus_df: pd.DataFrame,
+	content_size: int,
+	qa_creation_func: Callable,
+	output_filepath: Optional[str] = None,
+	upsert: bool = False,
+	random_state: int = 42,
+	cache_batch: int = 32,
+	**kwargs,
+) -> pd.DataFrame:
+	"""
+	Make single content (single-hop, single-document) QA dataset using given qa_creation_func.
+	It generates a single content QA dataset, which means its retrieval ground truth will be only one.
+	It is the most basic form of QA dataset.
+
+	:param corpus_df: The corpus dataframe to make QA dataset from.
+	:param content_size: This function will generate QA dataset for the given number of contents.
+	:param qa_creation_func: The function to create QA pairs.
+	    You can use like `generate_qa_llama_index` or `generate_qa_llama_index_by_ratio`.
+	    The input func must have `contents` parameter for the list of content string.
+	:param output_filepath: Optional filepath to save the parquet file.
+	    If None, the function will return the processed_data as pd.DataFrame, but do not save as parquet.
+	    File directory must exist. File extension must be .parquet
+	:param upsert: If true, the function will overwrite the existing file if it exists.
+	    Default is False.
+	:param random_state: The random state for sampling corpus from the given corpus_df.
+	:param cache_batch: The number of batches to use for caching the generated QA dataset.
+	    When the cache_batch size data is generated, the dataset will save to the designated output_filepath.
+	    If the cache_batch size is too small, the process time will be longer.
+	:param kwargs: The keyword arguments for qa_creation_func.
+	:return: QA dataset dataframe.
+	    You can save this as parquet file to use at AutoRAG.
+	"""
+	assert content_size > 0, "content_size must be greater than 0."
+	if content_size > len(corpus_df):
+		logger.warning(
+			f"content_size {content_size} is larger than the corpus size {len(corpus_df)}. "
+			"Setting content_size to the corpus size."
+		)
+		content_size = len(corpus_df)
+	sampled_corpus = corpus_df.sample(n=content_size, random_state=random_state)
+	sampled_corpus = sampled_corpus.reset_index(drop=True)
+
+	def make_query_generation_gt(row):
+		return row["qa"]["query"], row["qa"]["generation_gt"]
+
+	qa_data = pd.DataFrame()
+	for idx, i in tqdm(enumerate(range(0, len(sampled_corpus), cache_batch))):
+		qa = qa_creation_func(
+			contents=sampled_corpus["contents"].tolist()[i : i + cache_batch], **kwargs
+		)
+
+		temp_qa_data = pd.DataFrame(
+			{
+				"qa": qa,
+				"retrieval_gt": sampled_corpus["doc_id"].tolist()[i : i + cache_batch],
+			}
+		)
+		temp_qa_data = temp_qa_data.explode("qa", ignore_index=True)
+		temp_qa_data["qid"] = [str(uuid.uuid4()) for _ in range(len(temp_qa_data))]
+		temp_qa_data[["query", "generation_gt"]] = temp_qa_data.apply(
+			make_query_generation_gt, axis=1, result_type="expand"
+		)
+		temp_qa_data = temp_qa_data.drop(columns=["qa"])
+
+		temp_qa_data["retrieval_gt"] = temp_qa_data["retrieval_gt"].apply(
+			lambda x: [[x]]
+		)
+		temp_qa_data["generation_gt"] = temp_qa_data["generation_gt"].apply(
+			lambda x: [x]
+		)
+
+		if idx == 0:
+			qa_data = temp_qa_data
+		else:
+			qa_data = pd.concat([qa_data, temp_qa_data], ignore_index=True)
+		if output_filepath is not None:
+			save_parquet_safe(qa_data, output_filepath, upsert=upsert)
+
+	return qa_data
+
+
+def make_qa_with_existing_qa(
+	corpus_df: pd.DataFrame,
+	existing_query_df: pd.DataFrame,
+	content_size: int,
+	answer_creation_func: Optional[Callable] = None,
+	exist_gen_gt: Optional[bool] = False,
+	output_filepath: Optional[str] = None,
+	embedding_model: str = "openai_embed_3_large",
+	collection: Optional[chromadb.Collection] = None,
+	upsert: bool = False,
+	random_state: int = 42,
+	cache_batch: int = 32,
+	top_k: int = 3,
+	**kwargs,
+) -> pd.DataFrame:
+	"""
+	Make single-hop QA dataset using given qa_creation_func and existing queries.
+
+	:param corpus_df: The corpus dataframe to make QA dataset from.
+	:param existing_query_df: Dataframe containing existing queries to use for QA pair creation.
+	:param content_size: This function will generate QA dataset for the given number of contents.
+	:param answer_creation_func: Optional function to create answer with input query.
+	    If exist_gen_gt is False, this function must be given.
+	:param exist_gen_gt: Optional boolean to use existing generation_gt.
+	    If True, the existing_query_df must have 'generation_gt' column.
+	    If False, the answer_creation_func must be given.
+	:param output_filepath: Optional filepath to save the parquet file.
+	:param embedding_model: The embedding model to use for vectorization.
+	    You can add your own embedding model in the autorag.embedding_models.
+	    Please refer to how to add an embedding model in this doc: https://docs.auto-rag.com/local_model.html
+	    The default is 'openai_embed_3_large'.
+	:param collection: The chromadb collection to use for vector DB.
+	    You can make any chromadb collection and use it here.
+	    If you already ingested the corpus_df to the collection, the embedding process will not be repeated.
+	    The default is None. If None, it makes a temporary collection.
+	:param upsert: If true, the function will overwrite the existing file if it exists.
+	:param random_state: The random state for sampling corpus from the given corpus_df.
+	:param cache_batch: The number of batches to use for caching the generated QA dataset.
+	:param top_k: The number of sources to refer by model.
+	    Default is 3.
+	:param kwargs: The keyword arguments for qa_creation_func.
+	:return: QA dataset dataframe.
+	"""
+	raise DeprecationWarning("This function is deprecated.")
+	assert (
+		"query" in existing_query_df.columns
+	), "existing_query_df must have 'query' column."
+
+	if exist_gen_gt:
+		assert (
+			"generation_gt" in existing_query_df.columns
+		), "existing_query_df must have 'generation_gt' column."
+	else:
+		assert (
+			answer_creation_func is not None
+		), "answer_creation_func must be given when exist_gen_gt is False."
+
+	assert content_size > 0, "content_size must be greater than 0."
+	if content_size > len(corpus_df):
+		logger.warning(
+			f"content_size {content_size} is larger than the corpus size {len(corpus_df)}. "
+			"Setting content_size to the corpus size."
+		)
+		content_size = len(corpus_df)
+
+	logger.info("Loading local embedding model...")
+	embeddings = autorag.embedding_models[embedding_model]()
+
+	# Vector DB creation
+	if collection is None:
+		chroma_client = chromadb.Client()
+		collection_name = "auto-rag"
+		collection = chroma_client.get_or_create_collection(collection_name)
+
+	# embed corpus_df
+	vectordb_ingest(collection, corpus_df, embeddings)
+	query_embeddings = embeddings.get_text_embedding_batch(
+		existing_query_df["query"].tolist()
+	)
+
+	loop = get_event_loop()
+	tasks = [
+		vectordb_pure([query_embedding], top_k, collection)
+		for query_embedding in query_embeddings
+	]
+	results = loop.run_until_complete(process_batch(tasks, batch_size=cache_batch))
+	retrieved_ids = list(map(lambda x: x[0], results))
+
+	retrieved_contents: List[List[str]] = fetch_contents(corpus_df, retrieved_ids)
+	input_passage_strs: List[str] = list(
+		map(
+			lambda x: "\n".join(
+				[f"Document {i + 1}\n{content}" for i, content in enumerate(x)]
+			),
+			retrieved_contents,
+		)
+	)
+
+	retrieved_qa_df = pd.DataFrame(
+		{
+			"qid": [str(uuid.uuid4()) for _ in range(len(existing_query_df))],
+			"query": existing_query_df["query"].tolist(),
+			"retrieval_gt": list(map(lambda x: [x], retrieved_ids)),
+			"input_passage_str": input_passage_strs,
+		}
+	)
+
+	if exist_gen_gt:
+		generation_gt = existing_query_df["generation_gt"].tolist()
+		if isinstance(generation_gt[0], np.ndarray):
+			retrieved_qa_df["generation_gt"] = generation_gt
+		else:
+			raise ValueError(
+				"In existing_query_df, generation_gt (per query) must be in the form of List[str]."
+			)
+
+	sample_qa_df = retrieved_qa_df.sample(
+		n=min(content_size, len(retrieved_qa_df)), random_state=random_state
+	)
+
+	qa_df = sample_qa_df.copy(deep=True)
+	qa_df.drop(columns=["input_passage_str"], inplace=True)
+
+	if not exist_gen_gt:
+		generation_gt = answer_creation_func(
+			contents=sample_qa_df["input_passage_str"].tolist(),
+			queries=sample_qa_df["query"].tolist(),
+			batch=cache_batch,
+			**kwargs,
+		)
+		qa_df["generation_gt"] = generation_gt
+
+	if output_filepath is not None:
+		save_parquet_safe(qa_df, output_filepath, upsert=upsert)
+
+	return qa_df

autorag-workspace/autorag/data/legacy/qacreation/llama_index.py

@@ -0,0 +1,253 @@
+import os.path
+import random
+from typing import Optional, List, Dict, Any
+
+import pandas as pd
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.core.llms import LLM
+
+from autorag.utils.util import process_batch, get_event_loop
+
+package_dir = os.path.dirname(os.path.realpath(__file__))
+
+
+def generate_qa_llama_index(
+	llm: LLM,
+	contents: List[str],
+	prompt: Optional[str] = None,
+	question_num_per_content: int = 1,
+	max_retries: int = 3,
+	batch: int = 4,
+) -> List[List[Dict]]:
+	"""
+	Generate a qa set from the list of contents.
+	It uses a single prompt for all contents.
+	If you want to use more than one prompt for generating qa,
+	you can consider using generate_qa_llama_index_by_ratio.
+
+	:param llm: Llama index model
+	:param contents: List of content strings.
+	:param prompt: The prompt to use for the qa generation.
+	    The prompt must include the following placeholders:
+	    - {{text}}: The content string
+	    - {{num_questions}}: The number of questions to generate
+	    As default, the prompt is set to the default prompt for the question type.
+	:param question_num_per_content: Number of questions to generate for each content.
+	    Default is 1.
+	:param max_retries: The maximum number of retries when generated question number is not equal to the target number.
+	    Default is 3.
+	:param batch: The batch size to process asynchronously.
+	    Default is 4.
+	:return: 2-d list of dictionaries containing the query and generation_gt.
+	"""
+	# load default prompt
+	if prompt is None:
+		prompt = open(
+			os.path.join(package_dir, "llama_index_default_prompt.txt"), "r"
+		).read()
+
+	tasks = [
+		async_qa_gen_llama_index(
+			content, llm, prompt, question_num_per_content, max_retries
+		)
+		for content in contents
+	]
+	loops = get_event_loop()
+	results = loops.run_until_complete(process_batch(tasks, batch))
+	return results
+
+
+def generate_answers(
+	llm: LLM,
+	contents: List[str],
+	queries: List[str],
+	batch: int = 4,
+) -> List[List[Dict]]:
+	"""
+	Generate qa sets from the list of contents using existing queries.
+
+	:param llm: Llama index model
+	:param contents: List of content strings.
+	:param queries: List of existing queries.
+	:param batch: The batch size to process asynchronously.
+	:return: 2-d list of dictionaries containing the query and generation_gt.
+	"""
+
+	tasks = [
+		generate_basic_answer(llm, content, query)
+		for content, query in zip(contents, queries)
+	]
+	loops = get_event_loop()
+	results = loops.run_until_complete(process_batch(tasks, batch))
+	return results
+
+
+def generate_qa_llama_index_by_ratio(
+	llm: LLM,
+	contents: List[str],
+	prompts_ratio: Dict,
+	question_num_per_content: int = 1,
+	max_retries: int = 3,
+	random_state: int = 42,
+	batch: int = 4,
+) -> List[List[Dict]]:
+	"""
+	Generate a qa set from the list of contents.
+	You can set the ratio of prompts that you want to use for generating qa.
+	It distributes the number of questions to generate for each content by the ratio randomly.
+
+	:param llm: Llama index model
+	:param contents: List of content strings.
+	:param prompts_ratio: Dictionary of prompt paths and their ratios.
+	    Example: {"prompt/prompt1.txt": 0.5, "prompt/prompt2.txt": 0.5}
+	    The value sum doesn't have to be 1.
+	    The path must be the absolute path, and the file must exist.
+	    Plus, it has to be a text file which contains proper prompt.
+	    Each prompt must contain the following placeholders:
+	    - {{text}}: The content string
+	    - {{num_questions}}: The number of questions to generate
+	:param question_num_per_content: Number of questions to generate for each content.
+	    Default is 1.
+	:param max_retries: The maximum number of retries when generated question number is not equal to the target number.
+	    Default is 3.
+	:param random_state: Random seed
+	    Default is 42.
+	:param batch: The batch size to process asynchronously.
+	    Default is 4.
+	:return: 2-d list of dictionaries containing the query and generation_gt.
+	"""
+	prompts = list(map(lambda path: open(path, "r").read(), prompts_ratio.keys()))
+	assert all([validate_llama_index_prompt(prompt) for prompt in prompts])
+
+	content_indices = list(range(len(contents)))
+	random.seed(random_state)
+	random.shuffle(content_indices)
+
+	slice_content_indices: List[List[str]] = distribute_list_by_ratio(
+		content_indices, list(prompts_ratio.values())
+	)
+	temp_df = pd.DataFrame({"idx": slice_content_indices, "prompt": prompts})
+	temp_df = temp_df.explode("idx", ignore_index=True)
+	temp_df = temp_df.sort_values(by="idx", ascending=True)
+
+	final_df = pd.DataFrame({"content": contents, "prompt": temp_df["prompt"].tolist()})
+
+	tasks = [
+		async_qa_gen_llama_index(
+			content, llm, prompt, question_num_per_content, max_retries
+		)
+		for content, prompt in zip(
+			final_df["content"].tolist(), final_df["prompt"].tolist()
+		)
+	]
+
+	loops = get_event_loop()
+	results = loops.run_until_complete(process_batch(tasks, batch))
+
+	return results
+
+
+async def async_qa_gen_llama_index(
+	content: str,
+	llm: LLM,
+	prompt: str,
+	question_num: int = 1,
+	max_retries: int = 3,
+):
+	"""
+	Generate a qa set by using the given content and the llama index model.
+	You must select the question type.
+
+	:param content: Content string
+	:param llm: Llama index model
+	:param prompt: The prompt to use for the qa generation.
+	    The prompt must include the following placeholders:
+	    - {{text}}: The content string
+	    - {{num_questions}}: The number of questions to generate
+	:param question_num: The number of questions to generate
+	:param max_retries: Maximum number of retries when generated question number is not equal to the target number
+	:return: List of dictionaries containing the query and generation_gt
+	"""
+	validate_llama_index_prompt(prompt)
+
+	async def generate(content: str, llm: LLM):
+		for _ in range(max_retries):
+			output = await llm.acomplete(
+				prompt.replace("{{text}}", content).replace(
+					"{{num_questions}}", str(question_num)
+				)
+			)
+			result = parse_output(output.text)
+			if len(result) == question_num:
+				return result
+		raise InterruptedError(
+			f"Failed to generate output of length {question_num} after {max_retries} retries."
+		)
+
+	return await generate(content, llm)
+
+
+async def generate_basic_answer(llm: LLM, passage_str: str, query: str) -> str:
+	basic_answer_system_prompt = """You are an AI assistant to answer the given question in the provide evidence text.
+    You can find the evidence from the given text about question, and you have to write a proper answer to the given question.
+    You have to preserve the question's language at the answer.
+    For example, if the input question is Korean, the output answer must be in Korean.
+    """
+	user_prompt = f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:"
+
+	response = await llm.achat(
+		messages=[
+			ChatMessage(role=MessageRole.SYSTEM, content=basic_answer_system_prompt),
+			ChatMessage(role=MessageRole.USER, content=user_prompt),
+		],
+		temperature=1.0,
+	)
+	return response.message.content
+
+
+def validate_llama_index_prompt(prompt: str) -> bool:
+	"""
+	Validate the prompt for the llama index model.
+	The prompt must include the following placeholders:
+	- {{text}}: The content string
+	- {{num_questions}}: The number of questions to generate
+	"""
+	if "{{text}}" not in prompt:
+		raise ValueError("The prompt must include the placeholder {{text}}.")
+	if "{{num_questions}}" not in prompt:
+		raise ValueError("The prompt must include the placeholder {{num_questions}}.")
+	return True
+
+
+def parse_output(result: str) -> List[Dict]:
+	result = result.strip()
+	result = result.split("[Q]:")
+	final_result = list()
+	for res in result:
+		res = res.strip()
+		if res and "\n[A]:" in res:
+			qa = res.split("\n[A]:")
+			final_result.append(
+				{"query": qa[0].strip(), "generation_gt": qa[1].strip()}
+			)
+	return final_result
+
+
+def distribute_list_by_ratio(input_list, ratio) -> List[List[Any]]:
+	total_ratio = sum(ratio)
+	total_length = len(input_list)
+
+	# Calculate the length of each slice
+	slice_lengths = [int((r / total_ratio) * total_length) for r in ratio]
+
+	# Adjust the last slice in case of rounding issues
+	slice_lengths[-1] = total_length - sum(slice_lengths[:-1])
+
+	slices = []
+	start = 0
+	for length in slice_lengths:
+		end = start + length
+		slices.append(input_list[start:end])
+		start = end
+
+	return slices

autorag-workspace/autorag/data/legacy/qacreation/llama_index_default_prompt.txt

@@ -0,0 +1,54 @@
+You're an AI tasked to convert Text into a question and answer set.
+Cover as many details from Text as possible in the QnA set.
+
+Instructions:
+1. Both Questions and Answers MUST BE extracted from given Text
+2. Answers must be full sentences
+3. Questions should be as detailed as possible from Text
+4. Output must always have the provided number of QnAs
+5. Create questions that ask about information from the Text
+6. MUST include specific keywords from the Text.
+7. Do not mention any of these in the questions: "in the given text", "in the provided information", etc.
+
+Question examples:
+1. How do owen and riggs know each other?
+2. What does the word fore "mean" in golf?
+3. What makes charging bull in nyc popular to tourists?
+4. What kind of pistol does the army use?
+5. Who was the greatest violin virtuoso in the romantic period?
+<|separator|>
+
+Text:
+<|text_start|>
+Mark Hamill as Luke Skywalker : One of the last living Jedi , trained by Obi - Wan and Yoda , who is also a skilled X-wing fighter pilot allied with the Rebellion .
+Harrison Ford as Han Solo : A rogue smuggler , who aids the Rebellion against the Empire . Han is Luke and Leia 's friend , as well as Leia 's love interest .
+Carrie Fisher as Leia Organa : The former Princess of the destroyed planet Alderaan , who joins the Rebellion ; Luke 's twin sister , and Han 's love interest .
+Billy Dee Williams as Lando Calrissian : The former Baron Administrator of Cloud City and one of Han 's friends who aids the Rebellion .
+Anthony Daniels as C - 3PO : A humanoid protocol droid , who sides with the Rebellion .
+Peter Mayhew as Chewbacca : A Wookiee who is Han 's longtime friend , who takes part in the Rebellion .
+Kenny Baker as R2 - D2 : An astromech droid , bought by Luke ; and long - time friend to C - 3PO . He also portrays a GONK power droid in the background .
+Ian McDiarmid as the Emperor : The evil founding supreme ruler of the Galactic Empire , and Vader 's Sith Master .
+Frank Oz as Yoda : The wise , centuries - old Grand Master of the Jedi , who is Luke 's self - exiled Jedi Master living on Dagobah . After dying , he reappears to Luke as a Force - ghost . Yoda 's Puppetry was assisted by Mike Quinn .
+David Prowse as Darth Vader / Anakin Skywalker : A powerful Sith lord and the second in command of the Galactic Empire ; Luke and Leia 's father .
+<|text_end|>
+Output with 4 QnAs:
+<|separator|>
+
+[Q]: who played luke father in return of the jedi
+[A]: David Prowse acted as Darth Vader, a.k.a Anakin Skywalker, which is Luke and Leia's father.
+[Q]: Who is Han Solo's best friend? And what species is he?
+[A]: Han Solo's best friend is Chewbacca, who is a Wookiee.
+[Q]: Who played luke's teacher in the return of the jedi
+[A]: Yoda, the wise, centuries-old Grand Master of the Jedi, who is Luke's self-exiled Jedi Master living on Dagobah, was played by Frank Oz.
+Also, there is a mention of Obi-Wan Kenobi, who trained Luke Skywalker.
+But I can't find who played Obi-Wan Kenobi in the given text.
+[Q]: Where Yoda lives in the return of the jedi?
+[A]: Yoda, the Jedi Master, lives on Dagobah.
+<|separator|>
+
+Text:
+<|text_start|>
+{{text}}
+<|text_end|>
+Output with {{num_questions}} QnAs:
+<|separator|>

autorag-workspace/autorag/data/legacy/qacreation/ragas.py

@@ -0,0 +1,75 @@
+import uuid
+from typing import Optional
+
+import pandas as pd
+from langchain_core.embeddings import Embeddings
+from langchain_core.language_models import BaseChatModel
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+from autorag.data.utils.util import corpus_df_to_langchain_documents
+from autorag.utils import cast_qa_dataset
+
+
+def generate_qa_ragas(
+	corpus_df: pd.DataFrame,
+	test_size: int,
+	distributions: Optional[dict] = None,
+	generator_llm: Optional[BaseChatModel] = None,
+	critic_llm: Optional[BaseChatModel] = None,
+	embedding_model: Optional[Embeddings] = None,
+	**kwargs,
+) -> pd.DataFrame:
+	"""
+	QA dataset generation using RAGAS.
+	Returns qa dataset dataframe.
+
+	:param corpus_df: Corpus dataframe.
+	:param test_size: Number of queries to generate.
+	:param distributions: Distributions of different types of questions.
+	    Default is "simple is 0.5, multi_context is 0.4, and reasoning is 0.1."
+	    Each type of questions refers to Ragas evolution types.
+	:param generator_llm: Generator language model from Langchain.
+	:param critic_llm: Critic language model from Langchain.
+	:param embedding_model: Embedding model from Langchain.
+	:param kwargs: The additional option to pass to the 'generate_with_langchain_docs' method.
+	    You can input 'with_debugging_logs', 'is_async', 'raise_exceptions', and 'run_config'.
+	:return: QA dataset dataframe.
+	"""
+	from ragas.testset import TestsetGenerator
+	from ragas.testset.evolutions import simple, reasoning, multi_context
+
+	if generator_llm is None:
+		generator_llm = ChatOpenAI(model="gpt-3.5-turbo-16k")
+	if critic_llm is None:
+		critic_llm = ChatOpenAI(model="gpt-4-turbo")
+	if embedding_model is None:
+		embedding_model = OpenAIEmbeddings()
+	if distributions is None:
+		distributions = {simple: 0.5, multi_context: 0.4, reasoning: 0.1}
+
+	assert sum(list(distributions.values())) == 1.0, "Sum of distributions must be 1.0"
+
+	generator = TestsetGenerator.from_langchain(
+		generator_llm, critic_llm, embedding_model
+	)
+
+	langchain_docs = corpus_df_to_langchain_documents(corpus_df)
+
+	test_df = generator.generate_with_langchain_docs(
+		langchain_docs, test_size, distributions=distributions, **kwargs
+	).to_pandas()
+
+	result_df = pd.DataFrame(
+		{
+			"qid": [str(uuid.uuid4()) for _ in range(len(test_df))],
+			"query": test_df["question"].tolist(),
+			"generation_gt": list(map(lambda x: x, test_df["ground_truth"].tolist())),
+		}
+	)
+
+	result_df["retrieval_gt"] = test_df["metadata"].apply(
+		lambda x: list(map(lambda y: y["filename"], x))
+	)
+	result_df = cast_qa_dataset(result_df)
+
+	return result_df

autorag-workspace/autorag/data/legacy/qacreation/simple.py

@@ -0,0 +1,99 @@
+import os
+import pathlib
+import uuid
+from typing import Callable
+
+import pandas as pd
+
+
+def generate_qa_row(llm, corpus_data_row):
+	"""
+	this sample code to generate rag dataset using OpenAI chat model
+
+	:param llm: guidance model
+	:param corpus_data_row: need "contents" column
+	:return: should to be dict which has "query", "generation_gt" columns at least.
+	"""
+	from guidance import gen
+	import guidance
+
+	temp_llm = llm
+	with guidance.user():
+		temp_llm += f"""
+    You have to found a passge to solve "the problem".
+    You need to build a clean and clear set of (problem, passage, answer) in json format
+    so that you don't have to ask about "the problem" again.
+    problem need to end with question mark("?").
+    The process of approaching the answer based on the information of the given passage
+    must be clearly and neatly displayed in the answer.\n
+    \n
+    Here is set of (problem, passage, answer) in JSON format:\n
+    {{\n
+        "passage": {corpus_data_row["contents"]}\n
+        "problem":
+    """
+
+	with guidance.assistant():
+		temp_llm += gen("query", stop="?")
+	with guidance.user():
+		temp_llm += """
+        "answer":
+        """
+	with guidance.assistant():
+		temp_llm += gen("generation_gt")
+
+	corpus_data_row["metadata"]["qa_generation"] = "simple"
+
+	response = {"query": temp_llm["query"], "generation_gt": temp_llm["generation_gt"]}
+	return response
+
+
+def generate_simple_qa_dataset(
+	llm,
+	corpus_data: pd.DataFrame,
+	output_filepath: str,
+	generate_row_function: Callable,
+	**kwargs,
+):
+	"""
+	corpus_data to qa_dataset
+	qa_dataset will be saved to filepath(file_dir/filename)
+
+	:param llm: guidance.models.Model
+	:param corpus_data: pd.DataFrame. refer to the basic structure
+	:param output_filepath: file_dir must exist, filepath must not exist. file extension must be .parquet
+	:param generate_row_function: input(llm, corpus_data_row, kwargs) output(dict[columns contain "query" and "generation_gt"])
+	:param kwargs: if generate_row_function requires more args, use kwargs
+	:return: qa_dataset as pd.DataFrame
+	"""
+	output_file_dir = pathlib.PurePath(output_filepath).parent
+	if not os.path.isdir(output_file_dir):
+		raise NotADirectoryError(f"directory {output_file_dir}  not found.")
+	if not output_filepath.endswith("parquet"):
+		raise NameError(
+			f'file path: {output_filepath}  filename extension need to be ".parquet"'
+		)
+	if os.path.exists(output_filepath):
+		raise FileExistsError(
+			f"{output_filepath.split('/')[-1]} already exists in {output_file_dir}."
+		)
+
+	qa_data_lst = []
+	for _, corpus_data_row in corpus_data.iterrows():
+		response = generate_row_function(
+			llm=llm, corpus_data_row=corpus_data_row, **kwargs
+		)
+		qa_data_lst.append(
+			{
+				"qid": str(uuid.uuid4()),
+				"query": response["query"],
+				"retrieval_gt": [[corpus_data_row["doc_id"]]],
+				"generation_gt": [response["generation_gt"]],
+				"metadata": corpus_data_row["metadata"],
+			}
+		)
+
+	qa_dataset = pd.DataFrame(qa_data_lst)
+	qa_dataset.to_parquet(output_filepath, index=False)
+
+	return qa_dataset

autorag-workspace/autorag/data/parse/__init__.py

@@ -0,0 +1 @@
+from .langchain_parse import langchain_parse

autorag-workspace/autorag/data/parse/base.py

@@ -0,0 +1,79 @@
+import functools
+import logging
+from datetime import datetime
+from glob import glob
+from typing import Tuple, List, Optional
+import os
+
+from autorag.utils import result_to_dataframe
+from autorag.data.utils.util import get_file_metadata
+
+logger = logging.getLogger("AutoRAG")
+
+
+def parser_node(func):
+	@functools.wraps(func)
+	@result_to_dataframe(["texts", "path", "page", "last_modified_datetime"])
+	def wrapper(
+		data_path_glob: str,
+		file_type: str,
+		parse_method: Optional[str] = None,
+		**kwargs,
+	) -> Tuple[List[str], List[str], List[int], List[datetime]]:
+		logger.info(f"Running parser - {func.__name__} module...")
+
+		data_path_list = glob(data_path_glob)
+		if not data_path_list:
+			raise FileNotFoundError(f"data does not exits in {data_path_glob}")
+
+		assert file_type in [
+			"pdf",
+			"csv",
+			"json",
+			"md",
+			"html",
+			"xml",
+			"all_files",
+		], f"search type {file_type} is not supported"
+
+		# extract only files from data_path_list based on the file_type set in the YAML file
+		data_paths = (
+			[
+				data_path
+				for data_path in data_path_list
+				if os.path.basename(data_path).split(".")[-1] == file_type
+			]
+			if file_type != "all_files"
+			else data_path_list
+		)
+
+		if func.__name__ == "langchain_parse":
+			parse_method = parse_method.lower()
+			if parse_method == "directory":
+				path_split_list = data_path_glob.split("/")
+				glob_path = path_split_list.pop()
+				folder_path = "/".join(path_split_list)
+				kwargs.update({"glob": glob_path, "path": folder_path})
+				result = func(
+					data_path_list=data_paths, parse_method=parse_method, **kwargs
+				)
+			else:
+				result = func(
+					data_path_list=data_paths, parse_method=parse_method, **kwargs
+				)
+		elif func.__name__ in ["clova_ocr", "llama_parse", "table_hybrid_parse"]:
+			result = func(data_path_list=data_paths, **kwargs)
+		else:
+			raise ValueError(f"Unsupported module_type: {func.__name__}")
+		result = _add_last_modified_datetime(result)
+		return result
+
+	return wrapper
+
+
+def _add_last_modified_datetime(result):
+	last_modified_datetime_lst = list(
+		map(lambda x: get_file_metadata(x)["last_modified_datetime"], result[1])
+	)
+	result_with_dates = result + (last_modified_datetime_lst,)
+	return result_with_dates

autorag-workspace/autorag/data/parse/clova.py

@@ -0,0 +1,194 @@
+import base64
+import itertools
+import json
+import os
+from typing import List, Optional, Tuple
+
+import aiohttp
+import fitz  # PyMuPDF
+
+from autorag.data.parse.base import parser_node
+from autorag.utils.util import process_batch, get_event_loop
+
+
+@parser_node
+def clova_ocr(
+	data_path_list: List[str],
+	url: Optional[str] = None,
+	api_key: Optional[str] = None,
+	batch: int = 5,
+	table_detection: bool = False,
+) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use Naver Clova OCR.
+
+	:param data_path_list: The list of data paths to parse.
+	:param url: The URL for Clova OCR.
+		You can get the URL with the guide at https://guide.ncloud-docs.com/docs/clovaocr-example01
+		You can set the environment variable CLOVA_URL, or you can set it directly as a parameter.
+	:param api_key: The API key for Clova OCR.
+		You can get the API key with the guide at https://guide.ncloud-docs.com/docs/clovaocr-example01
+		You can set the environment variable CLOVA_API_KEY, or you can set it directly as a parameter.
+	:param batch: The batch size for parse documents. Default is 8.
+	:param table_detection: Whether to enable table detection. Default is False.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
+	url = os.getenv("CLOVA_URL", None) if url is None else url
+	if url is None:
+		raise KeyError(
+			"Please set the URL for Clova OCR in the environment variable CLOVA_URL "
+			"or directly set it on the config YAML file."
+		)
+
+	api_key = os.getenv("CLOVA_API_KEY", None) if api_key is None else api_key
+	if api_key is None:
+		raise KeyError(
+			"Please set the API key for Clova OCR in the environment variable CLOVA_API_KEY "
+			"or directly set it on the config YAML file."
+		)
+	if batch > 5:
+		raise ValueError("The batch size should be less than or equal to 5.")
+
+	image_data_lst = list(
+		map(lambda data_path: pdf_to_images(data_path), data_path_list)
+	)
+	image_info_lst = [
+		generate_image_info(pdf_path, len(image_data))
+		for pdf_path, image_data in zip(data_path_list, image_data_lst)
+	]
+
+	image_data_list = list(itertools.chain(*image_data_lst))
+	image_info_list = list(itertools.chain(*image_info_lst))
+
+	tasks = [
+		clova_ocr_pure(image_data, image_info, url, api_key, table_detection)
+		for image_data, image_info in zip(image_data_list, image_info_list)
+	]
+	loop = get_event_loop()
+	results = loop.run_until_complete(process_batch(tasks, batch))
+
+	texts, path, pages = zip(*results)
+	return list(texts), list(path), list(pages)
+
+
+async def clova_ocr_pure(
+	image_data: bytes,
+	image_info: dict,
+	url: str,
+	api_key: str,
+	table_detection: bool = False,
+) -> Tuple[str, str, int]:
+	session = aiohttp.ClientSession()
+	table_html = ""
+	headers = {"X-OCR-SECRET": api_key, "Content-Type": "application/json"}
+
+	# Convert image data to base64
+	image_base64 = base64.b64encode(image_data).decode("utf-8")
+
+	# Set data
+	data = {
+		"version": "V2",
+		"requestId": "sample_id",
+		"timestamp": 0,
+		"images": [{"format": "png", "name": "sample_image", "data": image_base64}],
+		"enableTableDetection": table_detection,
+	}
+
+	async with session.post(url, headers=headers, data=json.dumps(data)) as response:
+		resp_json = await response.json()
+		if "images" not in resp_json:
+			raise RuntimeError(
+				f"Invalid response from Clova API: {resp_json['detail']}"
+			)
+		if "tables" in resp_json["images"][0].keys():
+			table_html = json_to_html_table(
+				resp_json["images"][0]["tables"][0]["cells"]
+			)
+		page_text = extract_text_from_fields(resp_json["images"][0]["fields"])
+
+		if table_html:
+			page_text += f"\n\ntable html:\n{table_html}"
+
+		await session.close()
+		return page_text, image_info["pdf_path"], image_info["pdf_page"]
+
+
+def pdf_to_images(pdf_path: str) -> List[bytes]:
+	"""Convert each page of the PDF to an image and return the image data."""
+	pdf_document = fitz.open(pdf_path)
+	image_data_lst = []
+	for page_num in range(len(pdf_document)):
+		page = pdf_document.load_page(page_num)
+		pix = page.get_pixmap()
+		img_data = pix.tobytes("png")
+		image_data_lst.append(img_data)
+	return image_data_lst
+
+
+def generate_image_info(pdf_path: str, num_pages: int) -> List[dict]:
+	"""Generate image names based on the PDF file name and the number of pages."""
+	image_info_lst = [
+		{"pdf_path": pdf_path, "pdf_page": page_num + 1}
+		for page_num in range(num_pages)
+	]
+	return image_info_lst
+
+
+def extract_text_from_fields(fields):
+	text = ""
+	for field in fields:
+		text += field["inferText"]
+		if field["lineBreak"]:
+			text += "\n"
+		else:
+			text += " "
+	return text.strip()
+
+
+def json_to_html_table(json_data):
+	# Initialize the HTML table
+	html = '<table border="1">\n'
+	# Determine the number of rows and columns
+	max_row = max(cell["rowIndex"] + cell["rowSpan"] for cell in json_data)
+	max_col = max(cell["columnIndex"] + cell["columnSpan"] for cell in json_data)
+	# Create a 2D array to keep track of merged cells
+	table = [["" for _ in range(max_col)] for _ in range(max_row)]
+	# Fill the table with cell data
+	for cell in json_data:
+		row = cell["rowIndex"]
+		col = cell["columnIndex"]
+		row_span = cell["rowSpan"]
+		col_span = cell["columnSpan"]
+		cell_text = (
+			" ".join(
+				line["inferText"] for line in cell["cellTextLines"][0]["cellWords"]
+			)
+			if cell["cellTextLines"]
+			else ""
+		)
+		# Place the cell in the table
+		table[row][col] = {"text": cell_text, "rowSpan": row_span, "colSpan": col_span}
+		# Mark merged cells as occupied
+		for r in range(row, row + row_span):
+			for c in range(col, col + col_span):
+				if r != row or c != col:
+					table[r][c] = None
+	# Generate HTML from the table array
+	for row in table:
+		html += "  <tr>\n"
+		for cell in row:
+			if cell is None:
+				continue
+			if cell == "":
+				html += "    <td></td>\n"
+			else:
+				row_span_attr = (
+					f' rowspan="{cell["rowSpan"]}"' if cell["rowSpan"] > 1 else ""
+				)
+				col_span_attr = (
+					f' colspan="{cell["colSpan"]}"' if cell["colSpan"] > 1 else ""
+				)
+				html += f'    <td{row_span_attr}{col_span_attr}>{cell["text"]}</td>\n'
+		html += "  </tr>\n"
+	html += "</table>"
+	return html

autorag-workspace/autorag/data/parse/langchain_parse.py

@@ -0,0 +1,87 @@
+import multiprocessing as mp
+from itertools import chain
+from typing import List, Tuple
+
+from autorag.data import parse_modules
+from autorag.data.parse.base import parser_node
+
+
+@parser_node
+def langchain_parse(
+	data_path_list: List[str], parse_method: str, **kwargs
+) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use langchain document_loaders(parse) method
+
+	:param data_path_list: The list of data paths to parse.
+	:param parse_method: A langchain document_loaders(parse) method to use.
+	:param kwargs: The extra parameters for creating the langchain document_loaders(parse) instance.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
+	if parse_method in ["directory", "unstructured"]:
+		results = parse_all_files(data_path_list, parse_method, **kwargs)
+		texts, path = results[0], results[1]
+		pages = [-1] * len(texts)
+
+	else:
+		num_workers = mp.cpu_count()
+		# Execute parallel processing
+		with mp.Pool(num_workers) as pool:
+			results = pool.starmap(
+				langchain_parse_pure,
+				[(data_path, parse_method, kwargs) for data_path in data_path_list],
+			)
+
+		texts, path, pages = (list(chain.from_iterable(item)) for item in zip(*results))
+
+	return texts, path, pages
+
+
+def langchain_parse_pure(
+	data_path: str, parse_method: str, kwargs
+) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parses a single file using the specified parse method.
+
+	Args:
+	    data_path (str): The file path to parse.
+	    parse_method (str): The parsing method to use.
+	    kwargs (Dict): Additional keyword arguments for the parsing method.
+
+	Returns:
+	    Tuple[str, str]: A tuple containing the parsed text and the file path.
+	"""
+
+	parse_instance = parse_modules[parse_method](data_path, **kwargs)
+
+	# Load the text from the file
+	documents = parse_instance.load()
+
+	texts = list(map(lambda x: x.page_content, documents))
+	path = [data_path] * len(texts)
+	if parse_method in ["pymupdf", "pdfplumber", "pypdf", "pypdfium2"]:
+		pages = list(range(1, len(documents) + 1))
+	else:
+		pages = [-1] * len(texts)
+
+	# Clean up the parse instance
+	del parse_instance
+
+	return texts, path, pages
+
+
+def parse_all_files(
+	data_path_list: List[str], parse_method: str, **kwargs
+) -> Tuple[List[str], List[str]]:
+	if parse_method == "unstructured":
+		parse_instance = parse_modules[parse_method](data_path_list, **kwargs)
+	elif parse_method == "directory":
+		parse_instance = parse_modules[parse_method](**kwargs)
+	else:
+		raise ValueError(f"Unsupported parse method: {parse_method}")
+	docs = parse_instance.load()
+	texts = [doc.page_content for doc in docs]
+	file_names = [doc.metadata["source"] for doc in docs]
+
+	del parse_instance
+	return texts, file_names

autorag-workspace/autorag/data/parse/llamaparse.py

@@ -0,0 +1,126 @@
+import os
+from typing import List, Tuple
+from itertools import chain
+
+from llama_parse import LlamaParse
+
+from autorag.data.parse.base import parser_node
+from autorag.utils.util import process_batch, get_event_loop
+
+
+@parser_node
+def llama_parse(
+	data_path_list: List[str],
+	batch: int = 8,
+	use_vendor_multimodal_model: bool = False,
+	vendor_multimodal_model_name: str = "openai-gpt4o",
+	use_own_key: bool = False,
+	vendor_multimodal_api_key: str = None,
+	**kwargs,
+) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use llama_parse.
+	LLAMA_CLOUD_API_KEY environment variable should be set.
+	You can get the key from https://cloud.llamaindex.ai/api-key
+
+	:param data_path_list: The list of data paths to parse.
+	:param batch: The batch size for parse documents. Default is 8.
+	:param use_vendor_multimodal_model: Whether to use the vendor multimodal model. Default is False.
+	:param vendor_multimodal_model_name: The name of the vendor multimodal model. Default is "openai-gpt4o".
+	:param use_own_key: Whether to use the own API key. Default is False.
+	:param vendor_multimodal_api_key: The API key for the vendor multimodal model.
+	:param kwargs: The extra parameters for creating the llama_parse instance.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
+	if use_vendor_multimodal_model:
+		kwargs = _add_multimodal_params(
+			kwargs,
+			use_vendor_multimodal_model,
+			vendor_multimodal_model_name,
+			use_own_key,
+			vendor_multimodal_api_key,
+		)
+
+	parse_instance = LlamaParse(**kwargs)
+
+	tasks = [
+		llama_parse_pure(data_path, parse_instance) for data_path in data_path_list
+	]
+	loop = get_event_loop()
+	results = loop.run_until_complete(process_batch(tasks, batch))
+
+	del parse_instance
+
+	texts, path, pages = (list(chain.from_iterable(item)) for item in zip(*results))
+
+	return texts, path, pages
+
+
+async def llama_parse_pure(
+	data_path: str, parse_instance
+) -> Tuple[List[str], List[str], List[int]]:
+	documents = await parse_instance.aload_data(data_path)
+
+	texts = list(map(lambda x: x.text, documents))
+	path = [data_path] * len(texts)
+	pages = list(range(1, len(documents) + 1))
+
+	return texts, path, pages
+
+
+def _add_multimodal_params(
+	kwargs,
+	use_vendor_multimodal_model,
+	vendor_multimodal_model_name,
+	use_own_key,
+	vendor_multimodal_api_key,
+) -> dict:
+	kwargs["use_vendor_multimodal_model"] = use_vendor_multimodal_model
+	kwargs["vendor_multimodal_model_name"] = vendor_multimodal_model_name
+
+	def set_multimodal_api_key(
+		multimodal_model_name: str = "openai-gpt4o", _api_key: str = None
+	) -> str:
+		if multimodal_model_name in ["openai-gpt4o", "openai-gpt-4o-mini"]:
+			_api_key = (
+				os.getenv("OPENAI_API_KEY", None) if _api_key is None else _api_key
+			)
+			if _api_key is None:
+				raise KeyError(
+					"Please set the OPENAI_API_KEY in the environment variable OPENAI_API_KEY "
+					"or directly set it on the config YAML file."
+				)
+		elif multimodal_model_name in ["anthropic-sonnet-3.5"]:
+			_api_key = (
+				os.getenv("ANTHROPIC_API_KEY", None) if _api_key is None else _api_key
+			)
+			if _api_key is None:
+				raise KeyError(
+					"Please set the ANTHROPIC_API_KEY in the environment variable ANTHROPIC_API_KEY "
+					"or directly set it on the config YAML file."
+				)
+		elif multimodal_model_name in ["gemini-1.5-flash", "gemini-1.5-pro"]:
+			_api_key = (
+				os.getenv("GEMINI_API_KEY", None) if _api_key is None else _api_key
+			)
+			if _api_key is None:
+				raise KeyError(
+					"Please set the GEMINI_API_KEY in the environment variable GEMINI_API_KEY "
+					"or directly set it on the config YAML file."
+				)
+		elif multimodal_model_name in ["custom-azure-model"]:
+			raise NotImplementedError(
+				"Custom Azure multimodal model is not supported yet."
+			)
+		else:
+			raise ValueError("Invalid multimodal model name.")
+
+		return _api_key
+
+	if use_own_key:
+		api_key = set_multimodal_api_key(
+			vendor_multimodal_model_name, vendor_multimodal_api_key
+		)
+		kwargs["vendor_multimodal_api_key"] = api_key
+
+	return kwargs

autorag-workspace/autorag/data/parse/run.py

@@ -0,0 +1,141 @@
+import os
+from typing import List, Callable, Dict
+import pandas as pd
+from glob import glob
+
+from autorag.strategy import measure_speed
+from autorag.data.utils.util import get_param_combinations
+
+default_map = {
+	"pdf": {
+		"file_type": "pdf",
+		"module_type": "langchain_parse",
+		"parse_method": "pdfminer",
+	},
+	"csv": {
+		"file_type": "csv",
+		"module_type": "langchain_parse",
+		"parse_method": "csv",
+	},
+	"md": {
+		"file_type": "md",
+		"module_type": "langchain_parse",
+		"parse_method": "unstructuredmarkdown",
+	},
+	"html": {
+		"file_type": "html",
+		"module_type": "langchain_parse",
+		"parse_method": "bshtml",
+	},
+	"xml": {
+		"file_type": "xml",
+		"module_type": "langchain_parse",
+		"parse_method": "unstructuredxml",
+	},
+}
+
+
+def run_parser(
+	modules: List[Callable],
+	module_params: List[Dict],
+	data_path_glob: str,
+	project_dir: str,
+	all_files: bool,
+):
+	if not all_files:
+		# Set the parsing module to default if it is a file type in paths but not set in YAML.
+		data_path_list = glob(data_path_glob)
+		if not data_path_list:
+			raise FileNotFoundError(f"data does not exits in {data_path_glob}")
+
+		file_types = set(
+			[os.path.basename(data_path).split(".")[-1] for data_path in data_path_list]
+		)
+		set_file_types = set([module["file_type"] for module in module_params])
+
+		# Calculate the set difference once
+		file_types_to_remove = set_file_types - file_types
+
+		# Use list comprehension to filter out unwanted elements
+		module_params = [
+			param
+			for param in module_params
+			if param["file_type"] not in file_types_to_remove
+		]
+		modules = [
+			module
+			for module, param in zip(modules, module_params)
+			if param["file_type"] not in file_types_to_remove
+		]
+
+		# create a list of only those file_types that are in file_types but not in set_file_types
+		missing_file_types = list(file_types - set_file_types)
+
+		if missing_file_types:
+			add_modules_list = []
+			for missing_file_type in missing_file_types:
+				if missing_file_type == "json":
+					raise ValueError(
+						"JSON file type must have a jq_schema so you must set it in the YAML file."
+					)
+
+				add_modules_list.append(default_map[missing_file_type])
+
+			add_modules, add_params = get_param_combinations(add_modules_list)
+			modules.extend(add_modules)
+			module_params.extend(add_params)
+
+	results, execution_times = zip(
+		*map(
+			lambda x: measure_speed(x[0], data_path_glob=data_path_glob, **x[1]),
+			zip(modules, module_params),
+		)
+	)
+	average_times = list(map(lambda x: x / len(results[0]), execution_times))
+
+	# save results to parquet files
+	if all_files:
+		if len(module_params) > 1:
+			raise ValueError(
+				"All files is set to True, You can only use one parsing module."
+			)
+		filepaths = [os.path.join(project_dir, "parsed_result.parquet")]
+	else:
+		filepaths = list(
+			map(
+				lambda x: os.path.join(project_dir, f"{x['file_type']}.parquet"),
+				module_params,
+			)
+		)
+
+	_files = {}
+	for result, filepath in zip(results, filepaths):
+		_files[filepath].append(result) if filepath in _files.keys() else _files.update(
+			{filepath: [result]}
+		)
+	# Save files with a specific file type as Parquet files.
+	for filepath, value in _files.items():
+		pd.concat(value).to_parquet(filepath, index=False)
+
+	filenames = list(map(lambda x: os.path.basename(x), filepaths))
+
+	summary_df = pd.DataFrame(
+		{
+			"filename": filenames,
+			"module_name": list(map(lambda module: module.__name__, modules)),
+			"module_params": module_params,
+			"execution_time": average_times,
+		}
+	)
+	summary_df.to_csv(os.path.join(project_dir, "summary.csv"), index=False)
+
+	# concat all parquet files here if not all_files.
+	_filepaths = list(_files.keys())
+	if not all_files:
+		dataframes = [pd.read_parquet(file) for file in _filepaths]
+		combined_df = pd.concat(dataframes, ignore_index=True)
+		combined_df.to_parquet(
+			os.path.join(project_dir, "parsed_result.parquet"), index=False
+		)
+
+	return summary_df

autorag-workspace/autorag/data/parse/table_hybrid_parse.py

@@ -0,0 +1,134 @@
+import os
+import tempfile
+from glob import glob
+from typing import List, Tuple, Dict
+
+from PyPDF2 import PdfFileReader, PdfFileWriter
+import pdfplumber
+
+from autorag.support import get_support_modules
+from autorag.data.parse.base import parser_node
+
+
+@parser_node
+def table_hybrid_parse(
+	data_path_list: List[str],
+	text_parse_module: str,
+	text_params: Dict,
+	table_parse_module: str,
+	table_params: Dict,
+) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use table_hybrid_parse method.
+	The table_hybrid_parse method is a hybrid method that combines the parsing results of PDFs with and without tables.
+	It splits the PDF file into pages, separates pages with and without tables, and then parses and merges the results.
+
+	:param data_path_list: The list of data paths to parse.
+	:param text_parse_module: The text parsing module to use. The type should be a string.
+	:param text_params: The extra parameters for the text parsing module. The type should be a dictionary.
+	:param table_parse_module: The table parsing module to use. The type should be a string.
+	:param table_params: The extra parameters for the table parsing module. The type should be a dictionary.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
+	# make save folder directory
+	with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as save_dir:
+		text_dir = os.path.join(save_dir, "text")
+		table_dir = os.path.join(save_dir, "table")
+
+		os.makedirs(text_dir, exist_ok=True)
+		os.makedirs(table_dir, exist_ok=True)
+
+		# Split PDF file into pages and Save PDFs with and without tables
+		path_map_dict_lst = [
+			save_page_by_table(data_path, text_dir, table_dir)
+			for data_path in data_path_list
+		]
+		path_map_dict = {k: v for d in path_map_dict_lst for k, v in d.items()}
+
+		# Extract text pages
+		table_results, table_file_path = get_each_module_result(
+			table_parse_module, table_params, os.path.join(table_dir, "*")
+		)
+
+		# Extract table pages
+		text_results, text_file_path = get_each_module_result(
+			text_parse_module, text_params, os.path.join(text_dir, "*")
+		)
+
+		# Merge parsing results of PDFs with and without tables
+		texts = table_results + text_results
+		temp_path_lst = table_file_path + text_file_path
+
+		# Sort by file names
+		temp_path_lst, texts = zip(*sorted(zip(temp_path_lst, texts)))
+
+		# get original file path
+		path = list(map(lambda temp_path: path_map_dict[temp_path], temp_path_lst))
+
+		# get pages
+		pages = list(map(lambda x: get_page_from_path(x), temp_path_lst))
+
+		return list(texts), path, pages
+
+
+# Save PDFs with and without tables
+def save_page_by_table(data_path: str, text_dir: str, table_dir: str) -> Dict[str, str]:
+	file_name = os.path.basename(data_path).split(".pdf")[0]
+
+	with open(data_path, "rb") as input_data:
+		pdf_reader = PdfFileReader(input_data)
+		num_pages = pdf_reader.getNumPages()
+
+		path_map_dict = {}
+		for page_num in range(num_pages):
+			output_pdf_path = _get_output_path(
+				data_path, page_num, file_name, text_dir, table_dir
+			)
+			_save_single_page(pdf_reader, page_num, output_pdf_path)
+			path_map_dict.update({output_pdf_path: data_path})
+
+	return path_map_dict
+
+
+def _get_output_path(
+	data_path: str, page_num: int, file_name: str, text_dir: str, table_dir: str
+) -> str:
+	with pdfplumber.open(data_path) as pdf:
+		page = pdf.pages[page_num]
+		tables = page.extract_tables()
+		directory = table_dir if tables else text_dir
+		return os.path.join(directory, f"{file_name}_page_{page_num + 1}.pdf")
+
+
+def _save_single_page(pdf_reader: PdfFileReader, page_num: int, output_pdf_path: str):
+	pdf_writer = PdfFileWriter()
+	pdf_writer.addPage(pdf_reader.getPage(page_num))
+
+	with open(output_pdf_path, "wb") as output_file:
+		pdf_writer.write(output_file)
+
+
+def get_each_module_result(
+	module: str, module_params: Dict, data_path_glob: str
+) -> Tuple[List[str], List[str]]:
+	module_params["module_type"] = module
+
+	data_path_list = glob(data_path_glob)
+	if not data_path_list:
+		return [], []
+
+	module_name = module_params.pop("module_type")
+	module_callable = get_support_modules(module_name)
+	module_original = module_callable.__wrapped__
+	texts, path, _ = module_original(data_path_list, **module_params)
+
+	return texts, path
+
+
+def get_page_from_path(file_path: str) -> int:
+	file_name = os.path.basename(file_path)
+	split_result = file_name.rsplit("_page_", -1)
+	page_number_with_extension = split_result[1]
+	page_number, _ = page_number_with_extension.split(".")
+
+	return int(page_number)

autorag-workspace/autorag/data/qa/__init__.py

@@ -0,0 +1,3 @@
+# This is v2 version, the next version of data creation
+# The legacy (v1) version will be deprecated on AutoRAG version 0.3
+# The legacy (v1) version and new v2 data creation is not compatible with each other

autorag-workspace/autorag/data/qa/evolve/llama_index_query_evolve.py

@@ -0,0 +1,64 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatResponse, ChatMessage, MessageRole
+
+from autorag.data.qa.evolve.prompt import QUERY_EVOLVE_PROMPT
+
+
+async def llama_index_generate_base(
+	row: Dict,
+	llm: BaseLLM,
+	messages: List[ChatMessage],
+) -> Dict:
+	original_query = row["query"]
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "Text:\n" + "\n".join(
+		[f"{i + 1}. {c}" for i, c in enumerate(context)]
+	)
+	user_prompt = f"Question: {original_query}\nContext: {context_str}\nOutput: "
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	chat_response: ChatResponse = await llm.achat(messages=messages)
+	row["query"] = chat_response.message.content
+	return row
+
+
+async def conditional_evolve_ragas(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row,
+		llm,
+		QUERY_EVOLVE_PROMPT["conditional_evolve_ragas"][lang],
+	)
+
+
+async def reasoning_evolve_ragas(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row,
+		llm,
+		QUERY_EVOLVE_PROMPT["reasoning_evolve_ragas"][lang],
+	)
+
+
+async def compress_ragas(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	original_query = row["query"]
+	user_prompt = f"Question: {original_query}\nOutput: "
+	messages = QUERY_EVOLVE_PROMPT["compress_ragas"][lang]
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	chat_response: ChatResponse = await llm.achat(messages=messages)
+	row["query"] = chat_response.message.content
+	return row

autorag-workspace/autorag/data/qa/evolve/openai_query_evolve.py

@@ -0,0 +1,81 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.evolve.prompt import QUERY_EVOLVE_PROMPT
+
+
+class Response(BaseModel):
+	evolved_query: str
+
+
+async def query_evolve_openai_base(
+	row: Dict,
+	client: AsyncClient,
+	messages: List[ChatMessage],
+	model_name: str = "gpt-4o-2024-08-06",
+):
+	"""
+	Evolve the original query to a new evolved query using OpenAI structured outputs.
+	"""
+	original_query = row["query"]
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "Text:\n" + "\n".join(
+		[f"{i + 1}. {c}" for i, c in enumerate(context)]
+	)
+	user_prompt = f"Question: {original_query}\nContext: {context_str}\nOutput: "
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=Response,
+	)
+	row["query"] = completion.choices[0].message.parsed.evolved_query
+	return row
+
+
+async def conditional_evolve_ragas(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_evolve_openai_base(
+		row, client, QUERY_EVOLVE_PROMPT["conditional_evolve_ragas"][lang], model_name
+	)
+
+
+async def reasoning_evolve_ragas(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_evolve_openai_base(
+		row, client, QUERY_EVOLVE_PROMPT["reasoning_evolve_ragas"][lang], model_name
+	)
+
+
+async def compress_ragas(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	original_query = row["query"]
+	messages = QUERY_EVOLVE_PROMPT["compress_ragas"][lang]
+	user_prompt = f"Question: {original_query}\nOutput: "
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=Response,
+	)
+	row["query"] = completion.choices[0].message.parsed.evolved_query
+	return row

autorag-workspace/autorag/data/qa/evolve/prompt.py

@@ -0,0 +1,288 @@
+# The RAGAS prompts are coming from RAGAS under Apache-2.0 License. (English version) (the AutoRAG team translates Korean version prompt)
+# You can see the original prompts at the RAGAS library at https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/prompts.py
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+
+QUERY_EVOLVE_PROMPT = {
+	"conditional_evolve_ragas": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""Rewrite the provided question to increase its complexity by introducing a conditional element.
+The goal is to make the question more intricate by incorporating a scenario or condition that affects the context of the question.
+Follow the rules given below while rewriting the question.
+    1. The rewritten question should not be longer than 25 words. Use abbreviation wherever possible.
+    2. The rewritten question must be reasonable and must be understood and responded by humans.
+    3. The rewritten question must be fully answerable from information present context.
+    4. phrases like 'provided context','according to the context?',etc are not allowed to appear in the question.
+""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question : What is the function of the roots of a plant?
+Context : The roots of a plant absorb water and nutrients from the soil, anchor the plant in the ground, and store food.
+Output : """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="What dual purpose do plant roots serve concerning soil nutrients and stability?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question : How do vaccines protect against diseases?
+Context : Vaccines protect against diseases by stimulating the body's immune response to produce antibodies, which recognize and combat pathogens.
+Output : """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="How do vaccines utilize the body's immune system to defend against pathogens?",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""제공된 질문에 조건에 관련한 내용을 추가하여 복잡성을 높이세요.
+질문의 Context에 영향을 미치는 시나리오나 조건을 포함하여 질문을 더 복잡하게 만드는 것이 목표입니다.
+질문을 다시 작성할 때 다음 규칙을 따르십시오.
+    1. 다시 작성된 질문은 100자를 넘지 않아야 합니다. 가능한 경우 약어를 사용하십시오.
+    2. 다시 작성된 질문은 합리적이어야 하며 사람이 이해하고 응답할 수 있어야 합니다.
+    3. 다시 작성된 질문은 현재 Context에서 완전히 답변할 수 있어야 합니다.
+    4. '제공된 글', '단락에 따르면?', 'Context에 의하면' 등의 문구는 질문에 나타날 수 없습니다.
+    5. 한국어로 질문을 작성하세요.
+""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 식물의 뿌리 기능이 뭐야?
+Context: 식물의 뿌리는 토양에서 물과 영양분을 흡수하고, 식물을 땅에 고정하며, 영양분을 저장합니다.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="식물의 뿌리는 토양 영양분과 안정성에 대해 어떤 역할을 하나요?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 백신은 질병을 어떻게 예방하나요?
+Context: 백신은 신체의 면역 반응을 자극하여 병원체를 인식하고 싸우는 항체를 생성함으로써 질병으로부터 보호합니다.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="백신은 신체의 면역 체계를 어떻게 활용해서 질병을 예방합니까?",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""提供された質問に条件に関する内容を追加して、複雑さを高めます。
+質問のContextに影響を与えるシナリオや条件を含めて、質問をより複雑にすることが目標です。
+質問を再作成するときは、次のルールに従います。
+    1. 再作成された質問は100文字を超えてはいけません。 可能であれば略語を使ってください
+    2. 再作成された質問は合理的でなければならず、人が理解して回答できるものでなければなりません。
+    3. 再作成された質問は、現在のContextで完全に答えられる必要があります。
+    4. 「提供された文」、「段落によると?」、「Contextによると」などのフレーズは質問に表示されません。
+    5. 日本語で質問を書きましょう。
+""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 植物の根の機能は何ですか？
+Context: 植物の根は土壌から水や栄養分を吸収し、植物を地面に固定し、栄養分を蓄えます。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="植物の根は土壌栄養分と安定性に対してどのような役割をしますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: ワクチンは病気をどのように予防しますか?
+Context: ワクチンは、体の免疫反応を刺激して病原体を認識し、戦う抗体を生成することで病気から守ります。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="ワクチンは体の免疫システムをどのように活用して病気を予防しますか？",
+			),
+		],
+	},
+	"reasoning_evolve_ragas": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""Complicate the given question by rewriting question into a multi-hop reasoning question based on the provided context.
+Answering the question should require the reader to make multiple logical connections or inferences using the information available in given context.
+Rules to follow when rewriting question:
+1. Ensure that the rewritten question can be answered entirely from the information present in the contexts.
+2. Do not frame questions that contains more than 15 words. Use abbreviation wherever possible.
+3. Make sure the question is clear and unambiguous.
+4. phrases like 'based on the provided context','according to the context',etc are not allowed to appear in the question.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What is the capital of France?,
+Context: France is a country in Western Europe. It has several cities, including Paris, Lyon, and Marseille. Paris is not only known for its cultural landmarks like the Eiffel Tower and the Louvre Museum but also as the administrative center.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="Linking the Eiffel Tower and administrative center, which city stands as both?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What does the append() method do in Python?
+Context: In Python, lists are used to store multiple items in a single variable. Lists are one of 4 built-in data types used to store collections of data. The append() method adds a single item to the end of a list.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="If a list represents a variable collection, what method extends it by one item?",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""주어진 Context를 기반으로 기존 질문을 복잡하게 만들어 여러 논리적인 사고가 필요한 질문으로 다시 작성하세요.
+질문에 답하려면 주어진 Context의 정보를 사용해 여러 논리적 사고나 추론을 해야 합니다.
+질문을 다시 작성할 때 따라야 할 규칙:
+1. 다시 작성된 질문은 Context에 있는 정보만으로 완전히 답변할 수 있어야 합니다.
+2. 100자를 초과하는 질문을 작성하지 마세요. 가능한 경우 약어를 사용하세요.
+3. 질문이 명확하고 모호하지 않도록 하세요.
+4. '제공된 Context에 기반하여', '해당 단락에 따르면' 등의 문구는 질문에 포함되지 않아야 합니다.
+5. 한국어로 질문을 작성하세요.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 프랑스의 수도는 어디인가요?,
+Context: 프랑스는 서유럽에 있는 나라입니다. 파리, 리옹, 마르세유를 포함한 여러 도시가 있습니다. 파리는 에펠탑과 루브르 박물관 같은 문화적 랜드마크로 유명할 뿐만 아니라 행정 중심지로도 알려져 있습니다.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="에펠탑과 행정 중심지, 두 단어는 어떤 도시를 가리키나요?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""질문: Python에서 append() 메서드는 무엇을 하나요?
+컨텍스트: Python에서 리스트는 하나의 변수에 여러 항목을 저장하는 데 사용됩니다. 리스트는 데이터를 저장하는 데 사용되는 4가지 내장 데이터 유형 중 하나입니다. append() 메서드는 리스트의 끝에 새로운 항목을 추가합니다.
+출력: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="리스트가 변수들을 모아 놓은 것을 나타낸다면, 어떤 메서드를 사용해야 항목을 하나 더 추가할 수 있습니까?",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""与えられたContextに基づいて既存の質問を複雑にして、様々な論理的思考が必要な質問として書き直しましょう。
+質問に答えるためには、与えられたContextの情報を使って様々な論理的思考や推論をしなければなりません。
+質問を再作成するときに従うべきルール:
+1. 再作成された質問は、Contextにある情報だけで完全に答えられる必要があります。
+2. 100文字を超える質問を作成してはいけません。 可能であれば略語を使ってください。
+3. 質問が明確で曖昧にならないようにしましょう。
+4. 「提供されたContextに基づいて」、「当該段落によると」などのフレーズは、質問に含まれてはいけません。
+5. 日本語で質問を書きましょう。""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: フランスの首都はどこですか？,
+Context: フランスは西ヨーロッパにある国です。 パリ、リヨン、マルセイユを含むいくつかの都市があります。 パリはエッフェル塔やルーブル博物館のような文化的ランドマークとして有名なだけでなく、行政の中心地としても知られています。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="エッフェル塔と行政の中心地、二つの単語はどんな都市を指していますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: Pythonでappend() メソッドは何をしますか？
+Context: Pythonで、リストは 1 つの変数に複数の項目を保存するために使用されます。 リストは、データを保存するために使用される 4 つの組み込みデータ タイプの 1 つです。 append()メソッドは、リストの最後に新しい項目を追加します。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="リストが変数を集めたものである場合、どのメソッドを使えば項目を一つ追加することができますか？",
+			),
+		],
+	},
+	"compress_ragas": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""Rewrite the following question to make it more indirect and shorter while retaining the essence of the original question.
+    The goal is to create a question that conveys the same meaning but in a less direct manner. The rewritten question should shorter so use abbreviation wherever possible.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What is the distance between the Earth and the Moon?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="How far is the Moon from Earth?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What ingredients are required to bake a chocolate cake?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="What's needed for a chocolate cake?",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""주어진 질문을 더 간접적이고 짧게 다시 작성하세요.
+        목표는 질문을 원래 질문의 본질을 유지하면서 너무 직설적이지 않게 만드는 것입니다.
+        약어 등을 사용하여 질문을 더 짧게 만드세요.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 지구와 달 사이의 거리는 얼마입니까?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="달은 지구에서 얼마나 떨어져 있나요?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 초콜릿 케이크를 굽기 위해 필요한 재료는 무엇입니까?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="초콜릿 케이크에 필요한 것은 무엇인가요?",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""与えられた質問をより間接的かつ短く書き換えます。
+目標は、質問を元の質問の本質を保ちながら、あまりストレートにならないようにすることです。
+略語などを使用して、質問をより短くします。""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 地球と月の間の距離はどれくらいですか？
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="月は地球からどれくらい離れていますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: チョコレートケーキを焼くために必要な材料は何ですか？
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="チョコレートケーキに必要なものは何ですか？",
+			),
+		],
+	},
+}

autorag-workspace/autorag/data/qa/extract_evidence.py

@@ -0,0 +1 @@
+# This module is about extracting evidence from the given retrieval gt passage

autorag-workspace/autorag/data/qa/filter/dontknow.py

@@ -0,0 +1,117 @@
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatMessage, MessageRole, ChatResponse
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.filter.prompt import FILTER_PROMPT
+
+dont_know_phrases = {
+	"en": [
+		"I don't know",
+		"I do not know",
+		"Don't know",
+		"Do not know",
+	],
+	"ko": [
+		"몰라요",
+		"모르겠습니다",
+		"모르겠어요",
+		"몰라",
+		"내가 어떻게 알아?",
+		"모르겠소",
+		"몰라유",
+		"모르것는디",
+		"모르겠어유",
+		"모르겠네유",
+		"모르겠네요",
+	],
+	"ja": [
+		"知りません",
+		"わかりません",
+		"分かりません",
+		"知らないです",
+		"よく分かってません",
+		"わかりかねます",
+		"存じません",
+		"お答えいたしかねます",
+	],
+}
+
+
+def dontknow_filter_rule_based(row: Dict, lang: str = "en") -> bool:
+	assert (
+		"generation_gt" in row.keys()
+	), "generation_gt column is not in the DataFrame."
+	dont_know_phrase = dont_know_phrases[lang]
+	return not any(
+		phrase in s for phrase in dont_know_phrase for s in row["generation_gt"]
+	)
+
+
+class Response(BaseModel):
+	is_dont_know: bool
+
+
+async def dontknow_filter_openai(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-mini-2024-07-18",
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop rows that have a "don't know" answer.
+	It will drop unanswerable questions from the QA dataset.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	:param client: The OpenAI client.
+	:param model_name: The model name.
+		You have to use gpt-4o-2024-08-06 or gpt-4o-mini-2024-07-18.
+	:param lang: The supported language is en, ko or ja.
+	:return: False if the row generation_gt is a "don't know" meaning.
+	"""
+	assert "generation_gt" in row.keys(), "generation_gt column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["dontknow_filter"][lang]
+	result = []
+	for gen_gt in row["generation_gt"]:
+		completion = await client.beta.chat.completions.parse(
+			model=model_name,
+			messages=to_openai_message_dicts(
+				system_prompt + [ChatMessage(role=MessageRole.USER, content=gen_gt)]
+			),
+			response_format=Response,
+		)
+		result.append(completion.choices[0].message.parsed.is_dont_know)
+	return not any(result)
+
+
+async def dontknow_filter_llama_index(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop rows that have a "don't know" answer.
+	It will drop unanswerable questions from the QA dataset.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	:param llm: The Llama index llm instance.
+		It will be good if you set max tokens to low for saving tokens.
+	:param lang: The supported language is en, ko or ja.
+	:return: False if the row generation_gt is a "don't know" meaning.
+	"""
+	assert "generation_gt" in row.keys(), "generation_gt column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["dontknow_filter"][lang]
+	results = []
+	for gen_gt in row["generation_gt"]:
+		response: ChatResponse = await llm.achat(
+			messages=system_prompt
+			+ [ChatMessage(role=MessageRole.USER, content=gen_gt)]
+		)
+		result_str = response.message.content
+		results.append("true" in result_str.lower().strip())
+	return not any(results)

autorag-workspace/autorag/data/qa/filter/passage_dependency.py

@@ -0,0 +1,88 @@
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatMessage, MessageRole, ChatResponse
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.filter.prompt import FILTER_PROMPT
+
+
+class Response(BaseModel):
+	is_passage_dependent: bool
+
+
+async def passage_dependency_filter_openai(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-mini-2024-07-18",
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop passage-dependent question rows.
+	Passage-dependent questions are questions that the answer will change depending on what passage you choose.
+	The passage-dependent questions will not be good for RAG evaluation, because any retrieval system can't find the right passage with passage-dependent question.
+	For example, when someone asks "What is the highest score according to the table?" the answer will be different depending on the table.
+	And what is the table? The retrieval system can't find the right passage with this question.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	    :param client: The OpenAI client.
+	    :param model_name: The model name.
+	            You have to use gpt-4o-2024-08-06 or gpt-4o-mini-2024-07-18.
+	    :param lang: The supported language is en, ko or ja.
+	:return: False if the row question is a passage-dependent question (to be filtered).
+	"""
+	assert "query" in row.keys(), "query column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["passage_dependency"][lang]
+	query = row["query"]
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(
+			system_prompt
+			+ [
+				ChatMessage(
+					role=MessageRole.USER,
+					content=f"Question: {query}\nIs this the question passage dependent?",
+				)
+			]
+		),
+		response_format=Response,
+	)
+	return not completion.choices[0].message.parsed.is_passage_dependent
+
+
+async def passage_dependency_filter_llama_index(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop passage-dependent question rows.
+	Passage-dependent questions are questions that the answer will change depending on what passage you choose.
+	The passage-dependent questions will not be good for RAG evaluation, because any retrieval system can't find the right passage with passage-dependent question.
+	For example, when someone asks "What is the highest score according to the table?" the answer will be different depending on the table.
+	And what is the table? The retrieval system can't find the right passage with this question.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	:param llm: The Llama index llm instance.
+	            It will be good if you set max tokens to low for saving tokens.
+	    :param lang: The supported language is en, ko or ja.
+	:return: False if the row question is a passage-dependent question (to be filtered).
+	"""
+	assert "query" in row.keys(), "query column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["passage_dependency"][lang]
+	query = row["query"]
+	response: ChatResponse = await llm.achat(
+		messages=system_prompt
+		+ [
+			ChatMessage(
+				role=MessageRole.USER,
+				content=f"Question: {query}\nIs this the question passage dependent?",
+			)
+		]
+	)
+	result_str = response.message.content
+	return "true" not in result_str.lower().strip()

autorag-workspace/autorag/data/qa/filter/prompt.py

@@ -0,0 +1,73 @@
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+
+FILTER_PROMPT = {
+	"dontknow_filter": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""The following sentence is an answer about a question. You have to decide the answer implies 'I don't know'.
+If the answer implies 'I don't know', return True. If not, return False.""",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""다음 문장은 어떠한 질문에 대한 대답입니다. 해당 문장이 질문에 대해서 '모른다고' 답한 것인지 판단하십시오.
+만약 해당 문장이 '모른다고' 답한 것이라면, True를 반환하세요. 그렇지 않다면 False를 반환하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""次の文章はある質問に対する答えです。 該当文章が質問に対して「知らない」と答えたのか判断します。
+もし、その文章が「知らない」と答えたのであれば、Trueを返します。 そうでなければFalseを返します。""",
+			)
+		],
+	},
+	"passage_dependency": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""You are a classifier that recognize 'passage dependent' questions.
+The 'passage dependent' is the question that the answer will be change depending on what passage you choose.
+For example) 'What is the highest score according to the table?'
+This sentence is the passage dependent question because the answer will be different depending on the table.
+
+In contrast, the following sentence is not passage dependant.
+'What is the highest score of the KBO baseball history in one game?'
+'What is the capital of France?'
+These sentences will have the same answer regardless of the passage.
+
+Please return True if the input question is passage dependent. Else return False.""",
+			)
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""당신은 '단락 의존' 질문을 인식하는 분류기입니다.
+'단락 의존'이란 어떤 단락이 선택 되는지 따라 답이 달라지는 질문을 의미합니다.
+예를 들어, '주어진 표에 따르면 가장 높은 점수는 무엇인가요?'라는 질문은 단락 의존 질문입니다. 왜냐하면 표가 어떤 것인지에 따라 그 답이 달라지기 때문입니다.
+
+반면에, 다음 문장들은 단락 의존적이지 않습니다.
+'KBO 야구 역사상 한 경기에서 가장 높은 점수는 무엇인가요?' 또는 '프랑스의 수도는 무엇인가요?'
+이러한 문장은 단락에 관계 없이 동일한 답을 가집니다.
+
+입력된 질문이 단락 의존적이라면 True를 반환하고, 그렇지 않으면 False를 반환하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""あなたは「段落依存」の質問を認識する分類器です。
+「段落依存」とは、どの段落が選択されるかによって答えが変わる質問を意味します。
+たとえば、「与えられた表によると、最も高い点数は何ですか？」という質問は、段落依存の質問です。 なぜなら、表がどんなものかによってその答えが変わるからです。
+
+一方、次の文章は段落依存的ではありません。
+KBO野球史上1試合で最も高い点数は何ですか?またはフランスの首都は何ですか?'
+このような文章は段落に関係なく同じ答えを持ちます。
+
+入力された質問が段落依存的である場合はTrueを返し、そうでない場合はFalseを返します。""",
+			)
+		],
+	},
+}

autorag-workspace/autorag/data/qa/generation_gt/base.py

@@ -0,0 +1,16 @@
+from typing import Dict
+
+
+def add_gen_gt(row: Dict, new_gen_gt: str) -> Dict:
+	if "generation_gt" in list(row.keys()):
+		if isinstance(row["generation_gt"], list):
+			row["generation_gt"].append(new_gen_gt)
+		elif isinstance(row["generation_gt"], str):
+			row["generation_gt"] = [row["generation_gt"], new_gen_gt]
+		else:
+			raise ValueError(
+				"generation_gt should be either a string or a list of strings."
+			)
+		return row
+	row["generation_gt"] = [new_gen_gt]
+	return row

autorag-workspace/autorag/data/qa/generation_gt/llama_index_gen_gt.py

@@ -0,0 +1,41 @@
+import itertools
+from typing import Dict
+
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import MessageRole, ChatMessage
+
+from autorag.data.qa.generation_gt.base import add_gen_gt
+from autorag.data.qa.generation_gt.prompt import GEN_GT_SYSTEM_PROMPT
+
+
+async def make_gen_gt_llama_index(row: Dict, llm: BaseLLM, system_prompt: str) -> Dict:
+	retrieval_gt_contents = list(
+		itertools.chain.from_iterable(row["retrieval_gt_contents"])
+	)
+	query = row["query"]
+	passage_str = "\n".join(retrieval_gt_contents)
+	user_prompt = f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:"
+
+	response = await llm.achat(
+		messages=[
+			ChatMessage(role=MessageRole.SYSTEM, content=system_prompt),
+			ChatMessage(role=MessageRole.USER, content=user_prompt),
+		],
+		temperature=0.0,
+	)
+	return add_gen_gt(row, response.message.content)
+
+
+async def make_concise_gen_gt(row: Dict, llm: BaseLLM, lang: str = "en") -> Dict:
+	return await make_gen_gt_llama_index(
+		row, llm, GEN_GT_SYSTEM_PROMPT["concise"][lang]
+	)
+
+
+async def make_basic_gen_gt(row: Dict, llm: BaseLLM, lang: str = "en") -> Dict:
+	return await make_gen_gt_llama_index(row, llm, GEN_GT_SYSTEM_PROMPT["basic"][lang])
+
+
+async def make_custom_gen_gt(row: Dict, llm: BaseLLM, system_prompt: str) -> Dict:
+	return await make_gen_gt_llama_index(row, llm, system_prompt)

autorag-workspace/autorag/data/qa/generation_gt/openai_gen_gt.py

@@ -0,0 +1,84 @@
+import itertools
+from typing import Dict
+
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.generation_gt.base import add_gen_gt
+from autorag.data.qa.generation_gt.prompt import GEN_GT_SYSTEM_PROMPT
+
+
+class Response(BaseModel):
+	answer: str
+
+
+async def make_gen_gt_openai(
+	row: Dict,
+	client: AsyncClient,
+	system_prompt: str,
+	model_name: str = "gpt-4o-2024-08-06",
+):
+	retrieval_gt_contents = list(
+		itertools.chain.from_iterable(row["retrieval_gt_contents"])
+	)
+	query = row["query"]
+	passage_str = "\n".join(retrieval_gt_contents)
+	user_prompt = f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:"
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=[
+			{"role": "system", "content": system_prompt},
+			{"role": "user", "content": user_prompt},
+		],
+		temperature=0.0,
+		response_format=Response,
+	)
+	response: Response = completion.choices[0].message.parsed
+	return add_gen_gt(row, response.answer)
+
+
+async def make_concise_gen_gt(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+):
+	"""
+	Generate concise generation_gt using OpenAI Structured Output for preventing errors.
+	It generates a concise answer, so it is generally a word or just a phrase.
+
+	:param row: The input row of the qa dataframe.
+	:param client: The OpenAI async client.
+	:param model_name: The model name that supports structured output.
+	    It has to be "gpt-4o-2024-08-06" or "gpt-4o-mini-2024-07-18".
+	:param lang: The language code of the prompt.
+		Default is "en".
+	:return: The output row of the qa dataframe with added "generation_gt" in it.
+	"""
+	return await make_gen_gt_openai(
+		row, client, GEN_GT_SYSTEM_PROMPT["concise"][lang], model_name
+	)
+
+
+async def make_basic_gen_gt(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+):
+	"""
+	Generate basic generation_gt using OpenAI Structured Output for preventing errors.
+	It generates a "basic" answer, and its prompt is simple.
+
+	:param row: The input row of the qa dataframe.
+	:param client: The OpenAI async client.
+	:param model_name: The model name that supports structured output.
+	    It has to be "gpt-4o-2024-08-06" or "gpt-4o-mini-2024-07-18".
+	:param lang: The language code of the prompt.
+		Default is "en".
+	:return: The output row of the qa dataframe with added "generation_gt" in it.
+	"""
+	return await make_gen_gt_openai(
+		row, client, GEN_GT_SYSTEM_PROMPT["basic"][lang], model_name
+	)

autorag-workspace/autorag/data/qa/generation_gt/prompt.py

@@ -0,0 +1,27 @@
+GEN_GT_SYSTEM_PROMPT = {
+	"concise": {
+		"en": """You are an AI assistant to answer the given question in the provide evidence text.
+You can find the evidence from the given text about question, and you have to write a proper answer to the given question.
+Your answer have to be concise and relevant to the question.
+Do not make a verbose answer and make it super clear.
+It doesn't have to be an full sentence. It can be the answer is a word or a paraphrase.""",
+		"ko": """당신은 주어진 질문에 대해 제공된 Text 내에서 답을 찾는 AI 비서입니다.
+질문에 대한 답을 Text에서 찾아 적절한 답변을 작성하세요.
+답변은 간결하고 질문에 관련된 내용만 포함해야 합니다.
+불필요하게 길게 답변하지 말고, 명확하게 작성하세요.
+완전한 문장이 아니어도 되며, 답은 단어나 요약일 수 있습니다.""",
+		"ja": """
+あなたは与えられた質問に対して提供されたText内で答えを探すAI秘書です。
+質問に対する答えをTextで探して適切な答えを作成しましょう。
+回答は簡潔で、質問に関連する内容のみを含める必要があります。
+不必要に長く答えず、明確に作成しましょう。
+完全な文章でなくてもいいし、答えは単語や要約かもしれません。
+""",
+	},
+	"basic": {
+		"en": """You are an AI assistant to answer the given question in the provide evidence text.
+You can find the evidence from the given text about question, and you have to write a proper answer to the given question.""",
+		"ko": "당신은 주어진 질문에 대한 답을 제공된 Text 내에서 찾는 AI 비서입니다. 질문과 관련된 증거를 Text에서 찾아 적절한 답변을 작성하세요.",
+		"ja": "あなたは与えられた質問に対する答えを提供されたText内で探すAI秘書です。 質問に関する証拠をTextで探して適切な回答を作成しましょう。",
+	},
+}

autorag-workspace/autorag/data/qa/query/llama_gen_query.py

@@ -0,0 +1,82 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatResponse, ChatMessage, MessageRole
+
+from autorag.data.qa.query.prompt import QUERY_GEN_PROMPT, QUERY_GEN_PROMPT_EXTRA
+
+
+async def llama_index_generate_base(
+	row: Dict,
+	llm: BaseLLM,
+	messages: List[ChatMessage],
+) -> Dict:
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "\n".join([f"{i + 1}. {c}" for i, c in enumerate(context)])
+	user_prompt = f"Text:\n{context_str}\n\nGenerated Question from the Text:\n"
+	user_message = ChatMessage(role=MessageRole.USER, content=user_prompt)
+	new_messages = [*messages, user_message]
+	chat_response: ChatResponse = await llm.achat(messages=new_messages)
+	row["query"] = chat_response.message.content
+	return row
+
+
+async def factoid_query_gen(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row, llm, QUERY_GEN_PROMPT["factoid_single_hop"][lang]
+	)
+
+
+async def concept_completion_query_gen(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row, llm, QUERY_GEN_PROMPT["concept_completion"][lang]
+	)
+
+
+async def two_hop_incremental(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	messages = QUERY_GEN_PROMPT["two_hop_incremental"][lang]
+	passages = row["retrieval_gt_contents"]
+	assert (
+		len(passages) >= 2
+	), "You have to sample more than two passages for making two-hop questions."
+	context_str = f"Document 1: {passages[0][0]}\nDocument 2: {passages[1][0]}"
+	user_prompt = f"{context_str}\n\nGenerated two-hop Question from two Documents:\n"
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	chat_response: ChatResponse = await llm.achat(messages=messages)
+	response = chat_response.message.content
+	row["query"] = response.split(":")[-1].strip()
+	return row
+
+
+async def custom_query_gen(
+	row: Dict,
+	llm: BaseLLM,
+	messages: List[ChatMessage],
+) -> Dict:
+	return await llama_index_generate_base(row, llm, messages)
+
+
+# Experimental feature: can only use factoid_single_hop
+async def multiple_queries_gen(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+	n: int = 3,
+) -> Dict:
+	_messages = QUERY_GEN_PROMPT["factoid_single_hop"][lang]
+	_messages[0].content += QUERY_GEN_PROMPT_EXTRA["multiple_queries"][lang].format(n=n)
+	return await llama_index_generate_base(row, llm, _messages)

autorag-workspace/autorag/data/qa/query/openai_gen_query.py

@@ -0,0 +1,95 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.query.prompt import QUERY_GEN_PROMPT
+
+
+class Response(BaseModel):
+	query: str
+
+
+# Single hop QA generation OpenAI
+async def query_gen_openai_base(
+	row: Dict,
+	client: AsyncClient,
+	messages: List[ChatMessage],
+	model_name: str = "gpt-4o-2024-08-06",
+):
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "Text:\n" + "\n".join(
+		[f"{i + 1}. {c}" for i, c in enumerate(context)]
+	)
+	user_prompt = f"{context_str}\n\nGenerated Question from the Text:\n"
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=Response,
+	)
+	row["query"] = completion.choices[0].message.parsed.query
+	return row
+
+
+async def factoid_query_gen(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_gen_openai_base(
+		row, client, QUERY_GEN_PROMPT["factoid_single_hop"][lang], model_name
+	)
+
+
+async def concept_completion_query_gen(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_gen_openai_base(
+		row, client, QUERY_GEN_PROMPT["factoid_single_hop"][lang], model_name
+	)
+
+
+class TwoHopIncrementalResponse(BaseModel):
+	answer: str
+	one_hop_question: str
+	two_hop_question: str
+
+
+async def two_hop_incremental(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	"""
+	Create a two-hop question using incremental prompt.
+	Incremental prompt is more effective to create multi-hop question.
+	The input retrieval_gt has to include more than one passage.
+
+	:return: The two-hop question using openai incremental prompt
+	"""
+	messages = QUERY_GEN_PROMPT["two_hop_incremental"][lang]
+	passages = row["retrieval_gt_contents"]
+	assert (
+		len(passages) >= 2
+	), "You have to sample more than two passages for making two-hop questions."
+	context_str = f"Document 1: {passages[0][0]}\nDocument 2: {passages[1][0]}"
+	user_prompt = f"{context_str}\n\nGenerated two-hop Question from two Documents:\n"
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=TwoHopIncrementalResponse,
+	)
+	row["query"] = completion.choices[0].message.parsed.two_hop_question
+	return row

autorag-workspace/autorag/data/qa/query/prompt.py

@@ -0,0 +1,202 @@
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+
+QUERY_GEN_PROMPT = {
+	"factoid_single_hop": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""You're an AI tasked to convert Text into a factoid question.
+Factoid questions are those seeking brief, factual information that can be easily verified. They typically require a yes or no answer or a brief explanation and often inquire about specific details such as dates, names, places, or events.
+
+Examples of factoid questions include:
+
+- What is the capital of France?
+- Who invented the light bulb?
+- When was Wikipedia founded?
+
+Instructions:
+1. Questions MUST BE extracted from given Text
+2. Questions should be as detailed as possible from Text
+3. Create questions that ask about factual information from the Text
+4. Do not mention any of these in the questions: "in the given text", "in the provided information", etc.
+Users do not know the passage source of the question, so it should not be mentioned in the question.
+5. Do not ask about the file name or the file title. Ask about the content of the file.
+For example, avoid to write questions like `What is the file name of the document?`""",
+			)
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""당신은 주어진 Text를 '사실 질문'으로 변환하는 AI입니다.
+
+사실 질문(factoid questions)이란 사실적인 정보를 요구하는 질문으로, 쉽게 검증할 수 있는 답변을 필요로 합니다. 일반적으로 예/아니오 답변이나 간단한 설명을 요구하며, 날짜, 이름, 장소 또는 사건과 같은 구체적인 세부사항에 대해 묻는 질문입니다.
+
+사실 질문의 예는 다음과 같습니다:
+
+	•	프랑스의 수도는 어디입니까?
+	•	전구를 발명한 사람은 누구입니까?
+	•	위키피디아는 언제 설립되었습니까?
+
+지침:
+	1.	질문은 반드시 주어진 Text를 기반으로 작성되어야 합니다.
+	2.	질문은 Text를 기반으로 가능한 한 구체적으로 작성되어야 합니다.
+	3.	Text에서 사실적 정보를 요구하는 질문을 만들어야 합니다. 즉, Text를 기반으로 사실 질문을 만드세요.
+	4.	질문에 “주어진 Text에서” 또는 “제공된 단락에서”와 같은 표현을 포함해서는 안 됩니다.
+사용자는 질문의 출처가 Text라는 것을 모르기 때문에 반드시 그 출처를 언급해서는 안 됩니다.
+	5.	파일 이름이나 파일 제목에 대한 질문을 하지 마세요. 파일의 내용에 대해 물어보세요.
+예를 들어, '문서의 파일 이름은 무엇입니까?'와 같은 질문을 작성하지 마세요.
+	6.	질문을 한국어로 작성하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""あなたは与えられたTextを「実は質問」に変換するAIです。
+
+事実質問(factoid questions)とは、事実的な情報を求める質問であり、容易に検証できる回答を必要とします。 一般的に、「はい/いいえ」の返答や簡単な説明を要求し、日付、名前、場所、または事件のような具体的な詳細事項について尋ねる質問です。
+
+実は質問の例は次の通りです:
+
+	• フランスの首都はどこですか？
+	• 電球を発明したのは誰ですか？
+	• ウィキペディアはいつ設立されましたか？
+
+指針:
+	1. 質問は、必ず与えられたTextに基づいて作成されなければなりません。
+	2. 質問は、Textに基づいて可能な限り具体的に作成されなければなりません。
+	3. Textで事実的情報を要求する質問を作らなければなりません。 つまり、Textに基づいて質問を作成します。
+	4. 質問に「与えられたTextで」または「提供された段落で」のような表現を含めてはいけません。
+ユーザーは質問の出所がTextだということを知らないので、必ずしもその出所を言及してはいけません。
+	5. ファイル名やファイルタイトルを訊かないでください。ファイルの内容について聞いてください。
+例えば、「このドキュメントのファイル名は何ですか？
+	6. 質問を日本語で作成しなさい。""",
+			)
+		],
+	},
+	"concept_completion": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""You're an AI tasked to convert Text into a "Concept Completion" Question.
+A “concept completion” question asks directly about the essence or identity of a concept.
+
+Follow the following instructions.
+Instructions:
+1. Questions MUST BE extracted from given Text
+2. Questions should be as detailed as possible from Text
+3. Create questions that ask about information from the Text
+4. MUST include specific keywords from the Text.
+5. Do not mention any of these in the questions: "in the given text", "in the provided information", etc.
+Users do not know the passage source of the question, so it should not be mentioned in the question.
+6. Do not ask about the file name or the file title. Ask about the content of the file.
+For example, avoid to write questions like `What is the file name of the document?""",
+			)
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""당신은 Text를 “개념 완성” 질문으로 변환하는 AI입니다.
+"개념 완성" 질문은 개념의 본질이나 정체성에 대해 직접적으로 묻는 질문입니다.
+
+다음 지시사항을 따르세요.
+지시사항:
+1.	질문은 반드시 주어진 Text를 기반으로 작성되어야 합니다.
+2.	질문은 Text를 기반으로 가능한 한 자세하게 작성되어야 합니다.
+3.	Text에서 제공된 정보를 묻는 질문을 생성하세요.
+4.	Text의 특정 키워드를 반드시 질문에 포함하세요.
+5.	질문에 “주어진 Text에서” 또는 “제공된 단락에서”와 같은 표현을 포함해서는 안 됩니다.
+사용자는 질문의 출처가 Text라는 것을 모르기 때문에 반드시 그 출처를 언급해서는 안 됩니다.
+6.	파일 이름이나 파일 제목에 대한 질문을 하지 마세요. 파일의 내용에 대해 물어보세요.
+예를 들어, '문서의 파일 이름은 무엇입니까?'와 같은 질문을 작성하지 마세요.
+7.	질문을 한국어로 작성하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""あなたはTextを「概念完成」の質問に変換するAIです。
+「概念完成」の質問は概念の本質やアイデンティティについて直接的に尋ねる質問です。
+
+次の指示に従います。
+指示事項:
+1. 質問は、必ず与えられたTextに基づいて作成されなければなりません。
+2. 質問は、Textに基づいてできるだけ詳しく作成されなければなりません。
+3. Textで提供された情報を尋ねる質問を作成します。
+4. Textの特定のキーワードを必ず質問に含みます。
+5. 質問に「与えられたTextで」または「提供された段落で」のような表現を含めてはいけません。
+ユーザーは質問の出所がTextだということを知らないので、必ずしもその出所を言及してはいけません。
+6. ファイル名やファイルタイトルを訊かないでください。ファイルの内容について聞いてください。
+例えば、「このドキュメントのファイル名は何ですか？
+7. 質問を日本語で書きましょう。""",
+			)
+		],
+	},
+	"two_hop_incremental": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="Generate a multi-hop question for the given answer which requires reference to all of the given documents.",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Document 1: The Municipality of Nuevo Laredo is located in the Mexican state of Tamaulipas.
+Document 2: The Ciudad Deportiva (Sports City ¨ ¨) is a sports
+complex in Nuevo Laredo, Mexico. It is home to the Tecolotes de
+Nuevo Laredo Mexican Baseball League team and ...""",
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="""Answer: Tamaulipas
+One-hop question (using Document 1): In which Mexican state is Nuevo Laredo located?
+Two-hop question (using Document 2):  In which Mexican state can one find the Ciudad Deportiva, home to the Tecolotes de Nuevo Laredo?""",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="Generate a multi-hop question for the given answer which requires reference to all of the given documents.",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Document 1: The Municipality of Nuevo Laredo is located in the Mexican state of Tamaulipas.
+Document 2: The Ciudad Deportiva (Sports City ¨ ¨) is a sports
+complex in Nuevo Laredo, Mexico. It is home to the Tecolotes de
+Nuevo Laredo Mexican Baseball League team and ...""",
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="""Answer: Tamaulipas
+One-hop question (using Document 1): In which Mexican state is Nuevo Laredo located?
+Two-hop question (using Document 2):  In which Mexican state can one find the Ciudad Deportiva, home to the Tecolotes de Nuevo Laredo?""",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="与えられた答えに対するマルチホップ質問を生成し、与えられたすべての文書を参照する必要があります。",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Document 1: ヌエヴォ·ラレド自治体はメキシコのタマウリパス州にあります。
+Ciudad Deportiva（スポーツシティ ¨ ¨）はスポーツです
+メキシコのヌエボ·ラレドにある複合施設です。 テコロテス·デ·テコロテスの故郷です
+Nuevo Larredo メキシコ野球リーグのチームです···""",
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="""Answer: Tamaulipas
+One-hop question (using Document 1): ヌエヴォ·ラレド自治体はどのメキシコの州にありますか？
+Two-hop question (using Document 2): ヌエヴォ·ラレドのテコロテス·デ·テコロテスの故郷であるメキシコの州はどこですか？""",
+			),
+		],
+	},
+}
+
+# Experimental feature
+QUERY_GEN_PROMPT_EXTRA = {
+	"multiple_queries": {
+		"en": "\nAdditional instructions:\n  - Please make {n} questions.",
+		"ko": "\n추가 지침:\n  - 질문은 {n}개를 만드세요.",
+		"ja": "\n追加指示:\n  - 質問を{n}個作成してください。",
+	}
+}

autorag-workspace/autorag/data/qa/sample.py

@@ -0,0 +1,26 @@
+import uuid
+from typing import Iterable
+
+import pandas as pd
+
+
+def random_single_hop(
+	corpus_df: pd.DataFrame, n: int, random_state: int = 42
+) -> pd.DataFrame:
+	sample_df = corpus_df.sample(n, random_state=random_state)
+	return pd.DataFrame(
+		{
+			"qid": [str(uuid.uuid4()) for _ in range(len(sample_df))],
+			"retrieval_gt": [[[id_]] for id_ in sample_df["doc_id"].tolist()],
+		}
+	)
+
+
+def range_single_hop(corpus_df: pd.DataFrame, idx_range: Iterable):
+	sample_df = corpus_df.iloc[idx_range]
+	return pd.DataFrame(
+		{
+			"qid": [str(uuid.uuid4()) for _ in range(len(sample_df))],
+			"retrieval_gt": [[[id_]] for id_ in sample_df["doc_id"].tolist()],
+		}
+	)

autorag-workspace/autorag/data/qa/schema.py

@@ -0,0 +1,322 @@
+import logging
+from typing import Callable, Optional, Dict, Awaitable, Any, Tuple, List
+import uuid
+import pandas as pd
+from autorag.utils.util import process_batch, get_event_loop, fetch_contents
+
+from autorag.support import get_support_modules
+
+logger = logging.getLogger("AutoRAG")
+
+
+class Raw:
+	"""
+	The Raw class that stored document parsing results.
+	It can do chunking.
+	It has two column names, 'raw_id' and 'contents'.
+	"""
+
+	def __init__(self, raw_df: Optional[pd.DataFrame] = None):
+		self.data = raw_df
+
+	def batch_apply(
+		self, fn: Callable[[Dict, Any], Awaitable[Dict]], batch_size: int = 32, **kwargs
+	) -> "Raw":
+		raw_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(raw_dict, **kwargs) for raw_dict in raw_dicts]
+		results = loop.run_until_complete(process_batch(tasks, batch_size))
+		return Raw(pd.DataFrame(results))
+
+	def map(self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs) -> "Raw":
+		return Raw(fn(self.data, **kwargs))
+
+	def flatmap(self, fn: Callable, **kwargs) -> "Raw":
+		return fn(self.data, **kwargs)
+
+	def chunk(self, module_name: str, **module_params) -> "Corpus":
+		chunk_module = get_support_modules(module_name)
+		chunked_result = chunk_module(parsed_result=self.data, **module_params)
+		return Corpus(chunked_result, self)
+
+	def __add__(self, other):
+		assert isinstance(other, Raw), "You can only add Raw instances."
+		self.data = pd.concat([self.data, other.data], ignore_index=True).reset_index(
+			drop=True
+		)
+		return self
+
+
+class Corpus:
+	"""
+	The Corpus class that stored chunked passages.
+	It can generate qa set, linked with Raw instance.
+	"""
+
+	def __init__(
+		self,
+		corpus_df: Optional[pd.DataFrame] = None,
+		linked_raw: Optional[Raw] = None,
+	):
+		self.data = corpus_df
+		self._linked_raw = linked_raw
+
+	@property
+	def linked_raw(self) -> Raw:
+		return self._linked_raw
+
+	@linked_raw.setter
+	def linked_raw(self, raw: Raw):
+		raise NotImplementedError("linked_raw is read-only.")
+
+	def to_parquet(self, save_path: str):
+		"""
+		Save the corpus to the AutoRAG compatible parquet file.
+		It is not for the data creation, for running AutoRAG.
+		If you want to save it directly, use the below code.
+		`corpus.data.to_parquet(save_path)`
+
+		:param save_path: The path to save the corpus.
+		"""
+		if not save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		save_df = self.data.reset_index(drop=True)
+		save_df.to_parquet(save_path)
+
+	def batch_apply(
+		self, fn: Callable[[Dict, Any], Awaitable[Dict]], batch_size: int = 32, **kwargs
+	) -> "Corpus":
+		corpus_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(corpus_dict, **kwargs) for corpus_dict in corpus_dicts]
+		results = loop.run_until_complete(process_batch(tasks, batch_size))
+		return Corpus(pd.DataFrame(results), self.linked_raw)
+
+	def map(
+		self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs
+	) -> "Corpus":
+		return Corpus(fn(self.data, **kwargs), self.linked_raw)
+
+	def sample(self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs) -> "QA":
+		"""
+		Sample the corpus for making QA.
+		It selects the subset of the corpus and makes QA set from it.
+		You can generate questions from the created question.
+		It is the first step to make QA set from the corpus.
+		If you select just one passage from each passage, it will be a single-hop QA set.
+		If you select multiple passages from each passage, it will be a multi-hop QA set.
+
+		:param fn: The select function to perform.
+		It returns QA dataframe.
+		:return: QA instance that is selected.
+		It contains qid and retrieval_gt columns.
+		"""
+		return QA(fn(self.data, **kwargs), self)
+
+
+class QA:
+	def __init__(
+		self,
+		qa_df: Optional[pd.DataFrame] = None,
+		linked_corpus: Optional[Corpus] = None,
+	):
+		self.data = qa_df
+		self._linked_corpus = linked_corpus
+
+	@property
+	def linked_corpus(self) -> Corpus:
+		return self._linked_corpus
+
+	@linked_corpus.setter
+	def linked_corpus(self, corpus: Corpus):
+		raise NotImplementedError("linked_corpus is read-only.")
+
+	def batch_apply(
+		self, fn: Callable[[Dict, Any], Awaitable[Dict]], batch_size: int = 32, **kwargs
+	) -> "QA":
+		qa_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(qa_dict, **kwargs) for qa_dict in qa_dicts]
+		results = loop.run_until_complete(process_batch(tasks, batch_size))
+
+		# Experimental feature
+		if fn.__name__ == "multiple_queries_gen":
+			return self._process_multiple_queries_gen(results)
+
+		return QA(pd.DataFrame(results), self.linked_corpus)
+
+	def batch_filter(
+		self, fn: Callable[[Dict, Any], Awaitable[bool]], batch_size: int = 32, **kwargs
+	) -> "QA":
+		qa_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(qa_dict, **kwargs) for qa_dict in qa_dicts]
+		masks = loop.run_until_complete(process_batch(tasks, batch_size))
+		return QA(self.data[masks], self.linked_corpus)
+
+	def filter(self, fn: Callable[[Dict, Any], bool], **kwargs) -> "QA":
+		qa_dicts = self.data.to_dict(orient="records")
+		masks = [fn(qa_dict, **kwargs) for qa_dict in qa_dicts]
+		return QA(self.data[masks], self.linked_corpus)
+
+	def map(self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs) -> "QA":
+		return QA(fn(self.data, **kwargs), self.linked_corpus)
+
+	def make_retrieval_gt_contents(self) -> "QA":
+		"""
+		Make retrieval_gt_contents column from retrieval_gt column.
+		:return: The QA instance that has a retrieval_gt_contents column.
+		"""
+		self.data["retrieval_gt_contents"] = self.data["retrieval_gt"].apply(
+			lambda x: fetch_contents(self.linked_corpus.data, x)
+		)
+		return self
+
+	def to_parquet(self, qa_save_path: str, corpus_save_path: str):
+		"""
+		Save the qa and corpus to the AutoRAG compatible parquet file.
+		It is not for the data creation, for running AutoRAG.
+		If you want to save it directly, use the below code.
+		`qa.data.to_parquet(save_path)`
+
+		:param qa_save_path: The path to save the qa dataset.
+		:param corpus_save_path: The path to save the corpus.
+		"""
+		if not qa_save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		if not corpus_save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		save_df = self.data[
+			["qid", "query", "retrieval_gt", "generation_gt"]
+		].reset_index(drop=True)
+		save_df.to_parquet(qa_save_path)
+		self.linked_corpus.to_parquet(corpus_save_path)
+
+	def update_corpus(self, new_corpus: Corpus) -> "QA":
+		"""
+		Update linked corpus.
+		Not just replace linked_corpus to the new Corpus,
+		it replaces the whole `retrieval_gt` to the new corpus using `linked_raw`.
+		The QA data must have a `retrieval_gt` column.
+
+		:param new_corpus: Corpus that you want to replace.
+		    Must have valid `linked_raw` and `raw_id`, `raw_start_idx`, `raw_end_idx` columns.
+		:return: The QA instance that updated linked corpus.
+		"""
+		self.data["evidence_path"] = (
+			self.data["retrieval_gt"]
+			.apply(
+				lambda x: fetch_contents(
+					self.linked_corpus.data,
+					x,
+					column_name="path",
+				)
+			)
+			.tolist()
+		)
+		self.data["evidence_page"] = self.data["retrieval_gt"].apply(
+			lambda x: list(
+				map(
+					lambda lst: list(map(lambda x: x.get("page", -1), lst)),
+					fetch_contents(self.linked_corpus.data, x, column_name="metadata"),
+				)
+			)
+		)
+		if "evidence_start_end_idx" not in self.data.columns:
+			# make evidence start_end_idx
+			self.data["evidence_start_end_idx"] = (
+				self.data["retrieval_gt"]
+				.apply(
+					lambda x: fetch_contents(
+						self.linked_corpus.data,
+						x,
+						column_name="start_end_idx",
+					)
+				)
+				.tolist()
+			)
+
+		# matching the new corpus with the old corpus
+		path_corpus_dict = QA.__make_path_corpus_dict(new_corpus.data)
+		new_retrieval_gt = self.data.apply(
+			lambda row: QA.__match_index_row(
+				row["evidence_start_end_idx"],
+				row["evidence_path"],
+				row["evidence_page"],
+				path_corpus_dict,
+			),
+			axis=1,
+		).tolist()
+		new_qa = self.data.copy(deep=True)[["qid", "query", "generation_gt"]]
+		new_qa["retrieval_gt"] = new_retrieval_gt
+		return QA(new_qa, new_corpus)
+
+	@staticmethod
+	def __match_index(target_idx: Tuple[int, int], dst_idx: Tuple[int, int]) -> bool:
+		"""
+		Check if the target_idx is overlap by the dst_idx.
+		"""
+		target_start, target_end = target_idx
+		dst_start, dst_end = dst_idx
+		return (
+			dst_start <= target_start <= dst_end or dst_start <= target_end <= dst_end
+		)
+
+	@staticmethod
+	def __match_index_row(
+		evidence_indices: List[List[Tuple[int, int]]],
+		evidence_paths: List[List[str]],
+		evidence_pages: List[List[int]],
+		path_corpus_dict: Dict,
+	) -> List[List[str]]:
+		"""
+		Find the matched passage from new_corpus.
+
+		:param evidence_indices: The evidence indices at the corresponding Raw.
+		        Its shape is the same as the retrieval_gt.
+		:param evidence_paths: The evidence paths at the corresponding Raw.
+		        Its shape is the same as the retrieval_gt.
+		:param path_corpus_dict: The key is the path name, and the value is the corpus dataframe that only contains the path in the key.
+		        You can make it using `QA.__make_path_corpus_dict`.
+		:return:
+		"""
+		result = []
+		for i, idx_list in enumerate(evidence_indices):
+			sub_result = []
+			for j, idx in enumerate(idx_list):
+				path_corpus_df = path_corpus_dict[evidence_paths[i][j]]
+				if evidence_pages[i][j] >= 0:
+					path_corpus_df = path_corpus_df.loc[
+						path_corpus_df["metadata"].apply(lambda x: x.get("page", -1))
+						== evidence_pages[i][j]
+					]
+				matched_corpus = path_corpus_df.loc[
+					path_corpus_df["start_end_idx"].apply(
+						lambda x: QA.__match_index(idx, x)
+					)
+				]
+				sub_result.extend(matched_corpus["doc_id"].tolist())
+			result.append(sub_result)
+		return result
+
+	@staticmethod
+	def __make_path_corpus_dict(corpus_df: pd.DataFrame) -> Dict[str, pd.DataFrame]:
+		return {
+			path: corpus_df[corpus_df["path"] == path]
+			for path in corpus_df["path"].unique()
+		}
+
+	# Experimental feature
+	def _process_multiple_queries_gen(self, results: List[Dict]) -> "QA":
+		data = []
+		for result in results:
+			queries = result["query"].split("\n")
+			for query in queries:
+				new_result = {
+					key: (str(uuid.uuid4()) if key == "qid" else result[key])
+					for key in result.keys()
+				}
+				new_result["query"] = query
+				data.append(new_result)
+		df = pd.DataFrame(data)
+		return QA(df, self.linked_corpus)

autorag-workspace/autorag/data/utils/util.py

@@ -0,0 +1,103 @@
+import mimetypes
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Tuple, Callable
+
+import pandas as pd
+import yaml
+from langchain_core.documents import Document
+from llama_index.core.schema import NodeRelationship
+
+from autorag.schema import Module
+from autorag.utils.util import make_combinations, explode
+
+
+def get_file_metadata(file_path: str) -> Dict:
+	"""Get some handy metadate from filesystem.
+
+	Args:
+	    file_path: str: file path in str
+	"""
+	return {
+		"file_path": file_path,
+		"file_name": os.path.basename(file_path),
+		"file_type": mimetypes.guess_type(file_path)[0],
+		"file_size": os.path.getsize(file_path),
+		"creation_datetime": datetime.fromtimestamp(
+			Path(file_path).stat().st_ctime
+		).strftime("%Y-%m-%d"),
+		"last_modified_datetime": datetime.fromtimestamp(
+			Path(file_path).stat().st_mtime
+		).strftime("%Y-%m-%d"),
+		"last_accessed_datetime": datetime.fromtimestamp(
+			Path(file_path).stat().st_atime
+		).strftime("%Y-%m-%d"),
+	}
+
+
+def add_essential_metadata(metadata: Dict) -> Dict:
+	if "last_modified_datetime" not in metadata:
+		metadata["last_modified_datetime"] = datetime.now()
+	return metadata
+
+
+def corpus_df_to_langchain_documents(corpus_df: pd.DataFrame) -> List[Document]:
+	page_contents = corpus_df["contents"].tolist()
+	ids = corpus_df["doc_id"].tolist()
+	metadatas = corpus_df["metadata"].tolist()
+	return list(
+		map(
+			lambda x: Document(page_content=x[0], metadata={"filename": x[1], **x[2]}),
+			zip(page_contents, ids, metadatas),
+		)
+	)
+
+
+def add_essential_metadata_llama_text_node(metadata: Dict, relationships: Dict) -> Dict:
+	if "last_modified_datetime" not in metadata:
+		metadata["last_modified_datetime"] = datetime.now()
+
+	if "prev_id" not in metadata:
+		if NodeRelationship.PREVIOUS in relationships:
+			prev_node = relationships.get(NodeRelationship.PREVIOUS, None)
+			if prev_node:
+				metadata["prev_id"] = prev_node.node_id
+
+	if "next_id" not in metadata:
+		if NodeRelationship.NEXT in relationships:
+			next_node = relationships.get(NodeRelationship.NEXT, None)
+			if next_node:
+				metadata["next_id"] = next_node.node_id
+	return metadata
+
+
+def load_yaml(yaml_path: str):
+	if not os.path.exists(yaml_path):
+		raise ValueError(f"YAML file {yaml_path} does not exist.")
+	with open(yaml_path, "r", encoding="utf-8") as stream:
+		try:
+			yaml_dict = yaml.safe_load(stream)
+		except yaml.YAMLError as exc:
+			raise ValueError(f"YAML file {yaml_path} could not be loaded.") from exc
+	return yaml_dict["modules"]
+
+
+def get_param_combinations(modules: List[Dict]) -> Tuple[List[Callable], List[Dict]]:
+	module_callable_list, module_params_list = [], []
+	for module in modules:
+		module_instance = Module.from_dict(module)
+		module_params_list.append(module_instance.module_param)
+		module_callable_list.append(module_instance.module)
+
+	combinations = list(map(make_combinations, module_params_list))
+	module_list, combination_list = explode(module_callable_list, combinations)
+	return module_list, combination_list
+
+
+def get_start_end_idx(original_text: str, search_str: str) -> Tuple[int, int]:
+	start_idx = original_text.find(search_str)
+	if start_idx == -1:
+		return 0, 0
+	end_idx = start_idx + len(search_str)
+	return start_idx, end_idx - 1

autorag-workspace/autorag/deploy/__init__.py

@@ -0,0 +1,9 @@
+from .base import (
+	extract_node_line_names,
+	extract_node_strategy,
+	summary_df_to_yaml,
+	extract_best_config,
+	Runner,
+)
+from .api import ApiRunner
+from .gradio import GradioRunner

autorag-workspace/autorag/deploy/api.py

@@ -0,0 +1,293 @@
+import logging
+import os
+import pathlib
+import uuid
+from typing import Dict, Optional, List, Union, Literal
+
+import pandas as pd
+from quart import Quart, request, jsonify
+from quart.helpers import stream_with_context
+from pydantic import BaseModel, ValidationError
+
+from autorag.deploy.base import BaseRunner
+from autorag.nodes.generator.base import BaseGenerator
+from autorag.nodes.promptmaker.base import BasePromptMaker
+from autorag.utils.util import fetch_contents, to_list
+
+logger = logging.getLogger("AutoRAG")
+
+deploy_dir = pathlib.Path(__file__).parent
+root_dir = pathlib.Path(__file__).parent.parent
+
+VERSION_PATH = os.path.join(root_dir, "VERSION")
+
+
+class QueryRequest(BaseModel):
+	query: str
+	result_column: Optional[str] = "generated_texts"
+
+
+class RetrievedPassage(BaseModel):
+	content: str
+	doc_id: str
+	score: float
+	filepath: Optional[str] = None
+	file_page: Optional[int] = None
+	start_idx: Optional[int] = None
+	end_idx: Optional[int] = None
+
+
+class RunResponse(BaseModel):
+	result: Union[str, List[str]]
+	retrieved_passage: List[RetrievedPassage]
+
+
+class RetrievalResponse(BaseModel):
+	passages: List[RetrievedPassage]
+
+
+class StreamResponse(BaseModel):
+	"""
+	When the type is generated_text, only generated_text is returned. The other fields are None.
+	When the type is retrieved_passage, only retrieved_passage and passage_index are returned. The other fields are None.
+	"""
+
+	type: Literal["generated_text", "retrieved_passage"]
+	generated_text: Optional[str]
+	retrieved_passage: Optional[RetrievedPassage]
+	passage_index: Optional[int]
+
+
+class VersionResponse(BaseModel):
+	version: str
+
+
+class ApiRunner(BaseRunner):
+	def __init__(self, config: Dict, project_dir: Optional[str] = None):
+		super().__init__(config, project_dir)
+		self.app = Quart(__name__)
+
+		data_dir = os.path.join(project_dir, "data")
+		self.corpus_df = pd.read_parquet(
+			os.path.join(data_dir, "corpus.parquet"), engine="pyarrow"
+		)
+		self.__add_api_route()
+
+	def __add_api_route(self):
+		@self.app.route("/v1/run", methods=["POST"])
+		async def run_query():
+			try:
+				data = await request.get_json()
+				data = QueryRequest(**data)
+			except ValidationError as e:
+				return jsonify(e.errors()), 400
+
+			previous_result = pd.DataFrame(
+				{
+					"qid": str(uuid.uuid4()),
+					"query": [data.query],
+					"retrieval_gt": [[]],
+					"generation_gt": [""],
+				}
+			)  # pseudo qa data for execution
+			for module_instance, module_param in zip(
+				self.module_instances, self.module_params
+			):
+				new_result = module_instance.pure(
+					previous_result=previous_result, **module_param
+				)
+				duplicated_columns = previous_result.columns.intersection(
+					new_result.columns
+				)
+				drop_previous_result = previous_result.drop(columns=duplicated_columns)
+				previous_result = pd.concat([drop_previous_result, new_result], axis=1)
+
+			# Simulate processing the query
+			generated_text = previous_result[data.result_column].tolist()[0]
+			retrieved_passage = self.extract_retrieve_passage(previous_result)
+
+			response = RunResponse(
+				result=generated_text, retrieved_passage=retrieved_passage
+			)
+
+			return jsonify(response.model_dump()), 200
+
+		@self.app.route("/v1/retrieve", methods=["POST"])
+		async def run_retrieve_only():
+			data = await request.get_json()
+			query = data.get("query", None)
+			if query is None:
+				return jsonify(
+					{
+						"error": "Invalid request. You need to include 'query' in the request body."
+					}
+				), 400
+
+			previous_result = pd.DataFrame(
+				{
+					"qid": str(uuid.uuid4()),
+					"query": [query],
+					"retrieval_gt": [[]],
+					"generation_gt": [""],
+				}
+			)  # pseudo qa data for execution
+			for module_instance, module_param in zip(
+				self.module_instances, self.module_params
+			):
+				if isinstance(module_instance, BasePromptMaker) or isinstance(
+					module_instance, BaseGenerator
+				):
+					continue
+				new_result = module_instance.pure(
+					previous_result=previous_result, **module_param
+				)
+				duplicated_columns = previous_result.columns.intersection(
+					new_result.columns
+				)
+				drop_previous_result = previous_result.drop(columns=duplicated_columns)
+				previous_result = pd.concat([drop_previous_result, new_result], axis=1)
+
+			# Simulate processing the query
+			retrieved_passages = self.extract_retrieve_passage(previous_result)
+
+			retrieval_response = RetrievalResponse(passages=retrieved_passages)
+			return jsonify(retrieval_response.model_dump()), 200
+
+		@self.app.route("/v1/stream", methods=["POST"])
+		async def stream_query():
+			try:
+				data = await request.get_json()
+				data = QueryRequest(**data)
+			except ValidationError as e:
+				return jsonify(e.errors()), 400
+
+			@stream_with_context
+			async def generate():
+				previous_result = pd.DataFrame(
+					{
+						"qid": str(uuid.uuid4()),
+						"query": [data.query],
+						"retrieval_gt": [[]],
+						"generation_gt": [""],
+					}
+				)  # pseudo qa data for execution
+
+				for module_instance, module_param in zip(
+					self.module_instances, self.module_params
+				):
+					if not isinstance(module_instance, BaseGenerator):
+						new_result = module_instance.pure(
+							previous_result=previous_result, **module_param
+						)
+						duplicated_columns = previous_result.columns.intersection(
+							new_result.columns
+						)
+						drop_previous_result = previous_result.drop(
+							columns=duplicated_columns
+						)
+						previous_result = pd.concat(
+							[drop_previous_result, new_result], axis=1
+						)
+					else:
+						retrieved_passages = self.extract_retrieve_passage(
+							previous_result
+						)
+						for i, retrieved_passage in enumerate(retrieved_passages):
+							yield (
+								StreamResponse(
+									type="retrieved_passage",
+									generated_text=None,
+									retrieved_passage=retrieved_passage,
+									passage_index=i,
+								)
+								.model_dump_json()
+								.encode("utf-8")
+							)
+						# Start streaming of the result
+						assert len(previous_result) == 1
+						prompt: str = previous_result["prompts"].tolist()[0]
+						async for delta in module_instance.astream(
+							prompt=prompt, **module_param
+						):
+							response = StreamResponse(
+								type="generated_text",
+								generated_text=delta,
+								retrieved_passage=None,
+								passage_index=None,
+							)
+							yield response.model_dump_json().encode("utf-8")
+
+			return generate(), 200, {"X-Something": "value"}
+
+		@self.app.route("/version", methods=["GET"])
+		def get_version():
+			with open(VERSION_PATH, "r") as f:
+				version = f.read().strip()
+			response = VersionResponse(version=version)
+			return jsonify(response.model_dump()), 200
+
+	def run_api_server(
+		self, host: str = "0.0.0.0", port: int = 8000, remote: bool = True, **kwargs
+	):
+		"""
+		Run the pipeline as an api server.
+		Here is api endpoint documentation => https://docs.auto-rag.com/deploy/api_endpoint.html
+
+		:param host: The host of the api server.
+		:param port: The port of the api server.
+		:param remote: Whether to expose the api server to the public internet using ngrok.
+		:param kwargs: Other arguments for Flask app.run.
+		"""
+		logger.info(f"Run api server at {host}:{port}")
+		if remote:
+			from pyngrok import ngrok
+
+			http_tunnel = ngrok.connect(str(port), "http")
+			public_url = http_tunnel.public_url
+			logger.info(f"Public API URL: {public_url}")
+		self.app.run(host=host, port=port, **kwargs)
+
+	def extract_retrieve_passage(self, df: pd.DataFrame) -> List[RetrievedPassage]:
+		retrieved_ids: List[str] = df["retrieved_ids"].tolist()[0]
+		contents = fetch_contents(self.corpus_df, [retrieved_ids])[0]
+		scores = df["retrieve_scores"].tolist()[0]
+		if "path" in self.corpus_df.columns:
+			paths = fetch_contents(self.corpus_df, [retrieved_ids], column_name="path")[
+				0
+			]
+		else:
+			paths = [None] * len(retrieved_ids)
+		metadatas = fetch_contents(
+			self.corpus_df, [retrieved_ids], column_name="metadata"
+		)[0]
+		if "start_end_idx" in self.corpus_df.columns:
+			start_end_indices = fetch_contents(
+				self.corpus_df, [retrieved_ids], column_name="start_end_idx"
+			)[0]
+		else:
+			start_end_indices = [None] * len(retrieved_ids)
+		start_end_indices = to_list(start_end_indices)
+		return list(
+			map(
+				lambda content,
+				doc_id,
+				score,
+				path,
+				metadata,
+				start_end_idx: RetrievedPassage(
+					content=content,
+					doc_id=doc_id,
+					score=score,
+					filepath=path,
+					file_page=metadata.get("page", None),
+					start_idx=start_end_idx[0] if start_end_idx else None,
+					end_idx=start_end_idx[1] if start_end_idx else None,
+				),
+				contents,
+				retrieved_ids,
+				scores,
+				paths,
+				metadatas,
+				start_end_indices,
+			)
+		)

autorag-workspace/autorag/deploy/base.py

@@ -0,0 +1,235 @@
+import logging
+import os
+import pathlib
+import uuid
+from copy import deepcopy
+from typing import Optional, Dict, List
+
+import pandas as pd
+import yaml
+
+from autorag.support import get_support_modules
+from autorag.utils.util import load_summary_file, load_yaml_config
+
+logger = logging.getLogger("AutoRAG")
+
+
+def extract_node_line_names(config_dict: Dict) -> List[str]:
+	"""
+	Extract node line names with the given config dictionary order.
+
+	:param config_dict: The YAML configuration dict for the pipeline.
+	    You can load this to access trail_folder/config.yaml.
+	:return: The list of node line names.
+	    It is the order of the node line names in the pipeline.
+	"""
+	return [node_line["node_line_name"] for node_line in config_dict["node_lines"]]
+
+
+def extract_node_strategy(config_dict: Dict) -> Dict:
+	"""
+	Extract node strategies with the given config dictionary.
+	The return value is a dictionary of the node type and its strategy.
+
+	:param config_dict: The YAML configuration dict for the pipeline.
+	    You can load this to access trail_folder/config.yaml.
+	:return: Key is node_type and value is strategy dict.
+	"""
+	return {
+		node["node_type"]: node.get("strategy", {})
+		for node_line in config_dict["node_lines"]
+		for node in node_line["nodes"]
+	}
+
+
+def summary_df_to_yaml(summary_df: pd.DataFrame, config_dict: Dict) -> Dict:
+	"""
+	Convert trial summary dataframe to config yaml file.
+
+	:param summary_df: The trial summary dataframe of the evaluated trial.
+	:param config_dict: The yaml configuration dict for the pipeline.
+	    You can load this to access trail_folder/config.yaml.
+	:return: Dictionary of config yaml file.
+	    You can save this dictionary to yaml file.
+	"""
+
+	# summary_df columns : 'node_line_name', 'node_type', 'best_module_filename',
+	#                      'best_module_name', 'best_module_params', 'best_execution_time'
+	node_line_names = extract_node_line_names(config_dict)
+	node_strategies = extract_node_strategy(config_dict)
+	strategy_df = pd.DataFrame(
+		{
+			"node_type": list(node_strategies.keys()),
+			"strategy": list(node_strategies.values()),
+		}
+	)
+	summary_df = summary_df.merge(strategy_df, on="node_type", how="left")
+	summary_df["categorical_node_line_name"] = pd.Categorical(
+		summary_df["node_line_name"], categories=node_line_names, ordered=True
+	)
+	summary_df = summary_df.sort_values(by="categorical_node_line_name")
+	grouped = summary_df.groupby("categorical_node_line_name", observed=False)
+
+	node_lines = [
+		{
+			"node_line_name": node_line_name,
+			"nodes": [
+				{
+					"node_type": row["node_type"],
+					"strategy": row["strategy"],
+					"modules": [
+						{
+							"module_type": row["best_module_name"],
+							**row["best_module_params"],
+						}
+					],
+				}
+				for _, row in node_line.iterrows()
+			],
+		}
+		for node_line_name, node_line in grouped
+	]
+	return {"node_lines": node_lines}
+
+
+def extract_best_config(trial_path: str, output_path: Optional[str] = None) -> Dict:
+	"""
+	Extract the optimal pipeline from the evaluated trial.
+
+	:param trial_path: The path to the trial directory that you want to extract the pipeline from.
+	    Must already be evaluated.
+	:param output_path: Output path that pipeline yaml file will be saved.
+	    Must be .yaml or .yml file.
+	    If None, it does not save YAML file and just returns dict values.
+	    Default is None.
+	:return: The dictionary of the extracted pipeline.
+	"""
+	summary_path = os.path.join(trial_path, "summary.csv")
+	if not os.path.exists(summary_path):
+		raise ValueError(f"summary.csv does not exist in {trial_path}.")
+	trial_summary_df = load_summary_file(
+		summary_path, dict_columns=["best_module_params"]
+	)
+	config_yaml_path = os.path.join(trial_path, "config.yaml")
+	with open(config_yaml_path, "r") as f:
+		config_dict = yaml.safe_load(f)
+	yaml_dict = summary_df_to_yaml(trial_summary_df, config_dict)
+	yaml_dict["vectordb"] = extract_vectordb_config(trial_path)
+	if output_path is not None:
+		with open(output_path, "w") as f:
+			yaml.safe_dump(yaml_dict, f)
+	return yaml_dict
+
+
+def extract_vectordb_config(trial_path: str) -> List[Dict]:
+	# get vectordb.yaml file
+	project_dir = pathlib.PurePath(os.path.realpath(trial_path)).parent
+	vectordb_config_path = os.path.join(project_dir, "resources", "vectordb.yaml")
+	if not os.path.exists(vectordb_config_path):
+		raise ValueError(f"vectordb.yaml does not exist in {vectordb_config_path}.")
+	with open(vectordb_config_path, "r") as f:
+		vectordb_dict = yaml.safe_load(f)
+	result = vectordb_dict.get("vectordb", [])
+	if len(result) != 0:
+		return result
+	# return default setting of chroma
+	return [
+		{
+			"name": "default",
+			"db_type": "chroma",
+			"client_type": "persistent",
+			"embedding_model": "openai",
+			"collection_name": "openai",
+			"path": os.path.join(project_dir, "resources", "chroma"),
+		}
+	]
+
+
+class BaseRunner:
+	def __init__(self, config: Dict, project_dir: Optional[str] = None):
+		self.config = config
+		project_dir = os.getcwd() if project_dir is None else project_dir
+		os.environ["PROJECT_DIR"] = project_dir
+
+		# init modules
+		node_lines = deepcopy(self.config["node_lines"])
+		self.module_instances = []
+		self.module_params = []
+		for node_line in node_lines:
+			for node in node_line["nodes"]:
+				if len(node["modules"]) != 1:
+					raise ValueError(
+						"The number of modules in a node must be 1 for using runner."
+						"Please use extract_best_config method for extracting yaml file from evaluated trial."
+					)
+				module = node["modules"][0]
+				module_type = module.pop("module_type")
+				module_params = module
+				module_instance = get_support_modules(module_type)(
+					project_dir=project_dir,
+					**module_params,
+				)
+				self.module_instances.append(module_instance)
+				self.module_params.append(module_params)
+
+	@classmethod
+	def from_yaml(cls, yaml_path: str, project_dir: Optional[str] = None):
+		"""
+		Load Runner from the YAML file.
+		Must be extracted YAML file from the evaluated trial using the extract_best_config method.
+
+		:param yaml_path: The path of the YAML file.
+		:param project_dir: The path of the project directory.
+			Default is the current directory.
+		:return: Initialized Runner.
+		"""
+		config = load_yaml_config(yaml_path)
+		return cls(config, project_dir=project_dir)
+
+	@classmethod
+	def from_trial_folder(cls, trial_path: str):
+		"""
+		Load Runner from the evaluated trial folder.
+		Must already be evaluated using Evaluator class.
+		It sets the project_dir as the parent directory of the trial folder.
+
+		:param trial_path: The path of the trial folder.
+		:return: Initialized Runner.
+		"""
+		config = extract_best_config(trial_path)
+		return cls(config, project_dir=os.path.dirname(trial_path))
+
+
+class Runner(BaseRunner):
+	def run(self, query: str, result_column: str = "generated_texts"):
+		"""
+		Run the pipeline with query.
+		The loaded pipeline must start with a single query,
+		so the first module of the pipeline must be `query_expansion` or `retrieval` module.
+
+		:param query: The query of the user.
+		:param result_column: The result column name for the answer.
+		    Default is `generated_texts`, which is the output of the `generation` module.
+		:return: The result of the pipeline.
+		"""
+		previous_result = pd.DataFrame(
+			{
+				"qid": str(uuid.uuid4()),
+				"query": [query],
+				"retrieval_gt": [[]],
+				"generation_gt": [""],
+			}
+		)  # pseudo qa data for execution
+		for module_instance, module_param in zip(
+			self.module_instances, self.module_params
+		):
+			new_result = module_instance.pure(
+				previous_result=previous_result, **module_param
+			)
+			duplicated_columns = previous_result.columns.intersection(
+				new_result.columns
+			)
+			drop_previous_result = previous_result.drop(columns=duplicated_columns)
+			previous_result = pd.concat([drop_previous_result, new_result], axis=1)
+
+		return previous_result[result_column].tolist()[0]

autorag-workspace/autorag/deploy/gradio.py

@@ -0,0 +1,74 @@
+import logging
+import uuid
+
+import pandas as pd
+
+from autorag.deploy.base import BaseRunner
+
+import gradio as gr
+
+
+logger = logging.getLogger("AutoRAG")
+
+
+class GradioRunner(BaseRunner):
+	def run_web(
+		self,
+		server_name: str = "0.0.0.0",
+		server_port: int = 7680,
+		share: bool = False,
+		**kwargs,
+	):
+		"""
+		Run web interface to interact pipeline.
+		You can access the web interface at `http://server_name:server_port` in your browser
+
+		:param server_name: The host of the web. Default is 0.0.0.0.
+		:param server_port: The port of the web. Default is 7680.
+		:param share: Whether to create a publicly shareable link. Default is False.
+		:param kwargs: Other arguments for gr.ChatInterface.launch.
+		"""
+
+		logger.info(f"Run web interface at http://{server_name}:{server_port}")
+
+		def get_response(message, _):
+			return self.run(message)
+
+		gr.ChatInterface(
+			get_response, title="📚 AutoRAG", retry_btn=None, undo_btn=None
+		).launch(
+			server_name=server_name, server_port=server_port, share=share, **kwargs
+		)
+
+	def run(self, query: str, result_column: str = "generated_texts"):
+		"""
+		Run the pipeline with query.
+		The loaded pipeline must start with a single query,
+		so the first module of the pipeline must be `query_expansion` or `retrieval` module.
+
+		:param query: The query of the user.
+		:param result_column: The result column name for the answer.
+		    Default is `generated_texts`, which is the output of the `generation` module.
+		:return: The result of the pipeline.
+		"""
+		previous_result = pd.DataFrame(
+			{
+				"qid": str(uuid.uuid4()),
+				"query": [query],
+				"retrieval_gt": [[]],
+				"generation_gt": [""],
+			}
+		)  # pseudo qa data for execution
+		for module_instance, module_param in zip(
+			self.module_instances, self.module_params
+		):
+			new_result = module_instance.pure(
+				previous_result=previous_result, **module_param
+			)
+			duplicated_columns = previous_result.columns.intersection(
+				new_result.columns
+			)
+			drop_previous_result = previous_result.drop(columns=duplicated_columns)
+			previous_result = pd.concat([drop_previous_result, new_result], axis=1)
+
+		return previous_result[result_column].tolist()[0]

autorag-workspace/autorag/deploy/swagger.yml

@@ -0,0 +1,202 @@
+openapi: 3.0.0
+info:
+  title: Example API
+  version: 1.0.1
+paths:
+  /v1/run:
+    post:
+      summary: Run a query and get generated text with retrieved passages
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                query:
+                  type: string
+                  description: The query string
+                result_column:
+                  type: string
+                  description: The result column name
+                  default: generated_texts
+              required:
+                - query
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  type:
+                    type: string
+                    enum:
+                      - generated_text
+                      - retrieved_passage
+                    description: |
+                      When the type is "generated_text", only "generated_text" is returned.
+                      The other fields are None. When the type is "retrieved_passage", only
+                      "retrieved_passage" and "passage_index" are returned. The other fields are None.
+                  generated_text:
+                    type: string
+                    nullable: true
+                    description: |
+                      The generated text, only present when "type" is "generated_text".
+                  retrieved_passage:
+                    type: object
+                    nullable: true
+                    properties:
+                      content:
+                        type: string
+                      doc_id:
+                        type: string
+                      filepath:
+                        type: string
+                        nullable: true
+                      file_page:
+                        type: integer
+                        nullable: true
+                      start_idx:
+                        type: integer
+                        nullable: true
+                      end_idx:
+                        type: integer
+                        nullable: true
+                  passage_index:
+                    type: integer
+                    nullable: true
+                    description: |
+                      The index of the retrieved passage, only present when "type" is "retrieved_passage".
+                required:
+                  - type
+                oneOf:
+                  - required:
+                      - generated_text
+                  - required:
+                      - retrieved_passage
+                      - passage_index
+  /v1/retrieve:
+    post:
+      summary: Retrieve documents based on a query
+      operationId: runRetrieveOnly
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                query:
+                  type: string
+                  description: The query string to retrieve documents.
+              required:
+                - query
+      responses:
+        '200':
+          description: Successful retrieval of documents
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  passages:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        doc_id:
+                          type: string
+                          description: The unique identifier for the document.
+                        content:
+                          type: string
+                          description: The content of the retrieved document.
+                        score:
+                          type: number
+                          format: float
+                          description: The score of the retrieved document.
+        '400':
+          description: Invalid request due to missing query parameter
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  error:
+                    type: string
+                    description: Error message explaining the issue.
+  /v1/stream:
+    post:
+      summary: Stream generated text with retrieved passages
+      description: >
+        This endpoint streams the generated text line by line. The `retrieved_passage`
+        is sent first, followed by the `result` streamed incrementally.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                query:
+                  type: string
+                  description: The query string
+                result_column:
+                  type: string
+                  description: The result column name
+                  default: generated_texts
+              required:
+                - query
+      responses:
+        '200':
+          description: Successful response with streaming
+          content:
+            text/event-stream:
+              schema:
+                type: object
+                properties:
+                  result:
+                    oneOf:
+                      - type: string
+                      - type: array
+                        items:
+                          type: string
+                    description: The result text or list of texts (streamed line by line)
+                  retrieved_passage:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        content:
+                          type: string
+                        doc_id:
+                          type: string
+                        filepath:
+                          type: string
+                        nullable: true
+                        file_page:
+                          type: integer
+                          nullable: true
+                        start_idx:
+                          type: integer
+                          nullable: true
+                        end_idx:
+                          type: integer
+                          nullable: true
+
+  /version:
+    get:
+      summary: Get the API version
+      description: Returns the current version of the API as a string.
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  version:
+                    type: string
+                    description: The version of the API

autorag-workspace/autorag/embedding/base.py

@@ -0,0 +1,141 @@
+import logging
+import sys
+
+from random import random
+from typing import List, Union, Dict
+
+from llama_index.core.embeddings.mock_embed_model import MockEmbedding
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.embeddings.openai import OpenAIEmbeddingModelType
+from llama_index.embeddings.ollama import OllamaEmbedding
+from langchain_openai.embeddings import OpenAIEmbeddings
+
+from autorag import LazyInit
+
+logger = logging.getLogger("AutoRAG")
+
+
+class MockEmbeddingRandom(MockEmbedding):
+	"""Mock embedding with random vectors."""
+
+	def _get_vector(self) -> List[float]:
+		return [random() for _ in range(self.embed_dim)]
+
+
+embedding_models = {
+	# llama index
+	"openai": LazyInit(
+		OpenAIEmbedding
+	),  # default model is OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002
+	"openai_embed_3_large": LazyInit(
+		OpenAIEmbedding, model_name=OpenAIEmbeddingModelType.TEXT_EMBED_3_LARGE
+	),
+	"openai_embed_3_small": LazyInit(
+		OpenAIEmbedding, model_name=OpenAIEmbeddingModelType.TEXT_EMBED_3_SMALL
+	),
+	"mock": LazyInit(MockEmbeddingRandom, embed_dim=768),
+	# langchain
+	"openai_langchain": LazyInit(OpenAIEmbeddings),
+	"ollama": LazyInit(OllamaEmbedding),
+}
+
+try:
+	# you can use your own model in this way.
+	from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+
+	embedding_models["huggingface_baai_bge_small"] = LazyInit(
+		HuggingFaceEmbedding, model_name="BAAI/bge-small-en-v1.5"
+	)
+	embedding_models["huggingface_cointegrated_rubert_tiny2"] = LazyInit(
+		HuggingFaceEmbedding, model_name="cointegrated/rubert-tiny2"
+	)
+	embedding_models["huggingface_all_mpnet_base_v2"] = LazyInit(
+		HuggingFaceEmbedding,
+		model_name="sentence-transformers/all-mpnet-base-v2",
+		max_length=512,
+	)
+	embedding_models["huggingface_bge_m3"] = LazyInit(
+		HuggingFaceEmbedding, model_name="BAAI/bge-m3"
+	)
+	embedding_models["huggingface_multilingual_e5_large"] = LazyInit(
+		HuggingFaceEmbedding, model_name="intfloat/multilingual-e5-large-instruct"
+	)
+	embedding_models["huggingface_all_mpnet_base_v2"] = LazyInit(
+		HuggingFaceEmbedding, model_name="sentence-transformers/all-mpnet-base-v2"
+	) # 230313 추가 - 김용연
+	embedding_models["huggingface_KURE-v1"] = LazyInit(
+		HuggingFaceEmbedding, model_name="nlpai-lab/KURE-v1"
+	) # 230313 추가 - 김용연
+	embedding_models["huggingface_drangonku-v2-ko"] = LazyInit(
+		HuggingFaceEmbedding, model_name="dragonkue/snowflake-arctic-embed-l-v2.0-ko"
+	) # 230313 추가 - 김용연
+
+except ImportError:
+	logger.info(
+		"You are using API version of AutoRAG."
+		"To use local version, run pip install 'AutoRAG[gpu]'"
+	)
+
+
+class EmbeddingModel:
+	@staticmethod
+	def load(config: Union[str, Dict, List[Dict]]):
+		if isinstance(config, str):
+			return EmbeddingModel.load_from_str(config)
+		elif isinstance(config, dict):
+			return EmbeddingModel.load_from_dict(config)
+		elif isinstance(config, list):
+			return EmbeddingModel.load_from_list(config)
+		else:
+			raise ValueError("Invalid type of config")
+
+	@staticmethod
+	def load_from_str(name: str):
+		try:
+			return embedding_models[name]
+		except KeyError:
+			raise ValueError(f"Embedding model '{name}' is not supported")
+
+	@staticmethod
+	def load_from_list(option: List[dict]):
+		if len(option) != 1:
+			raise ValueError("Only one embedding model is supported")
+		return EmbeddingModel.load_from_dict(option[0])
+
+	@staticmethod
+	def load_from_dict(option: dict):
+		def _check_keys(target: dict):
+			if "type" not in target or "model_name" not in target:
+				raise ValueError("Both 'type' and 'model_name' must be provided")
+			if target["type"] not in ["openai", "huggingface", "mock", "ollama"]:
+				raise ValueError(
+					f"Embedding model type '{target['type']}' is not supported"
+				)
+
+		def _get_huggingface_class():
+			module = sys.modules.get("llama_index.embeddings.huggingface")
+			if not module:
+				logger.info(
+					"You are using API version of AutoRAG. "
+					"To use local version, run `pip install 'AutoRAG[gpu]'`."
+				)
+				return None
+			return getattr(module, "HuggingFaceEmbedding", None)
+
+		_check_keys(option)
+
+		model_options = option
+		model_type = model_options.pop("type")
+
+		embedding_map = {
+			"openai": OpenAIEmbedding,
+			"mock": MockEmbeddingRandom,
+			"huggingface": _get_huggingface_class(),
+			"ollama": OllamaEmbedding,
+		}
+
+		embedding_class = embedding_map.get(model_type)
+		if not embedding_class:
+			raise ValueError(f"Embedding model type '{model_type}' is not supported")
+
+		return LazyInit(embedding_class, **model_options)

autorag-workspace/autorag/evaluation/__init__.py

@@ -0,0 +1,3 @@
+from .retrieval import evaluate_retrieval
+from .generation import evaluate_generation
+from .retrieval_contents import evaluate_retrieval_contents

autorag-workspace/autorag/evaluation/generation.py

@@ -0,0 +1,86 @@
+import functools
+import warnings
+from typing import List, Callable, Union, Dict
+
+import pandas as pd
+
+from autorag.evaluation.metric.generation import (
+	bleu,
+	meteor,
+	rouge,
+	sem_score,
+	g_eval,
+	bert_score,
+	deepeval_faithfulness,
+)
+from autorag.evaluation.util import cast_metrics
+from autorag.schema.metricinput import MetricInput
+
+GENERATION_METRIC_FUNC_DICT = {
+	func.__name__: func
+	for func in [
+		bleu,
+		meteor,
+		rouge,
+		sem_score,
+		g_eval,
+		bert_score,
+		deepeval_faithfulness,
+	]
+}
+
+
+def evaluate_generation(
+	metric_inputs: List[MetricInput], metrics: Union[List[str], List[Dict]]
+):
+	def decorator_evaluate_generation(func: Callable):
+		@functools.wraps(func)
+		def wrapper(*args, **kwargs) -> pd.DataFrame:
+			generation_result = func(*args, **kwargs)
+			if type(generation_result) is tuple:
+				assert (
+					type(generation_result[0]) is list
+					and type(generation_result[0][0]) is str
+				), "Input func must return string list as generated answer at the first return value."
+				generated_str = generation_result[0]
+			elif type(generation_result) is list:
+				assert (
+					type(generation_result[0]) is str
+				), "Input func must return string list as generated answer at the first return value."
+				generated_str = generation_result
+			else:
+				raise ValueError(
+					"Input func must return string list as generated answer at the first return value."
+				)
+			for metric_input, generated_text in zip(metric_inputs, generated_str):
+				metric_input.generated_texts = generated_text
+
+			metric_scores = {}
+			metric_names, metric_params = cast_metrics(metrics)
+
+			for metric_name, metric_param in zip(metric_names, metric_params):
+				if metric_name not in GENERATION_METRIC_FUNC_DICT:
+					warnings.warn(
+						f"metric {metric_name} is not in supported metrics: {GENERATION_METRIC_FUNC_DICT.keys()}"
+						f"{metric_name} will be ignored."
+					)
+				else:
+					metric_scores[metric_name] = GENERATION_METRIC_FUNC_DICT[
+						metric_name
+					](
+						metric_inputs=metric_inputs,
+						**metric_param,
+					)
+
+			metric_result_df = pd.DataFrame(metric_scores)
+			execution_result_df = pd.DataFrame({"generated_texts": generated_str})
+			if type(generation_result) is tuple:
+				execution_result_df["generated_tokens"] = generation_result[1]
+				execution_result_df["generated_log_probs"] = generation_result[2]
+
+			result_df = pd.concat([execution_result_df, metric_result_df], axis=1)
+			return result_df
+
+		return wrapper
+
+	return decorator_evaluate_generation

autorag-workspace/autorag/evaluation/metric/__init__.py

@@ -0,0 +1,22 @@
+from .generation import (
+	bleu,
+	meteor,
+	rouge,
+	sem_score,
+	g_eval,
+	bert_score,
+	deepeval_faithfulness,
+)
+from .retrieval import (
+	retrieval_f1,
+	retrieval_recall,
+	retrieval_precision,
+	retrieval_mrr,
+	retrieval_ndcg,
+	retrieval_map,
+)
+from .retrieval_contents import (
+	retrieval_token_f1,
+	retrieval_token_precision,
+	retrieval_token_recall,
+)

autorag-workspace/autorag/evaluation/metric/deepeval_prompt.py

@@ -0,0 +1,322 @@
+class FaithfulnessTemplate:
+	@staticmethod
+	def generate_claims(text, lang: str = "en"):
+		if lang == "en":
+			return f"""Based on the given text, please generate a comprehensive list of FACTUAL claims that can inferred from the provided text.
+
+	Example:
+	Example Text:
+	"Einstein won the noble prize in 1968 for his discovery of the photoelectric effect."
+
+	Example JSON:
+	{{
+	    "claims": [
+	        "Einstein won the noble prize for his discovery of the photoelectric effect.",
+	        "Einstein won the noble prize in 1968."
+	    ]
+	}}
+	===== END OF EXAMPLE ======
+
+	**
+	IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
+	Only include claims that are factual, and the claims you extract should include the full context it was presented in, NOT cherry picked facts.
+	You should NOT include any prior knowledge, and take the text at face value when extracting claims.
+	**
+
+	Text:
+	{text}
+
+	JSON:
+	"""
+		elif lang == "ko":
+			return f"""주어진 텍스트에서 찾을 수 있는 사실적 정보들의 목록을 생성하세요.
+
+예시:
+예시 텍스트:
+“아인슈타인은 1968년에 광전 효과 발견으로 노벨상을 수상했다.”
+
+예시 JSON:
+{{
+“claims”: [
+“아인슈타인은 광전 효과 발견으로 노벨상을 수상했다.”,
+“아인슈타인은 1968년에 노벨상을 수상했다.”
+]
+}}
+===== 예시 끝 ======
+
+**
+중요: 오직 JSON 형식으로 “claims” 키가 문자열 목록으로 반환되도록 해야 합니다. 다른 단어나 설명은 필요하지 않습니다.
+사실에 기반한 주장만 포함하며, 추출한 주장은 전체 맥락을 유지해야 하며, 부분적으로 선택된 사실을 포함하지 않아야 합니다.
+사전 지식은 포함하지 말고, 텍스트에만 기초해 주장들을 추출해야 합니다.
+**
+
+텍스트:
+{text}
+
+JSON:
+"""
+		elif lang == "ja":
+			return f"""与えられたテキストに基づいて、そこから推測できる事実に基づく主張のリストを生成してください。
+
+例:
+例のテキスト:
+「アインシュタインは1968年に光電効果の発見でノーベル賞を受賞しました。」
+
+例のJSON:
+{{
+    "claims": [
+        "アインシュタインは光電効果の発見でノーベル賞を受賞しました。",
+        "アインシュタインは1968年にノーベル賞を受賞しました。"
+    ]
+}}
+===== 例の終わり ======
+
+**
+重要: 必ずJSON形式で"claims"キーが文字列のリストとして返されるようにしてください。説明や余計な言葉は不要です。
+事実に基づく主張のみを含め、抽出された主張は提示された文脈全体を含むものでなければなりません。一部の事実のみを抜粋することは避けてください。
+事前知識を使用せず、テキストに基づいて主張を抽出してください。
+**
+
+テキスト:
+{text}
+
+JSON:
+"""
+		else:
+			raise ValueError(f"Language {lang} is not supported.")
+
+	@staticmethod
+	def generate_truths(text, lang: str = "en"):
+		if lang == "en":
+			return f"""Based on the given text, please generate a comprehensive list of FACTUAL, undisputed truths that can inferred from the provided text.
+
+	Example:
+	Example Text:
+	"Einstein won the noble prize in 1968 for his discovery of the photoelectric effect."
+
+	Example JSON:
+	{{
+	    "truths": [
+	        "Einstein won the noble prize for his discovery of the photoelectric effect.",
+	        "Einstein won the noble prize in 1968."
+	    ]
+	}}
+	===== END OF EXAMPLE ======
+
+	**
+	IMPORTANT: Please make sure to only return in JSON format, with the "truths" key as a list of strings. No words or explanation is needed.
+	Only include truths that are factual.
+	**
+
+	Text:
+	{text}
+
+	JSON:
+	"""
+		elif lang == "ko":
+			return f"""주어진 텍스트에서 추출할 수 있는 사실적이고 논란이 없는 진실들의 목록을 생성하세요.
+
+예시:
+예시 텍스트:
+"아인슈타인은 1968년에 광전 효과 발견으로 노벨상을 수상했다."
+
+예시 JSON:
+{{
+    "truths": [
+        "아인슈타인은 광전 효과 발견으로 노벨상을 수상했다.",
+        "아인슈타인은 1968년에 노벨상을 수상했다."
+    ]
+}}
+===== 예시 끝 ======
+
+**
+중요: 오직 JSON 형식으로 "truths" 키가 문자열 목록으로 반환되도록 해야 합니다. 다른 단어나 설명은 필요하지 않습니다.
+사실에 기반한 진실만 포함해야 합니다.
+**
+
+텍스트:
+{text}
+
+JSON:
+"""
+		elif lang == "ja":
+			return f"""与えられたテキストに基づいて、そこから推測できる事実で議論の余地のない真実のリストを生成してください。
+
+例:
+例のテキスト:
+「アインシュタインは1968年に光電効果の発見でノーベル賞を受賞しました。」
+
+例のJSON:
+{{
+    "truths": [
+        "アインシュタインは光電効果の発見でノーベル賞を受賞しました。",
+        "アインシュタインは1968年にノーベル賞を受賞しました。"
+    ]
+}}
+===== 例の終わり ======
+
+**
+重要: 必ずJSON形式で"truths"キーが文字列のリストとして返されるようにしてください。説明や余計な言葉は不要です。
+事実に基づく真実のみを含めてください。
+**
+
+テキスト:
+{text}
+
+JSON:
+"""
+		else:
+			raise ValueError(f"Language {lang} is not supported.")
+
+	@staticmethod
+	def generate_verdicts(claims, retrieval_context, lang: str = "en"):
+		if lang == "en":
+			return f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
+	The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context.
+	Provide a 'reason' ONLY if the answer is 'no'.
+	The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
+
+	**
+	IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects.
+	Example retrieval contexts: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. Einstein won the Nobel Prize in 1968. Einstein is a German Scientist."
+	Example claims: ["Barack Obama is a caucasian male.", "Zurich is a city in London", "Einstein won the Nobel Prize for the discovery of the photoelectric effect which may have contributed to his fame.", "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect.", "Einstein was a Germen chef."]
+
+	Example:
+	{{
+	    "verdicts": [
+	        {{
+	            "verdict": "idk"
+	        }},
+	        {{
+	            "verdict": "idk"
+	        }},
+	        {{
+	            "verdict": "yes"
+	        }},
+	        {{
+	            "verdict": "no",
+	            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead."
+	        }},
+	        {{
+	            "verdict": "no",
+	            "reason": "The actual output claims Einstein is a Germen chef, which is not correct as the retrieval context states he was a German scientist instead."
+	        }},
+	    ]
+	}}
+	===== END OF EXAMPLE ======
+
+	The length of 'verdicts' SHOULD BE STRICTLY EQUAL to that of claims.
+	You DON'T have to provide a reason if the answer is 'yes' or 'idk'.
+	ONLY provide a 'no' answer if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGEMENT.
+	Claims made using vague, suggestive, speculative language such as 'may have', 'possibility due to', does NOT count as a contradiction.
+	Claims that is not backed up due to a lack of information/is not mentioned in the retrieval contexts MUST be answered 'idk', otherwise I WILL DIE.
+	**
+
+	Retrieval Contexts:
+	{retrieval_context}
+
+	Claims:
+	{claims}
+
+	JSON:
+	"""
+		elif lang == "ko":
+			return f"""주어진 주장에 대해, 각 주장이 주어진 문맥의 사실들과 모순되는지를 나타내는 JSON 객체 목록을 생성하세요. JSON은 두 개의 필드인 'verdict'와 'reason'으로 구성됩니다.
+'verdict'는 'yes', 'no', 또는 'idk' 중 하나여야 하며, 주어진 주장이 문맥과 일치하는지를 나타냅니다.
+'verdict'가 'no'인 경우에만 'reason'을 제공하세요. 'reason'에는 문맥에 따라 주장을 수정하는 내용이 포함되어야 합니다.
+
+**
+중요: 오직 JSON 형식으로 'verdicts' 키가 JSON 객체 목록으로 반환되도록 해야 합니다.
+예시 문맥: "아인슈타인은 광전 효과 발견으로 노벨상을 수상했다. 아인슈타인은 1968년에 노벨상을 수상했다. 아인슈타인은 독일 과학자이다."
+예시 주장: ["버락 오바마는 백인 남성이다.", "취리히는 런던에 있는 도시이다.", "아인슈타인은 광전 효과 발견으로 노벨상을 수상했으며, 이는 그의 명성에 기여했을 것이다.", "아인슈타인은 1969년에 광전 효과 발견으로 노벨상을 수상했다.", "아인슈타인은 독일 요리사였다."]
+
+예시:
+{{
+    "verdicts": [
+        {{
+            "verdict": "idk"
+        }},
+        {{
+            "verdict": "idk"
+        }},
+        {{
+            "verdict": "yes"
+        }},
+        {{
+            "verdict": "no",
+            "reason": "실제 출력은 아인슈타인이 1969년에 노벨상을 수상했다고 주장하지만, 문맥에서는 1968년이라고 명시되어 있습니다."
+        }},
+        {{
+            "verdict": "no",
+            "reason": "실제 출력은 아인슈타인이 독일 요리사라고 주장하지만, 문맥에서는 그가 독일 과학자라고 명시되어 있습니다."
+        }},
+    ]
+}}
+===== 예시 끝 ======
+
+'verdicts' 리스트의 길이는 반드시 주장들의 길이와 같아야 합니다.
+'yes' 또는 'idk'일 경우 'reason'을 제공할 필요가 없습니다.
+검색된 문맥과 직접적으로 모순되는 경우에만 'no' 답변을 제공하세요. 절대로 선험적인 지식을 사용하지 마세요.
+'~일 수 있다', '가능성이 있다'와 같은 모호한 표현은 모순으로 간주하지 마세요.
+문맥에 대한 정보 부족으로 뒷받침되지 않거나 언급되지 않은 주장은 반드시 'idk'로 답변하세요, 그렇지 않으면 내가 죽습니다.
+**
+
+주어진 문맥:
+{retrieval_context}
+
+주장:
+{claims}
+
+JSON:
+"""
+		elif lang == "ja":
+			return f"""与えられた主張について、それぞれの主張が取得された文脈の事実と矛盾しているかどうかを示すJSONオブジェクトのリストを生成してください。JSONには2つのフィールド、'verdict'と'reason'があります。
+'verdict'フィールドは、主張が文脈に一致するかどうかを示すため、厳密に'yes', 'no', 'idk'のいずれかを使用します。
+'verdict'が'no'の場合にのみ、'reason'を提供してください。'reason'には、文脈に基づいて主張を修正する内容が含まれている必要があります。
+
+**
+重要: 必ずJSON形式で'verdicts'キーがJSONオブジェクトのリストとして返されるようにしてください。
+例の文脈:「アインシュタインは光電効果の発見でノーベル賞を受賞しました。アインシュタインは1968年にノーベル賞を受賞しました。アインシュタインはドイツの科学者です。」
+例の主張: ["バラク・オバマは白人男性です。", "チューリッヒはロンドンにある都市です。", "アインシュタインは光電効果の発見でノーベル賞を受賞し、これが彼の名声に貢献したかもしれません。", "アインシュタインは1969年に光電効果の発見でノーベル賞を受賞しました。", "アインシュタインはドイツのシェフでした。"]
+
+例のJSON:
+{{
+    "verdicts": [
+        {{
+            "verdict": "idk"
+        }},
+        {{
+            "verdict": "idk"
+        }},
+        {{
+            "verdict": "yes"
+        }},
+        {{
+            "verdict": "no",
+            "reason": "実際の出力は、アインシュタインが1969年にノーベル賞を受賞したと主張していますが、文脈では1968年と述べられています。"
+        }},
+        {{
+            "verdict": "no",
+            "reason": "実際の出力は、アインシュタインがドイツのシェフだと主張していますが、文脈では彼がドイツの科学者であると述べられています。"
+        }},
+    ]
+}}
+===== 例の終わり ======
+
+'verdicts'のリストの長さは、主張のリストの長さと必ず等しくなければなりません。
+'yes'または'idk'の場合、'reason'を提供する必要はありません。
+文脈と直接矛盾する場合にのみ、'no'を提供してください。決して事前知識を使用しないでください。
+「〜かもしれない」や「〜の可能性がある」といった曖昧な表現は矛盾とは見なされません。
+情報が不足している、または文脈で言及されていない主張には必ず'idk'で答えてください。さもないと私は死んでしまいます。
+**
+
+文脈:
+{retrieval_context}
+
+主張:
+{claims}
+
+JSON:
+"""
+		else:
+			raise ValueError(f"Language {lang} is not supported.")

autorag-workspace/autorag/evaluation/metric/g_eval_prompts/coh_detailed.txt

@@ -0,0 +1,32 @@
+You will be given one summary written for a news article.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+Evaluation Criteria:
+
+Coherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby "the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic."
+
+Evaluation Steps:
+
+1. Read the news article carefully and identify the main topic and key points.
+2. Read the summary and compare it to the news article. Check if the summary covers the main topic and key points of the news article, and if it presents them in a clear and logical order.
+3. Assign a score for coherence on a scale of 1 to 5, where 1 is the lowest and 5 is the highest based on the Evaluation Criteria.
+
+
+Example:
+
+
+Source Text:
+
+{{Document}}
+
+Summary:
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Coherence:

autorag-workspace/autorag/evaluation/metric/g_eval_prompts/con_detailed.txt

@@ -0,0 +1,33 @@
+You will be given a news article. You will then be given one summary written for this article.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+
+Evaluation Criteria:
+
+Consistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts.
+
+Evaluation Steps:
+
+1. Read the news article carefully and identify the main facts and details it presents.
+2. Read the summary and compare it to the article. Check if the summary contains any factual errors that are not supported by the article.
+3. Assign a score for consistency based on the Evaluation Criteria.
+
+
+Example:
+
+
+Source Text:
+
+{{Document}}
+
+Summary:
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Consistency:

autorag-workspace/autorag/evaluation/metric/g_eval_prompts/flu_detailed.txt

@@ -0,0 +1,26 @@
+You will be given one summary written for a news article.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+
+Evaluation Criteria:
+
+Fluency (1-3): the quality of the summary in terms of grammar, spelling, punctuation, word choice, and sentence structure.
+
+- 1: Poor. The summary has many errors that make it hard to understand or sound unnatural.
+- 2: Fair. The summary has some errors that affect the clarity or smoothness of the text, but the main points are still comprehensible.
+- 3: Good. The summary has few or no errors and is easy to read and follow.
+
+
+Example:
+
+Summary:
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Fluency (1-3):

autorag-workspace/autorag/evaluation/metric/g_eval_prompts/rel_detailed.txt

@@ -0,0 +1,33 @@
+You will be given one summary written for a news article.
+
+Your task is to rate the summary on one metric.
+
+Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
+
+Evaluation Criteria:
+
+Relevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.
+
+Evaluation Steps:
+
+1. Read the summary and the source document carefully.
+2. Compare the summary to the source document and identify the main points of the article.
+3. Assess how well the summary covers the main points of the article, and how much irrelevant or redundant information it contains.
+4. Assign a relevance score from 1 to 5.
+
+
+Example:
+
+
+Source Text:
+
+{{Document}}
+
+Summary:
+
+{{Summary}}
+
+
+Evaluation Form (scores ONLY):
+
+- Relevance:

autorag-workspace/autorag/evaluation/metric/generation.py

@@ -0,0 +1,504 @@
+import asyncio
+import itertools
+import os
+from typing import List, Optional
+
+import evaluate
+import nltk
+import pandas as pd
+from llama_index.core.embeddings import BaseEmbedding
+from llama_index.embeddings.openai import OpenAIEmbedding
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+from rouge_score import tokenizers
+from rouge_score.rouge_scorer import RougeScorer
+from sacrebleu.metrics.bleu import BLEU
+
+from autorag.embedding.base import embedding_models
+from autorag.evaluation.metric.deepeval_prompt import FaithfulnessTemplate
+from autorag.evaluation.metric.util import (
+	autorag_metric_loop,
+	calculate_cosine_similarity,
+)
+from autorag.nodes.generator import OpenAILLM
+from autorag.nodes.generator.base import BaseGenerator
+from autorag.schema.metricinput import MetricInput
+from autorag.support import get_support_modules
+from autorag.utils.util import (
+	get_event_loop,
+	process_batch,
+	openai_truncate_by_token,
+	convert_inputs_to_list,
+	pop_params,
+	empty_cuda_cache,
+)
+
+
+@convert_inputs_to_list
+def huggingface_evaluate(
+	instance, key: str, metric_inputs: List[MetricInput], **kwargs
+) -> List[float]:
+	"""
+	Compute huggingface evaluate metric.
+
+	:param instance: The instance of huggingface evaluates metric.
+	:param key: The key to retrieve result score from huggingface evaluate result.
+	:param metric_inputs: A list of MetricInput schema
+	:param kwargs: The additional arguments for metric function.
+	:return: The list of scores.
+	"""
+
+	def compute_score(gt: List[str], pred: str) -> float:
+		return max(
+			list(
+				map(
+					lambda x: instance.compute(
+						predictions=[pred], references=[x], **kwargs
+					)[key],
+					gt,
+				)
+			)
+		)
+
+	result = list(
+		map(lambda x: compute_score(x.generation_gt, x.generated_texts), metric_inputs)
+	)
+	return result
+
+
+def make_generator_instance(generator_module_type: str, llm: str, **kwargs):
+	llm_class = get_support_modules(generator_module_type)
+	init_params = pop_params(llm_class.__init__, kwargs)
+	return llm_class(project_dir="", llm=llm, **init_params)
+
+
+@autorag_metric_loop(fields_to_check=["retrieval_gt_contents", "generated_texts"])
+def deepeval_faithfulness(
+	metric_inputs: List[MetricInput],
+	generator_module_type: str = "openai_llm",
+	lang: str = "en",
+	llm: str = "gpt-4o-2024-08-06",
+	batch: int = 16,
+	**kwargs,
+) -> List[float]:
+	"""
+	Compute deepeval faithfulness metric.
+	Its default model is gpt-4o-2024-08-06.
+	Since it uses OpenAI model, please be aware of the expensive cost.
+
+	:param metric_inputs: The list of MetricInput schema (Required Field -> "generation_gt", "generated_texts")
+	:param generator_module_type: Generator module type.
+	The default is "openai_llm".
+		You can use like "llama_index_llm" or "vllm".
+	:param lang: The prompt language that you want to use.
+	"en", "ko" and "ja" are supported.
+	Korean prompt is not officially supported by DeepEval, but it can be translated by AutoRAG developers.
+		Default is "en".
+	:param llm: The model name to use for generation.
+		Or llm if using llama_index_llm.
+		The default is "gpt-4o-2024-08-06".
+	:param batch: The batch size for processing.
+		Default is 16.
+	:param kwargs: The extra parameters for initializing the llm instance.
+	:return: The metric scores.
+	"""
+
+	class Truth(BaseModel):
+		truths: List[str]
+
+	class Claim(BaseModel):
+		claims: List[str]
+
+	class Verdict(BaseModel):
+		verdict: str
+		reason: Optional[str]
+
+	class FaithfulnessVerdicts(BaseModel):
+		verdicts: List[Verdict]
+
+	def calculate_score(verdicts: List[Verdict]) -> float:
+		number_of_verdicts = len(verdicts)
+		if number_of_verdicts == 0:
+			return 1
+
+		faithfulness_count = 0
+		for verdict in verdicts:
+			if verdict.verdict.strip().lower() != "no":
+				faithfulness_count += 1
+
+		score = faithfulness_count / number_of_verdicts
+		return score
+
+	retrieval_contexts = list(map(lambda x: x.retrieval_gt_contents, metric_inputs))
+	truth_prompts = list(
+		map(lambda x: FaithfulnessTemplate.generate_truths(x, lang), retrieval_contexts)
+	)
+
+	generated_texts = list(map(lambda x: x.generated_texts, metric_inputs))
+	claim_prompts = list(
+		map(lambda x: FaithfulnessTemplate.generate_claims(x, lang), generated_texts)
+	)
+
+	generator: BaseGenerator = make_generator_instance(
+		generator_module_type, llm=llm, batch=batch, **kwargs
+	)
+	if isinstance(generator, OpenAILLM):  # Because of the event loop error at the httpx
+		# TODO: Fix the httpx APIConnectionError at the many repetitive request to the OpenAILLM on the same instance
+		truth_responses: List[Truth] = generator.structured_output(truth_prompts, Truth)
+		claim_responses: List[Claim] = make_generator_instance(
+			generator_module_type, llm=llm, batch=batch, **kwargs
+		).structured_output(claim_prompts, Claim)
+		verdict_prompts = list(
+			map(
+				lambda claim, truth: FaithfulnessTemplate.generate_verdicts(
+					"\n\n".join(claim.claims), "\n\n".join(truth.truths), lang
+				),
+				claim_responses,
+				truth_responses,
+			)
+		)
+		verdict_responses: List[FaithfulnessVerdicts] = make_generator_instance(
+			generator_module_type, llm=llm, batch=batch, **kwargs
+		).structured_output(verdict_prompts, FaithfulnessVerdicts)
+	else:
+		truth_responses: List[Truth] = generator.structured_output(truth_prompts, Truth)
+		claim_responses: List[Claim] = generator.structured_output(claim_prompts, Claim)
+		verdict_prompts = list(
+			map(
+				lambda claim, truth: FaithfulnessTemplate.generate_verdicts(
+					"\n\n".join(claim.claims), "\n\n".join(truth.truths), lang
+				),
+				claim_responses,
+				truth_responses,
+			)
+		)
+		verdict_responses: List[FaithfulnessVerdicts] = generator.structured_output(
+			verdict_prompts, FaithfulnessVerdicts
+		)
+
+	result = list(map(lambda x: calculate_score(x.verdicts), verdict_responses))
+	return result
+
+
+@autorag_metric_loop(fields_to_check=["generation_gt", "generated_texts"])
+def bleu(
+	metric_inputs: List[MetricInput],
+	tokenize: Optional[str] = None,
+	smooth_method: str = "exp",
+	smooth_value: Optional[float] = None,
+	max_ngram_order: int = 4,
+	trg_lang: str = "",
+	effective_order: bool = True,
+	**kwargs,
+) -> List[float]:
+	"""
+	Computes the BLEU metric given pred and ground-truth.
+
+	:param metric_inputs: A list of MetricInput schema (Required Field -> "generation_gt", "generated_texts")
+	:param tokenize: The tokenizer to use. If None, defaults to language-specific tokenizers with '13a' as the fallback default. check #https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/metrics/bleu.py
+	:param smooth_method: The smoothing method to use ('floor', 'add-k', 'exp' or 'none').
+	:param smooth_value: The smoothing value for `floor` and `add-k` methods. `None` falls back to default value.
+	:param max_ngram_order: If given, it overrides the maximum n-gram order (default: 4) when computing precisions.
+	:param trg_lang: An optional language code to raise potential tokenizer warnings.
+	:param effective_order: If `True`, stop including n-gram orders for which precision is 0. This should be
+	`True`, if sentence-level BLEU will be computed.
+	"""
+	bleu_instance = BLEU(
+		tokenize=tokenize,
+		smooth_method=smooth_method,
+		smooth_value=smooth_value,
+		max_ngram_order=max_ngram_order,
+		trg_lang=trg_lang,
+		effective_order=effective_order,
+		**kwargs,
+	)
+
+	result = list(
+		map(
+			lambda x: bleu_instance.sentence_score(
+				x.generated_texts, x.generation_gt
+			).score,
+			metric_inputs,
+		)
+	)
+	return result
+
+
+@autorag_metric_loop(fields_to_check=["generation_gt", "generated_texts"])
+def meteor(
+	metric_inputs: List[MetricInput],
+	alpha: float = 0.9,
+	beta: float = 3.0,
+	gamma: float = 0.5,
+) -> List[float]:
+	"""
+	Compute meteor score for generation.
+
+	:param metric_inputs: A list of MetricInput schema (Required Field -> "generation_gt", "generated_texts")
+	:param alpha: Parameter for controlling relative weights of precision and recall.
+	    Default is 0.9.
+	:param beta: Parameter for controlling shape of penalty as a
+	    function of as a function of fragmentation.
+	    Default is 3.0.
+	:param gamma: Relative weight assigned to fragmentation penalty.
+	    Default is 0.5.
+	:return: A list of computed metric scores.
+	"""
+	nltk.download("punkt", quiet=True)
+	meteor_instance = evaluate.load("meteor")
+	result = huggingface_evaluate(
+		meteor_instance,
+		"meteor",
+		metric_inputs,
+		alpha=alpha,
+		beta=beta,
+		gamma=gamma,
+	)
+	del meteor_instance
+	return result
+
+
+@autorag_metric_loop(fields_to_check=["generation_gt", "generated_texts"])
+def rouge(
+	metric_inputs: List[MetricInput],
+	rouge_type: Optional[str] = "rougeL",
+	use_stemmer: bool = False,
+	split_summaries: bool = False,
+	batch: int = os.cpu_count(),
+) -> List[float]:
+	"""
+	Compute rouge score for generation.
+
+	:param metric_inputs: A list of MetricInput schema (Required Field -> "generation_gt", "generated_texts")
+	:param rouge_type: A rouge type to use for evaluation.
+	    Default is 'RougeL'.
+	    Choose between rouge1, rouge2, rougeL, and rougeLSum.
+	    - rouge1: unigram (1-gram) based scoring.
+	    - rouge2: bigram (2-gram) based scoring.
+	    - rougeL: Longest Common Subsequence based scoring.
+	    - rougeLSum: splits text using "\n"
+	:param use_stemmer: Bool indicating whether Porter stemmer should be used to
+	    strip word suffixes to improve matching. This arg is used in the
+	    DefaultTokenizer, but other tokenizers might or might not choose to
+	    use this. Default is False.
+	:param split_summaries: Whether to add newlines between sentences for rougeLsum.
+	    Default is False.
+	:param batch: The batch size for processing.
+	    Default is your cpu count.
+	:return: A list of computed metric scores.
+	"""
+	rouge_instance = RougeScorer(
+		rouge_types=[rouge_type],
+		use_stemmer=use_stemmer,
+		split_summaries=split_summaries,
+		tokenizer=tokenizers.DefaultTokenizer(use_stemmer),
+	)
+
+	async def compute(gt: List[str], pred: str) -> float:
+		return rouge_instance.score_multi(targets=gt, prediction=pred)[
+			rouge_type
+		].fmeasure
+
+	tasks = [
+		compute(metric_input.generation_gt, metric_input.generated_texts)
+		for metric_input in metric_inputs
+	]
+	loop = get_event_loop()
+	result = loop.run_until_complete(process_batch(tasks, batch_size=batch))
+
+	del rouge_instance
+	return result
+
+
+@autorag_metric_loop(fields_to_check=["generation_gt", "generated_texts"])
+def sem_score(
+	metric_inputs: List[MetricInput],
+	embedding_model: Optional[BaseEmbedding] = None,
+	batch: int = 128,
+) -> List[float]:
+	"""
+	Compute sem score between generation gt and pred with cosine similarity.
+
+	:param metric_inputs: A list of MetricInput schema (Required Field -> "generation_gt", "generated_texts")
+	:param embedding_model: Embedding model to use for compute cosine similarity.
+	    Default is all-mpnet-base-v2 embedding model.
+	    The paper used this embedding model.
+	:param batch: The batch size for processing.
+	    Default is 128
+	:return: A list of computed metric scores.
+	"""
+	generations = [metric_input.generated_texts for metric_input in metric_inputs]
+	generation_gt = [metric_input.generation_gt for metric_input in metric_inputs]
+	if embedding_model is None:
+		embedding_model = embedding_models.get("huggingface_all_mpnet_base_v2")()
+
+	embedding_model.embed_batch_size = batch
+
+	openai_embedding_max_length = 8000
+	if isinstance(embedding_model, OpenAIEmbedding):
+		generations = openai_truncate_by_token(
+			generations, openai_embedding_max_length, embedding_model.model_name
+		)
+
+	embedded_pred: List[List[float]] = embedding_model.get_text_embedding_batch(
+		generations, show_progress=True
+	)
+	gt_lengths = list(map(len, generation_gt))
+	flatten_gt = list(itertools.chain.from_iterable(generation_gt))
+	if isinstance(embedding_model, OpenAIEmbedding):
+		flatten_gt = openai_truncate_by_token(
+			flatten_gt, openai_embedding_max_length, embedding_model.model_name
+		)
+	embedded_gt_flatten = embedding_model.get_text_embedding_batch(
+		flatten_gt, show_progress=True
+	)
+	# re-group embedded_gt_flatten with gt_lengths
+	iterator = iter(embedded_gt_flatten)
+	embedded_gt: List[List[List[float]]] = [
+		list(itertools.islice(iterator, length)) for length in gt_lengths
+	]
+
+	result = []
+	for gt, pred in zip(embedded_gt, embedded_pred):
+		similarity_scores: List[float] = list(
+			map(lambda x: calculate_cosine_similarity(x, pred), gt)
+		)
+		result.append(max(similarity_scores))
+
+	del embedding_model
+	empty_cuda_cache()
+
+	return result
+
+
+@autorag_metric_loop(fields_to_check=["generation_gt", "generated_texts"])
+def g_eval(
+	metric_inputs: List[MetricInput],
+	metrics: Optional[List[str]] = None,
+	model: str = "gpt-4-0125-preview",
+	batch_size: int = 8,
+) -> List[float]:
+	"""
+	Calculate G-Eval score.
+	G-eval is a metric that uses high-performance LLM model to evaluate generation performance.
+	It evaluates the generation result by coherence, consistency, fluency, and relevance.
+	It uses only 'openai' model, and we recommend to use gpt-4 for evaluation accuracy.
+
+	:param metric_inputs: A list of MetricInput schema (Required Field -> "generation_gt", "generated_texts")
+	:param metrics: A list of metrics to use for evaluation.
+	    Default is all metrics, which is ['coherence', 'consistency', 'fluency', 'relevance'].
+	:param model: OpenAI model name.
+	    Default is 'gpt-4-0125-preview'.
+	:param batch_size: The batch size for processing.
+	    Default is 8.
+	:return: G-Eval score.
+	"""
+	generations = [metric_input.generated_texts for metric_input in metric_inputs]
+	generation_gt = [metric_input.generation_gt for metric_input in metric_inputs]
+	loop = get_event_loop()
+	tasks = [
+		async_g_eval(gt, pred, metrics, model)
+		for gt, pred in zip(generation_gt, generations)
+	]
+	result = loop.run_until_complete(process_batch(tasks, batch_size=batch_size))
+	return result
+
+
+async def async_g_eval(
+	generation_gt: List[str],
+	pred: str,
+	metrics: Optional[List[str]] = None,
+	model: str = "gpt-4-0125-preview",
+) -> float:
+	available_metrics = ["coherence", "consistency", "fluency", "relevance"]
+	if metrics is None:
+		metrics = available_metrics
+	else:
+		assert len(metrics) > 0, "metrics must be a list of string"
+		metrics = [metric for metric in metrics if metric in available_metrics]
+
+	current_path = os.path.dirname(os.path.realpath(__file__))
+	prompt_path = os.path.join(current_path, "g_eval_prompts")
+	g_eval_prompts = {
+		"coherence": open(os.path.join(prompt_path, "coh_detailed.txt")).read(),
+		"consistency": open(os.path.join(prompt_path, "con_detailed.txt")).read(),
+		"fluency": open(os.path.join(prompt_path, "flu_detailed.txt")).read(),
+		"relevance": open(os.path.join(prompt_path, "rel_detailed.txt")).read(),
+	}
+
+	client = AsyncOpenAI()
+
+	async def g_eval_score(prompt: str, gen_gt: List[str], pred: str):
+		scores = []
+		for gt in gen_gt:
+			input_prompt = prompt.replace("{{Document}}", gt).replace(
+				"{{Summary}}", pred
+			)
+			response = await client.chat.completions.create(
+				model=model,
+				messages=[{"role": "system", "content": input_prompt}],
+				logprobs=True,
+				top_logprobs=5,
+				temperature=0,
+				max_tokens=2,
+				frequency_penalty=0,
+				presence_penalty=0,
+				stop=None,
+				n=20,
+			)
+			if "(1-3):" in prompt:
+				scores.append(get_g_eval_score(response, max_score=3))
+			else:
+				scores.append(get_g_eval_score(response))
+		return max(scores)
+
+	def get_g_eval_score(responses, max_score: int = 5) -> int:
+		target_tokens = {str(i): 0 for i in range(1, max_score + 1)}
+		for choice in responses.choices:
+			first_top_log_probs = choice.logprobs.content[0].top_logprobs
+			for i, top_log_prob in enumerate(
+				list(map(lambda x: x.token, first_top_log_probs))
+			):
+				if top_log_prob in target_tokens:
+					target_tokens[top_log_prob] += 5 - i
+
+		return int(max(target_tokens, key=target_tokens.get))
+
+	g_eval_scores = await asyncio.gather(
+		*(g_eval_score(g_eval_prompts[x], generation_gt, pred) for x in metrics)
+	)
+	return sum(g_eval_scores) / len(g_eval_scores)
+
+
+@autorag_metric_loop(fields_to_check=["generation_gt", "generated_texts"])
+def bert_score(
+	metric_inputs: List[MetricInput],
+	lang: str = "en",
+	batch: int = 128,
+	n_threads: int = os.cpu_count(),
+) -> List[float]:
+	generations = [metric_input.generated_texts for metric_input in metric_inputs]
+	generation_gt = [metric_input.generation_gt for metric_input in metric_inputs]
+	evaluator = evaluate.load("bertscore")
+
+	df = pd.DataFrame(
+		{
+			"reference": generation_gt,
+			"prediction": generations,
+			"lang": lang,
+		}
+	)
+
+	df = df.explode("reference", ignore_index=False)
+	df["bert_score"] = evaluator.compute(
+		predictions=df["prediction"].tolist(),
+		references=df["reference"].tolist(),
+		lang=lang,
+		nthreads=n_threads,
+		batch_size=batch,
+	)["f1"]
+
+	del evaluator
+	empty_cuda_cache()
+
+	return df.groupby(level=0)["bert_score"].max().tolist()

autorag-workspace/autorag/evaluation/metric/retrieval.py

@@ -0,0 +1,115 @@
+import itertools
+import math
+
+from autorag.evaluation.metric.util import autorag_metric
+from autorag.schema.metricinput import MetricInput
+
+
+@autorag_metric(fields_to_check=["retrieval_gt", "retrieved_ids"])
+def retrieval_f1(metric_input: MetricInput):
+	"""
+	Compute f1 score for retrieval.
+
+	:param metric_input: The MetricInput schema for AutoRAG metric.
+	:return: The f1 score.
+	"""
+	recall_score = retrieval_recall.__wrapped__(metric_input)
+	precision_score = retrieval_precision.__wrapped__(metric_input)
+	if recall_score + precision_score == 0:
+		return 0
+	else:
+		return 2 * (recall_score * precision_score) / (recall_score + precision_score)
+
+
+@autorag_metric(fields_to_check=["retrieval_gt", "retrieved_ids"])
+def retrieval_recall(metric_input: MetricInput) -> float:
+	gt, pred = metric_input.retrieval_gt, metric_input.retrieved_ids
+
+	gt_sets = [frozenset(g) for g in gt]
+	pred_set = set(pred)
+	hits = sum(any(pred_id in gt_set for pred_id in pred_set) for gt_set in gt_sets)
+	recall = hits / len(gt) if len(gt) > 0 else 0.0
+	return recall
+
+
+@autorag_metric(fields_to_check=["retrieval_gt", "retrieved_ids"])
+def retrieval_precision(metric_input: MetricInput) -> float:
+	gt, pred = metric_input.retrieval_gt, metric_input.retrieved_ids
+
+	gt_sets = [frozenset(g) for g in gt]
+	pred_set = set(pred)
+	hits = sum(any(pred_id in gt_set for gt_set in gt_sets) for pred_id in pred_set)
+	precision = hits / len(pred) if len(pred) > 0 else 0.0
+	return precision
+
+
+@autorag_metric(fields_to_check=["retrieval_gt", "retrieved_ids"])
+def retrieval_ndcg(metric_input: MetricInput) -> float:
+	gt, pred = metric_input.retrieval_gt, metric_input.retrieved_ids
+
+	gt_sets = [frozenset(g) for g in gt]
+	pred_set = set(pred)
+	relevance_scores = {
+		pred_id: 1 if any(pred_id in gt_set for gt_set in gt_sets) else 0
+		for pred_id in pred_set
+	}
+
+	dcg = sum(
+		(2 ** relevance_scores[doc_id] - 1) / math.log2(i + 2)
+		for i, doc_id in enumerate(pred)
+	)
+
+	len_flatten_gt = len(list(itertools.chain.from_iterable(gt)))
+	len_pred = len(pred)
+	ideal_pred = [1] * min(len_flatten_gt, len_pred) + [0] * max(
+		0, len_pred - len_flatten_gt
+	)
+	idcg = sum(relevance / math.log2(i + 2) for i, relevance in enumerate(ideal_pred))
+
+	ndcg = dcg / idcg if idcg > 0 else 0
+	return ndcg
+
+
+@autorag_metric(fields_to_check=["retrieval_gt", "retrieved_ids"])
+def retrieval_mrr(metric_input: MetricInput) -> float:
+	"""
+	Reciprocal Rank (RR) is the reciprocal of the rank of the first relevant item.
+	Mean of RR in whole queries is MRR.
+	"""
+	gt, pred = metric_input.retrieval_gt, metric_input.retrieved_ids
+
+	# Flatten the ground truth list of lists into a single set of relevant documents
+	gt_sets = [frozenset(g) for g in gt]
+
+	rr_list = []
+	for gt_set in gt_sets:
+		for i, pred_id in enumerate(pred):
+			if pred_id in gt_set:
+				rr_list.append(1.0 / (i + 1))
+				break
+	return sum(rr_list) / len(gt_sets) if rr_list else 0.0
+
+
+@autorag_metric(fields_to_check=["retrieval_gt", "retrieved_ids"])
+def retrieval_map(metric_input: MetricInput) -> float:
+	"""
+	Mean Average Precision (MAP) is the mean of Average Precision (AP) for all queries.
+	"""
+	gt, pred = metric_input.retrieval_gt, metric_input.retrieved_ids
+
+	gt_sets = [frozenset(g) for g in gt]
+
+	ap_list = []
+
+	for gt_set in gt_sets:
+		pred_hits = [1 if pred_id in gt_set else 0 for pred_id in pred]
+		precision_list = [
+			sum(pred_hits[: i + 1]) / (i + 1)
+			for i, hit in enumerate(pred_hits)
+			if hit == 1
+		]
+		ap_list.append(
+			sum(precision_list) / len(precision_list) if precision_list else 0.0
+		)
+
+	return sum(ap_list) / len(gt_sets) if ap_list else 0.0

autorag-workspace/autorag/evaluation/metric/retrieval_contents.py

@@ -0,0 +1,65 @@
+"""
+This file contains the retrieval contents metric,
+which means calculate the metric based on the contents of the retrieved items.
+"""
+
+import itertools
+from collections import Counter
+
+import numpy as np
+
+from autorag.evaluation.metric.util import autorag_metric
+from autorag.schema.metricinput import MetricInput
+from autorag.utils.util import normalize_string
+
+
+def single_token_f1(ground_truth: str, prediction: str):
+	prediction_tokens = normalize_string(prediction).split()
+	ground_truth_tokens = normalize_string(ground_truth).split()
+	common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
+	num_same = sum(common.values())
+	if num_same == 0:
+		return 0, 0, 0
+	precision = 1.0 * num_same / len(prediction_tokens)
+	recall = 1.0 * num_same / len(ground_truth_tokens)
+	f1 = (2 * precision * recall) / (precision + recall)
+	return precision, recall, f1
+
+
+@autorag_metric(fields_to_check=["retrieved_contents", "retrieval_gt_contents"])
+def retrieval_token_f1(metric_input: MetricInput):
+	pred = metric_input.retrieved_contents
+	gt = itertools.chain.from_iterable(metric_input.retrieval_gt_contents)
+
+	calculated_results = list(
+		map(lambda x: single_token_f1(x[1], x[0]), list(itertools.product(pred, gt)))
+	)
+	_, _, result = zip(*calculated_results)
+	result_np = np.array(list(result)).reshape(len(pred), -1)
+	return result_np.max(axis=1).mean()
+
+
+@autorag_metric(fields_to_check=["retrieved_contents", "retrieval_gt_contents"])
+def retrieval_token_precision(metric_input: MetricInput):
+	pred = metric_input.retrieved_contents
+	gt = itertools.chain.from_iterable(metric_input.retrieval_gt_contents)
+
+	calculated_results = list(
+		map(lambda x: single_token_f1(x[1], x[0]), list(itertools.product(pred, gt)))
+	)
+	result, _, _ = zip(*calculated_results)
+	result_np = np.array(list(result)).reshape(len(pred), -1)
+	return result_np.max(axis=1).mean()
+
+
+@autorag_metric(fields_to_check=["retrieved_contents", "retrieval_gt_contents"])
+def retrieval_token_recall(metric_input: MetricInput):
+	pred = metric_input.retrieved_contents
+	gt = itertools.chain.from_iterable(metric_input.retrieval_gt_contents)
+
+	calculated_results = list(
+		map(lambda x: single_token_f1(x[1], x[0]), list(itertools.product(pred, gt)))
+	)
+	_, result, _ = zip(*calculated_results)
+	result_np = np.array(list(result)).reshape(len(pred), -1)
+	return result_np.max(axis=1).mean()

autorag-workspace/autorag/evaluation/metric/util.py

@@ -0,0 +1,88 @@
+import functools
+from typing import List
+
+import numpy as np
+
+from autorag.schema.metricinput import MetricInput
+from autorag.utils.util import convert_inputs_to_list
+
+
+def calculate_cosine_similarity(a, b):
+	dot_product = np.dot(a, b)
+	norm_a = np.linalg.norm(a)
+	norm_b = np.linalg.norm(b)
+	similarity = dot_product / (norm_a * norm_b)
+	return similarity
+
+
+def calculate_l2_distance(a, b):
+	return np.linalg.norm(a - b)
+
+
+def calculate_inner_product(a, b):
+	return np.dot(a, b)
+
+
+def autorag_metric(fields_to_check: List[str]):
+	def decorator_autorag_metric(func):
+		@functools.wraps(func)
+		@convert_inputs_to_list
+		def wrapper(metric_inputs: List[MetricInput], **kwargs) -> List[float]:
+			"""
+			Use 'for loop' to run each metric input.
+			Put the single metric input into the metric function.
+
+			:param metric_inputs: A list MetricInput schema for AutoRAG metric.
+			:param kwargs: The additional arguments for metric function.
+			:return: A list of computed metric scores.
+			"""
+			results = []
+			for metric_input in metric_inputs:
+				if metric_input.is_fields_notnone(fields_to_check=fields_to_check):
+					results.append(func(metric_input, **kwargs))
+				else:
+					results.append(None)
+			return results
+
+		return wrapper
+
+	return decorator_autorag_metric
+
+
+def autorag_metric_loop(fields_to_check: List[str]):
+	def decorator_autorag_generation_metric(func):
+		@functools.wraps(func)
+		@convert_inputs_to_list
+		def wrapper(metric_inputs: List[MetricInput], **kwargs) -> List[float]:
+			"""
+			Put the list of metric inputs into the metric function.
+
+			:param metric_inputs: A list MetricInput schema for AutoRAG metric.
+			:param kwargs: The additional arguments for metric function.
+			:return: A list of computed metric scores.
+			"""
+			bool_list = [
+				metric_input.is_fields_notnone(fields_to_check=fields_to_check)
+				for metric_input in metric_inputs
+			]
+			valid_inputs = [
+				metric_input
+				for metric_input, is_valid in zip(metric_inputs, bool_list)
+				if is_valid
+			]
+
+			results = [None] * len(metric_inputs)
+			if valid_inputs:
+				processed_valid = func(valid_inputs, **kwargs)
+
+				valid_index = 0
+				for i, is_valid in enumerate(bool_list):
+					if is_valid:
+						results[i] = processed_valid[valid_index]
+						valid_index += 1
+
+			return results
+
+		return wrapper
+
+	return decorator_autorag_generation_metric

autorag-workspace/autorag/evaluation/retrieval.py

@@ -0,0 +1,83 @@
+import functools
+import warnings
+from typing import List, Callable, Any, Tuple, Union, Dict
+
+import pandas as pd
+
+from autorag.evaluation.metric import (
+	retrieval_recall,
+	retrieval_precision,
+	retrieval_f1,
+	retrieval_ndcg,
+	retrieval_mrr,
+	retrieval_map,
+)
+from autorag.evaluation.util import cast_metrics
+from autorag.schema.metricinput import MetricInput
+
+RETRIEVAL_METRIC_FUNC_DICT = {
+	func.__name__: func
+	for func in [
+		retrieval_recall,
+		retrieval_precision,
+		retrieval_f1,
+		retrieval_ndcg,
+		retrieval_mrr,
+		retrieval_map,
+	]
+}
+
+
+def evaluate_retrieval(
+	metric_inputs: List[MetricInput],
+	metrics: Union[List[str], List[Dict]],
+):
+	def decorator_evaluate_retrieval(
+		func: Callable[
+			[Any], Tuple[List[List[str]], List[List[str]], List[List[float]]]
+		],
+	):
+		"""
+		Decorator for evaluating retrieval results.
+		You can use this decorator to any method that returns (contents, scores, ids),
+		which is the output of conventional retrieval modules.
+
+		:param func: Must return (contents, scores, ids)
+		:return: wrapper function that returns pd.DataFrame, which is the evaluation result.
+		"""
+
+		@functools.wraps(func)
+		def wrapper(*args, **kwargs) -> pd.DataFrame:
+			contents, pred_ids, scores = func(*args, **kwargs)
+			for metric_input, pred_id in zip(metric_inputs, pred_ids):
+				metric_input.retrieved_ids = pred_id
+
+			metric_scores = {}
+			metric_names, metric_params = cast_metrics(metrics)
+
+			for metric_name, metric_param in zip(metric_names, metric_params):
+				if metric_name in RETRIEVAL_METRIC_FUNC_DICT:
+					metric_func = RETRIEVAL_METRIC_FUNC_DICT[metric_name]
+					metric_scores[metric_name] = metric_func(
+						metric_inputs=metric_inputs, **metric_param
+					)
+				else:
+					warnings.warn(
+						f"metric {metric_name} is not in supported metrics: {RETRIEVAL_METRIC_FUNC_DICT.keys()}"
+						f"{metric_name} will be ignored."
+					)
+
+			metric_result_df = pd.DataFrame(metric_scores)
+			execution_result_df = pd.DataFrame(
+				{
+					"retrieved_contents": contents,
+					"retrieved_ids": pred_ids,
+					"retrieve_scores": scores,
+				}
+			)
+			result_df = pd.concat([execution_result_df, metric_result_df], axis=1)
+			return result_df
+
+		return wrapper
+
+	return decorator_evaluate_retrieval

autorag-workspace/autorag/evaluation/retrieval_contents.py

@@ -0,0 +1,65 @@
+import functools
+from typing import List, Callable, Any, Tuple
+
+import pandas as pd
+
+from autorag.evaluation.metric import (
+	retrieval_token_f1,
+	retrieval_token_precision,
+	retrieval_token_recall,
+)
+from autorag.schema.metricinput import MetricInput
+
+
+def evaluate_retrieval_contents(metric_inputs: List[MetricInput], metrics: List[str]):
+	def decorator_evaluate_retrieval_contents(
+		func: Callable[
+			[Any], Tuple[List[List[str]], List[List[str]], List[List[float]]]
+		],
+	):
+		"""
+		Decorator for evaluating retrieval contents.
+		You can use this decorator to any method that returns (contents, scores, ids),
+		which is the output of conventional retrieval modules.
+
+		:param func: Must return (contents, scores, ids)
+		:return: pd.DataFrame, which is the evaluation result and function result.
+		"""
+
+		@functools.wraps(func)
+		def wrapper(*args, **kwargs) -> pd.DataFrame:
+			contents, pred_ids, scores = func(*args, **kwargs)
+			metric_funcs = {
+				retrieval_token_recall.__name__: retrieval_token_recall,
+				retrieval_token_precision.__name__: retrieval_token_precision,
+				retrieval_token_f1.__name__: retrieval_token_f1,
+			}
+			for metric_input, content in zip(metric_inputs, contents):
+				metric_input.retrieved_contents = content
+
+			metrics_scores = {}
+			for metric in metrics:
+				if metric not in metric_funcs:
+					raise ValueError(
+						f"metric {metric} is not in supported metrics: {metric_funcs.keys()}"
+					)
+				else:
+					metric_func = metric_funcs[metric]
+					# Extract each required field from all payloads
+					metric_scores = metric_func(metric_inputs=metric_inputs)
+					metrics_scores[metric] = metric_scores
+
+			metric_result_df = pd.DataFrame(metrics_scores)
+			execution_result_df = pd.DataFrame(
+				{
+					"retrieved_contents": contents,
+					"retrieved_ids": pred_ids,
+					"retrieve_scores": scores,
+				}
+			)
+			result_df = pd.concat([execution_result_df, metric_result_df], axis=1)
+			return result_df
+
+		return wrapper
+
+	return decorator_evaluate_retrieval_contents

autorag-workspace/autorag/evaluation/util.py

@@ -0,0 +1,43 @@
+from copy import deepcopy
+from typing import Union, List, Dict, Tuple, Any
+
+from autorag.embedding.base import EmbeddingModel
+
+
+def cast_metrics(
+	metrics: Union[List[str], List[Dict]],
+) -> Tuple[List[str], List[Dict[str, Any]]]:
+	"""
+	 Turn metrics to list of metric names and parameter list.
+
+	:param metrics: List of string or dictionary.
+	:return: The list of metric names and dictionary list of metric parameters.
+	"""
+	metrics_copy = deepcopy(metrics)
+	if not isinstance(metrics_copy, list):
+		raise ValueError("metrics must be a list of string or dictionary.")
+	if isinstance(metrics_copy[0], str):
+		return metrics_copy, [{} for _ in metrics_copy]
+	elif isinstance(metrics_copy[0], dict):
+		# pop 'metric_name' key from dictionary
+		metric_names = list(map(lambda x: x.pop("metric_name"), metrics_copy))
+		metric_params = [
+			dict(
+				map(
+					lambda x, y: cast_embedding_model(x, y),
+					metric.keys(),
+					metric.values(),
+				)
+			)
+			for metric in metrics_copy
+		]
+		return metric_names, metric_params
+	else:
+		raise ValueError("metrics must be a list of string or dictionary.")
+
+
+def cast_embedding_model(key, value):
+	if key == "embedding_model":
+		return key, EmbeddingModel.load(value)()
+	else:
+		return key, value

autorag-workspace/autorag/evaluator.py

@@ -0,0 +1,560 @@
+import glob
+import json
+import logging
+import os
+import shutil
+from datetime import datetime
+from itertools import chain
+from typing import List, Dict, Optional
+from rich.progress import Progress, BarColumn, TimeElapsedColumn
+
+import pandas as pd
+import yaml
+
+from autorag.node_line import run_node_line
+from autorag.nodes.retrieval.base import get_bm25_pkl_name
+from autorag.nodes.retrieval.bm25 import bm25_ingest
+from autorag.nodes.retrieval.vectordb import (
+	vectordb_ingest,
+	filter_exist_ids,
+	filter_exist_ids_from_retrieval_gt,
+)
+from autorag.schema import Node
+from autorag.schema.node import (
+	module_type_exists,
+	extract_values_from_nodes,
+	extract_values_from_nodes_strategy,
+)
+from autorag.utils import (
+	cast_qa_dataset,
+	cast_corpus_dataset,
+	validate_qa_from_corpus_dataset,
+)
+from autorag.utils.util import (
+	load_summary_file,
+	explode,
+	load_yaml_config,
+	get_event_loop,
+)
+from autorag.vectordb import load_all_vectordb_from_yaml
+
+logger = logging.getLogger("AutoRAG")
+
+ascii_art = """
+                _        _____            _____
+     /\        | |      |  __ \     /\   / ____|
+    /  \  _   _| |_ ___ | |__) |   /  \ | |  __
+   / /\ \| | | | __/ _ \|  _  /   / /\ \| | |_ |
+  / ____ \ |_| | || (_) | | \ \  / ____ \ |__| |
+ /_/    \_\__,_|\__\___/|_|  \_\/_/    \_\_____|
+
+"""
+
+
+class Evaluator:
+	def __init__(
+		self,
+		qa_data_path: str,
+		corpus_data_path: str,
+		project_dir: Optional[str] = None,
+	):
+		"""
+		Initialize an Evaluator object.
+
+		:param qa_data_path: The path to the QA dataset.
+		    Must be parquet file.
+		:param corpus_data_path: The path to the corpus dataset.
+		    Must be parquet file.
+		:param project_dir: The path to the project directory.
+		    Default is the current directory.
+		"""
+		# validate data paths
+		if not os.path.exists(qa_data_path):
+			raise ValueError(f"QA data path {qa_data_path} does not exist.")
+		if not os.path.exists(corpus_data_path):
+			raise ValueError(f"Corpus data path {corpus_data_path} does not exist.")
+		if not qa_data_path.endswith(".parquet"):
+			raise ValueError(f"QA data path {qa_data_path} is not a parquet file.")
+		if not corpus_data_path.endswith(".parquet"):
+			raise ValueError(
+				f"Corpus data path {corpus_data_path} is not a parquet file."
+			)
+		self.qa_data_path = qa_data_path
+		self.corpus_data_path = corpus_data_path
+		self.qa_data = pd.read_parquet(qa_data_path, engine="pyarrow")
+		self.corpus_data = pd.read_parquet(corpus_data_path, engine="pyarrow")
+		self.qa_data = cast_qa_dataset(self.qa_data)
+		self.corpus_data = cast_corpus_dataset(self.corpus_data)
+		self.project_dir = project_dir if project_dir is not None else os.getcwd()
+		if not os.path.exists(self.project_dir):
+			os.makedirs(self.project_dir)
+
+		validate_qa_from_corpus_dataset(self.qa_data, self.corpus_data)
+
+		# copy dataset to the project directory
+		if not os.path.exists(os.path.join(self.project_dir, "data")):
+			os.makedirs(os.path.join(self.project_dir, "data"))
+		qa_path_in_project = os.path.join(self.project_dir, "data", "qa.parquet")
+		if not os.path.exists(qa_path_in_project):
+			self.qa_data.to_parquet(qa_path_in_project, index=False)
+		corpus_path_in_project = os.path.join(
+			self.project_dir, "data", "corpus.parquet"
+		)
+		if not os.path.exists(corpus_path_in_project):
+			self.corpus_data.to_parquet(corpus_path_in_project, index=False)
+
+	def start_trial(
+		self, yaml_path: str, skip_validation: bool = False, full_ingest: bool = True
+	):
+		"""
+		Start AutoRAG trial.
+		The trial means one experiment to optimize the RAG pipeline.
+		It consists of ingesting corpus data, running all nodes and modules, evaluating and finding the optimal modules.
+
+		:param yaml_path: The config YAML path
+		:param skip_validation: If True, it skips the validation step.
+			The validation step checks the input config YAML file is well formatted,
+			and there is any problem with the system settings.
+			Default is False.
+		:param full_ingest: If True, it checks the whole corpus data from corpus.parquet that exists in the Vector DB.
+			If your corpus is huge and don't want to check the whole vector DB, please set it to False.
+		:return: None
+		"""
+		# Make Resources directory
+		os.makedirs(os.path.join(self.project_dir, "resources"), exist_ok=True)
+
+		if not skip_validation:
+			logger.info(ascii_art)
+			logger.info(
+				"Start Validation input data and config YAML file first. "
+				"If you want to skip this, put the --skip_validation flag or "
+				"`skip_validation` at the start_trial function."
+			)
+			from autorag.validator import Validator  # resolve circular import
+
+			validator = Validator(
+				qa_data_path=self.qa_data_path, corpus_data_path=self.corpus_data_path
+			)
+			validator.validate(yaml_path)
+
+		os.environ["PROJECT_DIR"] = self.project_dir
+
+		trial_name = self.__get_new_trial_name()
+		self.__make_trial_dir(trial_name)
+
+		# copy YAML file to the trial directory
+		shutil.copy(
+			yaml_path, os.path.join(self.project_dir, trial_name, "config.yaml")
+		)
+		yaml_dict = load_yaml_config(yaml_path)
+		vectordb = yaml_dict.get("vectordb", [])
+
+		vectordb_config_path = os.path.join(
+			self.project_dir, "resources", "vectordb.yaml"
+		)
+		with open(vectordb_config_path, "w") as f:
+			yaml.safe_dump({"vectordb": vectordb}, f)
+
+		node_lines = self._load_node_lines(yaml_path)
+		self.__ingest_bm25_full(node_lines)
+
+		with Progress(
+			"[progress.description]{task.description}",
+			BarColumn(),
+			"[progress.percentage]{task.percentage:>3.0f}%",
+			"[progress.bar]{task.completed}/{task.total}",
+			TimeElapsedColumn(),
+		) as progress:
+			# Ingest VectorDB corpus
+			if any(
+				list(
+					map(
+						lambda nodes: module_type_exists(nodes, "vectordb"),
+						node_lines.values(),
+					)
+				)
+			):
+				task_ingest = progress.add_task("[cyan]Ingesting VectorDB...", total=1)
+
+				loop = get_event_loop()
+				loop.run_until_complete(self.__ingest_vectordb(yaml_path, full_ingest))
+
+				progress.update(task_ingest, completed=1)
+
+			trial_summary_df = pd.DataFrame(
+				columns=[
+					"node_line_name",
+					"node_type",
+					"best_module_filename",
+					"best_module_name",
+					"best_module_params",
+					"best_execution_time",
+				]
+			)
+			task_eval = progress.add_task(
+				"[cyan]Evaluating...", total=sum(map(len, node_lines.values()))
+			)
+
+			for i, (node_line_name, node_line) in enumerate(node_lines.items()):
+				node_line_dir = os.path.join(
+					self.project_dir, trial_name, node_line_name
+				)
+				os.makedirs(node_line_dir, exist_ok=False)
+				if i == 0:
+					previous_result = self.qa_data
+				logger.info(f"Running node line {node_line_name}...")
+				previous_result = run_node_line(
+					node_line, node_line_dir, previous_result, progress, task_eval
+				)
+
+				trial_summary_df = self._append_node_line_summary(
+					node_line_name, node_line_dir, trial_summary_df
+				)
+
+			trial_summary_df.to_csv(
+				os.path.join(self.project_dir, trial_name, "summary.csv"), index=False
+			)
+
+			logger.info("Evaluation complete.")
+
+	def __ingest_bm25_full(self, node_lines: Dict[str, List[Node]]):
+		if any(
+			list(
+				map(
+					lambda nodes: module_type_exists(nodes, "bm25"), node_lines.values()
+				)
+			)
+		):
+			logger.info("Embedding BM25 corpus...")
+			bm25_tokenizer_list = list(
+				chain.from_iterable(
+					map(
+						lambda nodes: self._find_bm25_tokenizer(nodes),
+						node_lines.values(),
+					)
+				)
+			)
+
+			if len(bm25_tokenizer_list) == 0:
+				bm25_tokenizer_list = ["porter_stemmer"]
+			for bm25_tokenizer in bm25_tokenizer_list:
+				bm25_dir = os.path.join(
+					self.project_dir, "resources", get_bm25_pkl_name(bm25_tokenizer)
+				)
+				if not os.path.exists(os.path.dirname(bm25_dir)):
+					os.makedirs(os.path.dirname(bm25_dir))
+				# ingest because bm25 supports update new corpus data
+				bm25_ingest(bm25_dir, self.corpus_data, bm25_tokenizer=bm25_tokenizer)
+			logger.info("BM25 corpus embedding complete.")
+
+	def __get_new_trial_name(self) -> str:
+		trial_json_path = os.path.join(self.project_dir, "trial.json")
+		if not os.path.exists(trial_json_path):
+			return "0"
+		with open(trial_json_path, "r") as f:
+			trial_json = json.load(f)
+		return str(int(trial_json[-1]["trial_name"]) + 1)
+
+	def __make_trial_dir(self, trial_name: str):
+		trial_json_path = os.path.join(self.project_dir, "trial.json")
+		if os.path.exists(trial_json_path):
+			with open(trial_json_path, "r") as f:
+				trial_json = json.load(f)
+		else:
+			trial_json = []
+
+		trial_json.append(
+			{
+				"trial_name": trial_name,
+				"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+			}
+		)
+		os.makedirs(os.path.join(self.project_dir, trial_name))
+		with open(trial_json_path, "w") as f:
+			json.dump(trial_json, f, indent=4)
+
+	@staticmethod
+	def _load_node_lines(yaml_path: str) -> Dict[str, List[Node]]:
+		yaml_dict = load_yaml_config(yaml_path)
+		node_lines = yaml_dict["node_lines"]
+		node_line_dict = {}
+		for node_line in node_lines:
+			node_line_dict[node_line["node_line_name"]] = list(
+				map(lambda node: Node.from_dict(node), node_line["nodes"])
+			)
+		return node_line_dict
+
+	def restart_trial(self, trial_path: str):
+		logger.info(ascii_art)
+		os.environ["PROJECT_DIR"] = self.project_dir
+		# Check if trial_path exists
+		if not os.path.exists(trial_path):
+			raise ValueError(f"Trial path {trial_path} does not exist.")
+		# Check if trial is completed
+		if os.path.exists(os.path.join(trial_path, "summary.csv")):
+			raise ValueError(f"Trial path {trial_path} is already completed.")
+
+		# Extract node lines from config.yaml
+		yaml_path = os.path.join(trial_path, "config.yaml")
+		node_lines = self._load_node_lines(yaml_path)
+
+		node_line_names = list(node_lines.keys())
+		nodes = list(node_lines.values())
+		node_names = list(
+			map(lambda node: list(map(lambda n: n.node_type, node)), nodes)
+		)
+
+		# If the First Node Line folder hasn't even been created, proceed to start_trial
+		if not os.path.exists(os.path.join(trial_path, node_line_names[0])):
+			self.start_trial(yaml_path)
+			return None
+
+		# Find conflict node line and node
+		conflict_line_name, conflict_node_name = self.__find_conflict_point(
+			trial_path, node_line_names, node_lines
+		)
+		node_dir = os.path.join(trial_path, conflict_line_name, conflict_node_name)
+		if os.path.exists(node_dir):
+			shutil.rmtree(node_dir)
+
+		# Set remain_nodes and remain_lines
+		remain_nodes, completed_node_names, remain_lines, remain_line_names = (
+			self._set_remain_nodes_and_lines(
+				node_line_names,
+				nodes,
+				node_names,
+				conflict_node_name,
+				conflict_line_name,
+			)
+		)
+		# Set previous_result
+		previous_result = self.__set_previous_result(
+			node_line_names, node_names, trial_path, conflict_node_name
+		)
+
+		# Run Node
+		if remain_nodes:
+			conflict_line_dir = os.path.join(trial_path, conflict_line_name)
+			summary_lst = []
+			# Get already run node summary and append to summary_lst
+			for completed_node_name in completed_node_names:
+				summary_lst = self._append_node_summary(
+					conflict_line_dir, completed_node_name, summary_lst
+				)
+			for node in remain_nodes:
+				previous_result = node.run(previous_result, conflict_line_dir)
+				summary_lst = self._append_node_summary(
+					conflict_line_dir, node.node_type, summary_lst
+				)
+			pd.DataFrame(summary_lst).to_csv(
+				os.path.join(conflict_line_dir, "summary.csv"), index=False
+			)
+
+		# Run node line
+		trial_summary_df = pd.DataFrame(
+			columns=[
+				"node_line_name",
+				"node_type",
+				"best_module_filename",
+				"best_module_name",
+				"best_module_params",
+				"best_execution_time",
+			]
+		)
+		completed_line_names = node_line_names[
+			: node_line_names.index(conflict_line_name)
+		]
+		# Get already run node line's summary and append to trial_summary_df
+		if completed_line_names:
+			for line_name in completed_line_names:
+				node_line_dir = os.path.join(trial_path, line_name)
+				trial_summary_df = self._append_node_line_summary(
+					line_name, node_line_dir, trial_summary_df
+				)
+		if remain_lines:
+			for node_line_name, node_line in zip(remain_line_names, remain_lines):
+				node_line_dir = os.path.join(trial_path, node_line_name)
+				if not os.path.exists(node_line_dir):
+					os.makedirs(node_line_dir)
+				logger.info(f"Running node line {node_line_name}...")
+				previous_result = run_node_line(
+					node_line, node_line_dir, previous_result
+				)
+				trial_summary_df = self._append_node_line_summary(
+					node_line_name, node_line_dir, trial_summary_df
+				)
+		trial_summary_df.to_csv(os.path.join(trial_path, "summary.csv"), index=False)
+
+		logger.info("Evaluation complete.")
+
+	def __find_conflict_point(
+		self,
+		trial_path: str,
+		node_line_names: List[str],
+		node_lines: Dict[str, List[Node]],
+	) -> tuple[str, str]:
+		for node_line_name in node_line_names:
+			node_line_dir = os.path.join(trial_path, node_line_name)
+			if not os.path.exists(node_line_dir):
+				return node_line_name, node_lines[node_line_name][0].node_type
+
+			if not os.path.exists(os.path.join(node_line_dir, "summary.csv")):
+				conflict_node_name = self._find_conflict_node_name(
+					node_line_dir, node_lines[node_line_name]
+				)
+				return node_line_name, conflict_node_name
+
+		raise ValueError(f"No error node line found in {trial_path}.")
+
+	@staticmethod
+	def _find_conflict_node_name(node_line_dir: str, node_line: List[Node]) -> str:
+		for node in node_line:
+			node_dir = os.path.join(node_line_dir, node.node_type)
+			if not os.path.exists(node_dir) or not os.path.exists(
+				os.path.join(node_dir, "summary.csv")
+			):
+				return node.node_type
+		raise TypeError("No conflict node name found.")
+
+	def __set_previous_result(
+		self,
+		node_line_names: List[str],
+		node_names: List[List[str]],
+		trial_path: str,
+		conflict_node_name: str,
+	):
+		exploded_node_line, exploded_node = explode(node_line_names, node_names)
+		conflict_node_index = exploded_node.index(conflict_node_name)
+		# Set previous_result
+		if conflict_node_index == 0:
+			previous_result = self.qa_data
+		else:
+			previous_node_line = exploded_node_line[conflict_node_index - 1]
+			previous_node = exploded_node[conflict_node_index - 1]
+
+			previous_node_dir = os.path.join(
+				trial_path, previous_node_line, previous_node
+			)
+			best_file_pattern = f"{previous_node_dir}/best_*.parquet"
+			previous_result = pd.read_parquet(
+				glob.glob(best_file_pattern)[0], engine="pyarrow"
+			)
+		return previous_result
+
+	@staticmethod
+	def _set_remain_nodes_and_lines(
+		node_line_names: List[str],
+		nodes: List[List[Node]],
+		node_names: List[List[str]],
+		conflict_node_name: str,
+		conflict_node_line_name: str,
+	):
+		conflict_node_line_index = node_line_names.index(conflict_node_line_name)
+		full_conflict_node_line_nodes = nodes[conflict_node_line_index]
+		full_conflict_node_line_node_names = node_names[conflict_node_line_index]
+
+		if conflict_node_name == full_conflict_node_line_node_names[0]:
+			remain_nodes = None
+			completed_node_names = None
+			remain_node_lines = nodes[conflict_node_line_index:]
+			remain_node_line_names = node_line_names[conflict_node_line_index:]
+		else:
+			conflict_node_index = full_conflict_node_line_node_names.index(
+				conflict_node_name
+			)
+			remain_nodes = full_conflict_node_line_nodes[conflict_node_index:]
+			completed_node_names = full_conflict_node_line_node_names[
+				:conflict_node_index
+			]
+			if conflict_node_line_index + 1 >= len(node_line_names):
+				remain_node_lines = None
+				remain_node_line_names = None
+			else:
+				remain_node_lines = nodes[conflict_node_line_index + 1 :]
+				remain_node_line_names = node_line_names[conflict_node_line_index + 1 :]
+		return (
+			remain_nodes,
+			completed_node_names,
+			remain_node_lines,
+			remain_node_line_names,
+		)
+
+	@staticmethod
+	def _append_node_line_summary(
+		node_line_name: str, node_line_dir: str, trial_summary_df: pd.DataFrame
+	):
+		summary_df = load_summary_file(
+			os.path.join(node_line_dir, "summary.csv"),
+			dict_columns=["best_module_params"],
+		)
+		summary_df = summary_df.assign(node_line_name=node_line_name)
+		summary_df = summary_df[list(trial_summary_df.columns)]
+		if len(trial_summary_df) <= 0:
+			trial_summary_df = summary_df
+		else:
+			trial_summary_df = pd.concat(
+				[trial_summary_df, summary_df], ignore_index=True
+			)
+		return trial_summary_df
+
+	@staticmethod
+	def _append_node_summary(
+		node_line_dir: str, node_name: str, summary_lst: List[Dict]
+	):
+		node_summary_df = load_summary_file(
+			os.path.join(node_line_dir, node_name, "summary.csv")
+		)
+		best_node_row = node_summary_df.loc[node_summary_df["is_best"]]
+		summary_lst.append(
+			{
+				"node_type": node_name,
+				"best_module_filename": best_node_row["filename"].values[0],
+				"best_module_name": best_node_row["module_name"].values[0],
+				"best_module_params": best_node_row["module_params"].values[0],
+				"best_execution_time": best_node_row["execution_time"].values[0],
+			}
+		)
+		return summary_lst
+
+	@staticmethod
+	def _find_bm25_tokenizer(nodes: List[Node]):
+		bm25_tokenizer_list = extract_values_from_nodes(nodes, "bm25_tokenizer")
+		strategy_tokenizer_list = list(
+			chain.from_iterable(
+				extract_values_from_nodes_strategy(nodes, "bm25_tokenizer")
+			)
+		)
+		return list(set(bm25_tokenizer_list + strategy_tokenizer_list))
+
+	@staticmethod
+	def _find_embedding_model(nodes: List[Node]):
+		embedding_models_list = extract_values_from_nodes(nodes, "embedding_model")
+		retrieval_module_dicts = extract_values_from_nodes_strategy(
+			nodes, "retrieval_modules"
+		)
+		for retrieval_modules in retrieval_module_dicts:
+			vectordb_modules = list(
+				filter(lambda x: x["module_type"] == "vectordb", retrieval_modules)
+			)
+			embedding_models_list.extend(
+				list(map(lambda x: x.get("embedding_model", None), vectordb_modules))
+			)
+		embedding_models_list = list(
+			filter(lambda x: x is not None, embedding_models_list)
+		)
+		return list(set(embedding_models_list))
+
+	async def __ingest_vectordb(self, yaml_path, full_ingest: bool):
+		vectordb_list = load_all_vectordb_from_yaml(yaml_path, self.project_dir)
+		if full_ingest is True:
+			# get the target ingest corpus from the whole corpus
+			for vectordb in vectordb_list:
+				target_corpus = await filter_exist_ids(vectordb, self.corpus_data)
+				await vectordb_ingest(vectordb, target_corpus)
+		else:
+			# get the target ingest corpus from the retrieval gt only
+			for vectordb in vectordb_list:
+				target_corpus = await filter_exist_ids_from_retrieval_gt(
+					vectordb, self.qa_data, self.corpus_data
+				)
+				await vectordb_ingest(vectordb, target_corpus)

autorag-workspace/autorag/generator_models.py

@@ -0,0 +1,4 @@
+import autorag
+from llama_index.llms.ollama import Ollama
+
+autorag.generator_models["ollama"] = Ollama

autorag-workspace/autorag/node_line.py

@@ -0,0 +1,73 @@
+import os
+import pathlib
+from typing import Dict, List, Optional
+from rich.progress import Progress
+
+import pandas as pd
+
+from autorag.schema import Node
+from autorag.utils.util import load_summary_file
+
+
+def make_node_lines(node_line_dict: Dict) -> List[Node]:
+	"""
+	This method makes a list of nodes from node line dictionary.
+	:param node_line_dict: Node_line_dict loaded from yaml file, or get from user input.
+	:return: List of Nodes inside this node line.
+	"""
+	nodes = node_line_dict.get("nodes")
+	if nodes is None:
+		raise ValueError("Node line must have 'nodes' key.")
+	node_objects = list(map(lambda x: Node.from_dict(x), nodes))
+	return node_objects
+
+
+def run_node_line(
+	nodes: List[Node],
+	node_line_dir: str,
+	previous_result: Optional[pd.DataFrame] = None,
+	progress: Progress = None,
+	task_eval: Progress.tasks = None,
+):
+	"""
+	Run the whole node line by running each node.
+
+	:param nodes: A list of nodes.
+	:param node_line_dir: This node line's directory.
+	:param previous_result: A result of the previous node line.
+	    If None, it loads qa data from data/qa.parquet.
+	:param progress: Rich Progress object.
+	:param task_eval: Progress task object
+	:return: The final result of the node line.
+	"""
+	if previous_result is None:
+		project_dir = pathlib.PurePath(node_line_dir).parent.parent
+		qa_path = os.path.join(project_dir, "data", "qa.parquet")
+		if not os.path.exists(qa_path):
+			raise ValueError(f"qa.parquet does not exist in {qa_path}.")
+		previous_result = pd.read_parquet(qa_path, engine="pyarrow")
+
+	summary_lst = []
+	for node in nodes:
+		previous_result = node.run(previous_result, node_line_dir)
+		node_summary_df = load_summary_file(
+			os.path.join(node_line_dir, node.node_type, "summary.csv")
+		)
+		best_node_row = node_summary_df.loc[node_summary_df["is_best"]]
+		summary_lst.append(
+			{
+				"node_type": node.node_type,
+				"best_module_filename": best_node_row["filename"].values[0],
+				"best_module_name": best_node_row["module_name"].values[0],
+				"best_module_params": best_node_row["module_params"].values[0],
+				"best_execution_time": best_node_row["execution_time"].values[0],
+			}
+		)
+		# Update progress for each node
+		if progress:
+			progress.update(task_eval, advance=1)
+
+	pd.DataFrame(summary_lst).to_csv(
+		os.path.join(node_line_dir, "summary.csv"), index=False
+	)
+	return previous_result

autorag-workspace/autorag/nodes/generator/__init__.py

@@ -0,0 +1,4 @@
+from .llama_index_llm import LlamaIndexLLM
+from .openai_llm import OpenAILLM
+from .vllm import Vllm
+from .vllm_api import VllmAPI

autorag-workspace/autorag/nodes/generator/base.py

@@ -0,0 +1,103 @@
+import abc
+import functools
+import logging
+from pathlib import Path
+from typing import Union, Tuple, List
+
+import pandas as pd
+from llama_index.core.output_parsers import PydanticOutputParser
+
+from autorag import generator_models
+from autorag.schema import BaseModule
+from autorag.utils import result_to_dataframe
+
+logger = logging.getLogger("AutoRAG")
+
+
+class BaseGenerator(BaseModule, metaclass=abc.ABCMeta):
+	def __init__(self, project_dir: str, llm: str, *args, **kwargs):
+		logger.info(f"Initialize generator node - {self.__class__.__name__}")
+		self.llm = llm
+
+	def __del__(self):
+		logger.info(f"Deleting generator module - {self.__class__.__name__}")
+
+	def cast_to_run(self, previous_result: pd.DataFrame, *args, **kwargs):
+		logger.info(f"Running generator node - {self.__class__.__name__} module...")
+		assert (
+			"prompts" in previous_result.columns
+		), "previous_result must contain prompts column."
+		prompts = previous_result["prompts"].tolist()
+		return prompts
+
+	def structured_output(self, prompts: List[str], output_cls):
+		response, _, _ = self._pure(prompts)
+		parser = PydanticOutputParser(output_cls)
+		result = []
+		for res in response:
+			try:
+				result.append(parser.parse(res))
+			except Exception as e:
+				logger.warning(
+					f"Error parsing response: {e} \nSo returning None instead in this case."
+				)
+				result.append(None)
+		return result
+
+	@abc.abstractmethod
+	async def astream(self, prompt: str, **kwargs):
+		pass
+
+	@abc.abstractmethod
+	def stream(self, prompt: str, **kwargs):
+		pass
+
+
+def generator_node(func):
+	@functools.wraps(func)
+	@result_to_dataframe(["generated_texts", "generated_tokens", "generated_log_probs"])
+	def wrapper(
+		project_dir: Union[str, Path], previous_result: pd.DataFrame, llm: str, **kwargs
+	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
+		"""
+		This decorator makes a generator module to be a node.
+		It automatically extracts prompts from previous_result and runs the generator function.
+		Plus, it retrieves the llm instance from autorag.generator_models.
+
+		:param project_dir: The project directory.
+		:param previous_result: The previous result that contains prompts,
+		:param llm: The llm name that you want to use.
+		:param kwargs: The extra parameters for initializing the llm instance.
+		:return: Pandas dataframe that contains generated texts, generated tokens, and generated log probs.
+		    Each column is "generated_texts", "generated_tokens", and "generated_log_probs".
+		"""
+		logger.info(f"Running generator node - {func.__name__} module...")
+		assert (
+			"prompts" in previous_result.columns
+		), "previous_result must contain prompts column."
+		prompts = previous_result["prompts"].tolist()
+		if func.__name__ == "llama_index_llm":
+			if llm not in generator_models:
+				raise ValueError(
+					f"{llm} is not a valid llm name. Please check the llm name."
+					"You can check valid llm names from autorag.generator_models."
+				)
+			batch = kwargs.pop("batch", 16)
+			if llm == "huggingfacellm":
+				model_name = kwargs.pop("model", None)
+				if model_name is not None:
+					kwargs["model_name"] = model_name
+				else:
+					if "model_name" not in kwargs.keys():
+						raise ValueError(
+							"`model` or `model_name` parameter must be provided for using huggingfacellm."
+						)
+				kwargs["tokenizer_name"] = kwargs["model_name"]
+			llm_instance = generator_models[llm](**kwargs)
+			result = func(prompts=prompts, llm=llm_instance, batch=batch)
+			del llm_instance
+			return result
+		else:
+			return func(prompts=prompts, llm=llm, **kwargs)
+
+	return wrapper

autorag-workspace/autorag/nodes/generator/llama_index_llm.py

@@ -0,0 +1,97 @@
+from typing import List, Tuple
+
+import pandas as pd
+from llama_index.core.base.llms.base import BaseLLM
+from transformers import AutoTokenizer
+
+from autorag import generator_models
+from autorag.nodes.generator.base import BaseGenerator
+from autorag.utils.util import (
+	get_event_loop,
+	process_batch,
+	result_to_dataframe,
+	pop_params,
+)
+
+
+class LlamaIndexLLM(BaseGenerator):
+	def __init__(self, project_dir: str, llm: str, batch: int = 16, *args, **kwargs):
+		"""
+		Initialize the Llama Index LLM module.
+
+		:param project_dir: The project directory.
+		:param llm: A llama index LLM instance.
+		:param batch: The batch size for llm.
+			Set low if you face some errors.
+			Default is 16.
+		:param kwargs: The extra parameters for initializing the llm instance.
+		"""
+		super().__init__(project_dir=project_dir, llm=llm)
+		if self.llm not in generator_models.keys():
+			raise ValueError(
+				f"{self.llm} is not a valid llm name. Please check the llm name."
+				"You can check valid llm names from autorag.generator_models."
+			)
+		self.batch = batch
+		llm_class = generator_models[self.llm]
+
+		if llm_class.class_name() in [
+			"HuggingFace_LLM",
+			"HuggingFaceInferenceAPI",
+			"TextGenerationInference",
+		]:
+			model_name = kwargs.pop("model", None)
+			if model_name is not None:
+				kwargs["model_name"] = model_name
+			else:
+				if "model_name" not in kwargs.keys():
+					raise ValueError(
+						"`model` or `model_name` parameter must be provided for using huggingfacellm."
+					)
+			kwargs["tokenizer_name"] = kwargs["model_name"]
+		self.llm_instance: BaseLLM = llm_class(**pop_params(llm_class.__init__, kwargs))
+
+	def __del__(self):
+		super().__del__()
+		del self.llm_instance
+
+	@result_to_dataframe(["generated_texts", "generated_tokens", "generated_log_probs"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		prompts = self.cast_to_run(previous_result=previous_result)
+		return self._pure(prompts)
+
+	def _pure(
+		self,
+		prompts: List[str],
+	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
+		"""
+		Llama Index LLM module.
+		It gets the LLM instance from llama index, and returns generated text by the input prompt.
+		It does not generate the right log probs, but it returns the pseudo log probs,
+		which are not meant to be used for other modules.
+
+		:param prompts: A list of prompts.
+		:return: A tuple of three elements.
+			The first element is a list of a generated text.
+			The second element is a list of generated text's token ids, used tokenizer is GPT2Tokenizer.
+			The third element is a list of generated text's pseudo log probs.
+		"""
+		tasks = [self.llm_instance.acomplete(prompt) for prompt in prompts]
+		loop = get_event_loop() # get_event_loop()
+		results = loop.run_until_complete(process_batch(tasks, batch_size=self.batch))
+
+		generated_texts = list(map(lambda x: x.text, results))
+		tokenizer = AutoTokenizer.from_pretrained("gpt2", use_fast=False)
+		tokenized_ids = tokenizer(generated_texts).data["input_ids"]
+		pseudo_log_probs = list(map(lambda x: [0.5] * len(x), tokenized_ids))
+		return generated_texts, tokenized_ids, pseudo_log_probs
+
+	async def astream(self, prompt: str, **kwargs):
+		async for completion_response in await self.llm_instance.astream_complete(
+			prompt
+		):
+			yield completion_response.text
+
+	def stream(self, prompt: str, **kwargs):
+		for completion_response in self.llm_instance.stream_complete(prompt):
+			yield completion_response.text

autorag-workspace/autorag/nodes/generator/openai_llm.py

@@ -0,0 +1,296 @@
+import logging
+from typing import List, Tuple
+
+import pandas as pd
+import tiktoken
+from openai import AsyncOpenAI
+from tiktoken import Encoding
+
+from autorag.nodes.generator.base import BaseGenerator
+from autorag.utils.util import (
+	get_event_loop,
+	process_batch,
+	pop_params,
+	result_to_dataframe,
+)
+
+logger = logging.getLogger("AutoRAG")
+
+MAX_TOKEN_DICT = {  # model name : token limit
+	"gpt-4.5-preview": 128_000,
+	"gpt-4.5-preview-2025-02-27": 128_000,
+	"o1": 200_000,
+	"o1-preview": 128_000,
+	"o1-preview-2024-09-12": 128_000,
+	"o1-mini": 128_000,
+	"o1-mini-2024-09-12": 128_000,
+	"o3-mini": 200_000,
+	"gpt-4o-mini": 128_000,
+	"gpt-4o-mini-2024-07-18": 128_000,
+	"gpt-4o": 128_000,
+	"gpt-4o-2024-08-06": 128_000,
+	"gpt-4o-2024-05-13": 128_000,
+	"chatgpt-4o-latest": 128_000,
+	"gpt-4-turbo": 128_000,
+	"gpt-4-turbo-2024-04-09": 128_000,
+	"gpt-4-turbo-preview": 128_000,
+	"gpt-4-0125-preview": 128_000,
+	"gpt-4-1106-preview": 128_000,
+	"gpt-4-vision-preview": 128_000,
+	"gpt-4-1106-vision-preview": 128_000,
+	"gpt-4": 8_192,
+	"gpt-4-0613": 8_192,
+	"gpt-4-32k": 32_768,
+	"gpt-4-32k-0613": 32_768,
+	"gpt-3.5-turbo-0125": 16_385,
+	"gpt-3.5-turbo": 16_385,
+	"gpt-3.5-turbo-1106": 16_385,
+	"gpt-3.5-turbo-instruct": 4_096,
+	"gpt-3.5-turbo-16k": 16_385,
+	"gpt-3.5-turbo-0613": 4_096,
+	"gpt-3.5-turbo-16k-0613": 16_385,
+}
+
+
+class OpenAILLM(BaseGenerator):
+	def __init__(self, project_dir, llm: str, batch: int = 16, *args, **kwargs):
+		super().__init__(project_dir, llm, *args, **kwargs)
+		assert batch > 0, "batch size must be greater than 0."
+		self.batch = batch
+
+		client_init_params = pop_params(AsyncOpenAI.__init__, kwargs)
+		self.client = AsyncOpenAI(**client_init_params)
+
+		if self.llm.startswith("gpt-4.5"):
+			self.tokenizer = tiktoken.get_encoding("o200k_base")
+		else:
+			self.tokenizer = tiktoken.encoding_for_model(self.llm)
+
+		self.max_token_size = (
+			MAX_TOKEN_DICT.get(self.llm) - 7
+		)  # because of chat token usage
+		if self.max_token_size is None:
+			raise ValueError(
+				f"Model {self.llm} does not supported. "
+				f"Please select the model between {list(MAX_TOKEN_DICT.keys())}"
+			)
+
+	@result_to_dataframe(["generated_texts", "generated_tokens", "generated_log_probs"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		prompts = self.cast_to_run(previous_result)
+		return self._pure(prompts, **kwargs)
+
+	def _pure(
+		self,
+		prompts: List[str],
+		truncate: bool = True,
+		**kwargs,
+	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
+		"""
+		OpenAI generator module.
+		Uses an official openai library for generating answer from the given prompt.
+		It returns real token ids and log probs, so you must use this for using token ids and log probs.
+
+		:param prompts: A list of prompts.
+		:param llm: A model name for openai.
+		    Default is gpt-3.5-turbo.
+		:param batch: Batch size for openai api call.
+		    If you get API limit errors, you should lower the batch size.
+		    Default is 16.
+		:param truncate: Whether to truncate the input prompt.
+		    Default is True.
+		:param api_key: OpenAI API key. You can set this by passing env variable `OPENAI_API_KEY`
+		:param kwargs: The optional parameter for openai api call `openai.chat.completion`
+		    See https://platform.openai.com/docs/api-reference/chat/create for more details.
+		:return: A tuple of three elements.
+		    The first element is a list of generated text.
+		    The second element is a list of generated text's token ids.
+		    The third element is a list of generated text's log probs.
+		"""
+		if kwargs.get("logprobs") is not None:
+			kwargs.pop("logprobs")
+			logger.warning(
+				"parameter logprob does not effective. It always set to True."
+			)
+		if kwargs.get("n") is not None:
+			kwargs.pop("n")
+			logger.warning("parameter n does not effective. It always set to 1.")
+
+		# TODO: fix this after updating tiktoken for the gpt-4.5 model. It is not yet supported yet.
+		if truncate:
+			prompts = list(
+				map(
+					lambda prompt: truncate_by_token(
+						prompt, self.tokenizer, self.max_token_size
+					),
+					prompts,
+				)
+			)
+
+		openai_chat_params = pop_params(self.client.chat.completions.create, kwargs)
+		loop = get_event_loop()
+		if self.llm.startswith("o1") or self.llm.startswith("o3"):
+			tasks = [
+				self.get_result_o1(prompt, **openai_chat_params) for prompt in prompts
+			]
+		else:
+			tasks = [
+				self.get_result(prompt, **openai_chat_params) for prompt in prompts
+			]
+		result = loop.run_until_complete(process_batch(tasks, self.batch))
+		answer_result = list(map(lambda x: x[0], result))
+		token_result = list(map(lambda x: x[1], result))
+		logprob_result = list(map(lambda x: x[2], result))
+		return answer_result, token_result, logprob_result
+
+	def structured_output(self, prompts: List[str], output_cls, **kwargs):
+		supported_models = [
+			"gpt-4o-mini-2024-07-18",
+			"gpt-4o-2024-08-06",
+		]
+		if self.llm not in supported_models:
+			raise ValueError(
+				f"{self.llm} is not a valid model name for structured output. "
+				f"Please select the model between {supported_models}"
+			)
+
+		if kwargs.get("logprobs") is not None:
+			kwargs.pop("logprobs")
+			logger.warning(
+				"parameter logprob does not effective. It always set to False."
+			)
+		if kwargs.get("n") is not None:
+			kwargs.pop("n")
+			logger.warning("parameter n does not effective. It always set to 1.")
+
+		# TODO: fix this after updating tiktoken for the gpt-4.5 model. It is not yet supported yet.
+		prompts = list(
+			map(
+				lambda prompt: truncate_by_token(
+					prompt, self.tokenizer, self.max_token_size
+				),
+				prompts,
+			)
+		)
+
+		openai_chat_params = pop_params(self.client.beta.chat.completions.parse, kwargs)
+		loop = get_event_loop()
+		tasks = [
+			self.get_structured_result(prompt, output_cls, **openai_chat_params)
+			for prompt in prompts
+		]
+		result = loop.run_until_complete(process_batch(tasks, self.batch))
+		return result
+
+	async def astream(self, prompt: str, **kwargs):
+		# TODO: gpt-4.5-preview does not support logprobs. It should be fixed after the openai update.
+		if kwargs.get("logprobs") is not None:
+			kwargs.pop("logprobs")
+			logger.warning(
+				"parameter logprob does not effective. It always set to False."
+			)
+		if kwargs.get("n") is not None:
+			kwargs.pop("n")
+			logger.warning("parameter n does not effective. It always set to 1.")
+
+		prompt = truncate_by_token(prompt, self.tokenizer, self.max_token_size)
+
+		openai_chat_params = pop_params(self.client.chat.completions.create, kwargs)
+
+		stream = await self.client.chat.completions.create(
+			model=self.llm,
+			messages=[
+				{"role": "user", "content": prompt},
+			],
+			logprobs=False,
+			n=1,
+			stream=True,
+			**openai_chat_params,
+		)
+		result = ""
+		async for chunk in stream:
+			if chunk.choices[0].delta.content is not None:
+				result += chunk.choices[0].delta.content
+				yield result
+
+	def stream(self, prompt: str, **kwargs):
+		raise NotImplementedError("stream method is not implemented yet.")
+
+	async def get_structured_result(self, prompt: str, output_cls, **kwargs):
+		logprobs = True
+		if self.llm.startswith("gpt-4.5"):
+			logprobs = False
+		response = await self.client.beta.chat.completions.parse(
+			model=self.llm,
+			messages=[
+				{"role": "user", "content": prompt},
+			],
+			response_format=output_cls,
+			logprobs=logprobs,
+			n=1,
+			**kwargs,
+		)
+		return response.choices[0].message.parsed
+
+	async def get_result(self, prompt: str, **kwargs):
+		# TODO: gpt-4.5-preview does not support logprobs. It should be fixed after the openai update.
+		logprobs = True
+		if self.llm.startswith("gpt-4.5"):
+			logprobs = False
+		response = await self.client.chat.completions.create(
+			model=self.llm,
+			messages=[
+				{"role": "user", "content": prompt},
+			],
+			logprobs=logprobs,
+			n=1,
+			**kwargs,
+		)
+		choice = response.choices[0]
+		answer = choice.message.content
+		# TODO: gpt-4.5-preview does not support logprobs. It should be fixed after the openai update.
+		if self.llm.startswith("gpt-4.5"):
+			tokens = self.tokenizer.encode(answer, allowed_special="all")
+			logprobs = [0.5] * len(tokens)
+			logger.warning("gpt-4.5-preview does not support logprobs yet.")
+		else:
+			logprobs = list(map(lambda x: x.logprob, choice.logprobs.content))
+			tokens = list(
+				map(
+					lambda x: self.tokenizer.encode(x.token, allowed_special="all")[0],
+					choice.logprobs.content,
+				)
+			)
+			assert len(tokens) == len(
+				logprobs
+			), "tokens and logprobs size is different."
+		return answer, tokens, logprobs
+
+	async def get_result_o1(self, prompt: str, **kwargs):
+		assert self.llm.startswith("o1") or self.llm.startswith(
+			"o3"
+		), "This function only supports o1 or o3 model."
+		# The default temperature for the o1 model is 1. 1 is only supported.
+		# See https://platform.openai.com/docs/guides/reasoning about beta limitation of o1 models.
+		kwargs["temperature"] = 1
+		kwargs["top_p"] = 1
+		kwargs["presence_penalty"] = 0
+		kwargs["frequency_penalty"] = 0
+		response = await self.client.chat.completions.create(
+			model=self.llm,
+			messages=[
+				{"role": "user", "content": prompt},
+			],
+			logprobs=False,
+			n=1,
+			**kwargs,
+		)
+		answer = response.choices[0].message.content
+		tokens = self.tokenizer.encode(answer, allowed_special="all")
+		pseudo_log_probs = [0.5] * len(tokens)
+		return answer, tokens, pseudo_log_probs
+
+
+def truncate_by_token(prompt: str, tokenizer: Encoding, max_token_size: int):
+	tokens = tokenizer.encode(prompt, allowed_special="all")
+	return tokenizer.decode(tokens[:max_token_size])

autorag-workspace/autorag/nodes/generator/run.py

@@ -0,0 +1,144 @@
+import os
+import pathlib
+from typing import List, Dict, Union
+
+import pandas as pd
+
+from autorag.evaluation import evaluate_generation
+from autorag.evaluation.util import cast_metrics
+from autorag.schema.metricinput import MetricInput
+from autorag.strategy import measure_speed, filter_by_threshold, select_best
+from autorag.utils.util import to_list
+
+
+def run_generator_node(
+	modules: List,
+	module_params: List[Dict],
+	previous_result: pd.DataFrame,
+	node_line_dir: str,
+	strategies: Dict,
+) -> pd.DataFrame:
+	"""
+	Run evaluation and select the best module among generator node results.
+	And save the results and summary to generator node directory.
+
+	:param modules: Generator modules to run.
+	:param module_params: Generator module parameters.
+	    Including node parameters, which is used for every module in this node.
+	:param previous_result: Previous result dataframe.
+	    Could be prompt maker node's result.
+	:param node_line_dir: This node line's directory.
+	:param strategies: Strategies for generator node.
+	:return: The best result dataframe.
+	    It contains previous result columns and generator node's result columns.
+	"""
+	if not os.path.exists(node_line_dir):
+		os.makedirs(node_line_dir)
+	project_dir = pathlib.PurePath(node_line_dir).parent.parent
+	node_dir = os.path.join(node_line_dir, "generator")  # node name
+	if not os.path.exists(node_dir):
+		os.makedirs(node_dir)
+	qa_data = pd.read_parquet(
+		os.path.join(project_dir, "data", "qa.parquet"), engine="pyarrow"
+	)
+	if "generation_gt" not in qa_data.columns:
+		raise ValueError("You must have 'generation_gt' column in qa.parquet.")
+
+	results, execution_times = zip(
+		*map(
+			lambda x: measure_speed(
+				x[0].run_evaluator,
+				project_dir=project_dir,
+				previous_result=previous_result,
+				**x[1],
+			),
+			zip(modules, module_params),
+		)
+	)
+	average_times = list(map(lambda x: x / len(results[0]), execution_times))
+
+	# get average token usage
+	token_usages = list(map(lambda x: x["generated_tokens"].apply(len).mean(), results))
+
+	# make rows to metric_inputs
+	generation_gt = to_list(qa_data["generation_gt"].tolist())
+
+	metric_inputs = [MetricInput(generation_gt=gen_gt) for gen_gt in generation_gt]
+
+	metric_names, metric_params = cast_metrics(strategies.get("metrics"))
+	if metric_names is None or len(metric_names) <= 0:
+		raise ValueError("You must at least one metrics for generator evaluation.")
+	results = list(
+		map(
+			lambda result: evaluate_generator_node(
+				result, metric_inputs, strategies.get("metrics")
+			),
+			results,
+		)
+	)
+
+	# save results to folder
+	filepaths = list(
+		map(lambda x: os.path.join(node_dir, f"{x}.parquet"), range(len(modules)))
+	)
+	list(
+		map(lambda x: x[0].to_parquet(x[1], index=False), zip(results, filepaths))
+	)  # execute save to parquet
+	filenames = list(map(lambda x: os.path.basename(x), filepaths))
+
+	summary_df = pd.DataFrame(
+		{
+			"filename": filenames,
+			"module_name": list(map(lambda module: module.__name__, modules)),
+			"module_params": module_params,
+			"execution_time": average_times,
+			"average_output_token": token_usages,
+			**{
+				metric: list(map(lambda x: x[metric].mean(), results))
+				for metric in metric_names
+			},
+		}
+	)
+
+	# filter by strategies
+	if strategies.get("speed_threshold") is not None:
+		results, filenames = filter_by_threshold(
+			results, average_times, strategies["speed_threshold"], filenames
+		)
+	if strategies.get("token_threshold") is not None:
+		results, filenames = filter_by_threshold(
+			results, token_usages, strategies["token_threshold"], filenames
+		)
+	selected_result, selected_filename = select_best(
+		results, metric_names, filenames, strategies.get("strategy", "mean")
+	)
+	best_result = pd.concat([previous_result, selected_result], axis=1)
+
+	# add 'is_best' column at summary file
+	summary_df["is_best"] = summary_df["filename"] == selected_filename
+
+	# save files
+	summary_df.to_csv(os.path.join(node_dir, "summary.csv"), index=False)
+	best_result.to_parquet(
+		os.path.join(
+			node_dir, f"best_{os.path.splitext(selected_filename)[0]}.parquet"
+		),
+		index=False,
+	)
+	return best_result
+
+
+def evaluate_generator_node(
+	result_df: pd.DataFrame,
+	metric_inputs: List[MetricInput],
+	metrics: Union[List[str], List[Dict]],
+):
+	@evaluate_generation(metric_inputs=metric_inputs, metrics=metrics)
+	def evaluate_generation_module(df: pd.DataFrame):
+		return (
+			df["generated_texts"].tolist(),
+			df["generated_tokens"].tolist(),
+			df["generated_log_probs"].tolist(),
+		)
+
+	return evaluate_generation_module(result_df)

autorag-workspace/autorag/nodes/generator/vllm.py

@@ -0,0 +1,121 @@
+import gc
+from copy import deepcopy
+from typing import List, Tuple
+
+import pandas as pd
+
+from autorag.nodes.generator.base import BaseGenerator
+from autorag.utils import result_to_dataframe
+from autorag.utils.util import pop_params, to_list
+
+
+class Vllm(BaseGenerator):
+	def __init__(self, project_dir: str, llm: str, **kwargs):
+		super().__init__(project_dir, llm, **kwargs)
+		try:
+			from vllm import SamplingParams, LLM
+		except ImportError:
+			raise ImportError(
+				"Please install vllm library. You can install it by running `pip install vllm`."
+			)
+
+		model_from_kwargs = kwargs.pop("model", None)
+		model = llm if model_from_kwargs is None else model_from_kwargs
+
+		input_kwargs = deepcopy(kwargs)
+		sampling_params_init_params = pop_params(
+			SamplingParams.from_optional, input_kwargs
+		)
+		self.vllm_model = LLM(model, **input_kwargs)
+
+		# delete not sampling param keys in the kwargs
+		kwargs_keys = list(kwargs.keys())
+		for key in kwargs_keys:
+			if key not in sampling_params_init_params:
+				kwargs.pop(key)
+
+	def __del__(self):
+		try:
+			import torch
+			import contextlib
+
+			if torch.cuda.is_available():
+				from vllm.distributed.parallel_state import (
+					destroy_model_parallel,
+					destroy_distributed_environment,
+				)
+
+				destroy_model_parallel()
+				destroy_distributed_environment()
+				del self.vllm_model.llm_engine.model_executor
+				del self.vllm_model
+				with contextlib.suppress(AssertionError):
+					torch.distributed.destroy_process_group()
+				gc.collect()
+				torch.cuda.empty_cache()
+				torch.cuda.synchronize()
+		except ImportError:
+			del self.vllm_model
+
+		super().__del__()
+
+	@result_to_dataframe(["generated_texts", "generated_tokens", "generated_log_probs"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		prompts = self.cast_to_run(previous_result)
+		return self._pure(prompts, **kwargs)
+
+	def _pure(
+		self, prompts: List[str], **kwargs
+	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
+		"""
+		Vllm module.
+		It gets the VLLM instance and returns generated texts by the input prompt.
+		You can set logprobs to get the log probs of the generated text.
+		Default logprobs is 1.
+
+		:param prompts: A list of prompts.
+		:param kwargs: The extra parameters for generating the text.
+		:return: A tuple of three elements.
+		    The first element is a list of generated text.
+		    The second element is a list of generated text's token ids.
+		    The third element is a list of generated text's log probs.
+		"""
+		try:
+			from vllm.outputs import RequestOutput
+			from vllm.sequence import SampleLogprobs
+			from vllm import SamplingParams
+		except ImportError:
+			raise ImportError(
+				"Please install vllm library. You can install it by running `pip install vllm`."
+			)
+
+		if "logprobs" not in kwargs:
+			kwargs["logprobs"] = 1
+
+		sampling_params = pop_params(SamplingParams.from_optional, kwargs)
+		generate_params = SamplingParams(**sampling_params)
+		results: List[RequestOutput] = self.vllm_model.generate(
+			prompts, generate_params
+		)
+		generated_texts = list(map(lambda x: x.outputs[0].text, results))
+		generated_token_ids = list(map(lambda x: x.outputs[0].token_ids, results))
+		log_probs: List[SampleLogprobs] = list(
+			map(lambda x: x.outputs[0].logprobs, results)
+		)
+		generated_log_probs = list(
+			map(
+				lambda x: list(map(lambda y: y[0][y[1]].logprob, zip(x[0], x[1]))),
+				zip(log_probs, generated_token_ids),
+			)
+		)
+		return (
+			to_list(generated_texts),
+			to_list(generated_token_ids),
+			to_list(generated_log_probs),
+		)
+
+	async def astream(self, prompt: str, **kwargs):
+		raise NotImplementedError
+
+	def stream(self, prompt: str, **kwargs):
+		raise NotImplementedError

autorag-workspace/autorag/nodes/generator/vllm_api.py

@@ -0,0 +1,176 @@
+import logging
+from typing import List, Tuple
+import time
+
+import pandas as pd
+import requests
+from asyncio import to_thread
+
+from autorag.nodes.generator.base import BaseGenerator
+from autorag.utils.util import get_event_loop, process_batch, result_to_dataframe
+
+logger = logging.getLogger("AutoRAG")
+
+DEFAULT_MAX_TOKENS = 4096  # Default token limit
+
+
+class VllmAPI(BaseGenerator):
+	def __init__(
+		self,
+		project_dir,
+		llm: str,
+		uri: str,
+		max_tokens: int = None,
+		batch: int = 16,
+		*args,
+		**kwargs,
+	):
+		"""
+		VLLM API Wrapper for OpenAI-compatible chat/completions format.
+
+		:param project_dir: Project directory.
+		:param llm: Model name (e.g., LLaMA model).
+		:param uri: VLLM API server URI.
+		:param max_tokens: Maximum token limit.
+		    Default is 4096.
+		:param batch: Request batch size.
+		    Default is 16.
+		"""
+		super().__init__(project_dir, llm, *args, **kwargs)
+		assert batch > 0, "Batch size must be greater than 0."
+		self.uri = uri.rstrip("/")  # Set API URI
+		self.batch = batch
+		# Use the provided max_tokens if available, otherwise use the default
+		self.max_token_size = max_tokens if max_tokens else DEFAULT_MAX_TOKENS
+		self.max_model_len = self.get_max_model_length()
+		logger.info(f"{llm} max model length: {self.max_model_len}")
+
+	@result_to_dataframe(["generated_texts", "generated_tokens", "generated_log_probs"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		prompts = self.cast_to_run(previous_result)
+		return self._pure(prompts, **kwargs)
+
+	def _pure(
+		self, prompts: List[str], truncate: bool = True, **kwargs
+	) -> Tuple[List[str], List[List[int]], List[List[float]]]:
+		"""
+		Method to call the VLLM API to generate text.
+
+		:param prompts: List of input prompts.
+		:param truncate: Whether to truncate input prompts to fit within the token limit.
+		:param kwargs: Additional options (e.g., temperature, top_p).
+		:return: Generated text, token lists, and log probability lists.
+		"""
+		if kwargs.get("logprobs") is not None:
+			kwargs.pop("logprobs")
+			logger.warning(
+				"parameter logprob does not effective. It always set to True."
+			)
+		if kwargs.get("n") is not None:
+			kwargs.pop("n")
+			logger.warning("parameter n does not effective. It always set to 1.")
+
+		if truncate:
+			prompts = list(map(lambda p: self.truncate_by_token(p), prompts))
+		loop = get_event_loop()
+		tasks = [to_thread(self.get_result, prompt, **kwargs) for prompt in prompts]
+		results = loop.run_until_complete(process_batch(tasks, self.batch))
+
+		answer_result = list(map(lambda x: x[0], results))
+		token_result = list(map(lambda x: x[1], results))
+		logprob_result = list(map(lambda x: x[2], results))
+		return answer_result, token_result, logprob_result
+
+	def truncate_by_token(self, prompt: str) -> str:
+		"""
+		Function to truncate prompts to fit within the maximum token limit.
+		"""
+		tokens = self.encoding_for_model(prompt)["tokens"]  # Simple tokenization
+		return self.decoding_for_model(tokens[: self.max_model_len])["prompt"]
+
+	def call_vllm_api(self, prompt: str, **kwargs) -> dict:
+		"""
+		Calls the VLLM API to get chat/completions responses.
+
+		:param prompt: Input prompt.
+		:param kwargs: Additional API options (e.g., temperature, max_tokens).
+		:return: API response.
+		"""
+		payload = {
+			"model": self.llm,
+			"messages": [{"role": "user", "content": prompt}],
+			"temperature": kwargs.get("temperature", 0.4),
+			"max_tokens": min(
+				kwargs.get("max_tokens", self.max_token_size), self.max_token_size
+			),
+			"logprobs": True,
+			"n": 1,
+		}
+		start_time = time.time()  # Record request start time
+		response = requests.post(f"{self.uri}/v1/chat/completions", json=payload)
+		end_time = time.time()  # Record request end time
+
+		response.raise_for_status()
+		elapsed_time = end_time - start_time  # Calculate elapsed time
+		logger.info(
+			f"Request chat completions to vllm server completed in {elapsed_time:.2f} seconds"
+		)
+		return response.json()
+
+	# Additional method: abstract method implementation
+	async def astream(self, prompt: str, **kwargs):
+		"""
+		Asynchronous streaming method not implemented.
+		"""
+		raise NotImplementedError("astream method is not implemented for VLLM API yet.")
+
+	def stream(self, prompt: str, **kwargs):
+		"""
+		Synchronous streaming method not implemented.
+		"""
+		raise NotImplementedError("stream method is not implemented for VLLM API yet.")
+
+	def get_result(self, prompt: str, **kwargs):
+		response = self.call_vllm_api(prompt, **kwargs)
+		choice = response["choices"][0]
+		answer = choice["message"]["content"]
+
+		# Handle cases where logprobs is None
+		if choice.get("logprobs") and "content" in choice["logprobs"]:
+			logprobs = list(map(lambda x: x["logprob"], choice["logprobs"]["content"]))
+			tokens = list(
+				map(
+					lambda x: self.encoding_for_model(x["token"])["tokens"],
+					choice["logprobs"]["content"],
+				)
+			)
+		else:
+			logprobs = []
+			tokens = []
+
+		return answer, tokens, logprobs
+
+	def encoding_for_model(self, answer_piece: str):
+		payload = {
+			"model": self.llm,
+			"prompt": answer_piece,
+			"add_special_tokens": True,
+		}
+		response = requests.post(f"{self.uri}/tokenize", json=payload)
+		response.raise_for_status()
+		return response.json()
+
+	def decoding_for_model(self, tokens: list[int]):
+		payload = {
+			"model": self.llm,
+			"tokens": tokens,
+		}
+		response = requests.post(f"{self.uri}/detokenize", json=payload)
+		response.raise_for_status()
+		return response.json()
+
+	def get_max_model_length(self):
+		response = requests.get(f"{self.uri}/v1/models")
+		response.raise_for_status()
+		json_data = response.json()
+		return json_data["data"][0]["max_model_len"]

autorag-workspace/autorag/nodes/passageaugmenter/__init__.py

@@ -0,0 +1,2 @@
+from .pass_passage_augmenter import PassPassageAugmenter
+from .prev_next_augmenter import PrevNextPassageAugmenter

autorag-workspace/autorag/nodes/passageaugmenter/base.py

@@ -0,0 +1,80 @@
+import abc
+import logging
+import os
+
+import pandas as pd
+
+from autorag.schema import BaseModule
+from autorag.utils import (
+	validate_qa_dataset,
+	sort_by_scores,
+	validate_corpus_dataset,
+	cast_corpus_dataset,
+)
+from autorag.utils.util import select_top_k
+
+logger = logging.getLogger("AutoRAG")
+
+
+class BasePassageAugmenter(BaseModule, metaclass=abc.ABCMeta):
+	def __init__(self, project_dir: str, *args, **kwargs):
+		logger.info(
+			f"Initialize passage augmenter node - {self.__class__.__name__} module..."
+		)
+		data_dir = os.path.join(project_dir, "data")
+		corpus_df = pd.read_parquet(
+			os.path.join(data_dir, "corpus.parquet"), engine="pyarrow"
+		)
+		validate_corpus_dataset(corpus_df)
+		corpus_df = cast_corpus_dataset(corpus_df)
+		self.corpus_df = corpus_df
+
+	def __del__(self):
+		logger.info(
+			f"Initialize passage augmenter node - {self.__class__.__name__} module..."
+		)
+
+	def cast_to_run(self, previous_result: pd.DataFrame, *args, **kwargs):
+		logger.info(
+			f"Running passage augmenter node - {self.__class__.__name__} module..."
+		)
+		validate_qa_dataset(previous_result)
+
+		# find ids columns
+		assert (
+			"retrieved_ids" in previous_result.columns
+		), "previous_result must have retrieved_ids column."
+		ids = previous_result["retrieved_ids"].tolist()
+
+		return ids
+
+	@staticmethod
+	def sort_by_scores(
+		augmented_contents,
+		augmented_ids,
+		augmented_scores,
+		top_k: int,
+		reverse: bool = True,
+	):
+		# sort by scores
+		df = pd.DataFrame(
+			{
+				"contents": augmented_contents,
+				"ids": augmented_ids,
+				"scores": augmented_scores,
+			}
+		)
+		df[["contents", "ids", "scores"]] = df.apply(
+			lambda row: sort_by_scores(row, reverse=reverse),
+			axis=1,
+			result_type="expand",
+		)
+
+		# select by top_k
+		results = select_top_k(df, ["contents", "ids", "scores"], top_k)
+
+		return (
+			results["contents"].tolist(),
+			results["ids"].tolist(),
+			results["scores"].tolist(),
+		)

autorag-workspace/autorag/nodes/passageaugmenter/pass_passage_augmenter.py

@@ -0,0 +1,43 @@
+from typing import List
+
+import pandas as pd
+
+from autorag.nodes.passageaugmenter.base import BasePassageAugmenter
+from autorag.utils import result_to_dataframe
+
+
+class PassPassageAugmenter(BasePassageAugmenter):
+	@result_to_dataframe(["retrieved_contents", "retrieved_ids", "retrieve_scores"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		"""
+		Run the passage augmenter node - PassPassageAugmenter module.
+
+		:param previous_result: The previous result Dataframe.
+		:param top_k: You must input the top_k value to get the top k results.
+		:param kwargs: Not affected.
+		:return: DataFrame with retrieved_contents, retrieved_ids, and retrieve_scores columns
+		"""
+		top_k = kwargs.pop("top_k")
+
+		ids = self.cast_to_run(previous_result)
+		contents = previous_result["retrieved_contents"].tolist()
+		scores = previous_result["retrieve_scores"].tolist()
+
+		augmented_ids, augmented_contents, augmented_scores = self._pure(
+			ids, contents, scores
+		)
+		return self.sort_by_scores(
+			augmented_contents, augmented_ids, augmented_scores, top_k
+		)
+
+	def _pure(
+		self,
+		ids_list: List[List[str]],
+		contents_list: List[List[str]],
+		scores_list: List[List[float]],
+	):
+		"""
+		Do not perform augmentation.
+		Return given passages, scores, and ids as is.
+		"""
+		return ids_list, contents_list, scores_list

autorag-workspace/autorag/nodes/passageaugmenter/prev_next_augmenter.py

@@ -0,0 +1,155 @@
+from typing import List, Union
+
+import numpy as np
+import pandas as pd
+
+from autorag.embedding.base import EmbeddingModel
+from autorag.evaluation.metric.util import calculate_cosine_similarity
+from autorag.nodes.passageaugmenter.base import BasePassageAugmenter
+from autorag.utils.util import (
+	filter_dict_keys,
+	fetch_contents,
+	embedding_query_content,
+	result_to_dataframe,
+	empty_cuda_cache,
+)
+
+
+class PrevNextPassageAugmenter(BasePassageAugmenter):
+	def __init__(
+		self,
+		project_dir: str,
+		embedding_model: Union[str, dict] = "openai",
+		*args,
+		**kwargs,
+	):
+		"""
+		Initialize the PrevNextPassageAugmenter module.
+
+		:param project_dir:
+		:param embedding_model: The embedding model name to use for calculating cosine similarity
+			Default is openai (text-embedding-ada-002)
+		:param kwargs:
+		"""
+		super().__init__(project_dir, *args, **kwargs)
+		slim_corpus_df = self.corpus_df[["doc_id", "metadata"]]
+		slim_corpus_df.loc[:, "metadata"] = slim_corpus_df["metadata"].apply(
+			filter_dict_keys, keys=["prev_id", "next_id"]
+		)
+		self.slim_corpus_df = slim_corpus_df
+
+		# init embedding model
+		self.embedding_model = EmbeddingModel.load(embedding_model)()
+
+	def __del__(self):
+		del self.embedding_model
+		empty_cuda_cache()
+		super().__del__()
+
+	@result_to_dataframe(["retrieved_contents", "retrieved_ids", "retrieve_scores"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		"""
+		Run the passage augmenter node - PrevNextPassageAugmenter module.
+
+		:param previous_result: The previous result Dataframe.
+		:param top_k: You must input the top_k value to get the top k results.
+		:param kwargs: Not affected.
+		:return: DataFrame with retrieved_contents, retrieved_ids, and retrieve_scores columns
+		"""
+		top_k = kwargs.pop("top_k")
+
+		ids = self.cast_to_run(previous_result)
+		# find queries columns
+		assert (
+			"query" in previous_result.columns
+		), "previous_result must have query column."
+		queries = previous_result["query"].tolist()
+
+		mode = kwargs.pop("mode", "both")
+		num_passages = kwargs.pop("num_passages", 1)
+		augmented_ids = self._pure(ids, num_passages, mode)
+
+		# fetch contents from corpus to use augmented ids
+		augmented_contents = fetch_contents(self.corpus_df, augmented_ids)
+
+		query_embeddings, contents_embeddings = embedding_query_content(
+			queries, augmented_contents, self.embedding_model, batch=128
+		)
+
+		# get scores from calculated cosine similarity
+		augmented_scores = [
+			np.array(
+				[
+					calculate_cosine_similarity(query_embedding, x)
+					for x in content_embeddings
+				]
+			).tolist()
+			for query_embedding, content_embeddings in zip(
+				query_embeddings, contents_embeddings
+			)
+		]
+		return self.sort_by_scores(
+			augmented_contents, augmented_ids, augmented_scores, top_k
+		)
+
+	def _pure(
+		self,
+		ids_list: List[List[str]],
+		num_passages: int = 1,
+		mode: str = "both",
+	) -> List[List[str]]:
+		"""
+		Add passages before and/or after the retrieved passage.
+		For more information, visit https://docs.llamaindex.ai/en/stable/examples/node_postprocessor/PrevNextPostprocessorDemo/.
+
+		:param ids_list: The list of lists of ids retrieved
+		:param num_passages: The number of passages to add before and after the retrieved passage
+		    Default is 1.
+		:param mode: The mode of augmentation
+		    'prev': add passages before the retrieved passage
+		    'next': add passages after the retrieved passage
+		    'both': add passages before and after the retrieved passage
+		    Default is 'next'.
+		:return: The list of lists of augmented ids
+		"""
+		if mode not in ["prev", "next", "both"]:
+			raise ValueError(f"mode must be 'prev', 'next', or 'both', but got {mode}")
+
+		augmented_ids = [
+			(
+				lambda ids: prev_next_augmenter_pure(
+					ids, self.slim_corpus_df, mode, num_passages
+				)
+			)(ids)
+			for ids in ids_list
+		]
+
+		return augmented_ids
+
+
+def prev_next_augmenter_pure(
+	ids: List[str], corpus_df: pd.DataFrame, mode: str, num_passages: int
+):
+	def fetch_id_sequence(start_id, key):
+		sequence = []
+		current_id = start_id
+		for _ in range(num_passages):
+			current_id = (
+				corpus_df.loc[corpus_df["doc_id"] == current_id]["metadata"]
+				.values[0]
+				.get(key)
+			)
+			if current_id is None:
+				break
+			sequence.append(current_id)
+		return sequence
+
+	augmented_group = []
+	for id_ in ids:
+		current_ids = [id_]
+		if mode in ["prev", "both"]:
+			current_ids = fetch_id_sequence(id_, "prev_id")[::-1] + current_ids
+		if mode in ["next", "both"]:
+			current_ids += fetch_id_sequence(id_, "next_id")
+		augmented_group.extend(current_ids)
+	return augmented_group

autorag-workspace/autorag/nodes/passageaugmenter/run.py

@@ -0,0 +1,131 @@
+import logging
+import os
+import pathlib
+from typing import List, Dict
+
+import pandas as pd
+
+from autorag.nodes.retrieval.run import evaluate_retrieval_node
+from autorag.schema.metricinput import MetricInput
+from autorag.strategy import measure_speed, filter_by_threshold, select_best
+from autorag.utils.util import apply_recursive, to_list
+
+logger = logging.getLogger("AutoRAG")
+
+
+def run_passage_augmenter_node(
+	modules: List,
+	module_params: List[Dict],
+	previous_result: pd.DataFrame,
+	node_line_dir: str,
+	strategies: Dict,
+) -> pd.DataFrame:
+	if not os.path.exists(node_line_dir):
+		os.makedirs(node_line_dir)
+	project_dir = pathlib.PurePath(node_line_dir).parent.parent
+	qa_df = pd.read_parquet(
+		os.path.join(project_dir, "data", "qa.parquet"), engine="pyarrow"
+	)
+	retrieval_gt = qa_df["retrieval_gt"].tolist()
+	retrieval_gt = apply_recursive(lambda x: str(x), to_list(retrieval_gt))
+
+	results, execution_times = zip(
+		*map(
+			lambda task: measure_speed(
+				task[0].run_evaluator,
+				project_dir=project_dir,
+				previous_result=previous_result,
+				**task[1],
+			),
+			zip(modules, module_params),
+		)
+	)
+	average_times = list(map(lambda x: x / len(results[0]), execution_times))
+	metric_inputs = [
+		MetricInput(retrieval_gt=ret_gt, query=query, generation_gt=gen_gt)
+		for ret_gt, query, gen_gt in zip(
+			retrieval_gt,
+			previous_result["query"].tolist(),
+			previous_result["generation_gt"].tolist(),
+		)
+	]
+
+	# run metrics before filtering
+	if strategies.get("metrics") is None:
+		raise ValueError(
+			"You must at least one metrics for passage_augmenter evaluation."
+		)
+	results = list(
+		map(
+			lambda x: evaluate_retrieval_node(
+				x,
+				metric_inputs,
+				strategies.get("metrics"),
+			),
+			results,
+		)
+	)
+
+	# save results to folder
+	save_dir = os.path.join(node_line_dir, "passage_augmenter")  # node name
+	if not os.path.exists(save_dir):
+		os.makedirs(save_dir)
+	filepaths = list(
+		map(lambda x: os.path.join(save_dir, f"{x}.parquet"), range(len(modules)))
+	)
+	list(
+		map(lambda x: x[0].to_parquet(x[1], index=False), zip(results, filepaths))
+	)  # execute save to parquet
+	filenames = list(map(lambda x: os.path.basename(x), filepaths))
+
+	summary_df = pd.DataFrame(
+		{
+			"filename": filenames,
+			"module_name": list(map(lambda module: module.__name__, modules)),
+			"module_params": module_params,
+			"execution_time": average_times,
+			**{
+				f"passage_augmenter_{metric}": list(
+					map(lambda result: result[metric].mean(), results)
+				)
+				for metric in strategies.get("metrics")
+			},
+		}
+	)
+
+	# filter by strategies
+	if strategies.get("speed_threshold") is not None:
+		results, filenames = filter_by_threshold(
+			results, average_times, strategies["speed_threshold"], filenames
+		)
+	selected_result, selected_filename = select_best(
+		results,
+		strategies.get("metrics"),
+		filenames,
+		strategies.get("strategy", "mean"),
+	)
+	# change metric name columns to passage_augmenter_metric_name
+	selected_result = selected_result.rename(
+		columns={
+			metric_name: f"passage_augmenter_{metric_name}"
+			for metric_name in strategies["metrics"]
+		}
+	)
+	# drop retrieval result columns in previous_result
+	previous_result = previous_result.drop(
+		columns=["retrieved_contents", "retrieved_ids", "retrieve_scores"]
+	)
+	best_result = pd.concat([previous_result, selected_result], axis=1)
+
+	# add 'is_best' column to summary file
+	summary_df["is_best"] = summary_df["filename"] == selected_filename
+
+	# save files
+	summary_df.to_csv(os.path.join(save_dir, "summary.csv"), index=False)
+	best_result.to_parquet(
+		os.path.join(
+			save_dir, f"best_{os.path.splitext(selected_filename)[0]}.parquet"
+		),
+		index=False,
+	)
+	return best_result

autorag-workspace/autorag/nodes/passagecompressor/__init__.py

@@ -0,0 +1,4 @@
+from .longllmlingua import LongLLMLingua
+from .pass_compressor import PassCompressor
+from .refine import Refine
+from .tree_summarize import TreeSummarize

autorag-workspace/autorag/nodes/passagecompressor/base.py

@@ -0,0 +1,83 @@
+import abc
+import logging
+from typing import Dict
+
+import pandas as pd
+from llama_index.core.llms import LLM
+
+from autorag import generator_models
+from autorag.schema import BaseModule
+from autorag.utils import result_to_dataframe
+
+logger = logging.getLogger("AutoRAG")
+
+
+class BasePassageCompressor(BaseModule, metaclass=abc.ABCMeta):
+	def __init__(self, project_dir: str, *args, **kwargs):
+		logger.info(
+			f"Initialize passage compressor node - {self.__class__.__name__} module..."
+		)
+
+	def __del__(self):
+		logger.info(
+			f"Deleting passage compressor node - {self.__class__.__name__} module..."
+		)
+
+	def cast_to_run(self, previous_result: pd.DataFrame, *args, **kwargs):
+		logger.info(
+			f"Running passage compressor node - {self.__class__.__name__} module..."
+		)
+		assert all(
+			[
+				column in previous_result.columns
+				for column in [
+					"query",
+					"retrieved_contents",
+				]
+			]
+		), "previous_result must have retrieved_contents, retrieved_ids, and retrieve_scores columns."
+		assert len(previous_result) > 0, "previous_result must have at least one row."
+
+		queries = previous_result["query"].tolist()
+		retrieved_contents = previous_result["retrieved_contents"].tolist()
+		return queries, retrieved_contents
+
+
+class LlamaIndexCompressor(BasePassageCompressor, metaclass=abc.ABCMeta):
+	param_list = ["prompt", "chat_prompt", "batch"]
+
+	def __init__(self, project_dir: str, **kwargs):
+		"""
+		Initialize passage compressor module.
+
+		:param project_dir: The project directory
+		:param llm: The llm name that will be used to summarize.
+			The LlamaIndex LLM model can be used in here.
+		:param kwargs: Extra parameter for init llm
+		"""
+		super().__init__(project_dir)
+		kwargs_dict = dict(
+			filter(lambda x: x[0] not in self.param_list, kwargs.items())
+		)
+		llm_name = kwargs_dict.pop("llm")
+		self.llm: LLM = make_llm(llm_name, kwargs_dict)
+
+	def __del__(self):
+		del self.llm
+		super().__del__()
+
+	@result_to_dataframe(["retrieved_contents"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		queries, retrieved_contents = self.cast_to_run(previous_result)
+		param_dict = dict(filter(lambda x: x[0] in self.param_list, kwargs.items()))
+		result = self._pure(queries, retrieved_contents, **param_dict)
+		return list(map(lambda x: [x], result))
+
+
+def make_llm(llm_name: str, kwargs: Dict) -> LLM:
+	if llm_name not in generator_models:
+		raise KeyError(
+			f"{llm_name} is not supported. "
+			"You can add it manually by calling autorag.generator_models."
+		)
+	return generator_models[llm_name](**kwargs)

autorag-workspace/autorag/nodes/passagecompressor/longllmlingua.py

@@ -0,0 +1,115 @@
+from typing import List, Optional
+
+import pandas as pd
+
+from autorag.nodes.passagecompressor.base import BasePassageCompressor
+from autorag.utils.util import pop_params, result_to_dataframe, empty_cuda_cache
+
+
+# TODO: Parallel Processing Refactoring at #460
+
+
+class LongLLMLingua(BasePassageCompressor):
+	def __init__(
+		self, project_dir: str, model_name: str = "NousResearch/Llama-2-7b-hf", **kwargs
+	):
+		try:
+			from llmlingua import PromptCompressor
+		except ImportError:
+			raise ImportError(
+				"LongLLMLingua is not installed. Please install it by running `pip install llmlingua`."
+			)
+
+		super().__init__(project_dir)
+		model_init_params = pop_params(PromptCompressor.__init__, kwargs)
+		self.llm_lingua = PromptCompressor(model_name=model_name, **model_init_params)
+
+	def __del__(self):
+		del self.llm_lingua
+		empty_cuda_cache()
+		super().__del__()
+
+	@result_to_dataframe(["retrieved_contents"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		queries, retrieved_contents = self.cast_to_run(previous_result)
+		results = self._pure(queries, retrieved_contents, **kwargs)
+		return list(map(lambda x: [x], results))
+
+	def _pure(
+		self,
+		queries: List[str],
+		contents: List[List[str]],
+		instructions: Optional[str] = None,
+		target_token: int = 300,
+		**kwargs,
+	) -> List[str]:
+		"""
+		Compresses the retrieved texts using LongLLMLingua.
+		For more information, visit https://github.com/microsoft/LLMLingua.
+
+		:param queries: The queries for retrieved passages.
+		:param contents: The contents of retrieved passages.
+		:param model_name: The model name to use for compression.
+		    The default is "NousResearch/Llama-2-7b-hf".
+		:param instructions: The instructions for compression.
+		    Default is None. When it is None, it will use default instructions.
+		:param target_token: The target token for compression.
+		    Default is 300.
+		:param kwargs: Additional keyword arguments.
+		:return: The list of compressed texts.
+		"""
+		if instructions is None:
+			instructions = "Given the context, please answer the final question"
+		results = [
+			llmlingua_pure(
+				query, contents_, self.llm_lingua, instructions, target_token, **kwargs
+			)
+			for query, contents_ in zip(queries, contents)
+		]
+
+		return results
+
+
+def llmlingua_pure(
+	query: str,
+	contents: List[str],
+	llm_lingua,
+	instructions: str,
+	target_token: int = 300,
+	**kwargs,
+) -> str:
+	"""
+	Return the compressed text.
+
+	:param query: The query for retrieved passages.
+	:param contents: The contents of retrieved passages.
+	:param llm_lingua: The llm instance, that will be used to compress.
+	:param instructions: The instructions for compression.
+	:param target_token: The target token for compression.
+	    Default is 300.
+	:param kwargs: Additional keyword arguments.
+	:return: The compressed text.
+	"""
+	try:
+		from llmlingua import PromptCompressor
+	except ImportError:
+		raise ImportError(
+			"LongLLMLingua is not installed. Please install it by running `pip install llmlingua`."
+		)
+	# split by "\n\n" (recommended by LongLLMLingua authors)
+	new_context_texts = [c for context in contents for c in context.split("\n\n")]
+	compress_prompt_params = pop_params(PromptCompressor.compress_prompt, kwargs)
+	compressed_prompt = llm_lingua.compress_prompt(
+		new_context_texts,
+		question=query,
+		instruction=instructions,
+		rank_method="longllmlingua",
+		target_token=target_token,
+		**compress_prompt_params,
+	)
+	compressed_prompt_txt = compressed_prompt["compressed_prompt"]
+
+	# separate out the question and instruction
+	result = "\n\n".join(compressed_prompt_txt.split("\n\n")[1:-1])
+
+	return result

autorag-workspace/autorag/nodes/passagecompressor/pass_compressor.py

@@ -0,0 +1,16 @@
+from typing import List
+
+import pandas as pd
+
+from autorag.nodes.passagecompressor.base import BasePassageCompressor
+from autorag.utils import result_to_dataframe
+
+
+class PassCompressor(BasePassageCompressor):
+	@result_to_dataframe(["retrieved_contents"])
+	def pure(self, previous_result: pd.DataFrame, *args, **kwargs):
+		_, contents = self.cast_to_run(previous_result)
+		return self._pure(contents)
+
+	def _pure(self, contents: List[List[str]]):
+		return contents

autorag-workspace/autorag/nodes/passagecompressor/refine.py

@@ -0,0 +1,54 @@
+from typing import List, Optional
+
+from llama_index.core import PromptTemplate
+from llama_index.core.prompts import PromptType
+from llama_index.core.prompts.utils import is_chat_model
+from llama_index.core.response_synthesizers import Refine as rf
+
+from autorag.nodes.passagecompressor.base import LlamaIndexCompressor
+from autorag.utils.util import get_event_loop, process_batch
+
+
+class Refine(LlamaIndexCompressor):
+	def _pure(
+		self,
+		queries: List[str],
+		contents: List[List[str]],
+		prompt: Optional[str] = None,
+		chat_prompt: Optional[str] = None,
+		batch: int = 16,
+	) -> List[str]:
+		"""
+		Refine a response to a query across text chunks.
+		This function is a wrapper for llama_index.response_synthesizers.Refine.
+		For more information, visit https://docs.llamaindex.ai/en/stable/examples/response_synthesizers/refine/.
+
+		:param queries: The queries for retrieved passages.
+		:param contents: The contents of retrieved passages.
+		:param prompt: The prompt template for refine.
+		    If you want to use chat prompt, you should pass chat_prompt instead.
+		    At prompt, you must specify where to put 'context_msg' and 'query_str'.
+		    Default is None. When it is None, it will use llama index default prompt.
+		:param chat_prompt: The chat prompt template for refine.
+		    If you want to use normal prompt, you should pass prompt instead.
+		    At prompt, you must specify where to put 'context_msg' and 'query_str'.
+		    Default is None. When it is None, it will use llama index default chat prompt.
+		:param batch: The batch size for llm.
+		    Set low if you face some errors.
+		    Default is 16.
+		:return: The list of compressed texts.
+		"""
+		if prompt is not None and not is_chat_model(self.llm):
+			refine_template = PromptTemplate(prompt, prompt_type=PromptType.REFINE)
+		elif chat_prompt is not None and is_chat_model(self.llm):
+			refine_template = PromptTemplate(chat_prompt, prompt_type=PromptType.REFINE)
+		else:
+			refine_template = None
+		summarizer = rf(llm=self.llm, refine_template=refine_template, verbose=True)
+		tasks = [
+			summarizer.aget_response(query, content)
+			for query, content in zip(queries, contents)
+		]
+		loop = get_event_loop()
+		results = loop.run_until_complete(process_batch(tasks, batch_size=batch))
+		return results