Fix Dockerfile build issue

autorag/data/qa/__init__.py

@@ -0,0 +1,3 @@
+# This is v2 version, the next version of data creation
+# The legacy (v1) version will be deprecated on AutoRAG version 0.3
+# The legacy (v1) version and new v2 data creation is not compatible with each other

autorag/data/qa/evolve/llama_index_query_evolve.py

@@ -0,0 +1,64 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatResponse, ChatMessage, MessageRole
+
+from autorag.data.qa.evolve.prompt import QUERY_EVOLVE_PROMPT
+
+
+async def llama_index_generate_base(
+	row: Dict,
+	llm: BaseLLM,
+	messages: List[ChatMessage],
+) -> Dict:
+	original_query = row["query"]
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "Text:\n" + "\n".join(
+		[f"{i + 1}. {c}" for i, c in enumerate(context)]
+	)
+	user_prompt = f"Question: {original_query}\nContext: {context_str}\nOutput: "
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	chat_response: ChatResponse = await llm.achat(messages=messages)
+	row["query"] = chat_response.message.content
+	return row
+
+
+async def conditional_evolve_ragas(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row,
+		llm,
+		QUERY_EVOLVE_PROMPT["conditional_evolve_ragas"][lang],
+	)
+
+
+async def reasoning_evolve_ragas(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row,
+		llm,
+		QUERY_EVOLVE_PROMPT["reasoning_evolve_ragas"][lang],
+	)
+
+
+async def compress_ragas(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	original_query = row["query"]
+	user_prompt = f"Question: {original_query}\nOutput: "
+	messages = QUERY_EVOLVE_PROMPT["compress_ragas"][lang]
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	chat_response: ChatResponse = await llm.achat(messages=messages)
+	row["query"] = chat_response.message.content
+	return row

autorag/data/qa/evolve/openai_query_evolve.py

@@ -0,0 +1,81 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.evolve.prompt import QUERY_EVOLVE_PROMPT
+
+
+class Response(BaseModel):
+	evolved_query: str
+
+
+async def query_evolve_openai_base(
+	row: Dict,
+	client: AsyncClient,
+	messages: List[ChatMessage],
+	model_name: str = "gpt-4o-2024-08-06",
+):
+	"""
+	Evolve the original query to a new evolved query using OpenAI structured outputs.
+	"""
+	original_query = row["query"]
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "Text:\n" + "\n".join(
+		[f"{i + 1}. {c}" for i, c in enumerate(context)]
+	)
+	user_prompt = f"Question: {original_query}\nContext: {context_str}\nOutput: "
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=Response,
+	)
+	row["query"] = completion.choices[0].message.parsed.evolved_query
+	return row
+
+
+async def conditional_evolve_ragas(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_evolve_openai_base(
+		row, client, QUERY_EVOLVE_PROMPT["conditional_evolve_ragas"][lang], model_name
+	)
+
+
+async def reasoning_evolve_ragas(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_evolve_openai_base(
+		row, client, QUERY_EVOLVE_PROMPT["reasoning_evolve_ragas"][lang], model_name
+	)
+
+
+async def compress_ragas(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	original_query = row["query"]
+	messages = QUERY_EVOLVE_PROMPT["compress_ragas"][lang]
+	user_prompt = f"Question: {original_query}\nOutput: "
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=Response,
+	)
+	row["query"] = completion.choices[0].message.parsed.evolved_query
+	return row

autorag/data/qa/evolve/prompt.py

@@ -0,0 +1,288 @@
+# The RAGAS prompts are coming from RAGAS under Apache-2.0 License. (English version) (the AutoRAG team translates Korean version prompt)
+# You can see the original prompts at the RAGAS library at https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/prompts.py
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+
+QUERY_EVOLVE_PROMPT = {
+	"conditional_evolve_ragas": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""Rewrite the provided question to increase its complexity by introducing a conditional element.
+The goal is to make the question more intricate by incorporating a scenario or condition that affects the context of the question.
+Follow the rules given below while rewriting the question.
+    1. The rewritten question should not be longer than 25 words. Use abbreviation wherever possible.
+    2. The rewritten question must be reasonable and must be understood and responded by humans.
+    3. The rewritten question must be fully answerable from information present context.
+    4. phrases like 'provided context','according to the context?',etc are not allowed to appear in the question.
+""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question : What is the function of the roots of a plant?
+Context : The roots of a plant absorb water and nutrients from the soil, anchor the plant in the ground, and store food.
+Output : """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="What dual purpose do plant roots serve concerning soil nutrients and stability?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question : How do vaccines protect against diseases?
+Context : Vaccines protect against diseases by stimulating the body's immune response to produce antibodies, which recognize and combat pathogens.
+Output : """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="How do vaccines utilize the body's immune system to defend against pathogens?",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""제공된 질문에 조건에 관련한 내용을 추가하여 복잡성을 높이세요.
+질문의 Context에 영향을 미치는 시나리오나 조건을 포함하여 질문을 더 복잡하게 만드는 것이 목표입니다.
+질문을 다시 작성할 때 다음 규칙을 따르십시오.
+    1. 다시 작성된 질문은 100자를 넘지 않아야 합니다. 가능한 경우 약어를 사용하십시오.
+    2. 다시 작성된 질문은 합리적이어야 하며 사람이 이해하고 응답할 수 있어야 합니다.
+    3. 다시 작성된 질문은 현재 Context에서 완전히 답변할 수 있어야 합니다.
+    4. '제공된 글', '단락에 따르면?', 'Context에 의하면' 등의 문구는 질문에 나타날 수 없습니다.
+    5. 한국어로 질문을 작성하세요.
+""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 식물의 뿌리 기능이 뭐야?
+Context: 식물의 뿌리는 토양에서 물과 영양분을 흡수하고, 식물을 땅에 고정하며, 영양분을 저장합니다.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="식물의 뿌리는 토양 영양분과 안정성에 대해 어떤 역할을 하나요?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 백신은 질병을 어떻게 예방하나요?
+Context: 백신은 신체의 면역 반응을 자극하여 병원체를 인식하고 싸우는 항체를 생성함으로써 질병으로부터 보호합니다.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="백신은 신체의 면역 체계를 어떻게 활용해서 질병을 예방합니까?",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""提供された質問に条件に関する内容を追加して、複雑さを高めます。
+質問のContextに影響を与えるシナリオや条件を含めて、質問をより複雑にすることが目標です。
+質問を再作成するときは、次のルールに従います。
+    1. 再作成された質問は100文字を超えてはいけません。 可能であれば略語を使ってください
+    2. 再作成された質問は合理的でなければならず、人が理解して回答できるものでなければなりません。
+    3. 再作成された質問は、現在のContextで完全に答えられる必要があります。
+    4. 「提供された文」、「段落によると?」、「Contextによると」などのフレーズは質問に表示されません。
+    5. 日本語で質問を書きましょう。
+""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 植物の根の機能は何ですか？
+Context: 植物の根は土壌から水や栄養分を吸収し、植物を地面に固定し、栄養分を蓄えます。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="植物の根は土壌栄養分と安定性に対してどのような役割をしますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: ワクチンは病気をどのように予防しますか?
+Context: ワクチンは、体の免疫反応を刺激して病原体を認識し、戦う抗体を生成することで病気から守ります。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="ワクチンは体の免疫システムをどのように活用して病気を予防しますか？",
+			),
+		],
+	},
+	"reasoning_evolve_ragas": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""Complicate the given question by rewriting question into a multi-hop reasoning question based on the provided context.
+Answering the question should require the reader to make multiple logical connections or inferences using the information available in given context.
+Rules to follow when rewriting question:
+1. Ensure that the rewritten question can be answered entirely from the information present in the contexts.
+2. Do not frame questions that contains more than 15 words. Use abbreviation wherever possible.
+3. Make sure the question is clear and unambiguous.
+4. phrases like 'based on the provided context','according to the context',etc are not allowed to appear in the question.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What is the capital of France?,
+Context: France is a country in Western Europe. It has several cities, including Paris, Lyon, and Marseille. Paris is not only known for its cultural landmarks like the Eiffel Tower and the Louvre Museum but also as the administrative center.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="Linking the Eiffel Tower and administrative center, which city stands as both?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What does the append() method do in Python?
+Context: In Python, lists are used to store multiple items in a single variable. Lists are one of 4 built-in data types used to store collections of data. The append() method adds a single item to the end of a list.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="If a list represents a variable collection, what method extends it by one item?",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""주어진 Context를 기반으로 기존 질문을 복잡하게 만들어 여러 논리적인 사고가 필요한 질문으로 다시 작성하세요.
+질문에 답하려면 주어진 Context의 정보를 사용해 여러 논리적 사고나 추론을 해야 합니다.
+질문을 다시 작성할 때 따라야 할 규칙:
+1. 다시 작성된 질문은 Context에 있는 정보만으로 완전히 답변할 수 있어야 합니다.
+2. 100자를 초과하는 질문을 작성하지 마세요. 가능한 경우 약어를 사용하세요.
+3. 질문이 명확하고 모호하지 않도록 하세요.
+4. '제공된 Context에 기반하여', '해당 단락에 따르면' 등의 문구는 질문에 포함되지 않아야 합니다.
+5. 한국어로 질문을 작성하세요.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 프랑스의 수도는 어디인가요?,
+Context: 프랑스는 서유럽에 있는 나라입니다. 파리, 리옹, 마르세유를 포함한 여러 도시가 있습니다. 파리는 에펠탑과 루브르 박물관 같은 문화적 랜드마크로 유명할 뿐만 아니라 행정 중심지로도 알려져 있습니다.
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="에펠탑과 행정 중심지, 두 단어는 어떤 도시를 가리키나요?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""질문: Python에서 append() 메서드는 무엇을 하나요?
+컨텍스트: Python에서 리스트는 하나의 변수에 여러 항목을 저장하는 데 사용됩니다. 리스트는 데이터를 저장하는 데 사용되는 4가지 내장 데이터 유형 중 하나입니다. append() 메서드는 리스트의 끝에 새로운 항목을 추가합니다.
+출력: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="리스트가 변수들을 모아 놓은 것을 나타낸다면, 어떤 메서드를 사용해야 항목을 하나 더 추가할 수 있습니까?",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""与えられたContextに基づいて既存の質問を複雑にして、様々な論理的思考が必要な質問として書き直しましょう。
+質問に答えるためには、与えられたContextの情報を使って様々な論理的思考や推論をしなければなりません。
+質問を再作成するときに従うべきルール:
+1. 再作成された質問は、Contextにある情報だけで完全に答えられる必要があります。
+2. 100文字を超える質問を作成してはいけません。 可能であれば略語を使ってください。
+3. 質問が明確で曖昧にならないようにしましょう。
+4. 「提供されたContextに基づいて」、「当該段落によると」などのフレーズは、質問に含まれてはいけません。
+5. 日本語で質問を書きましょう。""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: フランスの首都はどこですか？,
+Context: フランスは西ヨーロッパにある国です。 パリ、リヨン、マルセイユを含むいくつかの都市があります。 パリはエッフェル塔やルーブル博物館のような文化的ランドマークとして有名なだけでなく、行政の中心地としても知られています。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="エッフェル塔と行政の中心地、二つの単語はどんな都市を指していますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: Pythonでappend() メソッドは何をしますか？
+Context: Pythonで、リストは 1 つの変数に複数の項目を保存するために使用されます。 リストは、データを保存するために使用される 4 つの組み込みデータ タイプの 1 つです。 append()メソッドは、リストの最後に新しい項目を追加します。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="リストが変数を集めたものである場合、どのメソッドを使えば項目を一つ追加することができますか？",
+			),
+		],
+	},
+	"compress_ragas": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""Rewrite the following question to make it more indirect and shorter while retaining the essence of the original question.
+    The goal is to create a question that conveys the same meaning but in a less direct manner. The rewritten question should shorter so use abbreviation wherever possible.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What is the distance between the Earth and the Moon?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="How far is the Moon from Earth?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: What ingredients are required to bake a chocolate cake?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="What's needed for a chocolate cake?",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""주어진 질문을 더 간접적이고 짧게 다시 작성하세요.
+        목표는 질문을 원래 질문의 본질을 유지하면서 너무 직설적이지 않게 만드는 것입니다.
+        약어 등을 사용하여 질문을 더 짧게 만드세요.""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 지구와 달 사이의 거리는 얼마입니까?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="달은 지구에서 얼마나 떨어져 있나요?",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 초콜릿 케이크를 굽기 위해 필요한 재료는 무엇입니까?
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="초콜릿 케이크에 필요한 것은 무엇인가요?",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""与えられた質問をより間接的かつ短く書き換えます。
+目標は、質問を元の質問の本質を保ちながら、あまりストレートにならないようにすることです。
+略語などを使用して、質問をより短くします。""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 地球と月の間の距離はどれくらいですか？
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="月は地球からどれくらい離れていますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: チョコレートケーキを焼くために必要な材料は何ですか？
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="チョコレートケーキに必要なものは何ですか？",
+			),
+		],
+	},
+}

autorag/data/qa/extract_evidence.py

@@ -0,0 +1 @@
+# This module is about extracting evidence from the given retrieval gt passage

autorag/data/qa/filter/dontknow.py

@@ -0,0 +1,117 @@
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatMessage, MessageRole, ChatResponse
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.filter.prompt import FILTER_PROMPT
+
+dont_know_phrases = {
+	"en": [
+		"I don't know",
+		"I do not know",
+		"Don't know",
+		"Do not know",
+	],
+	"ko": [
+		"몰라요",
+		"모르겠습니다",
+		"모르겠어요",
+		"몰라",
+		"내가 어떻게 알아?",
+		"모르겠소",
+		"몰라유",
+		"모르것는디",
+		"모르겠어유",
+		"모르겠네유",
+		"모르겠네요",
+	],
+	"ja": [
+		"知りません",
+		"わかりません",
+		"分かりません",
+		"知らないです",
+		"よく分かってません",
+		"わかりかねます",
+		"存じません",
+		"お答えいたしかねます",
+	],
+}
+
+
+def dontknow_filter_rule_based(row: Dict, lang: str = "en") -> bool:
+	assert (
+		"generation_gt" in row.keys()
+	), "generation_gt column is not in the DataFrame."
+	dont_know_phrase = dont_know_phrases[lang]
+	return not any(
+		phrase in s for phrase in dont_know_phrase for s in row["generation_gt"]
+	)
+
+
+class Response(BaseModel):
+	is_dont_know: bool
+
+
+async def dontknow_filter_openai(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-mini-2024-07-18",
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop rows that have a "don't know" answer.
+	It will drop unanswerable questions from the QA dataset.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	:param client: The OpenAI client.
+	:param model_name: The model name.
+		You have to use gpt-4o-2024-08-06 or gpt-4o-mini-2024-07-18.
+	:param lang: The supported language is en, ko or ja.
+	:return: False if the row generation_gt is a "don't know" meaning.
+	"""
+	assert "generation_gt" in row.keys(), "generation_gt column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["dontknow_filter"][lang]
+	result = []
+	for gen_gt in row["generation_gt"]:
+		completion = await client.beta.chat.completions.parse(
+			model=model_name,
+			messages=to_openai_message_dicts(
+				system_prompt + [ChatMessage(role=MessageRole.USER, content=gen_gt)]
+			),
+			response_format=Response,
+		)
+		result.append(completion.choices[0].message.parsed.is_dont_know)
+	return not any(result)
+
+
+async def dontknow_filter_llama_index(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop rows that have a "don't know" answer.
+	It will drop unanswerable questions from the QA dataset.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	:param llm: The Llama index llm instance.
+		It will be good if you set max tokens to low for saving tokens.
+	:param lang: The supported language is en, ko or ja.
+	:return: False if the row generation_gt is a "don't know" meaning.
+	"""
+	assert "generation_gt" in row.keys(), "generation_gt column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["dontknow_filter"][lang]
+	results = []
+	for gen_gt in row["generation_gt"]:
+		response: ChatResponse = await llm.achat(
+			messages=system_prompt
+			+ [ChatMessage(role=MessageRole.USER, content=gen_gt)]
+		)
+		result_str = response.message.content
+		results.append("true" in result_str.lower().strip())
+	return not any(results)

autorag/data/qa/filter/passage_dependency.py

@@ -0,0 +1,88 @@
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatMessage, MessageRole, ChatResponse
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.filter.prompt import FILTER_PROMPT
+
+
+class Response(BaseModel):
+	is_passage_dependent: bool
+
+
+async def passage_dependency_filter_openai(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-mini-2024-07-18",
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop passage-dependent question rows.
+	Passage-dependent questions are questions that the answer will change depending on what passage you choose.
+	The passage-dependent questions will not be good for RAG evaluation, because any retrieval system can't find the right passage with passage-dependent question.
+	For example, when someone asks "What is the highest score according to the table?" the answer will be different depending on the table.
+	And what is the table? The retrieval system can't find the right passage with this question.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	    :param client: The OpenAI client.
+	    :param model_name: The model name.
+	            You have to use gpt-4o-2024-08-06 or gpt-4o-mini-2024-07-18.
+	    :param lang: The supported language is en, ko or ja.
+	:return: False if the row question is a passage-dependent question (to be filtered).
+	"""
+	assert "query" in row.keys(), "query column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["passage_dependency"][lang]
+	query = row["query"]
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(
+			system_prompt
+			+ [
+				ChatMessage(
+					role=MessageRole.USER,
+					content=f"Question: {query}\nIs this the question passage dependent?",
+				)
+			]
+		),
+		response_format=Response,
+	)
+	return not completion.choices[0].message.parsed.is_passage_dependent
+
+
+async def passage_dependency_filter_llama_index(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> bool:
+	"""
+	This will drop passage-dependent question rows.
+	Passage-dependent questions are questions that the answer will change depending on what passage you choose.
+	The passage-dependent questions will not be good for RAG evaluation, because any retrieval system can't find the right passage with passage-dependent question.
+	For example, when someone asks "What is the highest score according to the table?" the answer will be different depending on the table.
+	And what is the table? The retrieval system can't find the right passage with this question.
+	You can use this filter with the ` batch_filter ` function at `QA` class.
+
+	:param row: The row dict from QA dataset.
+	:param llm: The Llama index llm instance.
+	            It will be good if you set max tokens to low for saving tokens.
+	    :param lang: The supported language is en, ko or ja.
+	:return: False if the row question is a passage-dependent question (to be filtered).
+	"""
+	assert "query" in row.keys(), "query column is not in the row."
+	system_prompt: List[ChatMessage] = FILTER_PROMPT["passage_dependency"][lang]
+	query = row["query"]
+	response: ChatResponse = await llm.achat(
+		messages=system_prompt
+		+ [
+			ChatMessage(
+				role=MessageRole.USER,
+				content=f"Question: {query}\nIs this the question passage dependent?",
+			)
+		]
+	)
+	result_str = response.message.content
+	return "true" not in result_str.lower().strip()

autorag/data/qa/filter/prompt.py

@@ -0,0 +1,73 @@
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+
+FILTER_PROMPT = {
+	"dontknow_filter": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""The following sentence is an answer about a question. You have to decide the answer implies 'I don't know'.
+If the answer implies 'I don't know', return True. If not, return False.""",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""다음 문장은 어떠한 질문에 대한 대답입니다. 해당 문장이 질문에 대해서 '모른다고' 답한 것인지 판단하십시오.
+만약 해당 문장이 '모른다고' 답한 것이라면, True를 반환하세요. 그렇지 않다면 False를 반환하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""次の文章はある質問に対する答えです。 該当文章が質問に対して「知らない」と答えたのか判断します。
+もし、その文章が「知らない」と答えたのであれば、Trueを返します。 そうでなければFalseを返します。""",
+			)
+		],
+	},
+	"passage_dependency": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""You are a classifier that recognize 'passage dependent' questions.
+The 'passage dependent' is the question that the answer will be change depending on what passage you choose.
+For example) 'What is the highest score according to the table?'
+This sentence is the passage dependent question because the answer will be different depending on the table.
+
+In contrast, the following sentence is not passage dependant.
+'What is the highest score of the KBO baseball history in one game?'
+'What is the capital of France?'
+These sentences will have the same answer regardless of the passage.
+
+Please return True if the input question is passage dependent. Else return False.""",
+			)
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""당신은 '단락 의존' 질문을 인식하는 분류기입니다.
+'단락 의존'이란 어떤 단락이 선택 되는지 따라 답이 달라지는 질문을 의미합니다.
+예를 들어, '주어진 표에 따르면 가장 높은 점수는 무엇인가요?'라는 질문은 단락 의존 질문입니다. 왜냐하면 표가 어떤 것인지에 따라 그 답이 달라지기 때문입니다.
+
+반면에, 다음 문장들은 단락 의존적이지 않습니다.
+'KBO 야구 역사상 한 경기에서 가장 높은 점수는 무엇인가요?' 또는 '프랑스의 수도는 무엇인가요?'
+이러한 문장은 단락에 관계 없이 동일한 답을 가집니다.
+
+입력된 질문이 단락 의존적이라면 True를 반환하고, 그렇지 않으면 False를 반환하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""あなたは「段落依存」の質問を認識する分類器です。
+「段落依存」とは、どの段落が選択されるかによって答えが変わる質問を意味します。
+たとえば、「与えられた表によると、最も高い点数は何ですか？」という質問は、段落依存の質問です。 なぜなら、表がどんなものかによってその答えが変わるからです。
+
+一方、次の文章は段落依存的ではありません。
+KBO野球史上1試合で最も高い点数は何ですか?またはフランスの首都は何ですか?'
+このような文章は段落に関係なく同じ答えを持ちます。
+
+入力された質問が段落依存的である場合はTrueを返し、そうでない場合はFalseを返します。""",
+			)
+		],
+	},
+}

autorag/data/qa/generation_gt/base.py

@@ -0,0 +1,16 @@
+from typing import Dict
+
+
+def add_gen_gt(row: Dict, new_gen_gt: str) -> Dict:
+	if "generation_gt" in list(row.keys()):
+		if isinstance(row["generation_gt"], list):
+			row["generation_gt"].append(new_gen_gt)
+		elif isinstance(row["generation_gt"], str):
+			row["generation_gt"] = [row["generation_gt"], new_gen_gt]
+		else:
+			raise ValueError(
+				"generation_gt should be either a string or a list of strings."
+			)
+		return row
+	row["generation_gt"] = [new_gen_gt]
+	return row

autorag/data/qa/generation_gt/llama_index_gen_gt.py

@@ -0,0 +1,41 @@
+import itertools
+from typing import Dict
+
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import MessageRole, ChatMessage
+
+from autorag.data.qa.generation_gt.base import add_gen_gt
+from autorag.data.qa.generation_gt.prompt import GEN_GT_SYSTEM_PROMPT
+
+
+async def make_gen_gt_llama_index(row: Dict, llm: BaseLLM, system_prompt: str) -> Dict:
+	retrieval_gt_contents = list(
+		itertools.chain.from_iterable(row["retrieval_gt_contents"])
+	)
+	query = row["query"]
+	passage_str = "\n".join(retrieval_gt_contents)
+	user_prompt = f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:"
+
+	response = await llm.achat(
+		messages=[
+			ChatMessage(role=MessageRole.SYSTEM, content=system_prompt),
+			ChatMessage(role=MessageRole.USER, content=user_prompt),
+		],
+		temperature=0.0,
+	)
+	return add_gen_gt(row, response.message.content)
+
+
+async def make_concise_gen_gt(row: Dict, llm: BaseLLM, lang: str = "en") -> Dict:
+	return await make_gen_gt_llama_index(
+		row, llm, GEN_GT_SYSTEM_PROMPT["concise"][lang]
+	)
+
+
+async def make_basic_gen_gt(row: Dict, llm: BaseLLM, lang: str = "en") -> Dict:
+	return await make_gen_gt_llama_index(row, llm, GEN_GT_SYSTEM_PROMPT["basic"][lang])
+
+
+async def make_custom_gen_gt(row: Dict, llm: BaseLLM, system_prompt: str) -> Dict:
+	return await make_gen_gt_llama_index(row, llm, system_prompt)

autorag/data/qa/generation_gt/openai_gen_gt.py

@@ -0,0 +1,84 @@
+import itertools
+from typing import Dict
+
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.generation_gt.base import add_gen_gt
+from autorag.data.qa.generation_gt.prompt import GEN_GT_SYSTEM_PROMPT
+
+
+class Response(BaseModel):
+	answer: str
+
+
+async def make_gen_gt_openai(
+	row: Dict,
+	client: AsyncClient,
+	system_prompt: str,
+	model_name: str = "gpt-4o-2024-08-06",
+):
+	retrieval_gt_contents = list(
+		itertools.chain.from_iterable(row["retrieval_gt_contents"])
+	)
+	query = row["query"]
+	passage_str = "\n".join(retrieval_gt_contents)
+	user_prompt = f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:"
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=[
+			{"role": "system", "content": system_prompt},
+			{"role": "user", "content": user_prompt},
+		],
+		temperature=0.0,
+		response_format=Response,
+	)
+	response: Response = completion.choices[0].message.parsed
+	return add_gen_gt(row, response.answer)
+
+
+async def make_concise_gen_gt(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+):
+	"""
+	Generate concise generation_gt using OpenAI Structured Output for preventing errors.
+	It generates a concise answer, so it is generally a word or just a phrase.
+
+	:param row: The input row of the qa dataframe.
+	:param client: The OpenAI async client.
+	:param model_name: The model name that supports structured output.
+	    It has to be "gpt-4o-2024-08-06" or "gpt-4o-mini-2024-07-18".
+	:param lang: The language code of the prompt.
+		Default is "en".
+	:return: The output row of the qa dataframe with added "generation_gt" in it.
+	"""
+	return await make_gen_gt_openai(
+		row, client, GEN_GT_SYSTEM_PROMPT["concise"][lang], model_name
+	)
+
+
+async def make_basic_gen_gt(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+):
+	"""
+	Generate basic generation_gt using OpenAI Structured Output for preventing errors.
+	It generates a "basic" answer, and its prompt is simple.
+
+	:param row: The input row of the qa dataframe.
+	:param client: The OpenAI async client.
+	:param model_name: The model name that supports structured output.
+	    It has to be "gpt-4o-2024-08-06" or "gpt-4o-mini-2024-07-18".
+	:param lang: The language code of the prompt.
+		Default is "en".
+	:return: The output row of the qa dataframe with added "generation_gt" in it.
+	"""
+	return await make_gen_gt_openai(
+		row, client, GEN_GT_SYSTEM_PROMPT["basic"][lang], model_name
+	)

autorag/data/qa/generation_gt/prompt.py

@@ -0,0 +1,27 @@
+GEN_GT_SYSTEM_PROMPT = {
+	"concise": {
+		"en": """You are an AI assistant to answer the given question in the provide evidence text.
+You can find the evidence from the given text about question, and you have to write a proper answer to the given question.
+Your answer have to be concise and relevant to the question.
+Do not make a verbose answer and make it super clear.
+It doesn't have to be an full sentence. It can be the answer is a word or a paraphrase.""",
+		"ko": """당신은 주어진 질문에 대해 제공된 Text 내에서 답을 찾는 AI 비서입니다.
+질문에 대한 답을 Text에서 찾아 적절한 답변을 작성하세요.
+답변은 간결하고 질문에 관련된 내용만 포함해야 합니다.
+불필요하게 길게 답변하지 말고, 명확하게 작성하세요.
+완전한 문장이 아니어도 되며, 답은 단어나 요약일 수 있습니다.""",
+		"ja": """
+あなたは与えられた質問に対して提供されたText内で答えを探すAI秘書です。
+質問に対する答えをTextで探して適切な答えを作成しましょう。
+回答は簡潔で、質問に関連する内容のみを含める必要があります。
+不必要に長く答えず、明確に作成しましょう。
+完全な文章でなくてもいいし、答えは単語や要約かもしれません。
+""",
+	},
+	"basic": {
+		"en": """You are an AI assistant to answer the given question in the provide evidence text.
+You can find the evidence from the given text about question, and you have to write a proper answer to the given question.""",
+		"ko": "당신은 주어진 질문에 대한 답을 제공된 Text 내에서 찾는 AI 비서입니다. 질문과 관련된 증거를 Text에서 찾아 적절한 답변을 작성하세요.",
+		"ja": "あなたは与えられた質問に対する答えを提供されたText内で探すAI秘書です。 質問に関する証拠をTextで探して適切な回答を作成しましょう。",
+	},
+}

autorag/data/qa/query/llama_gen_query.py

@@ -0,0 +1,82 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.core.base.llms.types import ChatResponse, ChatMessage, MessageRole
+
+from autorag.data.qa.query.prompt import QUERY_GEN_PROMPT, QUERY_GEN_PROMPT_EXTRA
+
+
+async def llama_index_generate_base(
+	row: Dict,
+	llm: BaseLLM,
+	messages: List[ChatMessage],
+) -> Dict:
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "\n".join([f"{i + 1}. {c}" for i, c in enumerate(context)])
+	user_prompt = f"Text:\n{context_str}\n\nGenerated Question from the Text:\n"
+	user_message = ChatMessage(role=MessageRole.USER, content=user_prompt)
+	new_messages = [*messages, user_message]
+	chat_response: ChatResponse = await llm.achat(messages=new_messages)
+	row["query"] = chat_response.message.content
+	return row
+
+
+async def factoid_query_gen(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row, llm, QUERY_GEN_PROMPT["factoid_single_hop"][lang]
+	)
+
+
+async def concept_completion_query_gen(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	return await llama_index_generate_base(
+		row, llm, QUERY_GEN_PROMPT["concept_completion"][lang]
+	)
+
+
+async def two_hop_incremental(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+) -> Dict:
+	messages = QUERY_GEN_PROMPT["two_hop_incremental"][lang]
+	passages = row["retrieval_gt_contents"]
+	assert (
+		len(passages) >= 2
+	), "You have to sample more than two passages for making two-hop questions."
+	context_str = f"Document 1: {passages[0][0]}\nDocument 2: {passages[1][0]}"
+	user_prompt = f"{context_str}\n\nGenerated two-hop Question from two Documents:\n"
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	chat_response: ChatResponse = await llm.achat(messages=messages)
+	response = chat_response.message.content
+	row["query"] = response.split(":")[-1].strip()
+	return row
+
+
+async def custom_query_gen(
+	row: Dict,
+	llm: BaseLLM,
+	messages: List[ChatMessage],
+) -> Dict:
+	return await llama_index_generate_base(row, llm, messages)
+
+
+# Experimental feature: can only use factoid_single_hop
+async def multiple_queries_gen(
+	row: Dict,
+	llm: BaseLLM,
+	lang: str = "en",
+	n: int = 3,
+) -> Dict:
+	_messages = QUERY_GEN_PROMPT["factoid_single_hop"][lang]
+	_messages[0].content += QUERY_GEN_PROMPT_EXTRA["multiple_queries"][lang].format(n=n)
+	return await llama_index_generate_base(row, llm, _messages)

autorag/data/qa/query/openai_gen_query.py

@@ -0,0 +1,95 @@
+import itertools
+from typing import Dict, List
+
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.llms.openai.utils import to_openai_message_dicts
+from openai import AsyncClient
+from pydantic import BaseModel
+
+from autorag.data.qa.query.prompt import QUERY_GEN_PROMPT
+
+
+class Response(BaseModel):
+	query: str
+
+
+# Single hop QA generation OpenAI
+async def query_gen_openai_base(
+	row: Dict,
+	client: AsyncClient,
+	messages: List[ChatMessage],
+	model_name: str = "gpt-4o-2024-08-06",
+):
+	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
+	context_str = "Text:\n" + "\n".join(
+		[f"{i + 1}. {c}" for i, c in enumerate(context)]
+	)
+	user_prompt = f"{context_str}\n\nGenerated Question from the Text:\n"
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=Response,
+	)
+	row["query"] = completion.choices[0].message.parsed.query
+	return row
+
+
+async def factoid_query_gen(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_gen_openai_base(
+		row, client, QUERY_GEN_PROMPT["factoid_single_hop"][lang], model_name
+	)
+
+
+async def concept_completion_query_gen(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	return await query_gen_openai_base(
+		row, client, QUERY_GEN_PROMPT["factoid_single_hop"][lang], model_name
+	)
+
+
+class TwoHopIncrementalResponse(BaseModel):
+	answer: str
+	one_hop_question: str
+	two_hop_question: str
+
+
+async def two_hop_incremental(
+	row: Dict,
+	client: AsyncClient,
+	model_name: str = "gpt-4o-2024-08-06",
+	lang: str = "en",
+) -> Dict:
+	"""
+	Create a two-hop question using incremental prompt.
+	Incremental prompt is more effective to create multi-hop question.
+	The input retrieval_gt has to include more than one passage.
+
+	:return: The two-hop question using openai incremental prompt
+	"""
+	messages = QUERY_GEN_PROMPT["two_hop_incremental"][lang]
+	passages = row["retrieval_gt_contents"]
+	assert (
+		len(passages) >= 2
+	), "You have to sample more than two passages for making two-hop questions."
+	context_str = f"Document 1: {passages[0][0]}\nDocument 2: {passages[1][0]}"
+	user_prompt = f"{context_str}\n\nGenerated two-hop Question from two Documents:\n"
+	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
+
+	completion = await client.beta.chat.completions.parse(
+		model=model_name,
+		messages=to_openai_message_dicts(messages),
+		response_format=TwoHopIncrementalResponse,
+	)
+	row["query"] = completion.choices[0].message.parsed.two_hop_question
+	return row

autorag/data/qa/query/prompt.py

@@ -0,0 +1,202 @@
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+
+QUERY_GEN_PROMPT = {
+	"factoid_single_hop": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""You're an AI tasked to convert Text into a factoid question.
+Factoid questions are those seeking brief, factual information that can be easily verified. They typically require a yes or no answer or a brief explanation and often inquire about specific details such as dates, names, places, or events.
+
+Examples of factoid questions include:
+
+- What is the capital of France?
+- Who invented the light bulb?
+- When was Wikipedia founded?
+
+Instructions:
+1. Questions MUST BE extracted from given Text
+2. Questions should be as detailed as possible from Text
+3. Create questions that ask about factual information from the Text
+4. Do not mention any of these in the questions: "in the given text", "in the provided information", etc.
+Users do not know the passage source of the question, so it should not be mentioned in the question.
+5. Do not ask about the file name or the file title. Ask about the content of the file.
+For example, avoid to write questions like `What is the file name of the document?`""",
+			)
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""당신은 주어진 Text를 '사실 질문'으로 변환하는 AI입니다.
+
+사실 질문(factoid questions)이란 사실적인 정보를 요구하는 질문으로, 쉽게 검증할 수 있는 답변을 필요로 합니다. 일반적으로 예/아니오 답변이나 간단한 설명을 요구하며, 날짜, 이름, 장소 또는 사건과 같은 구체적인 세부사항에 대해 묻는 질문입니다.
+
+사실 질문의 예는 다음과 같습니다:
+
+	•	프랑스의 수도는 어디입니까?
+	•	전구를 발명한 사람은 누구입니까?
+	•	위키피디아는 언제 설립되었습니까?
+
+지침:
+	1.	질문은 반드시 주어진 Text를 기반으로 작성되어야 합니다.
+	2.	질문은 Text를 기반으로 가능한 한 구체적으로 작성되어야 합니다.
+	3.	Text에서 사실적 정보를 요구하는 질문을 만들어야 합니다. 즉, Text를 기반으로 사실 질문을 만드세요.
+	4.	질문에 “주어진 Text에서” 또는 “제공된 단락에서”와 같은 표현을 포함해서는 안 됩니다.
+사용자는 질문의 출처가 Text라는 것을 모르기 때문에 반드시 그 출처를 언급해서는 안 됩니다.
+	5.	파일 이름이나 파일 제목에 대한 질문을 하지 마세요. 파일의 내용에 대해 물어보세요.
+예를 들어, '문서의 파일 이름은 무엇입니까?'와 같은 질문을 작성하지 마세요.
+	6.	질문을 한국어로 작성하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""あなたは与えられたTextを「実は質問」に変換するAIです。
+
+事実質問(factoid questions)とは、事実的な情報を求める質問であり、容易に検証できる回答を必要とします。 一般的に、「はい/いいえ」の返答や簡単な説明を要求し、日付、名前、場所、または事件のような具体的な詳細事項について尋ねる質問です。
+
+実は質問の例は次の通りです:
+
+	• フランスの首都はどこですか？
+	• 電球を発明したのは誰ですか？
+	• ウィキペディアはいつ設立されましたか？
+
+指針:
+	1. 質問は、必ず与えられたTextに基づいて作成されなければなりません。
+	2. 質問は、Textに基づいて可能な限り具体的に作成されなければなりません。
+	3. Textで事実的情報を要求する質問を作らなければなりません。 つまり、Textに基づいて質問を作成します。
+	4. 質問に「与えられたTextで」または「提供された段落で」のような表現を含めてはいけません。
+ユーザーは質問の出所がTextだということを知らないので、必ずしもその出所を言及してはいけません。
+	5. ファイル名やファイルタイトルを訊かないでください。ファイルの内容について聞いてください。
+例えば、「このドキュメントのファイル名は何ですか？
+	6. 質問を日本語で作成しなさい。""",
+			)
+		],
+	},
+	"concept_completion": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""You're an AI tasked to convert Text into a "Concept Completion" Question.
+A “concept completion” question asks directly about the essence or identity of a concept.
+
+Follow the following instructions.
+Instructions:
+1. Questions MUST BE extracted from given Text
+2. Questions should be as detailed as possible from Text
+3. Create questions that ask about information from the Text
+4. MUST include specific keywords from the Text.
+5. Do not mention any of these in the questions: "in the given text", "in the provided information", etc.
+Users do not know the passage source of the question, so it should not be mentioned in the question.
+6. Do not ask about the file name or the file title. Ask about the content of the file.
+For example, avoid to write questions like `What is the file name of the document?""",
+			)
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""당신은 Text를 “개념 완성” 질문으로 변환하는 AI입니다.
+"개념 완성" 질문은 개념의 본질이나 정체성에 대해 직접적으로 묻는 질문입니다.
+
+다음 지시사항을 따르세요.
+지시사항:
+1.	질문은 반드시 주어진 Text를 기반으로 작성되어야 합니다.
+2.	질문은 Text를 기반으로 가능한 한 자세하게 작성되어야 합니다.
+3.	Text에서 제공된 정보를 묻는 질문을 생성하세요.
+4.	Text의 특정 키워드를 반드시 질문에 포함하세요.
+5.	질문에 “주어진 Text에서” 또는 “제공된 단락에서”와 같은 표현을 포함해서는 안 됩니다.
+사용자는 질문의 출처가 Text라는 것을 모르기 때문에 반드시 그 출처를 언급해서는 안 됩니다.
+6.	파일 이름이나 파일 제목에 대한 질문을 하지 마세요. 파일의 내용에 대해 물어보세요.
+예를 들어, '문서의 파일 이름은 무엇입니까?'와 같은 질문을 작성하지 마세요.
+7.	질문을 한국어로 작성하세요.""",
+			)
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""あなたはTextを「概念完成」の質問に変換するAIです。
+「概念完成」の質問は概念の本質やアイデンティティについて直接的に尋ねる質問です。
+
+次の指示に従います。
+指示事項:
+1. 質問は、必ず与えられたTextに基づいて作成されなければなりません。
+2. 質問は、Textに基づいてできるだけ詳しく作成されなければなりません。
+3. Textで提供された情報を尋ねる質問を作成します。
+4. Textの特定のキーワードを必ず質問に含みます。
+5. 質問に「与えられたTextで」または「提供された段落で」のような表現を含めてはいけません。
+ユーザーは質問の出所がTextだということを知らないので、必ずしもその出所を言及してはいけません。
+6. ファイル名やファイルタイトルを訊かないでください。ファイルの内容について聞いてください。
+例えば、「このドキュメントのファイル名は何ですか？
+7. 質問を日本語で書きましょう。""",
+			)
+		],
+	},
+	"two_hop_incremental": {
+		"en": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="Generate a multi-hop question for the given answer which requires reference to all of the given documents.",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Document 1: The Municipality of Nuevo Laredo is located in the Mexican state of Tamaulipas.
+Document 2: The Ciudad Deportiva (Sports City ¨ ¨) is a sports
+complex in Nuevo Laredo, Mexico. It is home to the Tecolotes de
+Nuevo Laredo Mexican Baseball League team and ...""",
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="""Answer: Tamaulipas
+One-hop question (using Document 1): In which Mexican state is Nuevo Laredo located?
+Two-hop question (using Document 2):  In which Mexican state can one find the Ciudad Deportiva, home to the Tecolotes de Nuevo Laredo?""",
+			),
+		],
+		"ko": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="Generate a multi-hop question for the given answer which requires reference to all of the given documents.",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Document 1: The Municipality of Nuevo Laredo is located in the Mexican state of Tamaulipas.
+Document 2: The Ciudad Deportiva (Sports City ¨ ¨) is a sports
+complex in Nuevo Laredo, Mexico. It is home to the Tecolotes de
+Nuevo Laredo Mexican Baseball League team and ...""",
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="""Answer: Tamaulipas
+One-hop question (using Document 1): In which Mexican state is Nuevo Laredo located?
+Two-hop question (using Document 2):  In which Mexican state can one find the Ciudad Deportiva, home to the Tecolotes de Nuevo Laredo?""",
+			),
+		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="与えられた答えに対するマルチホップ質問を生成し、与えられたすべての文書を参照する必要があります。",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Document 1: ヌエヴォ·ラレド自治体はメキシコのタマウリパス州にあります。
+Ciudad Deportiva（スポーツシティ ¨ ¨）はスポーツです
+メキシコのヌエボ·ラレドにある複合施設です。 テコロテス·デ·テコロテスの故郷です
+Nuevo Larredo メキシコ野球リーグのチームです···""",
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="""Answer: Tamaulipas
+One-hop question (using Document 1): ヌエヴォ·ラレド自治体はどのメキシコの州にありますか？
+Two-hop question (using Document 2): ヌエヴォ·ラレドのテコロテス·デ·テコロテスの故郷であるメキシコの州はどこですか？""",
+			),
+		],
+	},
+}
+
+# Experimental feature
+QUERY_GEN_PROMPT_EXTRA = {
+	"multiple_queries": {
+		"en": "\nAdditional instructions:\n  - Please make {n} questions.",
+		"ko": "\n추가 지침:\n  - 질문은 {n}개를 만드세요.",
+		"ja": "\n追加指示:\n  - 質問を{n}個作成してください。",
+	}
+}

autorag/data/qa/sample.py

@@ -0,0 +1,26 @@
+import uuid
+from typing import Iterable
+
+import pandas as pd
+
+
+def random_single_hop(
+	corpus_df: pd.DataFrame, n: int, random_state: int = 42
+) -> pd.DataFrame:
+	sample_df = corpus_df.sample(n, random_state=random_state)
+	return pd.DataFrame(
+		{
+			"qid": [str(uuid.uuid4()) for _ in range(len(sample_df))],
+			"retrieval_gt": [[[id_]] for id_ in sample_df["doc_id"].tolist()],
+		}
+	)
+
+
+def range_single_hop(corpus_df: pd.DataFrame, idx_range: Iterable):
+	sample_df = corpus_df.iloc[idx_range]
+	return pd.DataFrame(
+		{
+			"qid": [str(uuid.uuid4()) for _ in range(len(sample_df))],
+			"retrieval_gt": [[[id_]] for id_ in sample_df["doc_id"].tolist()],
+		}
+	)

autorag/data/qa/schema.py

@@ -0,0 +1,322 @@
+import logging
+from typing import Callable, Optional, Dict, Awaitable, Any, Tuple, List
+import uuid
+import pandas as pd
+from autorag.utils.util import process_batch, get_event_loop, fetch_contents
+
+from autorag.support import get_support_modules
+
+logger = logging.getLogger("AutoRAG")
+
+
+class Raw:
+	"""
+	The Raw class that stored document parsing results.
+	It can do chunking.
+	It has two column names, 'raw_id' and 'contents'.
+	"""
+
+	def __init__(self, raw_df: Optional[pd.DataFrame] = None):
+		self.data = raw_df
+
+	def batch_apply(
+		self, fn: Callable[[Dict, Any], Awaitable[Dict]], batch_size: int = 32, **kwargs
+	) -> "Raw":
+		raw_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(raw_dict, **kwargs) for raw_dict in raw_dicts]
+		results = loop.run_until_complete(process_batch(tasks, batch_size))
+		return Raw(pd.DataFrame(results))
+
+	def map(self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs) -> "Raw":
+		return Raw(fn(self.data, **kwargs))
+
+	def flatmap(self, fn: Callable, **kwargs) -> "Raw":
+		return fn(self.data, **kwargs)
+
+	def chunk(self, module_name: str, **module_params) -> "Corpus":
+		chunk_module = get_support_modules(module_name)
+		chunked_result = chunk_module(parsed_result=self.data, **module_params)
+		return Corpus(chunked_result, self)
+
+	def __add__(self, other):
+		assert isinstance(other, Raw), "You can only add Raw instances."
+		self.data = pd.concat([self.data, other.data], ignore_index=True).reset_index(
+			drop=True
+		)
+		return self
+
+
+class Corpus:
+	"""
+	The Corpus class that stored chunked passages.
+	It can generate qa set, linked with Raw instance.
+	"""
+
+	def __init__(
+		self,
+		corpus_df: Optional[pd.DataFrame] = None,
+		linked_raw: Optional[Raw] = None,
+	):
+		self.data = corpus_df
+		self._linked_raw = linked_raw
+
+	@property
+	def linked_raw(self) -> Raw:
+		return self._linked_raw
+
+	@linked_raw.setter
+	def linked_raw(self, raw: Raw):
+		raise NotImplementedError("linked_raw is read-only.")
+
+	def to_parquet(self, save_path: str):
+		"""
+		Save the corpus to the AutoRAG compatible parquet file.
+		It is not for the data creation, for running AutoRAG.
+		If you want to save it directly, use the below code.
+		`corpus.data.to_parquet(save_path)`
+
+		:param save_path: The path to save the corpus.
+		"""
+		if not save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		save_df = self.data.reset_index(drop=True)
+		save_df.to_parquet(save_path)
+
+	def batch_apply(
+		self, fn: Callable[[Dict, Any], Awaitable[Dict]], batch_size: int = 32, **kwargs
+	) -> "Corpus":
+		corpus_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(corpus_dict, **kwargs) for corpus_dict in corpus_dicts]
+		results = loop.run_until_complete(process_batch(tasks, batch_size))
+		return Corpus(pd.DataFrame(results), self.linked_raw)
+
+	def map(
+		self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs
+	) -> "Corpus":
+		return Corpus(fn(self.data, **kwargs), self.linked_raw)
+
+	def sample(self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs) -> "QA":
+		"""
+		Sample the corpus for making QA.
+		It selects the subset of the corpus and makes QA set from it.
+		You can generate questions from the created question.
+		It is the first step to make QA set from the corpus.
+		If you select just one passage from each passage, it will be a single-hop QA set.
+		If you select multiple passages from each passage, it will be a multi-hop QA set.
+
+		:param fn: The select function to perform.
+		It returns QA dataframe.
+		:return: QA instance that is selected.
+		It contains qid and retrieval_gt columns.
+		"""
+		return QA(fn(self.data, **kwargs), self)
+
+
+class QA:
+	def __init__(
+		self,
+		qa_df: Optional[pd.DataFrame] = None,
+		linked_corpus: Optional[Corpus] = None,
+	):
+		self.data = qa_df
+		self._linked_corpus = linked_corpus
+
+	@property
+	def linked_corpus(self) -> Corpus:
+		return self._linked_corpus
+
+	@linked_corpus.setter
+	def linked_corpus(self, corpus: Corpus):
+		raise NotImplementedError("linked_corpus is read-only.")
+
+	def batch_apply(
+		self, fn: Callable[[Dict, Any], Awaitable[Dict]], batch_size: int = 32, **kwargs
+	) -> "QA":
+		qa_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(qa_dict, **kwargs) for qa_dict in qa_dicts]
+		results = loop.run_until_complete(process_batch(tasks, batch_size))
+
+		# Experimental feature
+		if fn.__name__ == "multiple_queries_gen":
+			return self._process_multiple_queries_gen(results)
+
+		return QA(pd.DataFrame(results), self.linked_corpus)
+
+	def batch_filter(
+		self, fn: Callable[[Dict, Any], Awaitable[bool]], batch_size: int = 32, **kwargs
+	) -> "QA":
+		qa_dicts = self.data.to_dict(orient="records")
+		loop = get_event_loop()
+		tasks = [fn(qa_dict, **kwargs) for qa_dict in qa_dicts]
+		masks = loop.run_until_complete(process_batch(tasks, batch_size))
+		return QA(self.data[masks], self.linked_corpus)
+
+	def filter(self, fn: Callable[[Dict, Any], bool], **kwargs) -> "QA":
+		qa_dicts = self.data.to_dict(orient="records")
+		masks = [fn(qa_dict, **kwargs) for qa_dict in qa_dicts]
+		return QA(self.data[masks], self.linked_corpus)
+
+	def map(self, fn: Callable[[pd.DataFrame, Any], pd.DataFrame], **kwargs) -> "QA":
+		return QA(fn(self.data, **kwargs), self.linked_corpus)
+
+	def make_retrieval_gt_contents(self) -> "QA":
+		"""
+		Make retrieval_gt_contents column from retrieval_gt column.
+		:return: The QA instance that has a retrieval_gt_contents column.
+		"""
+		self.data["retrieval_gt_contents"] = self.data["retrieval_gt"].apply(
+			lambda x: fetch_contents(self.linked_corpus.data, x)
+		)
+		return self
+
+	def to_parquet(self, qa_save_path: str, corpus_save_path: str):
+		"""
+		Save the qa and corpus to the AutoRAG compatible parquet file.
+		It is not for the data creation, for running AutoRAG.
+		If you want to save it directly, use the below code.
+		`qa.data.to_parquet(save_path)`
+
+		:param qa_save_path: The path to save the qa dataset.
+		:param corpus_save_path: The path to save the corpus.
+		"""
+		if not qa_save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		if not corpus_save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		save_df = self.data[
+			["qid", "query", "retrieval_gt", "generation_gt"]
+		].reset_index(drop=True)
+		save_df.to_parquet(qa_save_path)
+		self.linked_corpus.to_parquet(corpus_save_path)
+
+	def update_corpus(self, new_corpus: Corpus) -> "QA":
+		"""
+		Update linked corpus.
+		Not just replace linked_corpus to the new Corpus,
+		it replaces the whole `retrieval_gt` to the new corpus using `linked_raw`.
+		The QA data must have a `retrieval_gt` column.
+
+		:param new_corpus: Corpus that you want to replace.
+		    Must have valid `linked_raw` and `raw_id`, `raw_start_idx`, `raw_end_idx` columns.
+		:return: The QA instance that updated linked corpus.
+		"""
+		self.data["evidence_path"] = (
+			self.data["retrieval_gt"]
+			.apply(
+				lambda x: fetch_contents(
+					self.linked_corpus.data,
+					x,
+					column_name="path",
+				)
+			)
+			.tolist()
+		)
+		self.data["evidence_page"] = self.data["retrieval_gt"].apply(
+			lambda x: list(
+				map(
+					lambda lst: list(map(lambda x: x.get("page", -1), lst)),
+					fetch_contents(self.linked_corpus.data, x, column_name="metadata"),
+				)
+			)
+		)
+		if "evidence_start_end_idx" not in self.data.columns:
+			# make evidence start_end_idx
+			self.data["evidence_start_end_idx"] = (
+				self.data["retrieval_gt"]
+				.apply(
+					lambda x: fetch_contents(
+						self.linked_corpus.data,
+						x,
+						column_name="start_end_idx",
+					)
+				)
+				.tolist()
+			)
+
+		# matching the new corpus with the old corpus
+		path_corpus_dict = QA.__make_path_corpus_dict(new_corpus.data)
+		new_retrieval_gt = self.data.apply(
+			lambda row: QA.__match_index_row(
+				row["evidence_start_end_idx"],
+				row["evidence_path"],
+				row["evidence_page"],
+				path_corpus_dict,
+			),
+			axis=1,
+		).tolist()
+		new_qa = self.data.copy(deep=True)[["qid", "query", "generation_gt"]]
+		new_qa["retrieval_gt"] = new_retrieval_gt
+		return QA(new_qa, new_corpus)
+
+	@staticmethod
+	def __match_index(target_idx: Tuple[int, int], dst_idx: Tuple[int, int]) -> bool:
+		"""
+		Check if the target_idx is overlap by the dst_idx.
+		"""
+		target_start, target_end = target_idx
+		dst_start, dst_end = dst_idx
+		return (
+			dst_start <= target_start <= dst_end or dst_start <= target_end <= dst_end
+		)
+
+	@staticmethod
+	def __match_index_row(
+		evidence_indices: List[List[Tuple[int, int]]],
+		evidence_paths: List[List[str]],
+		evidence_pages: List[List[int]],
+		path_corpus_dict: Dict,
+	) -> List[List[str]]:
+		"""
+		Find the matched passage from new_corpus.
+
+		:param evidence_indices: The evidence indices at the corresponding Raw.
+		        Its shape is the same as the retrieval_gt.
+		:param evidence_paths: The evidence paths at the corresponding Raw.
+		        Its shape is the same as the retrieval_gt.
+		:param path_corpus_dict: The key is the path name, and the value is the corpus dataframe that only contains the path in the key.
+		        You can make it using `QA.__make_path_corpus_dict`.
+		:return:
+		"""
+		result = []
+		for i, idx_list in enumerate(evidence_indices):
+			sub_result = []
+			for j, idx in enumerate(idx_list):
+				path_corpus_df = path_corpus_dict[evidence_paths[i][j]]
+				if evidence_pages[i][j] >= 0:
+					path_corpus_df = path_corpus_df.loc[
+						path_corpus_df["metadata"].apply(lambda x: x.get("page", -1))
+						== evidence_pages[i][j]
+					]
+				matched_corpus = path_corpus_df.loc[
+					path_corpus_df["start_end_idx"].apply(
+						lambda x: QA.__match_index(idx, x)
+					)
+				]
+				sub_result.extend(matched_corpus["doc_id"].tolist())
+			result.append(sub_result)
+		return result
+
+	@staticmethod
+	def __make_path_corpus_dict(corpus_df: pd.DataFrame) -> Dict[str, pd.DataFrame]:
+		return {
+			path: corpus_df[corpus_df["path"] == path]
+			for path in corpus_df["path"].unique()
+		}
+
+	# Experimental feature
+	def _process_multiple_queries_gen(self, results: List[Dict]) -> "QA":
+		data = []
+		for result in results:
+			queries = result["query"].split("\n")
+			for query in queries:
+				new_result = {
+					key: (str(uuid.uuid4()) if key == "qid" else result[key])
+					for key in result.keys()
+				}
+				new_result["query"] = query
+				data.append(new_result)
+		df = pd.DataFrame(data)
+		return QA(df, self.linked_corpus)