Initial commit

projects/example_01/chunk/chunk_config.yaml

@@ -0,0 +1,13 @@
+modules:
+  - module_type: llama_index_chunk
+    chunk_method: Token
+    chunk_size: [200, 1000]
+    chunk_overlap: [30, 200]
+    add_file_name: ko 
+  - module_type: llama_index_chunk
+    chunk_method: Semantic_llama_index
+    embed_model: hf_snowflake-arctic-embed-l-v2.0-ko
+    add_file_name: ko 
+  - module_type: langchain_chunk
+    chunk_method: recursivecharacter
+    separators: [ " ", "\n" ]

projects/example_01/chunk/summary.csv

@@ -0,0 +1,8 @@
+filename,module_name,module_params,execution_time
+0.parquet,llama_index_chunk,"{'chunk_method': 'Token', 'chunk_size': 200, 'chunk_overlap': 200, 'add_file_name': 'ko'}",9.001352617045057e-05
+1.parquet,llama_index_chunk,"{'chunk_method': 'Token', 'chunk_size': 200, 'chunk_overlap': 30, 'add_file_name': 'ko'}",3.7807608560139556e-05
+2.parquet,llama_index_chunk,"{'chunk_method': 'Token', 'chunk_size': 1000, 'chunk_overlap': 200, 'add_file_name': 'ko'}",2.4567047525651943e-05
+3.parquet,llama_index_chunk,"{'chunk_method': 'Token', 'chunk_size': 1000, 'chunk_overlap': 30, 'add_file_name': 'ko'}",2.697287288367227e-05
+4.parquet,llama_index_chunk,"{'chunk_method': 'Semantic_llama_index', 'embed_model': 'hf_snowflake-arctic-embed-l-v2.0-ko', 'add_file_name': 'ko'}",0.006124294066363658
+5.parquet,langchain_chunk,"{'chunk_method': 'recursivecharacter', 'separators': ' '}",1.7906897601052548e-06
+6.parquet,langchain_chunk,"{'chunk_method': 'recursivecharacter', 'separators': '\n'}",2.2023862824233143e-06