feat: add semantic chunker & centralize embeddings in rag-core-lib; helm + deps updates

infrastructure/rag/templates/admin-backend/deployment.yaml

@@ -23,6 +23,8 @@ spec:
           emptyDir: {}
         - name: tmp-dir
           emptyDir: {}
+        - name: nltk-data-dir
+          emptyDir: {}
       {{- if .Values.shared.imagePullSecret }}
       imagePullSecrets:
         - name: {{ .Values.shared.imagePullSecret.name }}
@@ -35,10 +37,16 @@ spec:
           - -c
           - |
             touch /app/services/admin-backend/log/logfile.log && \
-            chmod 600 /app/services/admin-backend/log/logfile.log
+            chmod 600 /app/services/admin-backend/log/logfile.log;
+            wget -q -O /tmp/punkt.zip https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt_tab.zip && \
+            unzip /tmp/punkt.zip -d /home/nonroot/nltk_data/tokenizers && \
+            wget -q -O /tmp/averaged_perceptron_tagger_eng.zip https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/taggers/averaged_perceptron_tagger_eng.zip && \
+            unzip /tmp/averaged_perceptron_tagger_eng.zip -d /home/nonroot/nltk_data/taggers;
         volumeMounts:
         - name: log-dir
           mountPath: /app/services/admin-backend/log
+        - name: nltk-data-dir
+          mountPath: /home/nonroot/nltk_data
       containers:
       - name: {{ .Values.adminBackend.name }}
         securityContext:
@@ -108,6 +116,8 @@ spec:
               name: {{ template "configmap.sourceUploaderName" . }}
           - configMapRef:
               name: {{ template "configmap.retryDecoratorName" . }}
+          - configMapRef:
+              name: {{ template "configmap.chunkerName" . }}
           - secretRef:
               name: {{ template "secret.langfuseName" . }}
           - secretRef:
@@ -116,6 +126,8 @@ spec:
               name: {{ template "secret.s3Name" . }}
           - secretRef:
               name: {{ template "secret.stackitVllmName" . }}
+          - secretRef:
+              name: {{ template "secret.stackitEmbedderName" . }}
         env:
           - name: PYTHONPATH
             value: {{ .Values.adminBackend.pythonPathEnv.PYTHONPATH }}

infrastructure/rag/values.yaml

@@ -338,8 +338,16 @@ adminBackend:
     ragapi:
       RAG_API_HOST: "http://backend:8080"
     chunker:
+      # Select which chunker implementation to use. Supported values: "semantic", "recursive"
+      # Defaults to "semantic" which leverages sentence-aware rebalancing.
+      CHUNKER_CLASS_TYPE_CHUNKER_TYPE: "recursive"
       CHUNKER_MAX_SIZE: 1000
-      CHUNKER_OVERLAP: 300
+      CHUNKER_OVERLAP: 100
+      # The following settings for the Chunker are only used when CHUNKER_CLASS_TYPE_CHUNKER_TYPE is set to "semantic".
+      CHUNKER_BREAKPOINT_THRESHOLD_TYPE: "percentile"
+      CHUNKER_BREAKPOINT_THRESHOLD_AMOUNT: 95
+      CHUNKER_BUFFER_SIZE: 1
+      CHUNKER_MIN_SIZE: 200
     keyValueStore:
       USECASE_KEYVALUE_PORT: 6379
       USECASE_KEYVALUE_HOST: "rag-keydb"

libs/admin-api-lib/pyproject.toml

@@ -27,9 +27,9 @@ per-file-ignores = """
   ./src/admin_api_lib/prompt_templates/summarize_prompt.py: E501,
   ./src/admin_api_lib/apis/admin_api.py: B008,WOT001,
   ./src/admin_api_lib/impl/admin_api.py: B008,
-  ./src/admin_api_lib/dependency_container.py: CCE002,CCE001,
+  ./src/admin_api_lib/dependency_container.py: CCE002,CCE001,WOT001,
   ./src/admin_api_lib/apis/admin_api_base.py: WOT001,
-  ./tests/*: S101,S106,D100,D103,PT011,N802
+  ./tests/*: S101,S106,D100,D103,PT011,N802,E501,
   ./src/admin_api_lib/impl/settings/confluence_settings.py: C901,N805,
   ./src/admin_api_lib/impl/utils/comma_separated_bool_list.py: R505,
   ./src/admin_api_lib/impl/utils/comma_separated_str_list.py: R505,
@@ -109,10 +109,11 @@ redis = "^6.0.0"
 pyyaml = "^6.0.2"
 python-multipart = "^0.0.20"
 starlette = ">=0.47.2,<0.49.0"
-langchain-text-splitters = ">=0.3.9"
+langchain-experimental = "^0.3.4"
+nltk = "^3.9.2"
 
 [tool.pytest.ini_options]
-log_cli = 1
+log_cli = true
 log_cli_level = "DEBUG"
 pythonpath = "src"
 testpaths = "src/tests"

libs/admin-api-lib/src/admin_api_lib/dependency_container.py

@@ -2,13 +2,9 @@
 
 from admin_api_lib.impl.api_endpoints.default_file_uploader import DefaultFileUploader
 from dependency_injector.containers import DeclarativeContainer
-from dependency_injector.providers import (  # noqa: WOT001
-    Configuration,
-    List,
-    Selector,
-    Singleton,
-)
+from dependency_injector.providers import Configuration, List, Selector, Singleton
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings import OllamaEmbeddings
 from langfuse import Langfuse
 
 from admin_api_lib.extractor_api_client.openapi_client.api.extractor_api import (
@@ -29,6 +25,7 @@
 from admin_api_lib.impl.api_endpoints.default_documents_status_retriever import (
     DefaultDocumentsStatusRetriever,
 )
+from admin_api_lib.impl.chunker.semantic_text_chunker import SemanticTextChunker
 from admin_api_lib.impl.chunker.text_chunker import TextChunker
 from admin_api_lib.impl.file_services.s3_service import S3Service
 from admin_api_lib.impl.information_enhancer.general_enhancer import GeneralEnhancer
@@ -41,6 +38,7 @@
 from admin_api_lib.impl.mapper.informationpiece2document import (
     InformationPiece2Document,
 )
+from admin_api_lib.impl.settings.chunker_class_type_settings import ChunkerClassTypeSettings
 from admin_api_lib.impl.settings.chunker_settings import ChunkerSettings
 from admin_api_lib.impl.settings.document_extractor_settings import (
     DocumentExtractorSettings,
@@ -59,12 +57,21 @@
 from admin_api_lib.rag_backend_client.openapi_client.configuration import (
     Configuration as RagConfiguration,
 )
+from rag_core_lib.impl.embeddings.langchain_community_embedder import (
+    LangchainCommunityEmbedder,
+)
+from rag_core_lib.impl.embeddings.stackit_embedder import StackitEmbedder
 from rag_core_lib.impl.langfuse_manager.langfuse_manager import LangfuseManager
 from rag_core_lib.impl.llms.llm_factory import chat_model_provider
+from rag_core_lib.impl.settings.embedder_class_type_settings import (
+    EmbedderClassTypeSettings,
+)
 from rag_core_lib.impl.settings.langfuse_settings import LangfuseSettings
+from rag_core_lib.impl.settings.ollama_embedder_settings import OllamaEmbedderSettings
 from rag_core_lib.impl.settings.ollama_llm_settings import OllamaSettings
 from rag_core_lib.impl.settings.rag_class_types_settings import RAGClassTypeSettings
 from rag_core_lib.impl.settings.retry_decorator_settings import RetryDecoratorSettings
+from rag_core_lib.impl.settings.stackit_embedder_settings import StackitEmbedderSettings
 from rag_core_lib.impl.settings.stackit_vllm_settings import StackitVllmSettings
 from rag_core_lib.impl.tracers.langfuse_traced_runnable import LangfuseTracedRunnable
 from rag_core_lib.impl.utils.async_threadsafe_semaphore import AsyncThreadsafeSemaphore
@@ -74,10 +81,14 @@ class DependencyContainer(DeclarativeContainer):
     """Dependency injection container for managing application dependencies."""
 
     class_selector_config = Configuration()
+    chunker_selector_config = Configuration()
 
     # Settings
     s3_settings = S3Settings()
     chunker_settings = ChunkerSettings()
+    chunker_embedder_type_settings = EmbedderClassTypeSettings()
+    stackit_chunker_embedder_settings = StackitEmbedderSettings()
+    ollama_chunker_embedder_settings = OllamaEmbedderSettings()
     ollama_settings = OllamaSettings()
     langfuse_settings = LangfuseSettings()
     stackit_vllm_settings = StackitVllmSettings()
@@ -88,6 +99,10 @@ class DependencyContainer(DeclarativeContainer):
     summarizer_settings = SummarizerSettings()
     source_uploader_settings = SourceUploaderSettings()
     retry_decorator_settings = RetryDecoratorSettings()
+    chunker_type_settings = ChunkerClassTypeSettings()
+
+    class_selector_config.from_dict(rag_class_type_settings.model_dump() | chunker_embedder_type_settings.model_dump())
+    chunker_selector_config.from_dict(chunker_type_settings.model_dump())
 
     key_value_store = Singleton(FileStatusKeyValueStore, key_value_store_settings)
     file_service = Singleton(S3Service, s3_settings=s3_settings)
@@ -96,7 +111,40 @@ class DependencyContainer(DeclarativeContainer):
         chunk_size=chunker_settings.max_size, chunk_overlap=chunker_settings.overlap
     )
 
-    chunker = Singleton(TextChunker, text_splitter)
+    semantic_chunker_embeddings = Selector(
+        class_selector_config.embedder_type,
+        stackit=Singleton(
+            StackitEmbedder,
+            stackit_chunker_embedder_settings,
+            retry_decorator_settings,
+        ),
+        ollama=Singleton(
+            LangchainCommunityEmbedder,
+            embedder=Singleton(
+                OllamaEmbeddings,
+                model=ollama_chunker_embedder_settings.model,
+                base_url=ollama_chunker_embedder_settings.base_url,
+            ),
+        ),
+    )
+
+    semantic_chunker = Singleton(
+        SemanticTextChunker,
+        embeddings=semantic_chunker_embeddings,
+        breakpoint_threshold_type=chunker_settings.breakpoint_threshold_type,
+        breakpoint_threshold_amount=chunker_settings.breakpoint_threshold_amount,
+        buffer_size=chunker_settings.buffer_size,
+        min_chunk_size=chunker_settings.min_size,
+        max_chunk_size=chunker_settings.max_size,
+        recursive_text_splitter=text_splitter,
+        overlap=chunker_settings.overlap,
+    )
+
+    chunker = Selector(
+        chunker_selector_config.chunker_type,
+        recursive=Singleton(TextChunker, text_splitter),
+        semantic=semantic_chunker,
+    )
     extractor_api_configuration = Singleton(ExtractorConfiguration, host=document_extractor_settings.host)
     document_extractor_api_client = Singleton(ApiClient, extractor_api_configuration)
     document_extractor = Singleton(ExtractorApi, document_extractor_api_client)

libs/admin-api-lib/src/admin_api_lib/impl/chunker/chunker_type.py

@@ -0,0 +1,11 @@
+"""Module containing the ChunkerType enumeration."""
+
+from enum import StrEnum, unique
+
+
+@unique
+class ChunkerType(StrEnum):
+    """An enumeration representing different types of chunkers."""
+
+    SEMANTIC = "semantic"
+    RECURSIVE = "recursive"