fix: unify Poetry virtualenv to /opt/.venv and add cache-busting for dev deps (Linux dev)

Tiltfile

@@ -13,6 +13,26 @@ backend_debug = cfg.get("debug", False)
 core_library_context = "./libs"
 
 
+def libs_ignore_except(include_libs):
+    """Return a list of paths to ignore for docker builds under the project root.
+
+    include_libs: list of folder names under ./libs that should NOT be ignored.
+    All other folders under ./libs will be ignored to avoid triggering rebuilds when
+    unrelated libraries change.
+    """
+    all_libs = [
+        "admin-api-lib",
+        "extractor-api-lib",
+        "rag-core-api",
+        "rag-core-lib",
+    ]
+    ignore = []
+    for lib in all_libs:
+        if lib not in include_libs:
+            ignore.append("libs/%s/" % lib)
+    return ignore
+
+
 def create_linter_command(folder_name, name):
     # Use TEST=1 for libs Dockerfile, dev=1 for service Dockerfiles
     build_arg = "TEST=1" if folder_name == "./libs" else "dev=1"
@@ -145,9 +165,41 @@ local_resource(
 ################################## build backend_rag image and do live update ##########################################
 ########################################################################################################################
 
+# Base ignore patterns applied to docker_builds that reference IGNORE_BASE.
+# These are files and directories that should not trigger rebuilds: build artifacts,
+# Python caches, test caches, virtualenvs, node_modules, and other OS/tooling files.
 IGNORE_BASE = [
+    # project directories we don't want to include in build context
     "infrastructure/",
     "services/frontend/",
+    # Python caches and bytecode
+    "**/__pycache__/",
+    "**/*.pyc",
+    "**/*.pyo",
+    "**/*.py[co]",
+    ".pytest_cache/",
+    "**/.pytest_cache/",
+    ".mypy_cache/",
+    # virtualenvs / local python envs
+    ".venv/",
+    "venv/",
+    "env/",
+    # build artifacts
+    "build/",
+    "dist/",
+    "*.egg-info/",
+    # tooling caches
+    "node_modules/",
+    "**/node_modules/",
+    "services/frontend/node_modules/",
+    # OS / editor files
+    ".DS_Store",
+    "*.swp",
+    "*.swo",
+    # pytest / test caches inside libs
+    "**/.pytest_cache/",
+    # nix / package manager locks and temp files (optional)
+    "**/.cache/",
 ]
 
 # NOTE: full image names should match the one in the helm chart values.yaml!
@@ -168,7 +220,7 @@ docker_build(
         sync(core_library_context+"/rag-core-lib", "/app/libs/rag-core-lib"),
     ],
     dockerfile=backend_context + "/Dockerfile",
-    ignore=IGNORE_BASE
+    ignore=IGNORE_BASE + libs_ignore_except(["rag-core-api", "rag-core-lib"])
 )
 
 # Add linter trigger
@@ -208,7 +260,7 @@ docker_build(
         sync(mcp_context, "/app/services/mcp-server"),
     ],
     dockerfile=mcp_context + "/Dockerfile",
-    ignore=IGNORE_BASE,
+    ignore=IGNORE_BASE + libs_ignore_except([]),
 )
 
 # Add linter trigger
@@ -239,11 +291,13 @@ docker_build(
     },
     live_update=[
         sync(admin_backend_context, "/app/services/admin-backend"),
-        sync(core_library_context + "/rag-core-lib", "/app/libs/rag-core-lib"),
-        sync(core_library_context + "/admin-api-lib", "/app/libs/admin-api-lib"),
+    sync(core_library_context + "/rag-core-lib", "/app/libs/rag-core-lib"),
+    sync(core_library_context + "/admin-api-lib", "/app/libs/admin-api-lib"),
     ],
     dockerfile=admin_backend_context + "/Dockerfile",
-    ignore=IGNORE_BASE,
+    # Ignore rag-core-api for this build context so changes in that library
+    # don't trigger admin-backend rebuilds (admin-backend doesn't COPY rag-core-api)
+    ignore=IGNORE_BASE + libs_ignore_except(["rag-core-lib", "admin-api-lib"]),
 )
 
 # Add linter trigger
@@ -287,7 +341,7 @@ docker_build(
         sync(core_library_context +"/extractor-api-lib", "/app/libs/extractor-api-lib"),
         ],
     dockerfile=extractor_context + "/Dockerfile",
-    ignore=IGNORE_BASE,
+    ignore=IGNORE_BASE + libs_ignore_except(["extractor-api-lib"]),
 )
 
 # Add linter trigger
@@ -329,10 +383,35 @@ docker_build(
         sync("./services/frontend/dist/libs", "/usr/share/nginx/html/libs"),
     ],
     ignore=[
+        # exclude non-frontend areas
+        "libs/",
+        "services/admin-backend/",
+        "services/rag-backend/",
+        "services/document-extractor/",
+        "services/mcp-server/",
         "infrastructure/",
+        "tools/",
+        "docs/",
+        ".github/",
+        ".vscode/",
+        # caches/artifacts
+        "**/__pycache__/",
+        "**/.pytest_cache/",
+        ".mypy_cache/",
+        ".venv/",
+        "venv/",
+        "env/",
+        "build/",
+        "dist/",
+        "*.egg-info/",
+        # frontend-specific caches
         "services/frontend/.nx/",
         "services/frontend/tmp/",
         "services/frontend/node_modules/",
+        # OS/editor files
+        ".DS_Store",
+        "*.swp",
+        "*.swo",
     ],
 )
 
@@ -352,10 +431,35 @@ docker_build(
         sync("./services/frontend/dist/libs", "/usr/share/nginx/html/libs"),
     ],
     ignore=[
+        # exclude non-frontend areas
+        "libs/",
+        "services/admin-backend/",
+        "services/rag-backend/",
+        "services/document-extractor/",
+        "services/mcp-server/",
         "infrastructure/",
+        "tools/",
+        "docs/",
+        ".github/",
+        ".vscode/",
+        # caches/artifacts
+        "**/__pycache__/",
+        "**/.pytest_cache/",
+        ".mypy_cache/",
+        ".venv/",
+        "venv/",
+        "env/",
+        "build/",
+        "dist/",
+        "*.egg-info/",
+        # frontend-specific caches
         "services/frontend/.nx/",
         "services/frontend/tmp/",
         "services/frontend/node_modules/",
+        # OS/editor files
+        ".DS_Store",
+        "*.swp",
+        "*.swo",
     ],
 )
 

services/admin-backend/Dockerfile

@@ -1,8 +1,12 @@
 FROM --platform=linux/amd64 python:3.13-bookworm AS build
 
 ARG dev=0
-ENV POETRY_VIRTUALENVS_PATH=/app/services/admin-backend/.venv
-ENV POETRY_VERSION=2.1.3
+ENV POETRY_VIRTUALENVS_PATH=/opt/.venv \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_VIRTUALENVS_IN_PROJECT=false \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH=/opt/.venv/bin:$PATH \
+    POETRY_VERSION=2.1.3
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential --no-install-recommends make && \
@@ -19,7 +23,8 @@ COPY services/admin-backend/pyproject.toml services/admin-backend/poetry.lock ./
 RUN mkdir log && chmod 700 log
 RUN touch /app/services/admin-backend/log/logfile.log && chmod 600 /app/services/admin-backend/log/logfile.log
 
-RUN poetry config virtualenvs.create false && \
+# Bust cache when dev flag changes
+RUN echo "CACHE_BUST_DEV=$dev" && poetry config virtualenvs.create false && \
     if [ "$dev" = "1" ]; then \
         poetry install --no-interaction --no-ansi --no-root --with dev; \
     else \
@@ -31,7 +36,7 @@ ARG dev=0
 
 RUN adduser --disabled-password --gecos "" --uid 10001 nonroot
 
-ENV POETRY_VIRTUALENVS_PATH=/app/services/admin-backend/.venv
+ENV POETRY_VIRTUALENVS_PATH=/opt/.venv
 COPY --from=build --chown=nonroot:nonroot ${POETRY_VIRTUALENVS_PATH} ${POETRY_VIRTUALENVS_PATH}
 COPY --from=build /usr/local/bin/ /usr/local/bin/
 COPY --from=build /usr/bin/make /usr/bin/make
@@ -44,6 +49,12 @@ WORKDIR /app/services/admin-backend
 
 COPY --chown=nonroot:nonroot services/admin-backend .
 
+# Ensure poetry reuses existing virtualenv when running as nonroot
+ENV POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_VIRTUALENVS_IN_PROJECT=false \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH=/opt/.venv/bin:$PATH
+
 # cleanup
 RUN apt-get clean autoclean
 RUN apt-get autoremove --yes
@@ -52,13 +63,12 @@ RUN if [ "$dev" = "0" ]; then \
         while read -r shell; do rm -f "$shell"; done < /etc/shells; \
         rm -rf /var/lib/{apt,dpkg,cache,log}/ \
     else \
-        echo "POETRY_VIRTUALENVS_PATH=/app/services/admin-backend/.venv" >> /etc/environment;\
-        export POETRY_VIRTUALENVS_PATH=/app/services/admin-backend/.venv;\
+        echo "POETRY_VIRTUALENVS_PATH=/opt/.venv" >> /etc/environment;\
+        export POETRY_VIRTUALENVS_PATH=/opt/.venv;\
         export PATH="${POETRY_VIRTUALENVS_PATH}/bin:$PATH";\
     fi
 
 
 USER nonroot
 COPY --from=build --chown=nonroot:nonroot  /app/services/admin-backend/log /app/services/admin-backend/log
 
-ENV PATH="${POETRY_VIRTUALENVS_PATH}/bin:${PATH}"

services/document-extractor/Dockerfile

@@ -1,8 +1,12 @@
 FROM --platform=linux/amd64 python:3.13-bookworm AS build
 
 ARG dev=0
-ENV POETRY_VIRTUALENVS_PATH=/app/services/document-extractor/.venv
-ENV POETRY_VERSION=2.1.3
+ENV POETRY_VIRTUALENVS_PATH=/opt/.venv \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_VIRTUALENVS_IN_PROJECT=false \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH=/opt/.venv/bin:$PATH \
+    POETRY_VERSION=2.1.3
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential --no-install-recommends make \
@@ -23,7 +27,8 @@ COPY services/document-extractor/pyproject.toml services/document-extractor/poet
 RUN mkdir log && chmod 700 log
 RUN touch /app/services/document-extractor/log/logfile.log && chmod 600 /app/services/document-extractor/log/logfile.log
 
-RUN poetry config virtualenvs.create false &&\
+# Cache bust with dev arg so dev dependencies install when dev=1
+RUN echo "CACHE_BUST_DEV=$dev" && poetry config virtualenvs.create false &&\
     if [ "$dev" = "1" ]; then \
         poetry install --no-interaction --no-ansi --no-root --with dev; \
     else \
@@ -35,7 +40,7 @@ ARG dev=0
 
 RUN adduser --disabled-password --gecos "" --uid 10001 nonroot
 
-ENV POETRY_VIRTUALENVS_PATH=/app/services/document-extractor/.venv
+ENV POETRY_VIRTUALENVS_PATH=/opt/.venv
 COPY --from=build  --chown=nonroot:nonroot ${POETRY_VIRTUALENVS_PATH} ${POETRY_VIRTUALENVS_PATH}
 COPY --from=build /usr/local/bin/ /usr/local/bin/
 COPY --from=build /usr/bin/ /usr/bin/
@@ -50,6 +55,12 @@ WORKDIR /app/services/document-extractor
 COPY --chown=nonroot:nonroot services/document-extractor .
 
 
+# Ensure poetry reuses existing virtualenv when running as nonroot
+ENV POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_VIRTUALENVS_IN_PROJECT=false \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH=/opt/.venv/bin:$PATH
+
 # cleanup
 RUN apt-get clean autoclean
 RUN apt-get autoremove --yes
@@ -58,13 +69,11 @@ RUN if [ "$dev" = "0" ]; then \
         while read -r shell; do rm -f "$shell"; done < /etc/shells; \
         rm -rf /var/lib/{apt,dpkg,cache,log}/ \
     else \
-        echo "POETRY_VIRTUALENVS_PATH=/app/services/document-extractor/.venv" >> /etc/environment;\
-        export POETRY_VIRTUALENVS_PATH=/app/services/document-extractor/.venv;\
+        echo "POETRY_VIRTUALENVS_PATH=/opt/.venv" >> /etc/environment;\
+        export POETRY_VIRTUALENVS_PATH=/opt/.venv;\
         export PATH="${POETRY_VIRTUALENVS_PATH}/bin:$PATH";\
     fi
 
 
 USER nonroot
 COPY --from=build --chown=nonroot:nonroot  /app/services/document-extractor/log /app/services/document-extractor/log
-
-ENV PATH="${POETRY_VIRTUALENVS_PATH}/bin:${PATH}"

services/mcp-server/Dockerfile

@@ -1,8 +1,12 @@
 FROM --platform=linux/amd64 python:3.13.7-bookworm AS build
 
 ARG dev=0
-ENV POETRY_VIRTUALENVS_PATH=/app/services/mcp-server/.venv
-ENV POETRY_VERSION=2.1.3
+ENV POETRY_VIRTUALENVS_PATH=/opt/.venv \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_VIRTUALENVS_IN_PROJECT=false \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH=/opt/.venv/bin:$PATH \
+    POETRY_VERSION=2.1.3
 
 WORKDIR /app
 
@@ -19,7 +23,8 @@ COPY services/mcp-server/pyproject.toml services/mcp-server/poetry.lock ./
 RUN mkdir log && chmod 700 log
 RUN touch /app/services/mcp-server/log/logfile.log && chmod 600 /app/services/mcp-server/log/logfile.log
 
-RUN poetry config virtualenvs.create false &&\
+# Cache bust with dev arg so dev dependencies install when dev=1
+RUN echo "CACHE_BUST_DEV=$dev" && poetry config virtualenvs.create false &&\
     if [ "$dev" = "1" ]; then \
     poetry install --no-interaction --no-ansi --no-root --with dev; \
     else \
@@ -31,7 +36,8 @@ ARG dev=0
 
 RUN adduser --disabled-password --gecos "" --uid 10001 nonroot
 
-ENV POETRY_VIRTUALENVS_PATH=/app/services/mcp-server/.venv
+ENV POETRY_VIRTUALENVS_PATH=/opt/.venv
+
 COPY --from=build --chown=nonroot:nonroot ${POETRY_VIRTUALENVS_PATH} ${POETRY_VIRTUALENVS_PATH}
 COPY --from=build /usr/local/bin/ /usr/local/bin/
 COPY --from=build /usr/bin/make /usr/bin/make
@@ -42,6 +48,13 @@ COPY --chown=nonroot:nonroot services/mcp-server/src ./src
 COPY --chown=nonroot:nonroot services/mcp-server/tests ./tests
 COPY --chown=nonroot:nonroot services/mcp-server/pyproject.toml services/mcp-server/poetry.lock ./
 COPY --chown=nonroot:nonroot services/mcp-server/Makefile ./
+
+# Ensure poetry reuses existing virtualenv when running as nonroot
+ENV POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_VIRTUALENVS_IN_PROJECT=false \
+    VIRTUAL_ENV=/opt/.venv \
+    PATH=/opt/.venv/bin:$PATH
+
 # cleanup
 RUN apt-get clean autoclean
 RUN apt-get autoremove --yes
@@ -50,15 +63,13 @@ RUN if [ "$dev" = "0" ]; then \
         while read -r shell; do rm -f "$shell"; done < /etc/shells; \
         rm -rf /var/lib/{apt,dpkg,cache,log}/ \
     else \
-        echo "POETRY_VIRTUALENVS_PATH=/app/services/mcp-server/.venv" >> /etc/environment;\
-        export POETRY_VIRTUALENVS_PATH=/app/services/mcp-server/.venv;\
+        echo "POETRY_VIRTUALENVS_PATH=/opt/.venv" >> /etc/environment;\
+        export POETRY_VIRTUALENVS_PATH=/opt/.venv;\
         export PATH="${POETRY_VIRTUALENVS_PATH}/bin:$PATH";\
     fi
 
 
 USER nonroot
 COPY --from=build --chown=nonroot:nonroot  /app/services/mcp-server/log /app/services/mcp-server/log
 
-ENV PATH="${POETRY_VIRTUALENVS_PATH}/bin:${PATH}"
-
 CMD [ "poetry", "run", "python", "src/main.py" ]