Release 0.4.58

.github/workflows/develop.yml

@@ -7,6 +7,10 @@ on:
 
 jobs:
   build-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        protocol: [http, non_http]
 
     runs-on:
     - benchmark
@@ -30,6 +34,7 @@ jobs:
 
     - name: Build and Run Tests
       env:
+        TEST_PROTOCOL: ${{ matrix.protocol }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -66,4 +71,3 @@ jobs:
         LENZ_TAG: dev
       run: BUILD_COMPLETE=true ./ci.sh
       shell: bash
-

.github/workflows/main.yml

@@ -7,6 +7,10 @@ on:
 
 jobs:
   build_and_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        protocol: [http, non_http]
 
     runs-on:
     - benchmark
@@ -30,6 +34,7 @@ jobs:
 
     - name: Build and Run Tests
       env:
+        TEST_PROTOCOL: ${{ matrix.protocol }}
         AWS_ACCESS_KEY_ID:     ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_DEFAULT_REGION:    ${{ secrets.AWS_DEFAULT_REGION }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/pr.yaml

@@ -7,6 +7,10 @@ on:
 
 jobs:
   run_test:
+    strategy:
+      fail-fast: false
+      matrix:
+        protocol: [http, non_http]
 
     runs-on:
     - gcp
@@ -34,11 +38,12 @@ jobs:
 
     - name: Build and Run Tests
       env:
+        TEST_PROTOCOL: ${{ matrix.protocol }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-        RUNNER_NAME: ${{ runner.name }}
+        RUNNER_NAME: ${{ runner.name }}_${{ matrix.protocol }}
         RUN_TESTS: true
         NO_PUSH: true
         BRANCH_NAME: ${{ github.event.pull_request.head.ref }}

.github/workflows/release.yaml

@@ -7,6 +7,10 @@ on:
 
 jobs:
   build-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        protocol: [http, non_http]
 
     runs-on:
     - self-hosted
@@ -35,10 +39,12 @@ jobs:
 
     - name: Build and Run Tests
       env:
+        TEST_PROTOCOL: ${{ matrix.protocol }}
         AWS_ACCESS_KEY_ID:     ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_DEFAULT_REGION:    ${{ secrets.AWS_DEFAULT_REGION }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        RUNNER_NAME: ${{ runner.name }}_${{ matrix.protocol }}
       run: RUN_TESTS=true NO_PUSH=true UPDATE_BRANCH=true ./ci.sh
       shell: bash
 
@@ -64,4 +70,4 @@ jobs:
 
     - name: Build Notebook Docker
       run: BUILD_COMPLETE=true NO_PUSH=true ./ci.sh
-      shell: bash
+      shell: bash

aperturedb/__init__.py

@@ -10,7 +10,7 @@
 import signal
 import sys
 
-__version__ = "0.4.57"
+__version__ = "0.4.58"
 
 logger = logging.getLogger(__name__)
 

ci.sh

@@ -186,9 +186,9 @@ push_aws_ecr(){
     PREFIX="aperturedata/"
     docker tag ${SRC_IMAGE} \
         684446431133.dkr.ecr.${REGION}.amazonaws.com/${DST_IMAGE}
-    aws ecr get-login-password --region ${REGION} | docker login --username AWS --password-stdin 684446431133.dkr.ecr.${REGION}.amazonaws.com
+    docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION -e AWS_SECRET_ACCESS_KEY amazon/aws-cli ecr get-login-password --region ${REGION} | docker login --username AWS --password-stdin 684446431133.dkr.ecr.${REGION}.amazonaws.com
 
-    aws ecr create-repository --repository-name ${ECR_REPO_NAME} --region us-west-2  || true
+    docker run --rm -e AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION -e AWS_SECRET_ACCESS_KEY amazon/aws-cli ecr create-repository --repository-name ${ECR_REPO_NAME} --region us-west-2  || true
 
     docker push 684446431133.dkr.ecr.${REGION}.amazonaws.com/${DST_IMAGE}
 }

robots.md

@@ -0,0 +1,69 @@
+# ApertureDB Python Testing Guide
+
+This guide is designed for automated agents and LLMs to quickly understand how to build and run the test suite for `py-aperturedb` locally.
+
+## Prerequisites
+The integration tests rely on local `aperturedb` + `lenz` containers orchestrated by Docker Compose. The environment runs a client container to execute the `pytest` suite against these spun-up backends.
+
+---
+
+## 1. Building the Test Environment Image
+
+To run the tests locally, you first need to build the `aperturedb-python-tests:latest` Docker image. From the workspace root (`/home/remis/src/ad/py-aperturedb`), populate the expected context directory and execute `docker build`:
+
+```bash
+# 1. Provide the expected context data for the test Dockerfile
+mkdir -p test/aperturedb/logs/runner_state
+mkdir -p docker/tests/aperturedata
+
+# 2. Copy the Python source and configurations over
+cp -r aperturedb pyproject.toml README.md docker/tests/aperturedata
+mkdir -m 777 -p docker/tests/aperturedata/test/aperturedb
+
+# 3. Copy test inputs and test scripts over
+cp -r test/*.py test/*.sh test/input docker/tests/aperturedata/test
+
+# 4. Build the test image locally
+docker build -t aperturedata/aperturedb-python-tests:latest -f docker/tests/Dockerfile .
+```
+
+---
+
+## 2. Running the Complete Test Suite
+
+Once the test image is built, use the `run_test_container.sh` wrapper script located in the `test/` directory to coordinate the backend integration environment and run the test suite.
+
+```bash
+cd test
+set -a && source .env && set +a
+bash run_test_container.sh
+```
+
+This script:
+1. Orchestrates spinning up `aperturedb`, `nginx`, `lenz` containers using `docker-compose.yml` on temporary networking bridges.
+2. Starts two parallel test suites (HTTP and non-HTTP modes).
+3. Checks exit codes for successes or failures and cleans up.
+
+---
+
+## 3. Running tests and bypassing AWS/GCP API requirements
+
+A subset of the `pytest` tests interact with external objects, such as fetching blob configurations directly from cloud providers (AWS S3, Google Storage - GS). If your local environment or the automated agent lacks valid AWS/GCP credentials (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `GCP_SERVICE_ACCOUNT_KEY`), you will hit authentication validation errors when resolving `storage.Client()` architectures or boto3 structures during execution.
+
+To dynamically skip external storage tests, you should inform `pytest` running inside the Docker sequence to skip functions interacting with S3 and Google Storage.
+
+Before running the test script, manually modify the `pytest` invocation inside `test/run_test.sh` to explicitly add `-k "not S3 and not GS"` to exclude any specific loading functions targeting S3 or GS, along with keeping away tests appropriately marked with `@pytest.mark.remote_credentials`.
+
+Use this quick `sed` command to patch the exclusion:
+
+```bash
+cd test
+# Modify the pytest invocation to ignore AWS/GCP specific loaders and remotely constrained mark configurations
+sed -i 's/pytest -m "$FILTER"/pytest -k "not test_S3ImageLoader and not test_GSImageLoader and not test_S3VideoLoader and not test_GSVideoLoader" -m "$FILTER and not remote_credentials and not external_network"/' run_test.sh
+
+# Then run the tests as usual
+set -a && source .env && set +a
+bash run_test_container.sh
+```
+
+By applying this change, any LLM/agent can successfully execute internal `aperturedb` connector + python-centric API tests without raising failures tied strictly to external cloud requirements.

test/run_test.sh

@@ -43,7 +43,7 @@ if [[ $RESULT != 0 ]]; then
 		ARCHIVE_NAME=logs.tar.gz
 		DESTINATION="s3://${BUCKET}/aperturedb-${NOW}-${FILTER// /_}.tgz"
 		tar czf ${ARCHIVE_NAME} ${APERTUREDB_LOG_PATH}/..
-		aws s3 cp ${ARCHIVE_NAME} $DESTINATION
+		docker run --rm -v $(pwd):/workspace -w /workspace -e AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION -e AWS_SECRET_ACCESS_KEY amazon/aws-cli s3 cp ${ARCHIVE_NAME} $DESTINATION
 		echo "Log output to $DESTINATION"
 	else
 		echo "Unable to output log, APERTUREDB_LOG_PATH not set."

test/run_test_container.sh

@@ -30,8 +30,16 @@ function run_aperturedb_instance(){
 
 IP_REGEX='[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}'
 
-GATEWAY_HTTP=$(run_aperturedb_instance "${RUNNER_NAME}_http" | grep $IP_REGEX )
-GATEWAY_NON_HTTP=$(run_aperturedb_instance "${RUNNER_NAME}_non_http" | grep $IP_REGEX )
+# Check if TEST_PROTOCOL is set, otherwise default to both
+TEST_PROTOCOL=${TEST_PROTOCOL:-"both"}
+
+if [ "$TEST_PROTOCOL" == "http" ] || [ "$TEST_PROTOCOL" == "both" ]; then
+    GATEWAY_HTTP=$(run_aperturedb_instance "${RUNNER_NAME}_http" | grep $IP_REGEX )
+fi
+
+if [ "$TEST_PROTOCOL" == "non_http" ] || [ "$TEST_PROTOCOL" == "both" ]; then
+    GATEWAY_NON_HTTP=$(run_aperturedb_instance "${RUNNER_NAME}_non_http" | grep $IP_REGEX )
+fi
 
 # The LOG_PATH and RUNNER_INFO_PATH are set to the current working directory
 LOG_PATH="$(pwd)/aperturedb/logs"
@@ -48,42 +56,57 @@ fi
 
 sleep 20 # wait for the containers to be up and running
 
-echo "running tests on docker image $REPOSITORY with $GATEWAY_HTTP and $GATEWAY_NON_HTTP"
-docker run \
-    -v $(pwd)/output:/aperturedata/test/output \
-    -v $(pwd)/${RUNNER_NAME}_http_ca:/ca \
-    --network=${RUNNER_NAME}_http_default \
-    -v "$LOG_PATH":"${TESTING_LOG_PATH}" \
-    -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-    -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION \
-    -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-    -e GCP_SERVICE_ACCOUNT_KEY="$GCP_SERVICE_ACCOUNT_KEY" \
-    -e APERTUREDB_LOG_PATH="${TESTING_LOG_PATH}" \
-    -e GATEWAY="nginx" \
-    -e FILTER="http" \
-    $REPOSITORY &
-
-pid1=$!
-docker run \
-    -v $(pwd)/output:/aperturedata/test/output \
-    -v $(pwd)/${RUNNER_NAME}_non_http_ca:/ca \
-    --network=${RUNNER_NAME}_non_http_default \
-    -v "$LOG_PATH":"${TESTING_LOG_PATH}" \
-    -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-    -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION \
-    -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-    -e GCP_SERVICE_ACCOUNT_KEY="$GCP_SERVICE_ACCOUNT_KEY" \
-    -e APERTUREDB_LOG_PATH="${TESTING_LOG_PATH}" \
-    -e GATEWAY="lenz" \
-    -e FILTER="not http" \
-    $REPOSITORY &
-
-pid2=$!
-
-wait $pid1
-exit_code1=$?
-wait $pid2
-exit_code2=$?
+pid1=0
+pid2=0
+
+if [ "$TEST_PROTOCOL" == "http" ] || [ "$TEST_PROTOCOL" == "both" ]; then
+    echo "running tests on docker image $REPOSITORY with $GATEWAY_HTTP"
+    docker run \
+        -v $(pwd)/output:/aperturedata/test/output \
+        -v $(pwd)/${RUNNER_NAME}_http_ca:/ca \
+        --network=${RUNNER_NAME}_http_default \
+        -v "$LOG_PATH":"${TESTING_LOG_PATH}" \
+        -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+        -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION \
+        -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+        -e GCP_SERVICE_ACCOUNT_KEY="$GCP_SERVICE_ACCOUNT_KEY" \
+        -e APERTUREDB_LOG_PATH="${TESTING_LOG_PATH}" \
+        -e GATEWAY="nginx" \
+        -e FILTER="http" \
+        $REPOSITORY &
+    pid1=$!
+fi
+
+if [ "$TEST_PROTOCOL" == "non_http" ] || [ "$TEST_PROTOCOL" == "both" ]; then
+    echo "running tests on docker image $REPOSITORY with $GATEWAY_NON_HTTP"
+    docker run \
+        -v $(pwd)/output:/aperturedata/test/output \
+        -v $(pwd)/${RUNNER_NAME}_non_http_ca:/ca \
+        --network=${RUNNER_NAME}_non_http_default \
+        -v "$LOG_PATH":"${TESTING_LOG_PATH}" \
+        -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+        -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION \
+        -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+        -e GCP_SERVICE_ACCOUNT_KEY="$GCP_SERVICE_ACCOUNT_KEY" \
+        -e APERTUREDB_LOG_PATH="${TESTING_LOG_PATH}" \
+        -e GATEWAY="lenz" \
+        -e FILTER="not http" \
+        $REPOSITORY &
+    pid2=$!
+fi
+
+exit_code1=0
+exit_code2=0
+
+if [ "$pid1" != "0" ]; then
+    wait $pid1
+    exit_code1=$?
+fi
+
+if [ "$pid2" != "0" ]; then
+    wait $pid2
+    exit_code2=$?
+fi
 
 if [ $exit_code1 -ne 0 ]; then
     echo "Tests failed for HTTP"
@@ -94,7 +117,6 @@ if [ $exit_code2 -ne 0 ]; then
     exit $exit_code2
 fi
 
-
 echo "Tests completed"
 echo " --- Runner name: ${RUNNER_NAME} ---"
 check_containers_networks