diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c57a3c02 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,156 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# VSCode +.vscode/ + +#Data files +*.adb.csv +*.jpg +*.npy +test/aperturedb/db*/ +test/input/blobs/ +docs/examples/ +examples/*/coco +examples/*/classification.txt +kaggleds/ +examples/*/kaggleds/ +docs/*/*.svg +test/aperturedb/log* +adb-python/* +/test/input/ +/test/input/images/ + +.aperturedb +test/data/ +test/aperturedb/certificate*/ +.devcontainer/aperturedb/ +.devcontainer/ca/ +test/*_ca/ \ No newline at end of file diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1310bd27..924ffc88 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,6 +13,10 @@ jobs: steps: + - name: Cleanup previous run + run: docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb/db*" + continue-on-error: true + - uses: actions/checkout@v3 - name: Login to DockerHub diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ea9097b6..c903f5d5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,6 +14,10 @@ jobs: steps: + - name: Cleanup previous run + run: docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb/db* /workspace/test/aperturedb/logs" + continue-on-error: true + - uses: actions/checkout@v3 - name: Login to DockerHub @@ -46,6 +50,10 @@ jobs: steps: + - name: Cleanup previous run + run: docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb/db* /workspace/test/aperturedb/logs" + continue-on-error: true + - uses: actions/checkout@v3 - name: Login to DockerHub @@ -56,4 +64,4 @@ jobs: - name: Build Notebook Docker run: BUILD_COMPLETE=true NO_PUSH=true ./ci.sh - shell: bash + shell: bash \ No newline at end of file diff --git a/aperturedb/Connector.py b/aperturedb/Connector.py index 80705f6d..b0f63a2f 100644 --- a/aperturedb/Connector.py +++ b/aperturedb/Connector.py @@ -236,8 +236,7 @@ def _send_msg(self, data): sent_len = struct.pack(MESSAGE_LENGTH_FORMAT, len(data)) # send size first - x = self.conn.send(sent_len + data) - return x == len(data) + MESSAGE_LENGTH_SIZE + self.conn.sendall(sent_len + data) def _recv_msg(self): recv_len = self.conn.recv(MESSAGE_LENGTH_SIZE) # get message size @@ -437,6 +436,7 @@ def _connect(self): except BaseException as e: self.conn.close() self.connected = False + self.authenticated = False raise self.connected = True @@ -451,7 +451,8 @@ def connect(self, details: str = None): self._connect() except socket.error as e: logger.error( - f"Error connecting to server: {self.config} \r\n{details}. {e=}", + f"Error connecting to server: " + f"{self.config} \r\n{details}. {e=}", exc_info=True, stack_info=True) @@ -477,18 +478,21 @@ def _query(self, query, blob_array = [], try_resume=True): # Serialize with protobuf and send data = query_msg.SerializeToString() + if self.conn is None: + self.connect(details="Initial connect from _query") + # this is for session refresh attempts tries = 0 while tries < self.config.retry_max_attempts: try: - if self._send_msg(data): - response = self._recv_msg() - if response is not None: - querRes = queryMessage.queryMessage() - queryMessage.ParseFromString(querRes, response) - response_blob_array = [b for b in querRes.blobs] - self.last_response = json.loads(querRes.json) - break + self._send_msg(data) + response = self._recv_msg() + if response is not None: + querRes = queryMessage.queryMessage() + queryMessage.ParseFromString(querRes, response) + response_blob_array = [b for b in querRes.blobs] + self.last_response = json.loads(querRes.json) + break except ssl.SSLEOFError as ssle: # this can happen when working in a notebook. # we log if this isn't the first try, or if @@ -527,6 +531,8 @@ def _query(self, query, blob_array = [], try_resume=True): self.conn.close() self.connected = False + self.authenticated = False + self.connect( details=f"Will retry in {self.config.retry_interval_seconds} seconds") if not self._ever_connected: @@ -537,7 +543,7 @@ def _query(self, query, blob_array = [], try_resume=True): # For example aperturedb server is restarted, or network is lost. # While this is useful bit of code, when executed in a refresh token # path, this can cause a deadlock. Hence the try_resume flag. - if try_resume: + if try_resume and self.connected: self._renew_session() if tries == self.config.retry_max_attempts: # We have tried enough times, and failed. Log some state info. @@ -565,6 +571,7 @@ def query(self, q, blobs=[]): Returns: _type_: _description_ """ + self._renew_session() if self.should_authenticate: self.authenticate( shared_data=self.shared_data, @@ -602,7 +609,9 @@ def _renew_session(self): break except UnauthorizedException as e: logger.warning( - f"[Attempt {count + 1} of {RENEW_SESSION_MAX_ATTEMPTS}] Failed to refresh token.", + f"[Attempt {count + 1} of " + f"{RENEW_SESSION_MAX_ATTEMPTS}] " + "Failed to refresh token.", exc_info=True, stack_info=True) time.sleep(RENEW_SESSION_RETRY_INTERVAL_SEC) diff --git a/aperturedb/__init__.py b/aperturedb/__init__.py index 3babebc7..60f933d9 100644 --- a/aperturedb/__init__.py +++ b/aperturedb/__init__.py @@ -10,7 +10,7 @@ import signal import sys -__version__ = "0.4.55" +__version__ = "0.4.56" logger = logging.getLogger(__name__) @@ -83,7 +83,7 @@ def emit(self, record): try: latest_version = json.loads(requests.get( - "https://pypi.org/pypi/aperturedb/json").text)["info"]["version"] + "https://pypi.org/pypi/aperturedb/json", timeout=1).text)["info"]["version"] except Exception as e: logger.warning( f"Failed to get latest version: {e}. You are using version {__version__}") diff --git a/test/.dockerignore b/test/.dockerignore index eb99c596..1bf3c10d 100644 --- a/test/.dockerignore +++ b/test/.dockerignore @@ -4,3 +4,4 @@ notebooks/ kaggleds/ __pycache__/ .pytest_cache/ +*_ca/ diff --git a/test/docker-compose.yml b/test/docker-compose.yml index a3be0ec1..a1395ad1 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -14,7 +14,7 @@ services: openssl req -new -key /cert/tls.key -out /ca/http.csr -days 3650 -subj \"/C=US/ST=NY/L=NYC/O=instance/OU=instanceDB/CN=${DB_HTTP_CN:-localhost}\" openssl x509 -req -CA /ca/ca.crt -CAkey /ca/ca.key -in /ca/http.csr -out /cert/http.crt -passin pass:1234" volumes: - - ./aperturedb/certificate:/cert + - ./aperturedb/certificate_${RUNNER_NAME}:/cert - ./${RUNNER_NAME}_ca:/ca lenz: @@ -38,7 +38,7 @@ services: LNZ_CERTIFICATE_PATH: /etc/lenz/certificate/tcp.crt LNZ_PRIVATE_KEY_PATH: /etc/lenz/certificate/tls.key volumes: - - ./aperturedb/certificate:/etc/lenz/certificate + - ./aperturedb/certificate_${RUNNER_NAME}:/etc/lenz/certificate aperturedb: image: $ADB_REPO:$ADB_TAG @@ -71,7 +71,7 @@ services: - source: nginx.conf target: /etc/nginx/conf.d/default.conf volumes: - - ./aperturedb/certificate:/etc/nginx/certificate + - ./aperturedb/certificate_${RUNNER_NAME}:/etc/nginx/certificate configs: nginx.conf: diff --git a/test/run_test_container.sh b/test/run_test_container.sh index 2d112041..c51c792f 100755 --- a/test/run_test_container.sh +++ b/test/run_test_container.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -u set -e function check_containers_networks(){ diff --git a/test/test_Session.py b/test/test_Session.py index 9b07ab32..91ae6a44 100644 --- a/test/test_Session.py +++ b/test/test_Session.py @@ -114,8 +114,7 @@ def mock_connect(host, port): # Check the exception is not an obscure one. assert "self.connected=False" in e.args[0] - # Check that we tried to connect 3 times. - assert connect_attempts == 3 + assert connect_attempts == 4 def test_socket_send_error_initial(self, monkeypatch): send_attempts = 0 @@ -124,7 +123,7 @@ def mock_send(x, buff): nonlocal send_attempts send_attempts += 1 raise socket.error("Connection broke when send") - monkeypatch.setattr(socket.socket, "send", mock_send) + monkeypatch.setattr(socket.socket, "sendall", mock_send) # Create new db connection. new_db = Connector( @@ -141,8 +140,8 @@ def mock_send(x, buff): # Check the exception is not an obscure one. assert "self.connected=False" in e.args[0] - # Check that we tried to send 5 (connect hello:2) + query:3) times. - assert send_attempts == 5 + # Should be exactly 7 attempts (1 initial + 3 retries, each doing query send + reconnect send) + assert send_attempts == 7 def test_socket_recv_error_initial(self, monkeypatch): connect_attempts = 0 @@ -170,11 +169,12 @@ def mock_recv(x, buff): # Check the exception is not an obscure one. assert "self.connected=False" in e.args[0] - # Check that we tried to connect 3 times. - assert connect_attempts == 3 + assert connect_attempts == 4 def test_con_close_on_send_query(self, db: Connector, monkeypatch): if not isinstance(db, ConnectorRest): + if db.conn is None: + db.connect() original_send_msg = db._send_msg count = 0 @@ -205,6 +205,8 @@ def mock_send_msg(msg): def test_con_close_on_recv_query(self, db: Connector, monkeypatch): if not isinstance(db, ConnectorRest): + if db.conn is None: + db.connect() original_recv_msg = db._recv_msg count = 0