From 57ec70c0aaaef0be0c8c49af99581ab52265a11d Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 14:28:14 -0500 Subject: [PATCH 01/34] Testing upgrades --- Dockerfile | 15 --------- requirements.txt | 88 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index baa46f51..72473ce1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,21 +26,6 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize rm /tmp/dockerize.tgz -# install mongodb -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 \ - && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends mongodb-org=3.6.11 mongodb-org-server=3.6.11 mongodb-org-shell=3.6.11 mongodb-org-mongos=3.6.11 mongodb-org-tools=3.6.11 \ - && apt-get install -y --no-install-recommends mongodb \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN echo "mongodb-org hold" | dpkg --set-selections \ - && echo "mongodb-org-server hold" | dpkg --set-selections \ - && echo "mongodb-org-shell hold" | dpkg --set-selections \ - && echo "mongodb-org-mongos hold" | dpkg --set-selections \ - && echo "mongodb-org-tools hold" | dpkg --set-selections - #Install Python3 and Libraries (source /root/miniconda/bin/activate) RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ && bash ~/miniconda.sh -b -p /miniconda-latest diff --git a/requirements.txt b/requirements.txt index be905f7b..3518f5f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,60 @@ -i https://pypi.org/simple/ -aiofiles==0.7.0 -aiohttp==3.7.4.post0 -cachetools==4.2.2 -codecov==2.1.13 -configparser==5.0.2 -confluent-kafka==1.9.2 -coverage==5.5 -docker==5.0.0 -gevent==21.12.0 -greenlet==1.1.0 -gunicorn==20.1.0 -htcondor==9.12.0 -Jinja2==3.0.1 -JSONRPCBase==0.2.0 -mock==4.0.3 -maps==5.1.1 -mongoengine==0.23.1 -psutil==5.8.0 -pymongo==3.12.0 -pytest==6.2.4 -pytest-cov==2.12.1 -python-dateutil==2.8.2 -python-dotenv==0.18.0 -requests==2.25.1 -requests-mock==1.9.3 -sanic==22.9.0 -slackclient==2.9.3 -toml==0.10.2 -urllib3==1.26.6 +aiofiles==24.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +aiohttp==3.9.5 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +aiosignal==1.3.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +attrs==23.2.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +cachetools==5.4.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +certifi==2024.7.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +cffi==1.16.0 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" +charset-normalizer==3.3.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +codecov==2.1.13 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +colorama==0.4.6 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" and sys_platform == "win32" +configparser==7.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +confluent-kafka==2.5.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +coverage==7.6.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +coverage[toml]==7.6.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +dnspython==2.6.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +docker==7.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +frozenlist==1.4.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +gevent==24.2.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +greenlet==3.0.3 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +gunicorn==22.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +htcondor==23.8.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +html5tagger==1.3.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +httptools==0.6.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +idna==3.7 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +iniconfig==2.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +jinja2==3.1.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +jsonrpcbase==0.2.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +maps==5.1.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +markupsafe==2.1.5 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +mock==5.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +mongoengine==0.28.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +multidict==6.0.5 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +packaging==24.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +pluggy==1.5.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +psutil==6.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +pycparser==2.22 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" +pymongo==4.8.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +pytest-cov==5.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +pytest==8.3.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +python-dateutil==2.9.0.post0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +python-dotenv==1.0.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +pywin32==306 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" and sys_platform == "win32" +requests-mock==1.12.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +requests==2.32.3 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +sanic-routing==23.12.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +sanic==24.6.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +setuptools==71.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +six==1.16.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +slackclient==2.9.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +toml==0.10.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +tracerite==1.1.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +typing-extensions==4.12.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +ujson==5.10.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" +urllib3==2.2.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +uvloop==0.19.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" +websockets==12.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +yarl==1.9.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +zope-event==5.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +zope-interface==6.4.post2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" From 6cc9d05c8ec4a6981cfffd597763b45dfe2db499 Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 14:40:41 -0500 Subject: [PATCH 02/34] Update mongo version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 46896e28..d94790a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -147,7 +147,7 @@ services: ZOOKEEPER_SERVERS: zookeeper:12888:13888 mongodb: - image: mongo:3.2 + image: mongo:7.0 environment: # provide your credentials here - MONGO_INITDB_ROOT_USERNAME=travis From d4a3656f49194c55bcf6ecacd66f1d001f5e300a Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 15:25:34 -0500 Subject: [PATCH 03/34] Update python --- .github/workflows/ee2-tests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index 67164438..c86e1c19 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.12" - name: flake8 Lint Lib uses: py-actions/flake8@v2 with: @@ -55,7 +55,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + - name: Set up Python environment + uses: actions/setup-python@v4 + with: + python-version: "3.12" - name: Install dependencies run: | pip install -r requirements.txt From 8f4aed40a2edc2fa6366444cc2366212d8318960 Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 15:26:27 -0500 Subject: [PATCH 04/34] Update python --- .github/workflows/ee2-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index c86e1c19..d63ab21c 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.12" + python-version: "3.12.4" - name: flake8 Lint Lib uses: py-actions/flake8@v2 with: @@ -59,7 +59,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.12" + python-version: "3.12.4" - name: Install dependencies run: | pip install -r requirements.txt From cde5ffcf56ec1fb0c23b5eb8f42bf95636a7dbc6 Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 16:54:35 -0500 Subject: [PATCH 05/34] 3.10.10 --- .github/workflows/ee2-tests.yml | 4 +- requirements.txt | 118 ++++++++++++++++---------------- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index d63ab21c..ada6b171 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.12.4" + python-version: "3.10.14" - name: flake8 Lint Lib uses: py-actions/flake8@v2 with: @@ -59,7 +59,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.12.4" + python-version: "3.10.14" - name: Install dependencies run: | pip install -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 3518f5f2..62cd8a0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,60 +1,60 @@ -i https://pypi.org/simple/ -aiofiles==24.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -aiohttp==3.9.5 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -aiosignal==1.3.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -attrs==23.2.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -cachetools==5.4.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -certifi==2024.7.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -cffi==1.16.0 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" -charset-normalizer==3.3.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -codecov==2.1.13 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -colorama==0.4.6 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" and sys_platform == "win32" -configparser==7.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -confluent-kafka==2.5.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -coverage==7.6.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -coverage[toml]==7.6.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -dnspython==2.6.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -docker==7.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -frozenlist==1.4.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -gevent==24.2.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -greenlet==3.0.3 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -gunicorn==22.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -htcondor==23.8.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -html5tagger==1.3.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -httptools==0.6.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -idna==3.7 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -iniconfig==2.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -jinja2==3.1.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -jsonrpcbase==0.2.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -maps==5.1.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -markupsafe==2.1.5 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -mock==5.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -mongoengine==0.28.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -multidict==6.0.5 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -packaging==24.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -pluggy==1.5.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -psutil==6.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -pycparser==2.22 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" -pymongo==4.8.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -pytest-cov==5.0.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -pytest==8.3.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -python-dateutil==2.9.0.post0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -python-dotenv==1.0.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -pywin32==306 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" and sys_platform == "win32" -requests-mock==1.12.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -requests==2.32.3 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -sanic-routing==23.12.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -sanic==24.6.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -setuptools==71.1.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -six==1.16.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -slackclient==2.9.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -toml==0.10.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -tracerite==1.1.1 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -typing-extensions==4.12.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -ujson==5.10.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" -urllib3==2.2.2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -uvloop==0.19.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_full_version >= "3.12.4" and python_full_version < "4.0.0" -websockets==12.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -yarl==1.9.4 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -zope-event==5.0 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" -zope-interface==6.4.post2 ; python_full_version >= "3.12.4" and python_full_version < "4.0.0" +aiofiles==24.1.0 ; python_version >= "3.10" and python_version < "4.0" +aiohttp==3.9.5 ; python_version >= "3.10" and python_version < "4.0" +aiosignal==1.3.1 ; python_version >= "3.10" and python_version < "4.0" +async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.11" +attrs==23.2.0 ; python_version >= "3.10" and python_version < "4.0" +cachetools==5.4.0 ; python_version >= "3.10" and python_version < "4.0" +certifi==2024.7.4 ; python_version >= "3.10" and python_version < "4.0" +cffi==1.16.0 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_version >= "3.10" and python_version < "4.0" +charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "4.0" +codecov==2.1.13 ; python_version >= "3.10" and python_version < "4.0" +colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "win32" +configparser==7.0.0 ; python_version >= "3.10" and python_version < "4.0" +confluent-kafka==2.5.0 ; python_version >= "3.10" and python_version < "4.0" +coverage==7.6.0 ; python_version >= "3.10" and python_version < "4.0" +coverage[toml]==7.6.0 ; python_version >= "3.10" and python_version < "4.0" +dnspython==2.6.1 ; python_version >= "3.10" and python_version < "4.0" +docker==7.1.0 ; python_version >= "3.10" and python_version < "4.0" +exceptiongroup==1.2.2 ; python_version >= "3.10" and python_version < "3.11" +frozenlist==1.4.1 ; python_version >= "3.10" and python_version < "4.0" +gevent==24.2.1 ; python_version >= "3.10" and python_version < "4.0" +greenlet==3.0.3 ; platform_python_implementation == "CPython" and python_version < "4.0" and python_version >= "3.10" +gunicorn==22.0.0 ; python_version >= "3.10" and python_version < "4.0" +htcondor==23.8.1 ; python_version >= "3.10" and python_version < "4.0" +html5tagger==1.3.0 ; python_version >= "3.10" and python_version < "4.0" +httptools==0.6.1 ; python_version >= "3.10" and python_version < "4.0" +idna==3.7 ; python_version >= "3.10" and python_version < "4.0" +iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "4.0" +jinja2==3.1.4 ; python_version >= "3.10" and python_version < "4.0" +markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "4.0" +mock==5.1.0 ; python_version >= "3.10" and python_version < "4.0" +mongoengine==0.28.2 ; python_version >= "3.10" and python_version < "4.0" +multidict==6.0.5 ; python_version >= "3.10" and python_version < "4.0" +packaging==24.1 ; python_version >= "3.10" and python_version < "4.0" +pluggy==1.5.0 ; python_version >= "3.10" and python_version < "4.0" +psutil==6.0.0 ; python_version >= "3.10" and python_version < "4.0" +pycparser==2.22 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_version >= "3.10" and python_version < "4.0" +pymongo==4.8.0 ; python_version >= "3.10" and python_version < "4.0" +pytest-cov==5.0.0 ; python_version >= "3.10" and python_version < "4.0" +pytest==8.3.2 ; python_version >= "3.10" and python_version < "4.0" +python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "4.0" +python-dotenv==1.0.1 ; python_version >= "3.10" and python_version < "4.0" +pywin32==306 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "win32" +requests==2.32.3 ; python_version >= "3.10" and python_version < "4.0" +sanic-routing==23.12.0 ; python_version >= "3.10" and python_version < "4.0" +sanic==24.6.0 ; python_version >= "3.10" and python_version < "4.0" +setuptools==71.1.0 ; python_version >= "3.10" and python_version < "4.0" +six==1.16.0 ; python_version >= "3.10" and python_version < "4.0" +slackclient==2.9.4 ; python_version >= "3.10" and python_version < "4.0" +toml==0.10.2 ; python_version >= "3.10" and python_version < "4.0" +tomli==2.0.1 ; python_version >= "3.10" and python_full_version <= "3.11.0a6" +tracerite==1.1.1 ; python_version >= "3.10" and python_version < "4.0" +typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "4.0" +ujson==5.10.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_version >= "3.10" and python_version < "4.0" +urllib3==2.2.2 ; python_version >= "3.10" and python_version < "4.0" +uvloop==0.19.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_version >= "3.10" and python_version < "4.0" +websockets==12.0 ; python_version >= "3.10" and python_version < "4.0" +yarl==1.9.4 ; python_version >= "3.10" and python_version < "4.0" +zope-event==5.0 ; python_version >= "3.10" and python_version < "4.0" +zope-interface==6.4.post2 ; python_version >= "3.10" and python_version < "4.0" \ No newline at end of file From 15a29013c30ab0eb5b5e5007b5ffc7402df94f30 Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 17:02:48 -0500 Subject: [PATCH 06/34] 3.10 --- requirements.txt | 88 ++++++++++++++++-------------------------------- 1 file changed, 29 insertions(+), 59 deletions(-) diff --git a/requirements.txt b/requirements.txt index 62cd8a0b..99c3ff17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,60 +1,30 @@ -i https://pypi.org/simple/ -aiofiles==24.1.0 ; python_version >= "3.10" and python_version < "4.0" -aiohttp==3.9.5 ; python_version >= "3.10" and python_version < "4.0" -aiosignal==1.3.1 ; python_version >= "3.10" and python_version < "4.0" -async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.11" -attrs==23.2.0 ; python_version >= "3.10" and python_version < "4.0" -cachetools==5.4.0 ; python_version >= "3.10" and python_version < "4.0" -certifi==2024.7.4 ; python_version >= "3.10" and python_version < "4.0" -cffi==1.16.0 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_version >= "3.10" and python_version < "4.0" -charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "4.0" -codecov==2.1.13 ; python_version >= "3.10" and python_version < "4.0" -colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "win32" -configparser==7.0.0 ; python_version >= "3.10" and python_version < "4.0" -confluent-kafka==2.5.0 ; python_version >= "3.10" and python_version < "4.0" -coverage==7.6.0 ; python_version >= "3.10" and python_version < "4.0" -coverage[toml]==7.6.0 ; python_version >= "3.10" and python_version < "4.0" -dnspython==2.6.1 ; python_version >= "3.10" and python_version < "4.0" -docker==7.1.0 ; python_version >= "3.10" and python_version < "4.0" -exceptiongroup==1.2.2 ; python_version >= "3.10" and python_version < "3.11" -frozenlist==1.4.1 ; python_version >= "3.10" and python_version < "4.0" -gevent==24.2.1 ; python_version >= "3.10" and python_version < "4.0" -greenlet==3.0.3 ; platform_python_implementation == "CPython" and python_version < "4.0" and python_version >= "3.10" -gunicorn==22.0.0 ; python_version >= "3.10" and python_version < "4.0" -htcondor==23.8.1 ; python_version >= "3.10" and python_version < "4.0" -html5tagger==1.3.0 ; python_version >= "3.10" and python_version < "4.0" -httptools==0.6.1 ; python_version >= "3.10" and python_version < "4.0" -idna==3.7 ; python_version >= "3.10" and python_version < "4.0" -iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "4.0" -jinja2==3.1.4 ; python_version >= "3.10" and python_version < "4.0" -markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "4.0" -mock==5.1.0 ; python_version >= "3.10" and python_version < "4.0" -mongoengine==0.28.2 ; python_version >= "3.10" and python_version < "4.0" -multidict==6.0.5 ; python_version >= "3.10" and python_version < "4.0" -packaging==24.1 ; python_version >= "3.10" and python_version < "4.0" -pluggy==1.5.0 ; python_version >= "3.10" and python_version < "4.0" -psutil==6.0.0 ; python_version >= "3.10" and python_version < "4.0" -pycparser==2.22 ; platform_python_implementation == "CPython" and sys_platform == "win32" and python_version >= "3.10" and python_version < "4.0" -pymongo==4.8.0 ; python_version >= "3.10" and python_version < "4.0" -pytest-cov==5.0.0 ; python_version >= "3.10" and python_version < "4.0" -pytest==8.3.2 ; python_version >= "3.10" and python_version < "4.0" -python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "4.0" -python-dotenv==1.0.1 ; python_version >= "3.10" and python_version < "4.0" -pywin32==306 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "win32" -requests==2.32.3 ; python_version >= "3.10" and python_version < "4.0" -sanic-routing==23.12.0 ; python_version >= "3.10" and python_version < "4.0" -sanic==24.6.0 ; python_version >= "3.10" and python_version < "4.0" -setuptools==71.1.0 ; python_version >= "3.10" and python_version < "4.0" -six==1.16.0 ; python_version >= "3.10" and python_version < "4.0" -slackclient==2.9.4 ; python_version >= "3.10" and python_version < "4.0" -toml==0.10.2 ; python_version >= "3.10" and python_version < "4.0" -tomli==2.0.1 ; python_version >= "3.10" and python_full_version <= "3.11.0a6" -tracerite==1.1.1 ; python_version >= "3.10" and python_version < "4.0" -typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "4.0" -ujson==5.10.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_version >= "3.10" and python_version < "4.0" -urllib3==2.2.2 ; python_version >= "3.10" and python_version < "4.0" -uvloop==0.19.0 ; sys_platform != "win32" and implementation_name == "cpython" and python_version >= "3.10" and python_version < "4.0" -websockets==12.0 ; python_version >= "3.10" and python_version < "4.0" -yarl==1.9.4 ; python_version >= "3.10" and python_version < "4.0" -zope-event==5.0 ; python_version >= "3.10" and python_version < "4.0" -zope-interface==6.4.post2 ; python_version >= "3.10" and python_version < "4.0" \ No newline at end of file +aiofiles==0.7.0 +aiohttp==3.7.4.post0 +cachetools==4.2.2 +codecov==2.1.13 +configparser==5.0.2 +confluent-kafka==1.9.2 +coverage==5.5 +docker==5.0.0 +gevent==21.12.0 +greenlet==1.1.0 +gunicorn==20.1.0 +htcondor==9.12.0 +Jinja2==3.0.1 +JSONRPCBase==0.2.0 +mock==4.0.3 +maps==5.1.1 +mongoengine==0.28.2 +psutil==5.8.0 +pymongo==4.8.0 +pytest==6.2.4 +pytest-cov==2.12.1 +python-dateutil==2.8.2 +python-dotenv==0.18.0 +requests==2.25.1 +requests-mock==1.9.3 +sanic==22.9.0 +slackclient==2.9.3 +toml==0.10.2 +urllib3==1.26.6 \ No newline at end of file From 0f4bb10819077e65ab512eae0503f7463e84207e Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 17:04:03 -0500 Subject: [PATCH 07/34] 3.10 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 99c3ff17..45604861 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ docker==5.0.0 gevent==21.12.0 greenlet==1.1.0 gunicorn==20.1.0 -htcondor==9.12.0 +htcondor==23.8.1 Jinja2==3.0.1 JSONRPCBase==0.2.0 mock==4.0.3 From 7161697afb4a303052416065501a198c103ec564 Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 17:08:06 -0500 Subject: [PATCH 08/34] Update greenlet --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 45604861..e95b54b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ confluent-kafka==1.9.2 coverage==5.5 docker==5.0.0 gevent==21.12.0 -greenlet==1.1.0 +greenlet==3.0.3 ; platform_python_implementation == "CPython" and python_version < "4.0" and python_version >= "3.10" gunicorn==20.1.0 htcondor==23.8.1 Jinja2==3.0.1 From 4d17158bc9db3b73ce2d4f23d53c1751c77858c2 Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 17:10:44 -0500 Subject: [PATCH 09/34] Update greenlet --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e95b54b2..87f61f27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ configparser==5.0.2 confluent-kafka==1.9.2 coverage==5.5 docker==5.0.0 -gevent==21.12.0 +gevent==24.2.1 ; python_version >= "3.10" and python_version < "4.0" greenlet==3.0.3 ; platform_python_implementation == "CPython" and python_version < "4.0" and python_version >= "3.10" gunicorn==20.1.0 htcondor==23.8.1 From 0b7b7eddf8fb92db84dcc9a3d72003ed085deaef Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 22:44:54 -0500 Subject: [PATCH 10/34] Update dockerfile to 3.10.10 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 72473ce1..76db5f3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize #Install Python3 and Libraries (source /root/miniconda/bin/activate) -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-x86_64.sh -O ~/miniconda.sh \ && bash ~/miniconda.sh -b -p /miniconda-latest # Setup Cron From f164e7f10b79cc74d9b70724f060b317a28ae49d Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 25 Jul 2024 23:04:38 -0500 Subject: [PATCH 11/34] Update greenlet --- .github/workflows/ee2-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index ada6b171..65aa813e 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -64,12 +64,16 @@ jobs: run: | pip install -r requirements.txt git clone https://github.com/kbase/jars /opt/jars + - name: Give MongoDB a chance to startup + run: | + docker-compose up -d mongodb - name: Build Docker Image run: | docker build . -t execution_engine2:test - name: Run Tests run: | docker-compose up -d + curl localhost:27017 | grep "It looks like you are trying to access MongoDB over HTTP on the native driver port." || exit 1 cp test/env/test.travis.env test.env make test-coverage codecov From 8894ce2f24daa0e243a1621908f3df1b1f60cfcc Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 00:56:37 -0500 Subject: [PATCH 12/34] Update startup script --- .../mongo/docker-entrypoint-initdb.d/mongo-init.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js index 9a9ec8bf..e87ed0c7 100755 --- a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js @@ -1,5 +1,3 @@ -db.auth('travis', 'travis') - db = db.getSiblingDB('ee2') db.createUser( @@ -13,4 +11,4 @@ db.createUser( } ] } -); \ No newline at end of file +); From db001a3493edaaa0f71a71e89f5b4c615d2f62bd Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 00:56:56 -0500 Subject: [PATCH 13/34] Update startup script --- test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js index e87ed0c7..1e4984cf 100755 --- a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js @@ -1,3 +1,4 @@ +print("Adding travis username to ee2 database") db = db.getSiblingDB('ee2') db.createUser( From 8e5eb4dd8aecc43dd4448699fa9a73c1608debbb Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 01:05:36 -0500 Subject: [PATCH 14/34] Update startup script --- .github/workflows/ee2-tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index 65aa813e..ada6b171 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -64,16 +64,12 @@ jobs: run: | pip install -r requirements.txt git clone https://github.com/kbase/jars /opt/jars - - name: Give MongoDB a chance to startup - run: | - docker-compose up -d mongodb - name: Build Docker Image run: | docker build . -t execution_engine2:test - name: Run Tests run: | docker-compose up -d - curl localhost:27017 | grep "It looks like you are trying to access MongoDB over HTTP on the native driver port." || exit 1 cp test/env/test.travis.env test.env make test-coverage codecov From 678db2ff4dafa3aa79742d8935694c18fa1d1d53 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 12:15:15 -0500 Subject: [PATCH 15/34] Update test error message --- test/tests_for_auth/ee2_admin_mode_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests_for_auth/ee2_admin_mode_test.py b/test/tests_for_auth/ee2_admin_mode_test.py index 15ee0682..042740a7 100644 --- a/test/tests_for_auth/ee2_admin_mode_test.py +++ b/test/tests_for_auth/ee2_admin_mode_test.py @@ -289,8 +289,8 @@ def test_no_user(self): job_params_1 = get_sample_job_params(method=method_1, wsid=self.ws_id) with self.assertRaisesRegex( - expected_exception=RuntimeError, - expected_regex=r"ServerError\('Token validation failed: Login failed! Server responded with code 401 Unauthorized'\)", + expected_exception=RuntimeError, + expected_regex=r"\('An error occurred while fetching user permissions from the Workspace', ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)\)", ): runner.run_job(params=job_params_1, as_admin=False) From e3d2beeb6bd84e00e3445cc1a70b2243392560ce Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 12:26:03 -0500 Subject: [PATCH 16/34] Update test error message --- lib/execution_engine2/sdk/EE2Runjob.py | 12 ++++++------ .../sdk/job_submission_parameters.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/execution_engine2/sdk/EE2Runjob.py b/lib/execution_engine2/sdk/EE2Runjob.py index 5a7a8a63..745788ea 100644 --- a/lib/execution_engine2/sdk/EE2Runjob.py +++ b/lib/execution_engine2/sdk/EE2Runjob.py @@ -483,10 +483,10 @@ def run_batch( :return: A list of condor job ids or a failure notification """ - if type(params) != list: + if not isinstance(params, list): raise IncorrectParamsException("params must be a list") - if type(batch_params) != dict: + if not isinstance(batch_params, dict): raise IncorrectParamsException("batch params must be a mapping") wsid = batch_params.get(_WORKSPACE_ID) @@ -586,7 +586,7 @@ def _check_job_requirements_vs_admin( def _check_is_string(self, putative_str, name): if not putative_str: return None - if type(putative_str) != str: + if not isinstance(putative_str, str): raise IncorrectParamsException(f"{name} must be a string") return putative_str @@ -924,7 +924,7 @@ def run( """ # TODO Test this - if type(params) != dict: + if not isinstance(params, dict): raise IncorrectParamsException("params must be a mapping") self._preflight(runjob_params=params, as_admin=as_admin) @@ -949,10 +949,10 @@ def _get_job_reqs_from_concierge_params( rl = concierge_params.get(_REQUIREMENTS_LIST) schd_reqs = {} if rl: - if type(rl) != list: + if not isinstance(rl, list): raise IncorrectParamsException(f"{_REQUIREMENTS_LIST} must be a list") for s in rl: - if type(s) != str or "=" not in s: + if not isinstance(s, str) or "=" not in s: raise IncorrectParamsException( f"Found illegal requirement in {_REQUIREMENTS_LIST}: {s}" ) diff --git a/lib/execution_engine2/sdk/job_submission_parameters.py b/lib/execution_engine2/sdk/job_submission_parameters.py index 90bc58d7..ea478a1d 100644 --- a/lib/execution_engine2/sdk/job_submission_parameters.py +++ b/lib/execution_engine2/sdk/job_submission_parameters.py @@ -145,7 +145,7 @@ def _params(self): ) def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return self._params() == ( other.cpus, other.memory_MB, @@ -214,7 +214,7 @@ def __init__( ) self.wsid = _gt_zero(wsid, "wsid", optional=True) source_ws_objects = source_ws_objects if source_ws_objects else [] - if type(source_ws_objects) != list: + if not isinstance(source_ws_objects, list): raise IncorrectParamsException("source_ws_objects must be a list") for i, ref in enumerate(source_ws_objects): upa, is_valid = _is_valid_UPA(ref) @@ -238,7 +238,7 @@ def _params(self): ) def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return self._params() == ( other.job_id, other.app_info, From 4defe6e3d38a45c720e48179d7dfdd767d32f845 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:11:51 -0500 Subject: [PATCH 17/34] Fix E721 --- lib/execution_engine2/utils/application_info.py | 2 +- .../utils/job_requirements_resolver.py | 12 ++++++------ lib/execution_engine2/utils/user_info.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/execution_engine2/utils/application_info.py b/lib/execution_engine2/utils/application_info.py index 5ebc6d02..a4290fee 100644 --- a/lib/execution_engine2/utils/application_info.py +++ b/lib/execution_engine2/utils/application_info.py @@ -116,7 +116,7 @@ def get_application_id(self) -> str: return self.application_module def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return ( self.module, self.method, diff --git a/lib/execution_engine2/utils/job_requirements_resolver.py b/lib/execution_engine2/utils/job_requirements_resolver.py index c0f17cdc..1bafa6dc 100644 --- a/lib/execution_engine2/utils/job_requirements_resolver.py +++ b/lib/execution_engine2/utils/job_requirements_resolver.py @@ -77,13 +77,13 @@ def _check_clientgroup(clientgroup, source): def _string_request(putative_string, name, source): - if type(putative_string) != str: + if isinstance(putative_string, str): _check_raise(name, putative_string, source) return putative_string.strip() def _int_request(putative_int, original, name, source): - if type(putative_int) == float: + if isinstance(putative_int, float): _check_raise(f"{name} request", original, source) try: return int(putative_int) @@ -96,7 +96,7 @@ def _check_cpus(cpus, source): def _check_memory(memory, source): - if type(memory) == int: + if isinstance(memory, int): return memory memory2 = _string_request(memory, "memory request", source) if memory2.endswith("M"): @@ -107,7 +107,7 @@ def _check_memory(memory, source): def _check_disk(disk, source): - if type(disk) == int: + if isinstance(disk, int): return disk disk2 = _string_request(disk, "disk request", source) if disk2.endswith("GB"): @@ -116,7 +116,7 @@ def _check_disk(disk, source): def _bool_request(putative_bool, name, source): - if type(putative_bool) == bool or type(putative_bool) == int: + if isinstance(putative_bool, bool) or isinstance(putative_bool, int): return bool(putative_bool) pbs = _string_request(putative_bool, name, source).lower() if pbs == "true": @@ -347,7 +347,7 @@ def normalize_job_reqs( def _has_value(cls, inc): if inc is None: return False - if type(inc) == str and not inc.strip(): + if isinstance(inc, str) and not inc.strip(): return False return True diff --git a/lib/execution_engine2/utils/user_info.py b/lib/execution_engine2/utils/user_info.py index 1a7a4e10..c59b1742 100644 --- a/lib/execution_engine2/utils/user_info.py +++ b/lib/execution_engine2/utils/user_info.py @@ -28,7 +28,7 @@ def __init__(self, username: str, token: str): self.token = _check_string(token, "token") def __eq__(self, other): - if type(self) == type(other): + if type(self) == type(other): # noqa E721 return (self.username, self.token) == (other.username, other.token) return False From 114b5ae5c01361767a1c34b89d7956649d1ad389 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:13:19 -0500 Subject: [PATCH 18/34] Fix E721 --- .../sdk/job_submission_parameters.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/execution_engine2/sdk/job_submission_parameters.py b/lib/execution_engine2/sdk/job_submission_parameters.py index ea478a1d..6408f310 100644 --- a/lib/execution_engine2/sdk/job_submission_parameters.py +++ b/lib/execution_engine2/sdk/job_submission_parameters.py @@ -124,9 +124,11 @@ def check_parameters( client_group, None if client_group_regex is None else bool(client_group_regex), _check_string(bill_to_user, "bill_to_user", optional=True), - None - if ignore_concurrency_limits is None - else bool(ignore_concurrency_limits), + ( + None + if ignore_concurrency_limits is None + else bool(ignore_concurrency_limits) + ), cls._check_scheduler_requirements(scheduler_requirements), None if debug_mode is None else bool(debug_mode), ) @@ -145,7 +147,7 @@ def _params(self): ) def __eq__(self, other): - if type(self) == type(other): # noqa E721 + if type(self) == type(other): # noqa E721 return self._params() == ( other.cpus, other.memory_MB, @@ -238,7 +240,7 @@ def _params(self): ) def __eq__(self, other): - if type(self) == type(other): # noqa E721 + if type(self) == type(other): # noqa E721 return self._params() == ( other.job_id, other.app_info, From 496b08eb3f0c8b51d8901d433723ff307a954107 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:19:11 -0500 Subject: [PATCH 19/34] Fix line length --- test/tests_for_auth/ee2_admin_mode_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/tests_for_auth/ee2_admin_mode_test.py b/test/tests_for_auth/ee2_admin_mode_test.py index 042740a7..536cca38 100644 --- a/test/tests_for_auth/ee2_admin_mode_test.py +++ b/test/tests_for_auth/ee2_admin_mode_test.py @@ -287,11 +287,11 @@ def test_no_user(self): runner = self.getRunner() method_1 = "module_name.function_name" job_params_1 = get_sample_job_params(method=method_1, wsid=self.ws_id) - - with self.assertRaisesRegex( - expected_exception=RuntimeError, - expected_regex=r"\('An error occurred while fetching user permissions from the Workspace', ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)\)", - ): + error_regex = ( + r"An error occurred while fetching user permissions from the Workspace" + r", ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)" + ) + with self.assertRaisesRegex(expected_exception=RuntimeError, expected_regex=error_regex): runner.run_job(params=job_params_1, as_admin=False) def test_admin_reader(self): From 7a62c786b1cfbb93c2fb3a0fece6ca853e533b75 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:19:53 -0500 Subject: [PATCH 20/34] Ran black --- lib/execution_engine2/utils/application_info.py | 2 +- lib/execution_engine2/utils/user_info.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/execution_engine2/utils/application_info.py b/lib/execution_engine2/utils/application_info.py index a4290fee..ce81e41c 100644 --- a/lib/execution_engine2/utils/application_info.py +++ b/lib/execution_engine2/utils/application_info.py @@ -116,7 +116,7 @@ def get_application_id(self) -> str: return self.application_module def __eq__(self, other): - if type(self) == type(other): # noqa E721 + if type(self) == type(other): # noqa E721 return ( self.module, self.method, diff --git a/lib/execution_engine2/utils/user_info.py b/lib/execution_engine2/utils/user_info.py index c59b1742..d1a2cbfb 100644 --- a/lib/execution_engine2/utils/user_info.py +++ b/lib/execution_engine2/utils/user_info.py @@ -28,7 +28,7 @@ def __init__(self, username: str, token: str): self.token = _check_string(token, "token") def __eq__(self, other): - if type(self) == type(other): # noqa E721 + if type(self) == type(other): # noqa E721 return (self.username, self.token) == (other.username, other.token) return False From 1643d11f0d97472d3e4dd237cbb05d78a03fde88 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:22:59 -0500 Subject: [PATCH 21/34] Ran black --- test/tests_for_auth/ee2_admin_mode_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/tests_for_auth/ee2_admin_mode_test.py b/test/tests_for_auth/ee2_admin_mode_test.py index 536cca38..8596195f 100644 --- a/test/tests_for_auth/ee2_admin_mode_test.py +++ b/test/tests_for_auth/ee2_admin_mode_test.py @@ -291,7 +291,9 @@ def test_no_user(self): r"An error occurred while fetching user permissions from the Workspace" r", ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)" ) - with self.assertRaisesRegex(expected_exception=RuntimeError, expected_regex=error_regex): + with self.assertRaisesRegex( + expected_exception=RuntimeError, expected_regex=error_regex + ): runner.run_job(params=job_params_1, as_admin=False) def test_admin_reader(self): From 3506b6d28bb2df4331a86164c89a15b881c77e3b Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:34:59 -0500 Subject: [PATCH 22/34] Fix noqa --- lib/execution_engine2/utils/job_requirements_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/execution_engine2/utils/job_requirements_resolver.py b/lib/execution_engine2/utils/job_requirements_resolver.py index 1bafa6dc..92c75308 100644 --- a/lib/execution_engine2/utils/job_requirements_resolver.py +++ b/lib/execution_engine2/utils/job_requirements_resolver.py @@ -77,7 +77,7 @@ def _check_clientgroup(clientgroup, source): def _string_request(putative_string, name, source): - if isinstance(putative_string, str): + if not isinstance(putative_string, str): _check_raise(name, putative_string, source) return putative_string.strip() From 97d7497df97f714a2bb371e1100428a605a8ba25 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:46:46 -0500 Subject: [PATCH 23/34] Fix error message --- test/tests_for_auth/ee2_admin_mode_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/tests_for_auth/ee2_admin_mode_test.py b/test/tests_for_auth/ee2_admin_mode_test.py index 8596195f..b556196d 100644 --- a/test/tests_for_auth/ee2_admin_mode_test.py +++ b/test/tests_for_auth/ee2_admin_mode_test.py @@ -288,13 +288,13 @@ def test_no_user(self): method_1 = "module_name.function_name" job_params_1 = get_sample_job_params(method=method_1, wsid=self.ws_id) error_regex = ( - r"An error occurred while fetching user permissions from the Workspace" - r", ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)" + r"\('An error occurred while fetching user permissions from the Workspace', " + r"ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)\)" ) with self.assertRaisesRegex( - expected_exception=RuntimeError, expected_regex=error_regex + expected_exception=RuntimeError, expected_regex=error_regex ): - runner.run_job(params=job_params_1, as_admin=False) + runner.run_job(params=job_params_1) def test_admin_reader(self): # Admin User with READ From 3dc56dd34c6332d8c6666fd8b6a5fa03f5b812a9 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 15:54:05 -0500 Subject: [PATCH 24/34] Fix error message --- test/tests_for_auth/ee2_admin_mode_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests_for_auth/ee2_admin_mode_test.py b/test/tests_for_auth/ee2_admin_mode_test.py index b556196d..b12c30e0 100644 --- a/test/tests_for_auth/ee2_admin_mode_test.py +++ b/test/tests_for_auth/ee2_admin_mode_test.py @@ -292,7 +292,7 @@ def test_no_user(self): r"ServerError\('Token validation failed: Auth service returned an error: 10020 Invalid token'\)\)" ) with self.assertRaisesRegex( - expected_exception=RuntimeError, expected_regex=error_regex + expected_exception=RuntimeError, expected_regex=error_regex ): runner.run_job(params=job_params_1) From 425007d334b3f1481e060ba84ace9b4ab5f1de3f Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 16:05:19 -0500 Subject: [PATCH 25/34] Fix error message --- .../mongo/docker-entrypoint-initdb.d/mongo-init.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js index 1e4984cf..982f264c 100755 --- a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js @@ -1,4 +1,8 @@ print("Adding travis username to ee2 database") + +db.auth('travis', 'travis') + + db = db.getSiblingDB('ee2') db.createUser( From b97fadd9b97797ef44d2f49affaff8c97b8de8cc Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 16:13:00 -0500 Subject: [PATCH 26/34] Fix error message --- test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js index 982f264c..280adc77 100755 --- a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js @@ -1,6 +1,6 @@ print("Adding travis username to ee2 database") -db.auth('travis', 'travis') +// db.auth('travis', 'travis') This no longer works on Mongo7 db = db.getSiblingDB('ee2') From acd9c64bf354826e9d17de445a8ef076d83c50a8 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 16:14:28 -0500 Subject: [PATCH 27/34] What happens on 3.12 --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 76db5f3a..d64b1147 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,9 +27,10 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize #Install Python3 and Libraries (source /root/miniconda/bin/activate) -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-x86_64.sh -O ~/miniconda.sh \ +RUN wget hhttps://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ && bash ~/miniconda.sh -b -p /miniconda-latest + # Setup Cron COPY ./bin/ee2_cronjobs /etc/cron.d/ee2_cronjobs From 918537d3bec4cc0fc3a95e4e7956b4d2839b5730 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 16:26:00 -0500 Subject: [PATCH 28/34] What happens on 3.12 --- Dockerfile | 2 +- requirements.txt | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d64b1147..b78670d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize #Install Python3 and Libraries (source /root/miniconda/bin/activate) -RUN wget hhttps://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ && bash ~/miniconda.sh -b -p /miniconda-latest diff --git a/requirements.txt b/requirements.txt index 87f61f27..c94b0591 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,10 @@ aiohttp==3.7.4.post0 cachetools==4.2.2 codecov==2.1.13 configparser==5.0.2 -confluent-kafka==1.9.2 +confluent-kafka==2.5.0 coverage==5.5 docker==5.0.0 gevent==24.2.1 ; python_version >= "3.10" and python_version < "4.0" -greenlet==3.0.3 ; platform_python_implementation == "CPython" and python_version < "4.0" and python_version >= "3.10" gunicorn==20.1.0 htcondor==23.8.1 Jinja2==3.0.1 @@ -27,4 +26,4 @@ requests-mock==1.9.3 sanic==22.9.0 slackclient==2.9.3 toml==0.10.2 -urllib3==1.26.6 \ No newline at end of file +urllib3==1.26.6 From 94b582e5bc02d3dd1f99ead1883055ad3ee9de5f Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 16:33:29 -0500 Subject: [PATCH 29/34] What happens on 3.12 --- .github/workflows/ee2-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index ada6b171..d63ab21c 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.10.14" + python-version: "3.12.4" - name: flake8 Lint Lib uses: py-actions/flake8@v2 with: @@ -59,7 +59,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.10.14" + python-version: "3.12.4" - name: Install dependencies run: | pip install -r requirements.txt From 0fa7a75d1cd710d136e08d6c04f052db44f17796 Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 26 Jul 2024 17:07:28 -0500 Subject: [PATCH 30/34] Back to 3.10 --- .github/workflows/ee2-tests.yml | 4 ++-- Dockerfile | 3 +-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index d63ab21c..ada6b171 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.12.4" + python-version: "3.10.14" - name: flake8 Lint Lib uses: py-actions/flake8@v2 with: @@ -59,7 +59,7 @@ jobs: - name: Set up Python environment uses: actions/setup-python@v4 with: - python-version: "3.12.4" + python-version: "3.10.14" - name: Install dependencies run: | pip install -r requirements.txt diff --git a/Dockerfile b/Dockerfile index b78670d9..76db5f3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,10 +27,9 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize #Install Python3 and Libraries (source /root/miniconda/bin/activate) -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \ +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-x86_64.sh -O ~/miniconda.sh \ && bash ~/miniconda.sh -b -p /miniconda-latest - # Setup Cron COPY ./bin/ee2_cronjobs /etc/cron.d/ee2_cronjobs diff --git a/requirements.txt b/requirements.txt index c94b0591..c5b928b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ aiohttp==3.7.4.post0 cachetools==4.2.2 codecov==2.1.13 configparser==5.0.2 -confluent-kafka==2.5.0 +confluent-kafka==1.9.2 coverage==5.5 docker==5.0.0 gevent==24.2.1 ; python_version >= "3.10" and python_version < "4.0" From 65c2f72d12bb41ed1e577b5c783cb699dfa91941 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 2 Aug 2024 15:21:39 -0500 Subject: [PATCH 31/34] Update ee2-tests.yml Update GHAs --- .github/workflows/ee2-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index ada6b171..bd2278b6 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest name: Lint With Black steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check --verbose" @@ -35,7 +35,7 @@ jobs: name: Lint With Flake8 steps: - name: Check out source repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python environment uses: actions/setup-python@v4 with: @@ -57,7 +57,7 @@ jobs: - name: Check out source repository uses: actions/checkout@v4 - name: Set up Python environment - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10.14" - name: Install dependencies From 13fce3589425ec22ea352ff0ebf37416f8168a9a Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 2 Aug 2024 15:33:47 -0500 Subject: [PATCH 32/34] Update ee2-tests.yml --- .github/workflows/ee2-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index bd2278b6..0e03c03e 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -69,7 +69,7 @@ jobs: docker build . -t execution_engine2:test - name: Run Tests run: | - docker-compose up -d + docker compose up -d cp test/env/test.travis.env test.env make test-coverage codecov From 07df69b5de781be377058d8a238bbeb8fadcd6ba Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 6 Aug 2024 11:15:15 -0500 Subject: [PATCH 33/34] SECURITY-36 Mongo3.6/Mongo7 Matrix (#484) * Update ee2-tests.yml to matrix --------- Co-authored-by: Boris --- .github/workflows/check_build.yml | 21 ++++ .github/workflows/ee2-tests.yml | 26 +++-- Makefile | 2 +- docker-compose.yml | 109 +++++++++--------- .../mongo-init.js | 10 +- .../mongo-init.js | 14 +++ 6 files changed, 111 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/check_build.yml rename test/dockerfiles/mongo/{docker-entrypoint-initdb.d => docker-entrypoint-initdb.d-3.6}/mongo-init.js (68%) create mode 100755 test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js diff --git a/.github/workflows/check_build.yml b/.github/workflows/check_build.yml new file mode 100644 index 00000000..5f13de01 --- /dev/null +++ b/.github/workflows/check_build.yml @@ -0,0 +1,21 @@ +name: Build EE2 Docker Image + +on: + pull_request + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and push + uses: docker/build-push-action@v6 + with: + push: false + tags: ee2/test:test \ No newline at end of file diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index 0e03c03e..c8397e64 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -1,12 +1,6 @@ # This workflow will install Python dependencies, run tests and lint # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -# To ssh into this build add the following: -#- name: Start SSH session -# uses: luchihoratiu/debug-via-ssh@main -# with: -# NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} -# SSH_PASS: ${{ secrets.SSH_PASS }} name: Execution Engine 2 Test Suite @@ -50,9 +44,17 @@ jobs: path: "./test" + Build_and_Run_Tests_and_CodeCov: name: Build and Run Tests and CodeCov runs-on: ubuntu-latest + strategy: + matrix: + mongo-config: + - version: "7.0" + init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/" + - version: "3.6" + init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/" steps: - name: Check out source repository uses: actions/checkout@v4 @@ -64,12 +66,16 @@ jobs: run: | pip install -r requirements.txt git clone https://github.com/kbase/jars /opt/jars - - name: Build Docker Image + - name: Set MongoDB Version and Init Path run: | - docker build . -t execution_engine2:test - - name: Run Tests + echo "MONGO_VERSION=${{ matrix.mongo-config.version }}" >> $GITHUB_ENV + echo "INIT_PATH=${{ matrix.mongo-config.init-path }}" >> $GITHUB_ENV + - name: Start Services and Run Tests run: | docker compose up -d cp test/env/test.travis.env test.env make test-coverage - codecov + codecov --token="${{ secrets.CODECOV_TOKEN }}" + - name: Cleanup + run: | + docker compose down diff --git a/Makefile b/Makefile index 0e3ad8c5..3eebfda6 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ setup-database: test-coverage: # Set up travis user in mongo @echo "Run tests for $(TESTS)" - PYTHONPATH=.:lib:test pytest --cov-report=xml --cov lib/execution_engine2/ --verbose $(TESTS) + PYTHONPATH=.:lib:test pytest --cov-report=xml --cov-report=term --cov lib/execution_engine2/ --verbose $(TESTS) build-condor-test-image: cd test/dockerfiles/condor && echo `pwd` && docker build -f Dockerfile . -t $(CONDOR_DOCKER_IMAGE_TAG_NAME) diff --git a/docker-compose.yml b/docker-compose.yml index d94790a4..996708d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,57 +2,58 @@ version: "3.1" services: + # For Local Dev # Requires a ` docker build . -t execution_engine2:test ` # Requires a ` make database ` - ee2: - image: execution_engine2:test - command: - - "-poll" - - "-template" - - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" - - "-template" - - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg" - - "-timeout" - - "120s" - - "-wait" - - "tcp://mongodb:27017" - - "-stdout" - - "/kb/deployment/jettybase/logs/request.log" - - "./scripts/entrypoint.sh" - entrypoint: [ "dockerize" ] - depends_on: ["mongodb","condor"] - environment: - - POOL_PASSWORD=weakpassword - env_file: test/deploy.cfg - volumes: - - ./:/ee2 - - ee2_with_ssh: - image: execution_engine2:test - command: - - "-poll" - - "-template" - - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" - - "-template" - - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg" - - "-timeout" - - "120s" - - "-wait" - - "tcp://mongodb:27017" - - "-stdout" - - "/kb/deployment/jettybase/logs/request.log" - - "./scripts/entrypoint.sh" - entrypoint: [ "dockerize" ] - depends_on: ["mongodb","condor"] - environment: - - POOL_PASSWORD=weakpassword - - LOCAL_DEV=1 - - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg - env_file: test/deploy.cfg - ports: ["23:22","5678:5678"] - volumes: - - ./:/ee2 - - /Users:/Users +# ee2: +# image: execution_engine2:test +# command: +# - "-poll" +# - "-template" +# - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" +# - "-template" +# - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg" +# - "-timeout" +# - "120s" +# - "-wait" +# - "tcp://mongodb:27017" +# - "-stdout" +# - "/kb/deployment/jettybase/logs/request.log" +# - "./scripts/entrypoint.sh" +# entrypoint: [ "dockerize" ] +# depends_on: ["mongodb","condor"] +# environment: +# - POOL_PASSWORD=weakpassword +# env_file: test/deploy.cfg +# volumes: +# - ./:/ee2 +# +# ee2_with_ssh: +# image: execution_engine2:test +# command: +# - "-poll" +# - "-template" +# - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config" +# - "-template" +# - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg" +# - "-timeout" +# - "120s" +# - "-wait" +# - "tcp://mongodb:27017" +# - "-stdout" +# - "/kb/deployment/jettybase/logs/request.log" +# - "./scripts/entrypoint.sh" +# entrypoint: [ "dockerize" ] +# depends_on: ["mongodb","condor"] +# environment: +# - POOL_PASSWORD=weakpassword +# - LOCAL_DEV=1 +# - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg +# env_file: test/deploy.cfg +# ports: ["23:22","5678:5678"] +# volumes: +# - ./:/ee2 +# - /Users:/Users @@ -146,13 +147,15 @@ services: ZOOKEEPER_SYNC_LIMIT: 2docker-compose rm ZOOKEEPER_SERVERS: zookeeper:12888:13888 + mongodb: - image: mongo:7.0 + image: mongo:${MONGO_VERSION:-7.0} # Use 7.0 as the default if MONGO_VERSION is not set environment: - # provide your credentials here - MONGO_INITDB_ROOT_USERNAME=travis - MONGO_INITDB_ROOT_PASSWORD=travis - MONGO_INITDB_DATABASE=ee2 - ports: ["27018:27017","27017:27017"] + ports: + - "27018:27017" + - "27017:27017" volumes: - - "./test/dockerfiles/mongo/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/" + - ${INIT_PATH:-./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/}:/docker-entrypoint-initdb.d/ diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/mongo-init.js similarity index 68% rename from test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js rename to test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/mongo-init.js index 280adc77..c9e30a38 100755 --- a/test/dockerfiles/mongo/docker-entrypoint-initdb.d/mongo-init.js +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/mongo-init.js @@ -1,10 +1,6 @@ -print("Adding travis username to ee2 database") - -// db.auth('travis', 'travis') This no longer works on Mongo7 - - +print("Adding travis username to ee2 database for mongo 3.6") +db.auth('travis', 'travis') db = db.getSiblingDB('ee2') - db.createUser( { user: "travis", @@ -16,4 +12,4 @@ db.createUser( } ] } -); +); \ No newline at end of file diff --git a/test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js new file mode 100755 index 00000000..72980892 --- /dev/null +++ b/test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/mongo-init.js @@ -0,0 +1,14 @@ +print("Adding travis username to ee2 database for mongo 7.0") +db = db.getSiblingDB('ee2') +db.createUser( + { + user: "travis", + pwd: "travis", + roles: [ + { + role: "dbOwner", + db: "ee2" + } + ] + } +); \ No newline at end of file From 5cb6cde6fadda964b882945d637d8b52c9e39dc2 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 6 Aug 2024 14:45:39 -0500 Subject: [PATCH 34/34] Update docker-compose.yml --- docker-compose.yml | 166 ++++++++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 996708d3..92e90082 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,95 +57,95 @@ services: - condor_worker_mini: - privileged: true - image: kbase/condor-worker:develop - hostname: condor_worker_mini - command: - - "-poll" - - "-template" - - "/kb/deployment/conf/.templates/condor_config_worker.templ:/etc/condor/condor_config.local" - - "-template" - - "/kb/deployment/conf/.templates/shared_port_config.templ:/etc/condor/config.d/shared_port_config" - - "-timeout" - - "120s" - - "-stdout" - - "/var/log/condor/ProcLog" - - "-stdout" - - "/var/log/condor/StartLog" - - "/kb/deployment/bin/start-condor.sh" - depends_on: ["condor"] - environment: - - POOL_PASSWORD=weakpassword - - COLLECTOR_HOST=condor:9618?sock=collector - - USE_TCP=True - - EXECUTE_SUFFIX=mini_kb - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /sys/fs/cgroup:/sys/fs/cgroup + # condor_worker_mini: + # privileged: true + # image: kbase/condor-worker:develop + # hostname: condor_worker_mini + # command: + # - "-poll" + # - "-template" + # - "/kb/deployment/conf/.templates/condor_config_worker.templ:/etc/condor/condor_config.local" + # - "-template" + # - "/kb/deployment/conf/.templates/shared_port_config.templ:/etc/condor/config.d/shared_port_config" + # - "-timeout" + # - "120s" + # - "-stdout" + # - "/var/log/condor/ProcLog" + # - "-stdout" + # - "/var/log/condor/StartLog" + # - "/kb/deployment/bin/start-condor.sh" + # depends_on: ["condor"] + # environment: + # - POOL_PASSWORD=weakpassword + # - COLLECTOR_HOST=condor:9618?sock=collector + # - USE_TCP=True + # - EXECUTE_SUFFIX=mini_kb + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # - /sys/fs/cgroup:/sys/fs/cgroup - condor: - image: kbase/condor:develop - hostname: condor - environment: - - POOL_PASSWORD=weakpassword - - USE_TCP=True - ports: - - "9618:9618" - entrypoint: - - "/usr/bin/dockerize" - command: - - "-poll" - - "-template" - - "/etc/condor/.templates/condor_config.local.templ:/etc/condor/condor_config.local" - - "-stdout" - - "/var/log/condor/MasterLog" - - "-stdout" - - "/var/log/condor/SchedLog" - - "/usr/sbin/start-condor.sh" - volumes: - - ./condor_shared/:/condor_shared/ - - ./condor_shared/cdr/:/cdr/ + # condor: + # image: kbase/condor:develop + # hostname: condor + # environment: + # - POOL_PASSWORD=weakpassword + # - USE_TCP=True + # ports: + # - "9618:9618" + # entrypoint: + # - "/usr/bin/dockerize" + # command: + # - "-poll" + # - "-template" + # - "/etc/condor/.templates/condor_config.local.templ:/etc/condor/condor_config.local" + # - "-stdout" + # - "/var/log/condor/MasterLog" + # - "-stdout" + # - "/var/log/condor/SchedLog" + # - "/usr/sbin/start-condor.sh" + # volumes: + # - ./condor_shared/:/condor_shared/ + # - ./condor_shared/cdr/:/cdr/ - manager: - image: kafkamanager/kafka-manager - hostname: manager - ports: - - "0.0.0.0:9000:9000" - depends_on: - - zookeeper - environment: - ZK_HOSTS: zookeeper + # manager: + # image: kafkamanager/kafka-manager + # hostname: manager + # ports: + # - "0.0.0.0:9000:9000" + # depends_on: + # - zookeeper + # environment: + # ZK_HOSTS: zookeeper - kafka: - image: confluentinc/cp-kafka:5.0.0 - hostname: kafka - ports: - - "0.0.0.0:9092:9092" - depends_on: - - zookeeper - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_HEAP_OPTS: -Xmx512m -Xms512m + # kafka: + # image: confluentinc/cp-kafka:5.0.0 + # hostname: kafka + # ports: + # - "0.0.0.0:9092:9092" + # depends_on: + # - zookeeper + # environment: + # KAFKA_BROKER_ID: 1 + # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + # KAFKA_HEAP_OPTS: -Xmx512m -Xms512m - # Required for kafka - zookeeper: - image: confluentinc/cp-zookeeper:latest - hostname: zookeeper - ports: - - "0.0.0.0:2181:2181" - environment: - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_INIT_LIMIT: 5 - ZOOKEEPER_SYNC_LIMIT: 2docker-compose rm - ZOOKEEPER_SERVERS: zookeeper:12888:13888 + # # Required for kafka + # zookeeper: + # image: confluentinc/cp-zookeeper:latest + # hostname: zookeeper + # ports: + # - "0.0.0.0:2181:2181" + # environment: + # ZOOKEEPER_SERVER_ID: 1 + # ZOOKEEPER_CLIENT_PORT: 2181 + # ZOOKEEPER_TICK_TIME: 2000 + # ZOOKEEPER_INIT_LIMIT: 5 + # ZOOKEEPER_SYNC_LIMIT: 2docker-compose rm + # ZOOKEEPER_SERVERS: zookeeper:12888:13888 mongodb: