SECURITY-36 3.10.10

.github/workflows/check_build.yml

@@ -0,0 +1,21 @@
+name: Build EE2 Docker Image
+
+on:
+  pull_request
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: false
+          tags: ee2/test:test

.github/workflows/ee2-tests.yml

@@ -1,12 +1,6 @@
 # This workflow will install Python dependencies, run tests and lint
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-# To ssh into this build add the following:
-#- name: Start SSH session
-#      uses: luchihoratiu/debug-via-ssh@main
-#      with:
-#        NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }}
-#        SSH_PASS: ${{ secrets.SSH_PASS }}
 
 name: Execution Engine 2 Test Suite
 
@@ -18,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Lint With Black
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           options: "--check --verbose"
@@ -35,11 +29,11 @@ jobs:
     name: Lint With Flake8
     steps:
       - name: Check out source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Python environment
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10.14"
       - name: flake8 Lint Lib
         uses: py-actions/flake8@v2
         with:
@@ -50,22 +44,38 @@ jobs:
           path: "./test"
 
 
+
   Build_and_Run_Tests_and_CodeCov:
     name: Build and Run Tests and CodeCov
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mongo-config:
+          - version: "7.0"
+            init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/"
+          - version: "3.6"
+            init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/"
     steps:
       - name: Check out source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      - name: Set up Python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10.14"
       - name: Install dependencies
         run: |
           pip install -r requirements.txt
           git clone https://github.com/kbase/jars /opt/jars
-      - name: Build Docker Image
+      - name: Set MongoDB Version and Init Path
         run: |
-          docker build . -t execution_engine2:test
-      - name: Run Tests
+          echo "MONGO_VERSION=${{ matrix.mongo-config.version }}" >> $GITHUB_ENV
+          echo "INIT_PATH=${{ matrix.mongo-config.init-path }}" >> $GITHUB_ENV
+      - name: Start Services and Run Tests
         run: |
-          docker-compose up -d
+          docker compose up -d
           cp test/env/test.travis.env test.env
           make test-coverage
-          codecov
+          codecov --token="${{ secrets.CODECOV_TOKEN }}"
+      - name: Cleanup
+        run: |
+          docker compose down

Dockerfile

@@ -26,23 +26,8 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize
       rm /tmp/dockerize.tgz
 
 
-# install mongodb
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 \
-    && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list  \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends mongodb-org=3.6.11 mongodb-org-server=3.6.11 mongodb-org-shell=3.6.11 mongodb-org-mongos=3.6.11 mongodb-org-tools=3.6.11 \
-    && apt-get install -y --no-install-recommends mongodb \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN echo "mongodb-org hold" | dpkg --set-selections \
-    && echo "mongodb-org-server hold" | dpkg --set-selections \
-    && echo "mongodb-org-shell hold" | dpkg --set-selections \
-    && echo "mongodb-org-mongos hold" | dpkg --set-selections \
-    && echo "mongodb-org-tools hold" | dpkg --set-selections
-
 #Install Python3 and Libraries (source /root/miniconda/bin/activate)
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-x86_64.sh -O ~/miniconda.sh \
 && bash ~/miniconda.sh -b -p /miniconda-latest
 
 # Setup Cron

Makefile

@@ -38,7 +38,7 @@ setup-database:
 test-coverage:
 	# Set up travis user in mongo
 	@echo "Run tests for $(TESTS)"
-	PYTHONPATH=.:lib:test pytest --cov-report=xml --cov lib/execution_engine2/ --verbose $(TESTS)
+	PYTHONPATH=.:lib:test pytest --cov-report=xml --cov-report=term --cov lib/execution_engine2/  --verbose $(TESTS)
 
 build-condor-test-image:
 	cd test/dockerfiles/condor && echo `pwd` && docker build -f Dockerfile . -t $(CONDOR_DOCKER_IMAGE_TAG_NAME)

RELEASE_NOTES.md

@@ -1,6 +1,14 @@
 # execution_engine2 (ee2) release notes
 =========================================
 
+## 0.0.14
+- Update clients to work with mongo7 by updating pymongo and mongoengine
+-  Fix linting issues from flake8 and black
+- Update outdated libraries that can no longer be pip installed
+- The latest Dockerfile build contains conda which installs python 3.12.4 so these changes will pin python to 3.10 and save us headache in the future
+- Speed up tests by moving docker build step out
+- Create matrix of tests to run against mongo3.6 and mongo7
+
 ## 0.0.13
 * Bump version in impl file and sync branches
 

docker-compose.yml

@@ -2,57 +2,58 @@ version: "3.1"
 
 services:
 
+  # For Local Dev
   # Requires a ` docker build . -t execution_engine2:test `
   # Requires a ` make database `
-  ee2:
-    image: execution_engine2:test
-    command:
-      - "-poll"
-      - "-template"
-      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
-      - "-template"
-      - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg"
-      - "-timeout"
-      - "120s"
-      - "-wait"
-      - "tcp://mongodb:27017"
-      - "-stdout"
-      - "/kb/deployment/jettybase/logs/request.log"
-      - "./scripts/entrypoint.sh"
-    entrypoint: [ "dockerize" ]
-    depends_on: ["mongodb","condor"]
-    environment:
-      - POOL_PASSWORD=weakpassword
-    env_file: test/deploy.cfg
-    volumes:
-      - ./:/ee2
-
-  ee2_with_ssh:
-    image: execution_engine2:test
-    command:
-      - "-poll"
-      - "-template"
-      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
-      - "-template"
-      - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg"
-      - "-timeout"
-      - "120s"
-      - "-wait"
-      - "tcp://mongodb:27017"
-      - "-stdout"
-      - "/kb/deployment/jettybase/logs/request.log"
-      - "./scripts/entrypoint.sh"
-    entrypoint: [ "dockerize" ]
-    depends_on: ["mongodb","condor"]
-    environment:
-      - POOL_PASSWORD=weakpassword
-      - LOCAL_DEV=1
-      - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg
-    env_file: test/deploy.cfg
-    ports: ["23:22","5678:5678"]
-    volumes:
-      - ./:/ee2
-      - /Users:/Users
+#  ee2:
+#    image: execution_engine2:test
+#    command:
+#      - "-poll"
+#      - "-template"
+#      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
+#      - "-template"
+#      - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg"
+#      - "-timeout"
+#      - "120s"
+#      - "-wait"
+#      - "tcp://mongodb:27017"
+#      - "-stdout"
+#      - "/kb/deployment/jettybase/logs/request.log"
+#      - "./scripts/entrypoint.sh"
+#    entrypoint: [ "dockerize" ]
+#    depends_on: ["mongodb","condor"]
+#    environment:
+#      - POOL_PASSWORD=weakpassword
+#    env_file: test/deploy.cfg
+#    volumes:
+#      - ./:/ee2
+#
+#  ee2_with_ssh:
+#    image: execution_engine2:test
+#    command:
+#      - "-poll"
+#      - "-template"
+#      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
+#      - "-template"
+#      - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg"
+#      - "-timeout"
+#      - "120s"
+#      - "-wait"
+#      - "tcp://mongodb:27017"
+#      - "-stdout"
+#      - "/kb/deployment/jettybase/logs/request.log"
+#      - "./scripts/entrypoint.sh"
+#    entrypoint: [ "dockerize" ]
+#    depends_on: ["mongodb","condor"]
+#    environment:
+#      - POOL_PASSWORD=weakpassword
+#      - LOCAL_DEV=1
+#      - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg
+#    env_file: test/deploy.cfg
+#    ports: ["23:22","5678:5678"]
+#    volumes:
+#      - ./:/ee2
+#      - /Users:/Users
 
 
 
@@ -146,13 +147,15 @@ services:
       ZOOKEEPER_SYNC_LIMIT: 2docker-compose rm
       ZOOKEEPER_SERVERS: zookeeper:12888:13888
 
+
   mongodb:
-    image: mongo:3.2
+    image: mongo:${MONGO_VERSION:-7.0} # Use 7.0 as the default if MONGO_VERSION is not set
     environment:
-      # provide your credentials here
       - MONGO_INITDB_ROOT_USERNAME=travis
       - MONGO_INITDB_ROOT_PASSWORD=travis
       - MONGO_INITDB_DATABASE=ee2
-    ports: ["27018:27017","27017:27017"]
+    ports:
+      - "27018:27017"
+      - "27017:27017"
     volumes:
-      - "./test/dockerfiles/mongo/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/"
+      - ${INIT_PATH:-./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/}:/docker-entrypoint-initdb.d/

kbase.yml

@@ -8,7 +8,7 @@ service-language:
     python
 
 module-version:
-    0.0.13
+    0.0.14
 
 owners:
     [bsadkhin, tgu2, wjriehl, gaprice]

lib/execution_engine2/sdk/EE2Runjob.py

@@ -483,10 +483,10 @@ def run_batch(
         :return: A list of condor job ids or a failure notification
         """
 
-        if type(params) != list:
+        if not isinstance(params, list):
             raise IncorrectParamsException("params must be a list")
 
-        if type(batch_params) != dict:
+        if not isinstance(batch_params, dict):
             raise IncorrectParamsException("batch params must be a mapping")
 
         wsid = batch_params.get(_WORKSPACE_ID)
@@ -586,7 +586,7 @@ def _check_job_requirements_vs_admin(
     def _check_is_string(self, putative_str, name):
         if not putative_str:
             return None
-        if type(putative_str) != str:
+        if not isinstance(putative_str, str):
             raise IncorrectParamsException(f"{name} must be a string")
         return putative_str
 
@@ -924,7 +924,7 @@ def run(
         """
 
         # TODO Test this
-        if type(params) != dict:
+        if not isinstance(params, dict):
             raise IncorrectParamsException("params must be a mapping")
 
         self._preflight(runjob_params=params, as_admin=as_admin)
@@ -949,10 +949,10 @@ def _get_job_reqs_from_concierge_params(
         rl = concierge_params.get(_REQUIREMENTS_LIST)
         schd_reqs = {}
         if rl:
-            if type(rl) != list:
+            if not isinstance(rl, list):
                 raise IncorrectParamsException(f"{_REQUIREMENTS_LIST} must be a list")
             for s in rl:
-                if type(s) != str or "=" not in s:
+                if not isinstance(s, str) or "=" not in s:
                     raise IncorrectParamsException(
                         f"Found illegal requirement in {_REQUIREMENTS_LIST}: {s}"
                     )

lib/execution_engine2/sdk/job_submission_parameters.py

@@ -124,9 +124,11 @@ def check_parameters(
             client_group,
             None if client_group_regex is None else bool(client_group_regex),
             _check_string(bill_to_user, "bill_to_user", optional=True),
-            None
-            if ignore_concurrency_limits is None
-            else bool(ignore_concurrency_limits),
+            (
+                None
+                if ignore_concurrency_limits is None
+                else bool(ignore_concurrency_limits)
+            ),
             cls._check_scheduler_requirements(scheduler_requirements),
             None if debug_mode is None else bool(debug_mode),
         )
@@ -145,7 +147,7 @@ def _params(self):
         )
 
     def __eq__(self, other):
-        if type(self) == type(other):
+        if type(self) == type(other):  # noqa E721
             return self._params() == (
                 other.cpus,
                 other.memory_MB,
@@ -214,7 +216,7 @@ def __init__(
         )
         self.wsid = _gt_zero(wsid, "wsid", optional=True)
         source_ws_objects = source_ws_objects if source_ws_objects else []
-        if type(source_ws_objects) != list:
+        if not isinstance(source_ws_objects, list):
             raise IncorrectParamsException("source_ws_objects must be a list")
         for i, ref in enumerate(source_ws_objects):
             upa, is_valid = _is_valid_UPA(ref)
@@ -238,7 +240,7 @@ def _params(self):
         )
 
     def __eq__(self, other):
-        if type(self) == type(other):
+        if type(self) == type(other):  # noqa E721
             return self._params() == (
                 other.job_id,
                 other.app_info,

lib/execution_engine2/utils/application_info.py

@@ -116,7 +116,7 @@ def get_application_id(self) -> str:
         return self.application_module
 
     def __eq__(self, other):
-        if type(self) == type(other):
+        if type(self) == type(other):  # noqa E721
             return (
                 self.module,
                 self.method,