Merge dev into main

.github/workflows/testing.yml

@@ -1,7 +1,6 @@
 name: Testrun test suite
 
 on:
-  push:
   pull_request:
   schedule:
     - cron: '0 13 * * *'
@@ -14,21 +13,69 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v2.3.4
-      - name: Run tests
+      - name: Install dependencies
+        shell: bash {0}
+        run: cmd/prepare
+      - name: Package Testrun
+        shell: bash {0}
+        run: cmd/package
+      - name: Install Testrun
+        shell: bash {0}
+        run: sudo dpkg -i testrun*.deb
+      - name: Run baseline tests
         shell: bash {0}
         run: testing/baseline/test_baseline
 
   testrun_tests:
     name:  Tests
     runs-on: ubuntu-20.04
     needs: testrun_baseline
-    timeout-minutes: 40
+    timeout-minutes: 45
     steps:
       - name: Checkout source
         uses: actions/checkout@v2.3.4
+      - name: Install dependencies
+        shell: bash {0}
+        run: cmd/prepare
+      - name: Package Testrun  
+        shell: bash {0}
+        run: cmd/package
+      - name: Install Testrun
+        shell: bash {0}
+        run: sudo dpkg -i testrun*.deb
       - name: Run tests
         shell: bash {0}
         run: testing/tests/test_tests
+      - name: Archive runtime results
+        if: ${{ always() }}
+        run: sudo tar --exclude-vcs -czf runtime.tgz /usr/local/testrun/runtime/
+      - name: Upload runtime results
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          if-no-files-found: error
+          name: runtime_${{ github.workflow }}_${{ github.run_id }}
+          path: runtime.tgz
+
+  testrun_api:
+    name:  API
+    runs-on: ubuntu-20.04
+    timeout-minutes: 40
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.4
+      - name: Install dependencies
+        shell: bash {0}
+        run: cmd/prepare
+      - name: Package Testrun
+        shell: bash {0}
+        run: cmd/package
+      - name: Install Testrun
+        shell: bash {0}
+        run: sudo dpkg -i testrun*.deb
+      - name: Run tests
+        shell: bash {0}
+        run: testing/api/test_api
 
   pylint:
     name:  Pylint
@@ -37,6 +84,22 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v2.3.4
-      - name: Run tests
+      - name: Run pylint
         shell: bash {0}
         run: testing/pylint/test_pylint
+
+  testrun_package:
+    name: Package
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2.3.4
+      - name: Package Testrun
+        shell: bash {0}
+        run: cmd/package
+      - name: Archive package
+        uses: actions/upload-artifact@v3
+        with:
+          name: Testrun Installer
+          path: testrun*.deb

.gitignore

@@ -6,3 +6,4 @@ pylint.out
 __pycache__/
 build/
 testing/unit_test/temp/
+*.deb

README.md

@@ -1,41 +1,42 @@
-  <img width="200" alt="Testrun logo" src="https://user-images.githubusercontent.com/7399056/221927867-4190a4e8-a571-4e40-9c2b-65780ad9264c.png" alt="Test Run">
+  <img width="200" alt="Testrun logo" src="https://user-images.githubusercontent.com/7399056/221927867-4190a4e8-a571-4e40-9c2b-65780ad9264c.png" alt="Testrun">
 
 ## Introduction :wave:
-Test Run is a tool to automate the validation of network-based functionality of IoT devices. Any device which is capable of receiving an IP address via DHCP is considered an IoT device by Test Run and can be tested.
+Testrun automates specific test cases to verify network and security functionality in IoT devices. It is an open source tool which allows external manufacturers to test their devices for the purposes of Device Qualification within the BOS program.
 
 ## Motivation :bulb:
-Without tools like Test Run, testing labs may be maintaining a large and complex network using equipment such as: A managed layer 3 switch, an enterprise-grade network router, virtualized or physical servers to provide DNS, NTP, 802.1x etc. With this amount of moving parts, all with dynamic configuration files and constant software updates, more time is likely to be spent on preparation and clean up of functinality or penetration testing - not forgetting the number of software tools required to perform the testing. The major issues which can and should be solved:
+Without tools like Testrun, testing labs may be maintaining a large and complex network using equipment such as: A managed layer 3 switch, an enterprise-grade network router, virtualized or physical servers to provide DNS, NTP, 802.1x etc. With this amount of moving parts, all with dynamic configuration files and constant software updates, more time is likely to be spent on preparation and clean up of functinality or penetration testing - not forgetting the number of software tools required to perform the testing. The major issues which can and should be solved:
  1) The complexity of managing a testing network
  2) The time required to perform testing of network functionality
  3) The accuracy and consistency of testing network functionality
 
 ## How it works :triangular_ruler:
-Test Run creates an isolated and controlled network environment to fully simulate enterprise network deployments in your device testing lab. 
+Testrun creates an isolated and controlled network environment to fully simulate enterprise network deployments in your device testing lab. 
 This removes the necessity for complex hardware, advanced knowledge and networking experience whilst enabling semi-technical engineers to validate device 
 behaviour against industry cyber standards. 
 
-Two runtime modes will be supported by Test Run:
+Two runtime modes are supported by Testrun:
 
-1) <strong>Automated Testing</strong>
+1) <strong>Automated testing</strong>
 
 Once the device has become operational (steady state), automated testing of the device under test will begin. Containerized test modules will then execute against the device (one module at a time). Once all test modules have completed execution, a final test report will be produced - presenting the results and further description of findings.
 
 2) <strong>Lab network</strong>
 
-Test Run cannot automate everything, and so additional manual testing may be required (or configuration changes may be required on the device). Rather than having to maintain a separate but idential lab network, Test Run will provide the network and some tools to assist an engineer performing the additional testing. At the same time, packet captures of the device behaviour will be recorded, alongside logs for each network service, for further debugging.
+Testrun cannot automate everything, and so additional manual testing may be required (or configuration changes may be required on the device). Rather than having to maintain a separate but idential lab network, Testrun will provide the network and some tools to assist an engineer performing the additional testing. At the same time, packet captures of the device behaviour will be recorded, alongside logs for each network service, for further debugging.
 
-## Minimum Requirements :computer:
+## Minimum requirements :computer:
 ### Hardware
  - PC running Ubuntu LTS (laptop or desktop)
  - 2x USB ethernet adapter (One may be built in ethernet)
  - Internet connection
 ### Software
- - Python 3 (Already available on Ubuntu LTS)
- - Docker - [Install guide](https://docs.docker.com/engine/install/ubuntu/)
- - Open vSwitch ``sudo apt-get install openvswitch-common openvswitch-switch``
+- Docker - installation guide: [https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository)
+
+## Get started ▶️
+Once you have met the hardware and software requirements, you can get started with Testrun by following the [Get started guide](docs/get_started.md).
 
 ## Roadmap :chart_with_upwards_trend:
-Test Run will constantly evolve to further support end-users by automating device network behaviour against industry standards.
+Testrun will constantly evolve to further support end-users by automating device network behaviour against industry standards.
 
 ## Issue reporting :triangular_flag_on_post:
 If the application has come across a problem at any point during setup or use, please raise an issue under the [issues tab](https://github.com/auto-iot/test-run/issues). Issue templates exist for both bug reports and feature requests. If neither of these are appropriate for your issue, raise a blank issue instead.
@@ -44,25 +45,26 @@ If the application has come across a problem at any point during setup or use, p
 The contributing requirements can be found in [CONTRIBUTING.md](CONTRIBUTING.md). In short, checkout the [Google CLA](https://cla.developers.google.com/) site to get started.
 
 ## FAQ :raising_hand:
-1) What device networking functionality is validated by Test Run?
+1) What device networking functionality is validated by Testrun?
 
   Best practices and requirements for IoT devices are constantly changing due to technological advances and discovery of vulnerabilities. 
   The current expectations for IoT devices on Google deployments can be found in the [Application Security Requirements for IoT Devices](https://partner-security.withgoogle.com/docs/iot_requirements).
-  Test Run aims to automate as much of the Application Security Requirements as possible.
+  Testrun aims to automate as much of the Application Security Requirements as possible.
 
 2) What services are provided on the virtual network?
 
   The following are network services that are containerized and accessible to the device under test though are likely to change over time:
  - DHCP in failover configuration with internet connectivity
- - DNS (and DNS over HTTPS)
+ - IPv6 SLAAC
+ - DNS
  - NTPv4
  - 802.1x Port Based Authentication
 
-3) Can I run Test Run on a virtual machine?
+3) Can I run Testrun on a virtual machine?
 
-  Probably. Provided that the required 2x USB ethernet adapters are passed to the virtual machine as USB devices rather than network adapters, Test Run should
-  still work. We will look to test and approve the use of virtualisation to run Test Run in the future.
+  Probably. Provided that the required 2x USB ethernet adapters are passed to the virtual machine as USB devices rather than network adapters, Testrun should
+  still work. We will look to test and approve the use of virtualisation to run Testrun in the future.
 
- 4) Can I connect multiple devices to Test Run?
+ 4) Can I connect multiple devices to Testrun?
 
   In short, Yes you can. The way in which multiple devices could be tested simultaneously is yet to be decided. However, if you simply want to add field/peer devices during runtime (even another laptop performing manual testing) then you may connect the USB ethernet adapter to an unmanaged switch.

bin/testrun

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 if [[ "$EUID" -ne 0 ]]; then
-    echo "Must run as root. Use sudo testrun"
+    echo "Must run as root. Use sudo $0"
     exit 1
 fi
 
@@ -26,17 +26,20 @@ fi
 # Ensure that /var/run/netns folder exists
 sudo mkdir -p /var/run/netns
 
+export TESTRUNPATH=/usr/local/testrun
+cd $TESTRUNPATH
+
 # Create device folder if it doesn't exist
 mkdir -p local/devices
 
-# Check if Python modules exist. Install if not
-[ ! -d "venv" ] && sudo cmd/install
+# Remove existing runtime data
+rm -rf runtime/*
 
 # Activate Python virtual environment
 source venv/bin/activate
 
 # Set the PYTHONPATH to include the "src" directory
-export PYTHONPATH="$PWD/framework/python/src"
-python -u framework/python/src/core/test_runner.py $@
+export PYTHONPATH="$TESTRUNPATH/framework/python/src"
+python -u framework/python/src/core/test_runner.py $@ 2>&1 | tee testrun.log
 
 deactivate

cmd/build

@@ -0,0 +1,52 @@
+#!/bin/bash -e
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds all docker images
+echo Building docker images
+
+# Build user interface
+echo Building user interface
+mkdir -p build/ui
+docker build -t test-run/ui -f modules/ui/ui.Dockerfile . 2>&1 | tee build/ui/ui.log
+
+# Build network modules
+echo Building network modules
+mkdir -p build/network
+for dir in modules/network/* ; do
+  module=$(basename $dir)
+  echo Building network module $module...
+  docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/network/$module.log
+done
+
+# Build validators
+echo Building network validators
+mkdir -p build/devices
+for dir in modules/devices/* ; do
+  module=$(basename $dir)
+  echo Building validator module $module...
+  docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . 2>&1 | tee build/devices/$module.log
+done
+
+# Build test modules
+echo Building test modules
+mkdir -p build/test
+for dir in modules/test/* ; do
+  module=$(basename $dir)
+  echo Building test module $module...
+  docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . 2>&1 | tee build/test/$module.log
+done
+
+echo Finished building modules

cmd/install

@@ -14,10 +14,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+echo Installing application dependencies
+
+TESTRUN_DIR=/usr/local/testrun
+cd $TESTRUN_DIR
+
 python3 -m venv venv
 
 source venv/bin/activate
 
 pip3 install -r framework/requirements.txt
 
+# Copy the default configuration
+cp -n local/system.json.example local/system.json
+
 deactivate
+
+# Build docker images
+sudo cmd/build
+
+echo Finished installing Testrun

cmd/package

@@ -0,0 +1,56 @@
+#!/bin/bash -e
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Creates a package for Testrun
+
+MAKE_SRC_DIR=make
+
+# Copy testrun script to /bin
+mkdir -p $MAKE_SRC_DIR/bin
+cp bin/testrun $MAKE_SRC_DIR/bin/testrun
+
+# Create testrun folder
+mkdir -p $MAKE_SRC_DIR/usr/local/testrun
+
+# Create postinst script
+cp cmd/install $MAKE_SRC_DIR/DEBIAN/postinst
+
+# Copy other commands
+mkdir -p $MAKE_SRC_DIR/usr/local/testrun/cmd
+cp cmd/{prepare,build} $MAKE_SRC_DIR/usr/local/testrun/cmd
+
+# Copy resources
+cp -r resources $MAKE_SRC_DIR/usr/local/testrun/
+
+# Create local folder
+mkdir -p $MAKE_SRC_DIR/usr/local/testrun/local
+cp local/system.json.example $MAKE_SRC_DIR/usr/local/testrun/local/system.json.example
+
+# Create device repository
+mkdir -p $MAKE_SRC_DIR/usr/local/testrun/local/devices
+
+# Copy root_certs folder
+mkdir -p local/root_certs
+cp -r local/root_certs $MAKE_SRC_DIR/usr/local/testrun/local/root_certs
+
+# Copy framework and modules into testrun folder
+cp -r {framework,modules} $MAKE_SRC_DIR/usr/local/testrun
+
+# Build .deb file
+dpkg-deb --build --root-owner-group make
+
+# Rename the .deb file
+mv make.deb testrun_1-0_amd64.deb

cmd/prepare

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Optional script to prepare your system for use with Testrun.
+# Installs system dependencies
+
+echo Installing system dependencies
+
+sudo apt-get install openvswitch-common openvswitch-switch python3 libpangocairo-1.0-0
+
+echo Finished installing system dependencies