Test driven py3, for merge, with full CI

.github/workflows/main.yml

@@ -1,50 +1,80 @@
+# actions can be run locally using act and docker, on Fedora 37 also with podman, using:
+# https://github.com/nektos/act
+# sudo dnf install -y act-cli podman-docker
+# act --bind --container-daemon-socket $XDG_RUNTIME_DIR/podman/podman.sock -W .github/workflows/main.yml
+
 name: Unit tests
 
+# Checks can be skipped by adding "skip-checks: true" to a commit message,
+# or requested by adding "request-checks: true" if disabled by default for pushes:
+# https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks#skipping-and-requesting-checks-for-individual-commits
 on: [push, pull_request]
+env:
+  PYTHONWARNINGS: "ignore"
+  PIP_ROOT_USER_ACTION: "ignore"   # For local testing using act-cli
+  PIP_NO_WARN_SCRIPT_LOCATION: "0" # For local testing using act-cli
 
 jobs:
-  test_py2:
-    runs-on: ubuntu-20.04
+  test:
+    strategy:
+      # max-parallel: 1  # See: https://github.com/xenserver/python-libs/pull/26#discussion_r1179482169
+      matrix:
+        include:
+        - python-version: '3.11'
+          os: ubuntu-latest
+        - python-version: '3.10'
+          os: ubuntu-latest
+        - python-version: '2.7'
+          os: ubuntu-20.04
+        - python-version: '3.9'
+          os: ubuntu-latest
+        - python-version: '3.8'
+          os: ubuntu-latest
+        - python-version: '3.7'
+          os: ubuntu-latest
+        - python-version: '3.6'
+          os: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
-      - name: Set up Python 2.7
-        uses: actions/setup-python@v2
+          fetch-depth: 0  # Needed by diff-cover to get the changed lines: origin/master..HEAD
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
         with:
-          python-version: '2.7'
+          python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
+      - name: Provide Pylint Summary and Report on the GitHub Action's info page
+        if: ${{ matrix.python-version == '3.10' }}
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
-          # FIXME: branding.py still has no permanent home
-          curl https://gist.github.com/ydirson/3c36a7e19d762cc529a6c82340894ccc/raw/5ca39f621b1feab813e171f535c1aad1bd483f1d/branding.py -O -L
-          pip install pyliblzma
-          pip install -e .
-          command -v xz
+          pip install -q pylint pandas tabulate 2>&1|grep -v root ||: # Hide warning for root in local container
+          tree=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/tree/$GITHUB_REF_NAME
+          [ -n "$GITHUB_STEP_SUMMARY" ] || GITHUB_STEP_SUMMARY=pylint-summary-table.md
+          ./gen_gh_pylint_report.py xcp $GITHUB_STEP_SUMMARY $tree
 
       - name: Test
         run: |
-          pytest --cov -rP
-          coverage xml
-          coverage html
-          coverage html -d htmlcov-tests --include="tests/*"
-          diff-cover --html-report coverage-diff.html coverage.xml
+          rm -f coverage.xml
+          pip -q install tox tox-gh-actions
+          tox --recreate
 
-      - name: Pylint
-        run: |
-          pylint --version
-          pylint --exit-zero xcp/ tests/ setup.py
-          pylint --exit-zero --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" xcp/ tests/ setup.py > pylint.txt
-          diff-quality --violations=pylint --html-report pylint-diff.html pylint.txt
+      - name: Upload coverage reports to Codecov
+        if: ${{ matrix.python-version == '3.11' }}
+        uses: codecov/codecov-action@v3
 
       - uses: actions/upload-artifact@v3
+        # Skip coverage upload during local actions testing using act-cli:
+        # For logging the contents of the context, see:
+        # https://docs.github.com/en/actions/learn-github-actions/contexts#example-printing-context-information-to-the-log
+        if: ${{ matrix.python-version == '3.11' && !startsWith(github.actor, 'nektos/act') }}
         with:
           name: Coverage and pylint reports
           path: |
-            coverage-diff.html
-            pylint-diff.html
-            htmlcov
-            htmlcov-tests
+            coverage.xml
+             .tox/py311-cov/log/htmlcov/
+             .tox/py311-cov/log/htmlcov-tests/
+             .tox/py311-cov/log/coverage-diff.html
+             .tox/py311-cov/log/pylint-diff.html
+             .tox/py311-cov/log/pylint.txt
+             .tox/py311-cov/log/.coverage

README.md

@@ -0,0 +1,230 @@
+Common XenServer/XCP-ng Python classes
+======================================
+
+The `xcp` directory contains the Common XenServer and XCP-ng Python packages.
+They are intented for use in XenServer and XCP-ng Dom0 only and deal with logging,
+Hardware/PCI, networking, and other Dom0 tasks.
+
+The pip package name is `python-libs` which is also the rpm package name in XenServer.
+XCP-ng packages it as [xcp-python-libs](https://github.com/xcp-ng-rpms/xcp-python-libs)
+([koji](https://koji.xcp-ng.org/packageinfo?packageID=400)).
+
+It supports Python 2.7 and is currently in progress to get further fixes for >= 3.6.
+It depends on `six`, and on Python 2.7, also `configparser` and `pyliblzma`.
+
+Pylint results from GitHub CI in GitHub Actions page
+----------------------------------------------------
+A step of the GitHub workflow produces a browser-friendly `pylint` report:
+From the [Actions tab](https://github.com/xenserver/python-libs/actions),
+open a recent workflow run the latest and scroll down until you see the tables!
+
+Testing locally and in GitHub CI using tox
+------------------------------------------
+
+`pytest` runs tests. Checks by `pylint` and `mypy`. With `tox`, developers can
+run the full test suite for Python 2.7 and 3.x. Unit tests are passing but there are
+ many Python3 issues which it does not uncover yet.
+
+> Intro: Managing a Project's Virtualenvs with tox -
+> A comprehensive beginner's introduction to tox.
+> https://www.seanh.cc/2018/09/01/tox-tutorial/
+
+To run the tests for all supported and installed python versions, run:
+```yaml
+# The latest versions of tox need 1.11.0, so sensure that you have the latest py-1.11:
+pip3 install 'py>=1.11.0' tox; tox
+```
+You can run tox with just the python versions you have using `tox -e py27-test -e py3.11-mypy`.
+The syntax is `-e py<pvthon-version>-<factor1>[-factor2]` The currently supported factors
+are:
+- `test`: runs pytest
+- `cov`: runs pytest --cov and generate XML and HTML reports in `.tox/py<ver>-cov/logs/`
+- `mypy`: runs mypy
+- `fox`: runs like `cov` but then opens the HTML reports in Firefox!
+
+The list of `virtualenvs` can be shown using this command: `tox -av -e py312-fox`
+```yaml
+using tox-3.28.0 from /usr/lib/python3.11/site-packages/tox/__init__.py (pid 157772)
+default environments:
+py27-test  -> Run in a python2.7 virtualenv: pytest
+py36-test  -> Run in a python3.6 virtualenv: pytest
+py37-test  -> Run in a python3.7 virtualenv: pytest
+py38-test  -> Run in a python3.8 virtualenv: pytest
+py39-test  -> Run in a python3.9 virtualenv: pytest
+py310-test -> Run in a python3.10 virtualenv: pytest
+py311-mypy -> Run in a python3.11 virtualenv: mypy static analyis
+py311-cov  -> Run in a python3.11 virtualenv: generate coverage html reports
+py312-test -> Run in a python3.12 virtualenv: pytest
+
+additional environments:
+py312-fox  -> Run in a python3.12 virtualenv: generate coverage html reports and open them in firefox
+```
+If you have just one version of Python3, that will be enough, just use `tox -e py<ver>-test`.
+
+Installation of additional python versions for testing different versions:
+- Fedora 37: `sudo dnf install tox` installs all Python versions, even 3.12a7.
+- On Ubuntu, the deadsnakes/ppa is broken(except for 3.12), so conda or pyenv has to be used.
+  For full instructions, see https://realpython.com/intro-to-pyenv/, E.g install on Ubuntu:
+  ```yaml
+  sudo apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev
+                          libreadline-dev libsqlite3-dev xz-utils libffi-dev liblzma-dev
+   curl https://pyenv.run | bash # and add the displayed commands to .bashrc
+   pyenv install 3.{6,7,8,9} && pyenv local 3.{6,7,8,9} # builds and adds them
+   ```
+- Note: `virtualenv-20.22` broke creating the `py27` venv with tox, at least in some setups.
+  As a workaround, downgrade it to 20.21 if that happens: `pip3 install -U 'virtualenv<20.22'`
+- For testing on newer Ubuntu hosts which have `python2-dev`, but not `pip2`, install `pip2` this way:
+  ```json
+  curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py;sudo python2 get-pip.py
+  ```
+
+Static analysis using mypy and pyright
+--------------------------------------
+The preconditions for using static analysis with `mypy` (which passes now but has
+only a few type comments) and `pyright` are present now and `mypy` is enabled in `tox`
+which runs the tests in GitHub CI as well. But of course, because they code is largely
+still not yet typed, no strict checks can be enabled so far. However, every checker
+which is possible now, is enabled.
+
+Checking the contents of untyped functions is enabled for all but four modules which
+would need more work. Look for `check_untyped_defs = false` in `pytproject.toml`.
+
+The goal or final benefit would be to have it to ensure internal type correctness
+and code quality but also to use static analysis to check the interoperability with
+the calling code.
+
+Type annotations: Use Type comments for now!
+--------------------------------------------
+Python2.7 can't support the type annotation syntax, but until all users are migrated,
+annotations in comments (type comments) can be used. They are supported by
+tools like `mypy` and `pyright` (VS Code):
+
+Quoting from https://stackoverflow.com/questions/53306458/python-3-type-hints-in-python-2:
+
+> Function annotations were introduced in [PEP 3107](https://www.python.org/dev/peps/pep-3107/) for Python 3.0. The usage of annotations as type hints was formalized in in [PEP 484](https://www.python.org/dev/peps/pep-0484/) for Python 3.5+.
+>
+> Any version before 3.0 then will not support the syntax you are using for type hints at all. However, PEP 484 [offers a workaround](https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code), which some editors may choose to honor. In your case, the hints would look like this:
+```py
+def get_default_device(use_gpu=True):
+    # type: (bool) -> cl.Device
+    ...
+```
+> or more verbosely,
+```py
+def get_default_device(use_gpu=True  # type: bool
+                      ):
+    # type: (...) -> cl.Device
+    ...
+```
+> The PEP explicitly states that this form of type hinting should work for any version of Python.
+
+As proof, these examples show how the comment below triggers the checks:
+```diff
+--- a/xcp/xmlunwrap.py
++++ b/xcp/xmlunwrap.py
+@@ -29,1 +29,2 @@ class XmlUnwrapError(Exception):
+ def getText(nodelist):
++    # type:(Element) -> str
+```
+mypy:
+```py
+$ mypy xcp/xmlunwrap.py
+xcp/xmlunwrap.py:31: error: Name "Element" is not defined
+xcp/xmlunwrap.py:38: error: Incompatible return value type (got "bytes", expected "str")
+```
+pyright (used by VS Code by default):
+```py
+$ pyright xcp/xmlunwrap.py|sed "s|$PWD/||"
+...
+pyright 1.1.295
+xcp/xmlunwrap.py
+  xcp/xmlunwrap.py:32:13 - error: "Element" is not defined (reportUndefinedVariable)
+  xcp/xmlunwrap.py:38:12 - error: Expression of type "Unknown | bytes" cannot be assigned to return type "str"
+    Type "Unknown | bytes" cannot be assigned to type "str"
+      "bytes" is incompatible with "str" (reportGeneralTypeIssues)
+  xcp/xmlunwrap.py:81:38 - error: Argument of type "Unknown | None" cannot be assigned to parameter "default" of type "str" in function "getStrAttribute"
+    Type "Unknown | None" cannot be assigned to type "str"
+      Type "None" cannot be assigned to type "str" (reportGeneralTypeIssues)
+3 errors, 0 warnings, 0 informations
+Completed in 0.604sec
+```
+See https://github.com/xenserver/python-libs/pull/23 for the context of this example.
+
+Special open TODOs:
+-------------------
+
+Charset encoding/string handling:
+* With Python3, `read()` on files `open()`ed without specifying binary mode will attempt
+  to decode the data into the Python3 Unicode string type, which will fail for all
+  binary data. Thus all `open()` calls which might open binary files have to be converted
+  to binary mode by default unless the caller is sure he is opening an ASCII file,
+  even then, enabling an error handle to handle decoding errors is recommended.
+* With Python3, the `stdin`, `stdout` and `stderr` pipes for `Popen()` default to
+  `bytes`(binary mode.) Binary mode is much safer because it foregoes the encode/decode. The existing users need to be able to enable text mode (when safe, it will attempt
+  to decode and encode!) or preferably be able to use bytes (which is the type behind Python2 strings too) instead. See these PRs for details:
+  * https://github.com/xenserver/python-libs/pull/22
+  * https://github.com/xenserver/python-libs/pull/23
+  * https://github.com/xenserver/python-libs/pull/24
+  * What's more: When code is called from a xapi plugin (such as ACK), when such code
+    attempts to read text files like the `pciids` file, and there is a Unicode char
+    it int, and the locale is not set up to be UTF-8 (because xapi plugins are started
+    from xapi), the UTF-8 decoder has to be explicitly enabled for these files,
+    bese by adding `encoding="utf-8"` to the arguments of these specific `open()` calls,
+    to have valid Unicode text strings, e.g. `xcp.pci`, for regular text processing.
+  * TODO: More to be opened for all remaining `open()` and `Popen()` users,
+    as well as ensuring that users of `urllib` are able to work with they bytes
+    it returns (there is no option to use text mode, data may be gzip-encoded!)
+
+Users
+-----
+
+* https://github.com/xenserver/host-installer
+    * /opt/xensource/installer/ (has copies of `cpiofile.py`, `repository.py` (with `accessor.py`)
+* https://github.com/xcp-ng-rpms/host-upgrade-plugin ([koji](https://koji.xcp-ng.org/packageinfo?packageID=104)):
+    * /etc/xapi.d/plugins/prepare_host_upgrade.py
+* https://github.com/xapi-project/xen-api (`xapi-core.rpm` and `xenopsd.rpm`)
+    * /etc/xapi.d/extensions/pool_update.apply
+    * /etc/xapi.d/extensions/pool_update.precheck
+    * /etc/xapi.d/plugins/disk-space
+    * /etc/xapi.d/plugins/install-supp-pack
+    * /opt/xensource/libexec/host-display
+    * /opt/xensource/libexec/mail-alarm
+    * /opt/xensource/libexec/usb_reset.py
+    * /opt/xensource/libexec/usb_scan.py
+    * /usr/libexec/xenopsd/igmp_query_injector.py
+* xenserver-release-config/[xcp-ng-release-config](https://koji.xcp-ng.org/rpminfo?rpmID=10250)
+    * /opt/xensource/libexec/fcoe_driver
+    * /opt/xensource/libexec/xen-cmdline
+* https://github.com/xcp-ng-rpms/interface-rename:
+    * /etc/sysconfig/network-scripts/interface-rename.py
+    * /opt/xensource/bin/interface-rename
+* pvsproxy (Proprietary)
+    * /usr/libexec/xapi-storage-script/volume/org.xen.xapi.storage.tmpfs/memoryhelper.py
+* https://github.com/xenserver/linux-guest-loader (not installed by default anymore)
+    * /opt/xensource/libexec/eliloader.py
+* https://github.com/xcp-ng-rpms/vcputune
+    * /opt/xensource/bin/host-cpu-tune
+* The ACK xenapi plugin see: https://github.com/xenserver/python-libs/pull/21
+
+Verification:
+```ps
+# rpm -qf $(grep -r import /usr/libexec/ /usr/bin /etc/xapi.d/ /opt/xensource/|grep xcp|cut -d: -f1|grep -v Binary) --qf '%{name}\n'|sort -u|tee xcp-python-libs-importers.txt
+host-upgrade-plugin
+interface-rename
+pvsproxy
+vcputune
+xapi-core
+xenopsd
+xenserver-release-config
+# grep -s import $(rpm -ql xapi-core)|grep xcp|cut -d: -f1
+/etc/xapi.d/extensions/pool_update.apply
+/etc/xapi.d/extensions/pool_update.precheck
+/etc/xapi.d/plugins/disk-space
+/etc/xapi.d/plugins/disk-space
+/etc/xapi.d/plugins/install-supp-pack
+/opt/xensource/libexec/host-display
+/opt/xensource/libexec/mail-alarm
+/opt/xensource/libexec/usb_reset.py
+/opt/xensource/libexec/usb_scan.py
+```
+

branding.py

@@ -0,0 +1,36 @@
+BRAND_CONSOLE_URL = "https://xcp-ng.org"
+BRAND_CONSOLE = "XCP-ng Center"
+BRAND_GUEST_SHORT = "VM"
+BRAND_GUESTS_SHORT = "VMs"
+BRAND_GUESTS = "Virtual Machines"
+BRAND_GUEST = "Virtual Machine"
+BRAND_SERVERS = "XCP-ng Hosts"
+BRAND_SERVER = "XCP-ng Host"
+BRAND_VDI = ""
+COMPANY_DOMAIN = "xcp-ng.org"
+COMPANY_NAME_LEGAL = "Open Source"
+COMPANY_NAME = "Open Source"
+COMPANY_NAME_SHORT = "Open Source"
+COMPANY = "Open Source"
+COMPANY_PRODUCT_BRAND = "XCP-ng"
+COMPANY_WEBSITE = "https://xcp-ng.org"
+COPYRIGHT_YEARS = "2018-2022"
+ISO_PV_TOOLS_COPYRIGHT = "XCP-ng"
+ISO_PV_TOOLS_LABEL = "XCP-ng VM Tools"
+ISO_PV_TOOLS_PUBLISHER = "XCP-ng"
+PLATFORM_MAJOR_VERSION = "3"
+PLATFORM_MICRO_VERSION = "1"
+PLATFORM_MINOR_VERSION = "2"
+PLATFORM_NAME = "XCP"
+PLATFORM_ORGANISATION = "xen.org"
+PLATFORM_VERSION = "3.2.1"
+PLATFORM_WEBSITE = "www.xen.org"
+PRODUCT_BRAND = "XCP-ng"
+PRODUCT_BRAND_DASHED = "XCP-ng"
+PRODUCT_MAJOR_VERSION = "8"
+PRODUCT_MICRO_VERSION = "1"
+PRODUCT_MINOR_VERSION = "2"
+PRODUCT_NAME = "xenenterprise"
+PRODUCT_VERSION = "8.2.1"
+PRODUCT_VERSION_TEXT = "8.2"
+PRODUCT_VERSION_TEXT_SHORT = "8.2"