From 55c5a4eaa4ca5c20e3d565a21ebf2b4fe1c732f5 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Wed, 17 Jul 2024 16:22:27 -0400 Subject: [PATCH 1/9] orbit: radius: markdown: inline handle_md `handle_md` does not need to be a separate function. It is only called once, and it doesn't offer any substantial encapsulation since it has preconditions that would require substantial work for any caller. Just combining the code that verifies that the markdown path is valid with the code that actually uses it makes it easier to tell that it is correct, not harder. While we are at it, using a regex is super overkill for this purpose, cgit is checked first in the `application` function so lines starting with `/cgit` will be directed to that handler before ever reaching the default case which calls this function, so we really only need to check whether or not the path ends with `.md` and there is `str.endswith` which does precisely that and does it faster. --- orbit/radius.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/orbit/radius.py b/orbit/radius.py index a6d5aa0f..c6724f3d 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -484,21 +484,17 @@ def handle_error(rocket): return rocket.respond(error_description) -def handle_md(rocket, md_path): - with open(md_path, 'r', newline='') as f: - content = markdown.markdown(f.read(), - extensions=['tables', 'fenced_code', - 'footnotes']) - return rocket.respond(content) - - def handle_try_md(rocket): - md_path = f'{config.doc_root}{rocket.path_info}' - if re.match("^(?!/cgit)(.*\\.md)$", rocket.path_info) \ - and os.access(md_path, os.R_OK): - return handle_md(rocket, md_path) - else: + if not rocket.path_info.endswith('.md'): + return rocket.raw_respond(HTTPStatus.NOT_FOUND) + path = f'{config.doc_root}{rocket.path_info}' + if not os.access(path, os.R_OK): return rocket.raw_respond(HTTPStatus.NOT_FOUND) + with open(path) as file: + md = file.read() + html = markdown.markdown(md, extensions=['tables', 'fenced_code', + 'footnotes']) + return rocket.respond(html) def application(env, SR): From a7ac7168a1f1892a1e20697cfad2c1bb6b631d60 Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:26:41 -0400 Subject: [PATCH 2/9] orbit: radius: application: check for equality for most routes Most routes are not patterns, we should just compare the path info with the exact expected string and avoid bloated regex invocations. While we are at it, match offers a more DRY solution to for checking rocket.path_info against many values. --- orbit/radius.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/orbit/radius.py b/orbit/radius.py index c6724f3d..99bf5cc2 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -6,7 +6,6 @@ import html import markdown import os -import re import subprocess import sys import secrets @@ -503,24 +502,27 @@ def application(env, SR): return rocket.raw_respond(HTTPStatus.METHOD_NOT_ALLOWED) # routes supporting get and post - if re.match("^/login", rocket.path_info): - return handle_login(rocket) - elif re.match("^/register", rocket.path_info): - return handle_register(rocket) - else: - if rocket.method != 'GET': - return rocket.raw_respond(HTTPStatus.METHOD_NOT_ALLOWED) + match rocket.path_info: + case '/login': + return handle_login(rocket) + case '/register': + return handle_register(rocket) + case _: + if rocket.method != 'GET': + return rocket.raw_respond(HTTPStatus.METHOD_NOT_ALLOWED) # routes supporting only get - if re.match("^/logout", rocket.path_info): - return handle_logout(rocket) - elif re.match("^/mail_auth", rocket.path_info): - return handle_mail_auth(rocket) - elif re.match("^/dashboard", rocket.path_info): - return handle_dashboard(rocket) - elif re.match("^/cgit", rocket.path_info): - return handle_cgit(rocket) - elif re.match("^/error", rocket.path_info): - return handle_error(rocket) - else: - return handle_try_md(rocket) + match rocket.path_info: + case '/logout': + return handle_logout(rocket) + case '/mail_auth': + return handle_mail_auth(rocket) + case '/dashboard': + return handle_dashboard(rocket) + case '/error': + return handle_error(rocket) + case p: + if p.startswith('/cgit'): + return handle_cgit(rocket) + else: + return handle_try_md(rocket) From fc5d8a4e03eb966812ed723fcd4848aff94cec95 Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:35:34 -0400 Subject: [PATCH 3/9] readme: overhaul We can remove some of the complexity about podman compose now that the important features for local development are merged upstream. We can convert all the sections to use lists instead of paragraph form to make them easier to read. Bulleted lists also make more sense than numbered lists for this purpose since there is some commentary and not just do this step and then this step. --- README.md | 195 ++++++++++++++++++++++++++---------------------------- 1 file changed, 93 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index 3fb3e5f8..3e7699f7 100644 --- a/README.md +++ b/README.md @@ -4,153 +4,144 @@ The singularity at the center of the KDLP infrastructure black hole. Section 1: Podman Setup -- + - Unfortunately, this project uses features that have been merged into podman-compose but as of June 2024, +have not yet been made part of a tagged release, so the version of podman-compose provided by any distro +is unlikely to work. -0. Make sure you have `podman`, `python3`, `pip`, `socat`, and `git` installed on your host machine. + - Fortunately, as podman-compose is written in python, it is simple to just clone a newer version of the +source code and add a symlink to your path to run the newer version of the script. Python will find any +libraries needed to run the code from the system packages installed as dependencies of the system podman-compose +package. -0. Clone the KDLP podman-compose repo. -We maintain a fork of podman compose with fixes and support for features -in the container-compose spec that are not yet in upstream podman-compose. + - Make sure you have `git`, `podman`, and `podman-compose` installed on your host machine. +On fedora these packages can be obtained by running `sudo dnf install -y git podman podman-compose`. - ```sh - git clone --depth=1 --single-branch --branch=kdlp https://github.com/underground-software/podman-compose.git - ``` + - Clone the upstream podman-compose repo. We recommend creating folder `mkdir -p ~/.local/src/` where the repo +can be placed. The appropriate command is `git -C ~/.local/src clone https://github.com/containers/podman-compose.git` -0. Get the podman-compose depdencies. Here are two options. + - Create a symlink in `~/.local/bin` named `podman-compose` which points to the `podman_compose.py` script in +the cloned repository. The appropriate command is `ln -s ../src/podman-compose/podman_compose.py ~/.local/bin/podman-compose` - 1. Set up a python virtual environment. + - Verify that `which podman-compose` outputs `~/.local/bin/podman-compose`. If it does not, you may need to refresh the cached +path for the program (for bash that is `hash podman-compose`), or potentially add `$HOME/.local/bin` to `$PATH` if it is +not already present. The exact details will vary depending on your shell and are outside the scope of this document. - - Create a new venv. - - ```sh - python3 -m venv kdlp-venv - source kdlp-venv/bin/activate - ``` +Section 2: Singularity Setup +-- - - Install podman-compose's requirements. + - Clone the singularity repo: `git clone https://github.com/underground-software/singularity.git`. - ```sh - pip install -r requirements.txt - ``` + - Create an empty `docs` folder within the repository: `mkdir docs`. - - If you install depenencies this way, make sure the venv is active (via `source kdlp-venv/bin/activate`) whenever you use our `podman-compose.py` script + - Build the containers: `podman-compose build`. - 1. Get the dependencies from you system package manager: - - Install the podman-compose packaged by your distribution. - - It should install the appropriate python dependencies as global python packages. - - NOTE: Running `podman-compose` in your terminal will invoke - the unpatched version installed by your system package manager - which is not compatible with singularity. - You must invoke our patched `podman_compose.py` script directly - unless you create your own symlink or alias. + - Launch singularity: `podman-compose up -d`. -### NOTE: From this point on, whenever we say `podman-compose`, treat this as an invocation of our patched version as described above. + - At this point, the application is listening on three unix sockets located in the `socks` ~~drawer~~ directory. -Section 2: Singularity Setup +Section 3: Testing and basic functionality -- -0. Clone the singularity repo. + - In order to verify that singularity is behaving correctly, you should run the test suite. + + - You will need to install some host packages needed by the testing script: flake8, shellcheck, jq, and curl. +On fedora these packages can be obtained by running `sudo dnf install -y flake8 shellcheck jq curl`. + + - Now you can run `./test.sh && echo 'tests passed'`. If you followed the directions, the tests should pass. + + - You will not be able to access the services using a normal web browser or email client without one more step as the +services listening on unix sockets instead of TCP ports. - ```sh - git clone https://github.com/underground-software/singularity.git - ``` + - Fortunately, it is easy to proxy the relevant TCP ports to the unix sockets so that the services can be accessed. +You will need the host program socat. On fedora it can be obtained by running `sudo dnf install -y socat`. -0. Create an empty `docs` folder - ```sh - mkdir docs - ``` + - Run `./dev_sockets.sh &` or `sudo ./dev_sockets.sh &` to spawn three instances of socat in the background. -0. Build the containers. + - Each socat instance will bind a specific TCP port and redirect connections to the unix socket for the corresponding service. - ```sh - podman-compose build - ``` + - When run without root privileges they will use ports above the threshold restricted for system services +(as configured in `/proc/sys/net/ipv4/ip_unprivileged_port_start`, 1024 by default): -0. Launch singularity. + - When run with root privileges it will use the traditional ports for the services +[as specified by the IANA](https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.txt). - ```sh - podman-compose up - ``` + - The test script will have created a user named `resu` whose password is `ssap` that you can use for manual testing. -0. Open another terminal and run the tests. If you followed the directions, they should pass. + - Try opening the website in your browser and/or the email services with your email client and/or the matrix server with your matrix client. - ```sh - ./test.sh - ``` + - Since singularity can only create self signed certificates by itself, you may need to accept warnings about them before you can proceed. -0. At this point, the application is listening on three unix sockets located in the `socks` ~~drawer~~ directory. + - Make sure to specify the protocol (https, smtps, pop3s) and port (443 or 1443, 465 or 1465, 995 or 1995) explicitly or your client may +try to use the wrong port or the unencrypted version of the protocol. - tl;dr run `sudo ./dev_sockets.sh &` to bind the services to the normal TCP ports. + - There is no web content by default so the homepage of the website will give a 404 error, however the links to the course +services in the navigation bar should work once signed in. - The `./dev_sockets.sh` script will spawn three instances of [`socat`](https://linux.die.net/man/1/socat). - Each instance proxies requests on a TCP port to a corresponding unix socket. - When run without privileges, it will listen on ports above the default threshold of 1024 - (as configured in `/proc/sys/net/ipv4/ip_unprivileged_port_start`), - i.e. `1443` for https, `1465` for smtps, and `1995` for pop3s. - This is suitable for local testing however you must take care - to specify the port and protocol in any URLs that access these services, - e.g. `https://localhost:1443` to access the local website deployment in your browser. + - Once finished: - When run with privileges, `socat` will bind to the normal, privileged ports for each service, - i.e. `443` for https, `465` for smtps, and `995` for pop3s - [as specifed by the IANA](https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.txt). - - ### NOTE: Singularity uses self-signed certificates by default. Accept any warning you see about the security certificate. + - Terminate the socat instances by bringing the script to the foreground `fg` and pressing `ctrl+c` -0. Terminate the singularity containers when you are finished. + - Terminate the singularity containers with `podman-compose down` when you are finished. - ``` - podman-compose down - ``` + - Clean up any volumes with changes made by the test script by pruning unused volumes `podman volume prune` -Section 3: Adding web content + - NOTE: if you have other dangling podman volumes that you do not want to lose, + you can instead manually delete each of the volumes for this project using `podman volume rm`. -By default no static web content is included with the repo. It goes in the docs folder. -Markdown files with the `.md` extension will be converted to html automatically. -Other static content will be served as is. You can edit `index.md` in docs to set the -homepage that shows up when you visit the website without specifying a path. + - They can be found in the output of `podman volume ls`. This project's volume names all start with `singularity`. -Section 4: Development Configuration for Live Editiing + +Section 4: Adding web content -- -By default, your edits to the web content in the repo are not reflected on the live website until you rebuild the container. -However, you can setup your local environment to enable immediate live editing of the website. + - By default no static web content is included with the repo. It goes in the `docs` folder. -If you set the following environment variable and rebuild the containers, they support live editing. + - Markdown files with the `.md` extension will be converted to HTML automatically. -```sh -export COMPOSE_FILE="container-compose.yml:container-compose-dev.yml" -``` + - Other static content will be served as is (e.g. images, css, etc). -For security reasons, be sure to `unset COMPOSE_FILE` before production deployment. + - You can edit `index.md` in docs to set the homepage that shows up when you visit the website without specifying a path. -Section 5: Production Deployment +Section 5: Development Configuration for Live Editing -- -To publish an instance of singularity on the internet, you must configure the hostname. + - By default, your edits to the web content in the repo are not reflected on the live website until you rebuild the container. +However, you can setup your local environment to enable immediate live editing of the website content without needing to rebuild. + + - If you set the following environment variable and rebuild the containers, they will support live editing: +`export COMPOSE_FILE="container-compose.yml:container-compose-dev.yml"` + + - For security reasons, be sure to `unset COMPOSE_FILE` and rebuild before a production deployment. + +Section 5: Production Deployment +-- -```sh -export SINGULARITY_HOSTNAME=singularity.example.com -``` + - To publish an instance of singularity on the internet, you must configure the hostname. +You may also want to remove or change the "(in development)" label from the footer of the website. -You may want to remove the "(in development)" label from the footer of the website. + - To do this, edit the `.env` file in the project directory, or make a copy of it and specify +that new file with `--env-file` when running podman-compose. -```sh -export SINGUALRITY_DEPLOYMENT_STATUS="" -``` -You can alternativelty set this to whatever text you'd like, e.g. "(in staging)". + - podman-compose currently does not properly override values specified in `.env` with the values +of the corresponding environment variables as required by the compose specification, however +we maintain our own fork of the project with a not yet merged pr applied to fix it. You can add +[our repository](https://github.com/underground-software/podman-compose.git) as a remote in your +clone of `podman-compose` and check out the `kdlp` branch. You can then export values in your shell +(e.g. `export SINGULARITY_HOSTNAME=my.real.domain.name`) and the containers will pick them up instead +of the defaults in `.env`. -For the simple case of a single instance deployment, + - For the simple case of a single instance deployment, you can run `sudo ./dev_sockets.sh &` to directly map -the TCP ports on your host to this singulariy instance's unix sockets. +the TCP ports on your host to this singularity instance's unix sockets. -Alternatively, you could configure an existing reverse proxy on the host such as `nginx` to proxy requests from the host to this container. + - Alternatively, you could configure an existing reverse proxy on the host +such as `nginx` to proxy requests from the host ports to the unix sockets. -You should obtain and deploy real SSL certificates. -The details of obtaining these certs are beyond the scope of these instructions. -We use -[letsencrypt's certbot](https://certbot.eff.org/). + - You should obtain and deploy real SSL certificates. We use +[letsencrypt's certbot](https://certbot.eff.org/), but the +details of obtaining certs are beyond the scope of these instructions. -To install your real certificates into an instance of singularity, -create a tarball containing the approriate `fullchain.pem` and `privkey.pem`, + - To install your real certificates into an instance of singularity, +create a `.tar` file containing the appropriate `fullchain.pem` and `privkey.pem`, and then run `podman volume import singularity_ssl-certs /path/to/tarball` -followed by `podman exec singularity_nginx_1 nginx -s reload`. +followed by `podman-compose exec nginx nginx -s reload`. From 25a5ed0cf3e577f454c45e59bbc81b087718915b Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:39:22 -0400 Subject: [PATCH 4/9] orbit: hyperspace: fix typo in help message canot -> cannot Fixes: 5222f97 ("orbit: hyperspace: add the ability clear a password hash") --- orbit/hyperspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orbit/hyperspace.py b/orbit/hyperspace.py index 5a119f9a..f53b499d 100755 --- a/orbit/hyperspace.py +++ b/orbit/hyperspace.py @@ -119,7 +119,7 @@ def hyperspace_main(raw_args): help='Change password for supplied username to supplied password', # NOQA: E501 dest='do', const=do_change_password) actions.add_argument('-c', '--clearpassword', action='store_const', - help='clear password for supplied username so they canot login', # NOQA: E501 + help='clear password for supplied username so they cannot login', # NOQA: E501 dest='do', const=do_reset_password) actions.add_argument('-w', '--withdrawuser', action='store_const', help='Delete ("withdraw") the supplied username', From 4aca0dd9bd499057ff5c6f75666a1b8dcd9e1c07 Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:49:32 -0400 Subject: [PATCH 5/9] pop: restrict access: do not update limit when denying access If the user already has a limit imposed we should bail without changing it if we are trying to deny access. Passing `XATTR_CREATE` will ensure that fsetxattr enforces this. --- pop/restrict_access.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pop/restrict_access.c b/pop/restrict_access.c index 67640f24..c4a77dd4 100644 --- a/pop/restrict_access.c +++ b/pop/restrict_access.c @@ -22,7 +22,7 @@ int main(int argc, char **argv) errx(1, "Username \"%s\" is too long", *username); int ret; if(denying) - ret = fsetxattr(journalfd, attr_buf, &limit, sizeof limit, 0); + ret = fsetxattr(journalfd, attr_buf, &limit, sizeof limit, XATTR_CREATE); else ret = fremovexattr(journalfd, attr_buf); if(0 > ret) From 4af56303393677a73f2fa22cabe535ab9760afbb Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:54:26 -0400 Subject: [PATCH 6/9] pop: restrict access: allow updating limit the new `-u` option provides the ability to atomically update the limit that was the old behavior of `-d` before the previous commit. --- pop/restrict_access.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/pop/restrict_access.c b/pop/restrict_access.c index c4a77dd4..4e0ea667 100644 --- a/pop/restrict_access.c +++ b/pop/restrict_access.c @@ -7,24 +7,35 @@ static char attr_buf[4096]; int main(int argc, char **argv) { - if(argc < 4 || argv[2][0]!='-' || (argv[2][1]!='a' && argv[2][1]!='d') || argv[2][2] != '\0') - errx(1, "Usage: %s [journal file] [-a|-d] username(s)...", argv[0]); + char action; + if(argc < 4 || argv[2][0]!='-' || !(action = argv[2][1]) || argv[2][2] != '\0') +usage: + errx(1, "Usage: %s [journal file] [-a|-d|-u] username(s)...", argv[0]); int journalfd = open(argv[1], O_RDWR); if(0 > journalfd) err(1, "Unable to open journal file \"%s\"", argv[1]); off_t limit; if(sizeof limit != fgetxattr(journalfd, "user.data_end", &limit, sizeof limit)) err(1, "Unable to read end of data marker from journal file"); - bool denying = argv[2][1]=='d'; for(char **username = &argv[3]; *username; ++username) { if(sizeof attr_buf <= (size_t)snprintf(attr_buf, sizeof attr_buf, "user.%s_limit", *username)) errx(1, "Username \"%s\" is too long", *username); int ret; - if(denying) + switch(action) + { + case 'u': + ret = fsetxattr(journalfd, attr_buf, &limit, sizeof limit, 0); + break; + case 'd': ret = fsetxattr(journalfd, attr_buf, &limit, sizeof limit, XATTR_CREATE); - else + break; + case 'a': ret = fremovexattr(journalfd, attr_buf); + break; + default: + goto usage; + } if(0 > ret) err(1, "Unable to modify attribute \"%s\"", attr_buf); } From cc4bb8f466b15c79f33eaff6dbdc95fbfc96b7e1 Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:01:18 -0400 Subject: [PATCH 7/9] denis: submit: use `--keep` with git am This keeps version numbers in patch subjects. --- denis/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/denis/submit.py b/denis/submit.py index c10f0f0b..94a94a56 100755 --- a/denis/submit.py +++ b/denis/submit.py @@ -84,7 +84,7 @@ def main(argv): maildir = Path(MAIL_DIR_ABSPATH) author_args = ["-c", "user.name=Denis", "-c", "user.email=daemon@denis.d"] - git_am_args = ["git", *author_args, "am"] + git_am_args = ["git", *author_args, "am", "--keep"] whitespace_errors = [] for i, patch in enumerate(patches): patch_abspath = str(maildir / patch.msg_id) From aa1cbf1e252ad7b1089c455cdef921dffd862a32 Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:02:34 -0400 Subject: [PATCH 8/9] denis: submit: remove unused time import flake8 pointed out that it was never used. No idea why I imported it originally... Fixes: ba2e68d ("denis: submit: refactor patchset processing logic to improve clarity") --- denis/submit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/denis/submit.py b/denis/submit.py index 94a94a56..92043714 100755 --- a/denis/submit.py +++ b/denis/submit.py @@ -5,7 +5,6 @@ import sys import git import tempfile -import time import db From 231db7181481292e8920e91a6f06c85d7adc3516 Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:06:05 -0400 Subject: [PATCH 9/9] smtp: include header with session id in each email. This will make for easy lookups based on submission id. --- smtp/smtp.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/smtp/smtp.c b/smtp/smtp.c index 3ccf116b..262577ce 100644 --- a/smtp/smtp.c +++ b/smtp/smtp.c @@ -559,8 +559,10 @@ static void handle_mail(enum state *state) } dprintf(CURR_EMAIL_FD, "Received: by " HOSTNAME " ; %s\r\n" - "Message-ID: <%s@" HOSTNAME ">\r\nFrom: <%.*s@" HOSTNAME ">\r\n", - now(), message_id, (int)username_size, username); + "X-KDLP-Submission-ID: <%s@" HOSTNAME ">\r\n" + "Message-ID: <%s@" HOSTNAME ">\r\n" + "From: <%.*s@" HOSTNAME ">\r\n", + now(), session_id, message_id, (int)username_size, username); *state = MAIL; REPLY("250 OK") case GREET: