From 349a2521ccca9c373156bcd5c24ca36fda0fb86f Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Sat, 25 May 2024 15:02:25 -0400 Subject: [PATCH 1/3] chronus: initial version This container manages the list of assignments and their due dates. On startup and when reloaded, its main program reads the list of all assignments, and spawns children proccesses to wait for the passage of each of their initial and final submission due dates and run a script then they occur. It skips due dates that are already in the past, so this container can be restarted safely without needing to make the scripts it runs idempotent. Currently, the scripts are just stubs that print their arguments. There is a `configure.sh` program similar to warpdrive that runs `configure.py` within the container to manage the assignments and reload the daemon after succesfully changing them. --- chronus/Containerfile | 31 ++++++++++++++ chronus/configure.py | 97 +++++++++++++++++++++++++++++++++++++++++++ chronus/configure.sh | 7 ++++ chronus/db.py | 20 +++++++++ chronus/final.py | 2 + chronus/initial.py | 2 + chronus/start.py | 56 +++++++++++++++++++++++++ container-compose.yml | 15 +++++++ 8 files changed, 230 insertions(+) create mode 100644 chronus/Containerfile create mode 100755 chronus/configure.py create mode 100755 chronus/configure.sh create mode 100755 chronus/db.py create mode 100755 chronus/final.py create mode 100755 chronus/initial.py create mode 100755 chronus/start.py diff --git a/chronus/Containerfile b/chronus/Containerfile new file mode 100644 index 00000000..a2b9f19e --- /dev/null +++ b/chronus/Containerfile @@ -0,0 +1,31 @@ +FROM alpine:3.19 AS build +RUN apk add \ + clang \ + make \ + ; + +COPY --from=run_at_source . /run-at + +RUN make -C /run-at CC='clang -static' + +FROM alpine:3.19 AS chronus + +RUN apk add \ + py3-peewee \ + ; + +WORKDIR /usr/local/share/chronus + +COPY . . + +RUN mkdir -p /var/lib/chronus && \ + ./db.py \ + : + +RUN chown -R 100:100 /var/lib/chronus + +COPY --from=build /run-at/run-at /usr/local/bin/run-at + +USER 100:100 + +ENTRYPOINT ["/usr/local/share/chronus/start.py"] diff --git a/chronus/configure.py b/chronus/configure.py new file mode 100755 index 00000000..94ec12bd --- /dev/null +++ b/chronus/configure.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +from argparse import ArgumentParser as ap + +import db + + +def main(): + parser = ap(prog='configure', description='Configure assignments') + + def add_assignment(parser, required=True): + parser.add_argument('-a', '--assignment', + help='Assignment to operate on', + required=required) + + def add_initial(parser, required=True): + parser.add_argument('-i', '--initial', + type=int, + help='Initial submission due date timestamp', + required=required) + + def add_final(parser, required=True): + parser.add_argument('-f', '--final', + type=int, + help='Final submission due date timetamp', + required=required) + + command_parsers = parser.add_subparsers(dest='command', required=True) + + create_parser = command_parsers.add_parser('create') + add_assignment(create_parser) + add_initial(create_parser) + add_final(create_parser) + + alter_parser = command_parsers.add_parser('alter') + add_assignment(alter_parser) + add_initial(alter_parser, required=False) + add_final(alter_parser, required=False) + + remove_parser = command_parsers.add_parser('remove') + add_assignment(remove_parser) + + command_parsers.add_parser('dump') + command_parsers.add_parser('reload') + + kwargs = vars(parser.parse_args()) + globals()[kwargs.pop('command')](**kwargs) + + +def create(assignment, initial, final): + try: + db.Assignment.create(name=assignment, + initial_due_date=initial, + final_due_date=final) + except db.peewee.IntegrityError: + print('cannot create assignment with duplicate name') + + +def alter(assignment, initial, final): + alterations = {} + if initial is not None: + alterations[db.Assignment.initial_due_date] = initial + if final is not None: + alterations[db.Assignment.final_due_date] = final + if not alterations: + return print('At least one new date must be specified') + query = (db.Assignment + .update(alterations) + .where(db.Assignment.name == assignment)) + if query.execute() < 1: + print(f'no such assignment {assignment}') + + +def remove(assignment): + query = (db.Assignment + .delete() + .where(db.Assignment.name == assignment)) + if query.execute() < 1: + print(f'no such assignment {assignment}') + + +def dump(): + print(' --- Assignments ---') + for asn in db.Assignment.select(): + print(f'''{asn.name}: +\tInitial: {asn.initial_due_date} +\tFinal: {asn.final_due_date}''') + + +def reload(): + import os + import signal + os.kill(1, signal.SIGUSR1) + + +if __name__ == '__main__': + exit(main()) diff --git a/chronus/configure.sh b/chronus/configure.sh new file mode 100755 index 00000000..82cb730c --- /dev/null +++ b/chronus/configure.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e + +COMPOSE=${COMPOSE:-podman-compose} + +${COMPOSE} exec chronus ./configure.py "$@" + diff --git a/chronus/db.py b/chronus/db.py new file mode 100755 index 00000000..d10cfb06 --- /dev/null +++ b/chronus/db.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +import peewee + +DB = peewee.SqliteDatabase("/var/lib/chronus/assignments.db") + + +class BaseModel(peewee.Model): + class Meta: + database = DB + strict_tables = True + + +class Assignment(BaseModel): + name = peewee.TextField(unique=True) + initial_due_date = peewee.IntegerField() + final_due_date = peewee.IntegerField() + + +if __name__ == '__main__': + DB.create_tables(BaseModel.__subclasses__()) diff --git a/chronus/final.py b/chronus/final.py new file mode 100755 index 00000000..05494ac1 --- /dev/null +++ b/chronus/final.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +print(__import__('sys').argv) diff --git a/chronus/initial.py b/chronus/initial.py new file mode 100755 index 00000000..05494ac1 --- /dev/null +++ b/chronus/initial.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +print(__import__('sys').argv) diff --git a/chronus/start.py b/chronus/start.py new file mode 100755 index 00000000..a3279b00 --- /dev/null +++ b/chronus/start.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import datetime +import signal +import subprocess +import sys + +import db + +def spawn_waiter(timestamp, name, script): + return subprocess.Popen(['/usr/local/bin/run-at', timestamp, script, name]) + + +def in_the_future(ts): + return datetime.datetime.now() < datetime.datetime.fromtimestamp(ts) + + +def main(): + # neither of these handlers should actually run, + # but because we are pid 1 in container we need + # to register handlers or they wont be delivered + def signal_handler(*_): + assert False + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGUSR1, signal_handler) + + again = True + while again: + procs = [] + for assignment in db.Assignment.select(): + name = assignment.name + initial = assignment.initial_due_date + final = assignment.final_due_date + + if in_the_future(initial): + procs.append(spawn_waiter(str(initial), name, './initial.py')) + else: + print(f'skipping initial for {name}', file=sys.stderr) + + if in_the_future(final): + procs.append(spawn_waiter(str(final), name, './final.py')) + else: + print(f'skipping final for {name}', file=sys.stderr) + + # send SIGUSR1 to reload with new due dates, SIGTERM to exit + if signal.SIGUSR1 == signal.sigwait([signal.SIGUSR1, signal.SIGTERM]): + print('reloading', file=sys.stderr) + else: + again = False + + for proc in procs: + proc.terminate() + + +if __name__ == '__main__': + exit(main()) diff --git a/container-compose.yml b/container-compose.yml index 61464212..62b39f82 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -137,12 +137,26 @@ services: - smtp networks: - denis + chronus: + build: + context: chronus + dockerfile: Containerfile + additional_contexts: + - run_at_source=./run-at + volumes: + - type: volume + source: chronus-db + target: /var/lib/chronus + read_only: false + networks: + - chronus networks: orbit: smtp: pop: submatrix: denis: + chronus: volumes: ssl-certs: email-mail: @@ -151,3 +165,4 @@ volumes: orbit-db: grades-db: submatrix-data: + chronus-db: From 96998c5650b0d251dd5f16f952bb904ac643d50b Mon Sep 17 00:00:00 2001 From: charliemirabile <46761267+charliemirabile@users.noreply.github.com> Date: Sun, 26 May 2024 17:08:04 -0400 Subject: [PATCH 2/3] chronus: restrict/allow access based on whether students submitted This is bootleg, but it basically works. --- chronus/Containerfile | 7 +++++++ chronus/final.py | 10 +++++++++- chronus/initial.py | 19 ++++++++++++++++++- container-compose.yml | 15 +++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/chronus/Containerfile b/chronus/Containerfile index a2b9f19e..a3867bb0 100644 --- a/chronus/Containerfile +++ b/chronus/Containerfile @@ -8,6 +8,10 @@ COPY --from=run_at_source . /run-at RUN make -C /run-at CC='clang -static' +COPY --from=pop_source . /pop + +RUN make -C /pop CC='clang -static' restrict_access + FROM alpine:3.19 AS chronus RUN apk add \ @@ -17,6 +21,8 @@ RUN apk add \ WORKDIR /usr/local/share/chronus COPY . . +COPY --from=denis_source . ./denis +COPY --from=orbit_source . ./orbit RUN mkdir -p /var/lib/chronus && \ ./db.py \ @@ -25,6 +31,7 @@ RUN mkdir -p /var/lib/chronus && \ RUN chown -R 100:100 /var/lib/chronus COPY --from=build /run-at/run-at /usr/local/bin/run-at +COPY --from=build /pop/restrict_access /usr/local/bin/restrict_access USER 100:100 diff --git a/chronus/final.py b/chronus/final.py index 05494ac1..38865560 100755 --- a/chronus/final.py +++ b/chronus/final.py @@ -1,2 +1,10 @@ #!/usr/bin/env python3 -print(__import__('sys').argv) + +import os + +import orbit.db + +# Block all students from seeing emails sent until +# we allow again after initial sub deadline +for user in orbit.db.User.select(): + os.system(f'restrict_access /var/lib/email/journal/journal -d {user.username}') diff --git a/chronus/initial.py b/chronus/initial.py index 05494ac1..3a40eb2d 100755 --- a/chronus/initial.py +++ b/chronus/initial.py @@ -1,2 +1,19 @@ #!/usr/bin/env python3 -print(__import__('sys').argv) + +import os +import sys + +import denis.db +import orbit.db + + +# this is passed from start.py via run-at +assignment = sys.argv[1] + +for user in orbit.db.User.select(): + sub = (denis.db.Submission + .get_or_none((denis.db.Submission.user == user.username) & + (denis.db.Submission.assignment == assignment))) + # let them see emails that have been sent since last final due date + if sub is not None: + os.system(f'restrict_access /var/lib/email/journal/journal -a {user.username}') diff --git a/container-compose.yml b/container-compose.yml index 62b39f82..8f817a35 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -143,11 +143,26 @@ services: dockerfile: Containerfile additional_contexts: - run_at_source=./run-at + - pop_source=./pop + - denis_source=./denis + - orbit_source=./orbit volumes: - type: volume source: chronus-db target: /var/lib/chronus read_only: false + - type: volume + source: email-journal + target: /var/lib/email/journal + read_only: false + - type: volume + source: orbit-db + target: /var/lib/orbit + read_only: true + - type: volume + source: grades-db + target: /var/lib/denis + read_only: true networks: - chronus networks: From 0687a09ee872ce26b43418b45d6b0bc13efea16e Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Wed, 29 May 2024 17:21:18 -0400 Subject: [PATCH 3/3] chronus: generate and send peer review Based on the list of students who submitted, we can form a cycle of peers to review each other's patches. --- chronus/Containerfile | 14 +++- chronus/config.py | 1 + chronus/initial.py | 178 ++++++++++++++++++++++++++++++++++++++++-- container-compose.yml | 3 + 4 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 chronus/config.py diff --git a/chronus/Containerfile b/chronus/Containerfile index a3867bb0..ce02f382 100644 --- a/chronus/Containerfile +++ b/chronus/Containerfile @@ -2,6 +2,7 @@ FROM alpine:3.19 AS build RUN apk add \ clang \ make \ + envsubst \ ; COPY --from=run_at_source . /run-at @@ -12,15 +13,26 @@ COPY --from=pop_source . /pop RUN make -C /pop CC='clang -static' restrict_access +WORKDIR /usr/local/share/chronus +COPY . . + +ARG CHRONUS_HOSTNAME +RUN test -n "$CHRONUS_HOSTNAME " || (echo 'CHRONUS_HOSTNAME is not set' && false) && \ + mv config.py config.py.template && \ + envsubst '$CHRONUS_HOSTNAME' < config.py.template > config.py && \ + rm config.py.template \ + ; + FROM alpine:3.19 AS chronus RUN apk add \ py3-peewee \ + py3-curl \ ; WORKDIR /usr/local/share/chronus -COPY . . +COPY --from=build /usr/local/share/chronus . COPY --from=denis_source . ./denis COPY --from=orbit_source . ./orbit diff --git a/chronus/config.py b/chronus/config.py new file mode 100644 index 00000000..100312aa --- /dev/null +++ b/chronus/config.py @@ -0,0 +1 @@ +hostname = '${CHRONUS_HOSTNAME}' diff --git a/chronus/initial.py b/chronus/initial.py index 3a40eb2d..cf39a0a0 100755 --- a/chronus/initial.py +++ b/chronus/initial.py @@ -1,19 +1,183 @@ #!/usr/bin/env python3 +import io import os +import pycurl +import random import sys +import config import denis.db import orbit.db +def generate_peer_review_email(assignment, review_table): + return f'''\ +Subject: Peer review assignments for {assignment} + +Hello everyone, + +For peer review, find the row with your name in the left column +and review the patches submitted any others in that row: + + -- Begin Table -- +{review_table} + -- End Table -- + +Begin each review by creating a new branch based off the latest commit +to the master branch of upstream ILKD_submissions repository. + +Your review must involve applying all the submitted patches in +sequential order and conducting the following tests: + +- You must verify each patch applies cleanly + + - This means no corrupt or missing patches + +- You must verify that no patch adds whitespace errors i.e.: + + - No whitespace at the end of any lines + + - No extra blank lines at the end of a file + + - Ensure there is a newline at the end of every file + +- You must verify that the diffstat output right after the `---` + in each patch seems reasonable, for example: + + - If this is patch 2/4, everything that the assignment directions + specify to include in patch 2 is present and nothing else + + - If the directions say to put the files into a folder named after + the assignment, all files added are in such a folder + + - If the directions say to add your code and a makefile, a file named + `Makefile` and at least one file with the `.c` extension are added + + - No stray files unrelated to the assignment are included (e.g. code + from other assignments or `.patch` files from previous attempts + +- You must verify that the actual contents of the files + added or modified by each patch are sane, for example: + + - If the patch adds code, the code should compile without + errors/warnings and not immediately crash if you run it + + - If the patch adds the output of a command, is there anything + actually in the file? Does it look like the kind of output + one can expect from the command? + + - If the patch includes another patch, is it corrupt? + + - If the patch answers provided questions, is each one answered? + +Refer to the particular assignment requirements for {assignment} +and the general submission procedures on the course website for details. + +Document any issues you find in detail in your reply to the cover letter +of the submission. + +Your reply should end with a trailer in the style of your "Signed-off-by:" +(DCO) line, either "Acked-by:" for approval or "Peer-reviewed-by:" if +any issues are encountered. + +General Tips: + +- Append this line to your `.muttrc` file to wire hitting the `l` key when + in the mutt index to having mutt run `git am` in the directory where you + invoked `mutt` and piping in the currently highlighted email patch + + macro index l '| git am'\\n + +- This macro will not work when an email is open for viewing (unless you + add another similar line that binds the key inside of the 'pager' menu + instead of the 'index' menu) + +- If patch application fails, subsequent use of this macro will fail since + git is in a "running `git am`" state. You must abort this existing + failed `git am` by running `git am --abort` in the repository. + You can do that without needing to leave mutt by pressing `!` and + entering `git am --abort` and pressing enter (The `!` shortcut + also works for running any other shell command from within mutt). + +- A cover letter has no diff so it is _not_ a patch! Don't try to apply it + - An attempt to apply the cover letter will require a `git am --abort` +''' # this is passed from start.py via run-at assignment = sys.argv[1] -for user in orbit.db.User.select(): - sub = (denis.db.Submission - .get_or_none((denis.db.Submission.user == user.username) & - (denis.db.Submission.assignment == assignment))) - # let them see emails that have been sent since last final due date - if sub is not None: - os.system(f'restrict_access /var/lib/email/journal/journal -a {user.username}') +students_who_submitted = [user.username for user in orbit.db.User.select() + if denis.db.Submission.get_or_none((denis.db.Submission.user + == user.username) & + (denis.db.Submission.assignment + == assignment)) is not None] + +# let them see emails that have been sent since last final due date +for student in students_who_submitted: + os.system(f'restrict_access /var/lib/email/journal/journal -a {student}') + + +# We want peer review assignments where everyone gives two reviews +# and recieves two reviews i.e. a graph structure where each node has +# indeg and outdeg 2. There are many such graphs possible, but an easy +# option is to form review assignments from adjacent triplets in a +# cycle formed from all the names in a random order. + +# put the names in a random order +random.shuffle(students_who_submitted) + +# Grab adjacent triplets. We can use negative indices in python to easily +# get the wrapping around behavior we want for forming cycles. Edge case +# is the situation where there are fewer than 3 students total and it is +# impossible to have any triplets. In that case we can form two pairs, +# one singleton, or the empty list which is why we have min(len, 3) +reviews = [[students_who_submitted[i+j] for j in + range(-min(len(students_who_submitted), 3), 0)] + for i in range(len(students_who_submitted))] + +# To make it easier for the student to find their row, we can sort the +# list. This will alphabetize based on the first column (and only the +# first column because we know that each row has a unique value) +reviews.sort() + + +# To further regularize the output, we can sort the names of the peers +# within each row, keeping the student who will review in the first slot +# while we are at it, we can convert each row to a space separated string +review_rows = [' '.join([s, *sorted(p)]) for [s, *p] in reviews] + +# Combine into final table +review_table = '\n'.join(review_rows) + +email_contents = generate_peer_review_email(assignment, review_table) + +client = pycurl.Curl() +client.setopt(client.URL, f'smtp://smtp:1465') + +# Client must log in, so the server knows their username +# but the password is not verified by the upstream server, +# checking creds is handled by nginx, which we are bypassing +client.setopt(client.USERNAME, 'peer_review') +client.setopt(client.PASSWORD, 'password') + +# The upstream server only supports auth plain sent with +# credentials immediately (immediate response form of sasl) +# cURL cannot detect whether an SMTP server supports that +# form of SASL auth (other protocols advertise whether it +# is supported), so it cURL defaults to the theoretically +# more compatible form where the type of authentication is +# sent followed by the actual credentials as a separate +# command. This form is not supported by our server so we +# need to tell cURL that it can and should go all at once. +client.setopt(client.SASL_IR, True) + +client.setopt(client.MAIL_FROM, f'peer_review@{config.hostname}') +client.setopt(client.MAIL_RCPT, [f'peer_review@{config.hostname}']) + +client.setopt(client.UPLOAD, True) +client.setopt(client.READFUNCTION, io.BytesIO(email_contents.encode()).read) + +# If this throws, we can just crash +client.perform() +client.close() +print(f'Peer review for {assignment} sent') diff --git a/container-compose.yml b/container-compose.yml index 8f817a35..b7f079d2 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -146,6 +146,8 @@ services: - pop_source=./pop - denis_source=./denis - orbit_source=./orbit + args: + CHRONUS_HOSTNAME: ${SINGULARITY_HOSTNAME} volumes: - type: volume source: chronus-db @@ -165,6 +167,7 @@ services: read_only: true networks: - chronus + - smtp networks: orbit: smtp: