diff --git a/backup/volumes_list b/backup/volumes_list index 73e639e3..dae063c5 100644 --- a/backup/volumes_list +++ b/backup/volumes_list @@ -5,5 +5,6 @@ singularity_email-journal singularity_orbit-db singularity_ssl-certs singularity_submatrix-data -singularity_grades-db +singularity_submissions-db +singularity_denis-db ' diff --git a/chronus/Containerfile b/chronus/Containerfile deleted file mode 100644 index ce02f382..00000000 --- a/chronus/Containerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM alpine:3.19 AS build -RUN apk add \ - clang \ - make \ - envsubst \ - ; - -COPY --from=run_at_source . /run-at - -RUN make -C /run-at CC='clang -static' - -COPY --from=pop_source . /pop - -RUN make -C /pop CC='clang -static' restrict_access - -WORKDIR /usr/local/share/chronus -COPY . . - -ARG CHRONUS_HOSTNAME -RUN test -n "$CHRONUS_HOSTNAME " || (echo 'CHRONUS_HOSTNAME is not set' && false) && \ - mv config.py config.py.template && \ - envsubst '$CHRONUS_HOSTNAME' < config.py.template > config.py && \ - rm config.py.template \ - ; - -FROM alpine:3.19 AS chronus - -RUN apk add \ - py3-peewee \ - py3-curl \ - ; - -WORKDIR /usr/local/share/chronus - -COPY --from=build /usr/local/share/chronus . -COPY --from=denis_source . ./denis -COPY --from=orbit_source . ./orbit - -RUN mkdir -p /var/lib/chronus && \ - ./db.py \ - : - -RUN chown -R 100:100 /var/lib/chronus - -COPY --from=build /run-at/run-at /usr/local/bin/run-at -COPY --from=build /pop/restrict_access /usr/local/bin/restrict_access - -USER 100:100 - -ENTRYPOINT ["/usr/local/share/chronus/start.py"] diff --git a/chronus/db.py b/chronus/db.py deleted file mode 100755 index d10cfb06..00000000 --- a/chronus/db.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 -import peewee - -DB = peewee.SqliteDatabase("/var/lib/chronus/assignments.db") - - -class BaseModel(peewee.Model): - class Meta: - database = DB - strict_tables = True - - -class Assignment(BaseModel): - name = peewee.TextField(unique=True) - initial_due_date = peewee.IntegerField() - final_due_date = peewee.IntegerField() - - -if __name__ == '__main__': - DB.create_tables(BaseModel.__subclasses__()) diff --git a/container-compose.yml b/container-compose.yml index b7f079d2..199f5284 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -40,7 +40,7 @@ services: - orbit_singularity_git_dir=./.git - orbit_docs_source=./docs - orbit_repos_source=./cgit_repos - - denis_source=./denis + - mailman_source=./mailman target: orbit args: orbit_version_info: "singularity ${SINGULARITY_VERSION} ${SINGULARITY_DEPLOYMENT_STATUS} https://github.com/underground-software/singularity" @@ -50,8 +50,8 @@ services: target: /var/lib/orbit read_only: false - type: volume - source: grades-db - target: /var/lib/denis + source: submissions-db + target: /var/lib/mailman read_only: true networks: - orbit @@ -114,9 +114,9 @@ services: networks: - submatrix - orbit - denis: + mailman: build: - context: denis + context: mailman dockerfile: Containerfile additional_contexts: - watcher_source=./watcher @@ -130,28 +130,28 @@ services: target: /var/lib/email/logs read_only: true - type: volume - source: grades-db - target: /var/lib/denis + source: submissions-db + target: /var/lib/mailman read_only: false depends_on: - smtp networks: - - denis - chronus: + - mailman + denis: build: - context: chronus + context: denis dockerfile: Containerfile additional_contexts: - run_at_source=./run-at - pop_source=./pop - - denis_source=./denis + - mailman_source=./mailman - orbit_source=./orbit args: CHRONUS_HOSTNAME: ${SINGULARITY_HOSTNAME} volumes: - type: volume - source: chronus-db - target: /var/lib/chronus + source: denis-db + target: /var/lib/denis read_only: false - type: volume source: email-journal @@ -162,25 +162,25 @@ services: target: /var/lib/orbit read_only: true - type: volume - source: grades-db - target: /var/lib/denis + source: submissions-db + target: /var/lib/mailman read_only: true networks: - - chronus + - denis - smtp networks: orbit: smtp: pop: submatrix: + mailman: denis: - chronus: volumes: ssl-certs: email-mail: email-logs: email-journal: orbit-db: - grades-db: + submissions-db: submatrix-data: - chronus-db: + denis-db: diff --git a/denis/Containerfile b/denis/Containerfile index f60383c5..89849ccb 100644 --- a/denis/Containerfile +++ b/denis/Containerfile @@ -2,34 +2,49 @@ FROM alpine:3.19 AS build RUN apk add \ clang \ make \ + envsubst \ ; -COPY --from=watcher_source . /watcher +COPY --from=run_at_source . /run-at -RUN make -C /watcher CC='clang -static' +RUN make -C /run-at CC='clang -static' + +COPY --from=pop_source . /pop + +RUN make -C /pop CC='clang -static' restrict_access + +WORKDIR /usr/local/share/denis +COPY . . + +ARG CHRONUS_HOSTNAME +RUN test -n "$CHRONUS_HOSTNAME " || (echo 'CHRONUS_HOSTNAME is not set' && false) && \ + mv config.py config.py.template && \ + envsubst '$CHRONUS_HOSTNAME' < config.py.template > config.py && \ + rm config.py.template \ + ; FROM alpine:3.19 AS denis RUN apk add \ py3-peewee \ - py3-gitpython \ + py3-curl \ ; - WORKDIR /usr/local/share/denis -COPY . . +COPY --from=build /usr/local/share/denis . +COPY --from=mailman_source . ./mailman +COPY --from=orbit_source . ./orbit -RUN mkdir -p /var/lib/denis/ && \ +RUN mkdir -p /var/lib/denis && \ ./db.py \ : RUN chown -R 100:100 /var/lib/denis -COPY --from=build /watcher/watcher /usr/local/bin/watcher - -VOLUME /var/lib/email/ +COPY --from=build /run-at/run-at /usr/local/bin/run-at +COPY --from=build /pop/restrict_access /usr/local/bin/restrict_access USER 100:100 -ENTRYPOINT ["/usr/local/bin/watcher", "/var/lib/email/logs", "submit.py"] +ENTRYPOINT ["/usr/local/share/denis/start.py"] diff --git a/chronus/config.py b/denis/config.py similarity index 100% rename from chronus/config.py rename to denis/config.py diff --git a/chronus/configure.py b/denis/configure.py similarity index 100% rename from chronus/configure.py rename to denis/configure.py diff --git a/chronus/configure.sh b/denis/configure.sh similarity index 55% rename from chronus/configure.sh rename to denis/configure.sh index 82cb730c..f22f4950 100755 --- a/chronus/configure.sh +++ b/denis/configure.sh @@ -3,5 +3,5 @@ set -e COMPOSE=${COMPOSE:-podman-compose} -${COMPOSE} exec chronus ./configure.py "$@" +${COMPOSE} exec denis ./configure.py "$@" diff --git a/denis/db.py b/denis/db.py index 8c2651f2..b1573e71 100755 --- a/denis/db.py +++ b/denis/db.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import peewee -DB = peewee.SqliteDatabase("/var/lib/denis/grades.db") +DB = peewee.SqliteDatabase("/var/lib/denis/assignments.db") class BaseModel(peewee.Model): @@ -11,12 +10,10 @@ class Meta: strict_tables = True -class Submission(BaseModel): - submission_id = peewee.TextField(unique=True) - assignment = peewee.TextField() - timestamp = peewee.IntegerField() - user = peewee.TextField() - status = peewee.TextField() +class Assignment(BaseModel): + name = peewee.TextField(unique=True) + initial_due_date = peewee.IntegerField() + final_due_date = peewee.IntegerField() if __name__ == '__main__': diff --git a/chronus/final.py b/denis/final.py similarity index 88% rename from chronus/final.py rename to denis/final.py index 38865560..d9098a70 100755 --- a/chronus/final.py +++ b/denis/final.py @@ -7,4 +7,4 @@ # Block all students from seeing emails sent until # we allow again after initial sub deadline for user in orbit.db.User.select(): - os.system(f'restrict_access /var/lib/email/journal/journal -d {user.username}') + os.system(f'restrict_access /var/lib/email/journal/journal -d {user.username}') # NOQA: 501 diff --git a/chronus/initial.py b/denis/initial.py similarity index 92% rename from chronus/initial.py rename to denis/initial.py index cf39a0a0..55695fe8 100755 --- a/chronus/initial.py +++ b/denis/initial.py @@ -7,9 +7,10 @@ import sys import config -import denis.db +import mailman.db import orbit.db + def generate_peer_review_email(assignment, review_table): return f'''\ Subject: Peer review assignments for {assignment} @@ -103,14 +104,13 @@ def generate_peer_review_email(assignment, review_table): - An attempt to apply the cover letter will require a `git am --abort` ''' + # this is passed from start.py via run-at assignment = sys.argv[1] students_who_submitted = [user.username for user in orbit.db.User.select() - if denis.db.Submission.get_or_none((denis.db.Submission.user - == user.username) & - (denis.db.Submission.assignment - == assignment)) is not None] + if mailman.db.Submission.get_or_none((mailman.db.Submission.user == user.username) & # NOQA: E501 + (mailman.db.Submission.assignment == assignment)) is not None] # NOQA: E501 # let them see emails that have been sent since last final due date for student in students_who_submitted: @@ -131,9 +131,9 @@ def generate_peer_review_email(assignment, review_table): # is the situation where there are fewer than 3 students total and it is # impossible to have any triplets. In that case we can form two pairs, # one singleton, or the empty list which is why we have min(len, 3) -reviews = [[students_who_submitted[i+j] for j in - range(-min(len(students_who_submitted), 3), 0)] - for i in range(len(students_who_submitted))] +reviews = [[students_who_submitted[i+j] + for j in range(-min(len(students_who_submitted), 3), 0)] + for i in range(len(students_who_submitted))] # To make it easier for the student to find their row, we can sort the # list. This will alphabetize based on the first column (and only the @@ -152,7 +152,7 @@ def generate_peer_review_email(assignment, review_table): email_contents = generate_peer_review_email(assignment, review_table) client = pycurl.Curl() -client.setopt(client.URL, f'smtp://smtp:1465') +client.setopt(client.URL, 'smtp://smtp:1465') # Client must log in, so the server knows their username # but the password is not verified by the upstream server, diff --git a/chronus/start.py b/denis/start.py similarity index 99% rename from chronus/start.py rename to denis/start.py index a3279b00..8d9ddd7e 100755 --- a/chronus/start.py +++ b/denis/start.py @@ -7,6 +7,7 @@ import db + def spawn_waiter(timestamp, name, script): return subprocess.Popen(['/usr/local/bin/run-at', timestamp, script, name]) diff --git a/mailman/Containerfile b/mailman/Containerfile new file mode 100644 index 00000000..9f600bff --- /dev/null +++ b/mailman/Containerfile @@ -0,0 +1,35 @@ +FROM alpine:3.19 AS build +RUN apk add \ + clang \ + make \ + ; + +COPY --from=watcher_source . /watcher + +RUN make -C /watcher CC='clang -static' + +FROM alpine:3.19 AS mailman + +RUN apk add \ + py3-peewee \ + py3-gitpython \ + ; + + +WORKDIR /usr/local/share/mailman + +COPY . . + +RUN mkdir -p /var/lib/mailman/ && \ + ./db.py \ + : + +RUN chown -R 100:100 /var/lib/mailman + +COPY --from=build /watcher/watcher /usr/local/bin/watcher + +VOLUME /var/lib/email/ + +USER 100:100 + +ENTRYPOINT ["/usr/local/bin/watcher", "/var/lib/email/logs", "submit.py"] diff --git a/mailman/db.py b/mailman/db.py new file mode 100755 index 00000000..8c6a4f16 --- /dev/null +++ b/mailman/db.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import peewee + +DB = peewee.SqliteDatabase("/var/lib/mailman/submissions.db") + + +class BaseModel(peewee.Model): + class Meta: + database = DB + strict_tables = True + + +class Submission(BaseModel): + submission_id = peewee.TextField(unique=True) + assignment = peewee.TextField() + timestamp = peewee.IntegerField() + user = peewee.TextField() + status = peewee.TextField() + + +if __name__ == '__main__': + DB.create_tables(BaseModel.__subclasses__()) diff --git a/denis/submit.py b/mailman/submit.py similarity index 98% rename from denis/submit.py rename to mailman/submit.py index 92043714..da290cae 100755 --- a/denis/submit.py +++ b/mailman/submit.py @@ -82,7 +82,7 @@ def main(argv): repo = git.Repo.init(repo_path) maildir = Path(MAIL_DIR_ABSPATH) author_args = ["-c", "user.name=Denis", "-c", - "user.email=daemon@denis.d"] + "user.email=daemon@mailman.d"] git_am_args = ["git", *author_args, "am", "--keep"] whitespace_errors = [] for i, patch in enumerate(patches): diff --git a/orbit/Containerfile b/orbit/Containerfile index 62fc973f..008c5b41 100644 --- a/orbit/Containerfile +++ b/orbit/Containerfile @@ -29,7 +29,7 @@ WORKDIR /usr/local/share/orbit COPY --from=build /usr/local/share/orbit /usr/local/share/orbit COPY --from=orbit_docs_source . ./docs -COPY --from=denis_source . ./denis +COPY --from=mailman_source . ./mailman COPY --from=orbit_singularity_git_dir . /var/git/singularity COPY --from=orbit_repos_source . /etc/cgit diff --git a/orbit/radius.py b/orbit/radius.py index 99bf5cc2..e7bed0bb 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -16,7 +16,7 @@ # === internal imports & constants === import config import db -import denis.db +import mailman.db sec_per_min = 60 min_per_ses = config.minutes_each_session_token_is_valid @@ -382,9 +382,9 @@ def handle_dashboard(rocket): if not rocket.session: return rocket.raw_respond(HTTPStatus.UNAUTHORIZED) - submissions = (denis.db.Submission.select() - .where(denis.db.Submission.user == rocket.session.username) - .order_by(- denis.db.Submission.timestamp)) + submissions = (mailman.db.Submission.select() + .where(mailman.db.Submission.user == rocket.session.username) # NOQA: E501 + .order_by(- mailman.db.Submission.timestamp)) def submission_fields(sub): return (datetime.fromtimestamp(sub.timestamp).isoformat(), diff --git a/orbit/test-style.sh b/orbit/test-style.sh deleted file mode 100755 index c7518fdf..00000000 --- a/orbit/test-style.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } -require flake8 - -set -ex - -flake8 radius.py -flake8 config.py -flake8 db.py -flake8 hyperspace.py diff --git a/script-lint.sh b/script-lint.sh index eb2f18cf..83983097 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -7,7 +7,6 @@ set -ex shellcheck script-lint.sh shellcheck test.sh -shellcheck orbit/test-style.sh shellcheck orbit/warpdrive.sh # -x needed to make shellcheck follow `source` command diff --git a/test.sh b/test.sh index 61bc08a4..b73aa576 100755 --- a/test.sh +++ b/test.sh @@ -21,12 +21,8 @@ require "${DOCKER}" # Check for shell script style compliance with shellcheck ./script-lint.sh -# Check python style compliance with flake8 -pushd orbit -./test-style.sh -popd - -flake8 submatrix/orbit_auth.py +# Check python style compliance +flake8 # Create test dir if it does not exist yet mkdir -p test @@ -215,3 +211,5 @@ curl --url "https://$SINGULARITY_HOSTNAME/_matrix/client/r0/login" \ }" \ | tee test/matrix_login_invalid \ | grep '{"errcode":"M_FORBIDDEN","error":"Invalid username or password"}' + +echo "ALL TESTS PASS"