Submission dashboard

container-compose.yml

@@ -40,6 +40,7 @@ services:
         - orbit_singularity_git_dir=./.git
         - orbit_docs_source=./docs
         - orbit_repos_source=./cgit_repos
+        - denis_source=./denis
       target: orbit
       args:
         orbit_version_info: "singularity ${SINGULARITY_VERSION} ${SINGULARITY_DEPLOYMENT_STATUS} https://github.com/underground-software/singularity"
@@ -48,6 +49,10 @@ services:
         source: orbit-db
         target: /var/orbit
         read_only: false
+      - type: volume
+        source: grades-db
+        target: /var/lib/denis
+        read_only: true
     networks:
       - orbit
   smtp:
@@ -101,13 +106,32 @@ services:
     networks:
       - submatrix
       - orbit
+  denis:
+    build:
+      context: denis
+      dockerfile: Containerfile
+      additional_contexts:
+        - watchtower_source=./watchtower
+    volumes:
+      - type: volume
+        source: email
+        target: /mnt/email_data
+        read_only: true
+      - type: volume
+        source: grades-db
+        target: /var/lib/denis
+        read_only: false
+    networks:
+      - denis
 networks:
   orbit:
   smtp:
   pop:
   submatrix:
+  denis:
 volumes:
   ssl-certs:
   email:
   orbit-db:
+  grades-db:
   submatrix-data:

denis/Containerfile

@@ -0,0 +1,35 @@
+FROM alpine:3.19 AS build
+RUN apk add \
+	clang \
+	make \
+	;
+
+COPY --from=watchtower_source . /watchtower
+
+RUN make -C /watchtower CC='clang -static' watcher
+
+FROM alpine:3.19 AS denis
+
+RUN apk add \
+	py3-peewee \
+	py3-gitpython \
+	;
+
+
+WORKDIR /usr/local/share/denis
+
+COPY . .
+
+RUN mkdir -p /var/lib/denis/ && \
+	./db.py \
+	:
+
+RUN chown -R 100:100 /var/lib/denis
+
+COPY --from=build /watchtower/watcher /usr/local/bin/watcher
+
+VOLUME /mnt/email_data/
+
+USER 100:100
+
+ENTRYPOINT ["/usr/local/bin/watcher", "/mnt/email_data/logs", "submit.py"]

denis/db.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import peewee
+
+DB = peewee.SqliteDatabase("/var/lib/denis/grades.db")
+
+
+class BaseModel(peewee.Model):
+    class Meta:
+        database = DB
+        strict_tables = True
+
+
+class Submission(BaseModel):
+    submission_id = peewee.TextField(unique=True)
+    assignment = peewee.TextField()
+    timestamp = peewee.IntegerField()
+    user = peewee.TextField()
+    status = peewee.TextField()
+
+
+if __name__ == '__main__':
+    DB.create_tables(BaseModel.__subclasses__())

denis/submit.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+
+import collections
+import db
+from pathlib import Path
+import sys
+import git
+import tempfile
+import time
+
+ASSIGNMENT_LIST = ["introductions",
+                   "exercise0", "exercise1", "exercise2",
+                   "programming0", "programming1", "programming2",
+                   "final0", "final1"]
+
+MAIL_DIR_ABSPATH = "/mnt/email_data/mail"
+
+REMOTE_URL = "http://host.containers.internal:3366/cgi-bin/git-receive-pack/grading.git"  # NOQA: E501
+
+
+def try_or_false(do, exc):
+    try:
+        do()
+        return True
+    except exc as e:
+        print(e, file=sys.stderr)
+        return False
+
+
+Email = collections.namedtuple('Email', ['rcpt', 'msg_id'])
+
+
+def email_from_log_line(line):
+    recipient, message_id = line.split()
+    return Email(rcpt=recipient, msg_id=message_id)
+
+
+# We assume inputs are correct as a precondition
+# Otherwise we simply crash
+def main(argv):
+    _, logdir, logfile = argv
+    with open(Path(logdir) / logfile) as log:
+        header, *email_lines = log.readlines()
+    timestamp, user = header.split()
+
+    emails = [email_from_log_line(line) for line in email_lines]
+
+    # no emails in session, just logged in and didn't send anything
+    if not emails:
+        return 0
+
+    cover_letter, *patches = emails
+
+    # if the 'cover letter' is not addressed to
+    # an assignment inbox, this email session
+    # isn't a patchset at all
+    if cover_letter.rcpt not in ASSIGNMENT_LIST:
+        # TODO process peer review
+        return 0
+
+    sub = db.Submission(submission_id=logfile, assignment=cover_letter.rcpt,
+                        timestamp=timestamp, user=user, status='new')
+
+    # only one email
+    if not patches:
+        # only one patch, but addressed to an
+        # assignment inbox. This cannot be valid.
+        sub.status = 'no patches or no cover letter'
+        sub.save()
+        return 0
+
+    mis_addressed_patches = [str(i+1) for i, patch in enumerate(patches)
+                             if patch.rcpt != cover_letter.rcpt]
+
+    if mis_addressed_patches:
+        sub.status = (f'patch(es) {",".join(mis_addressed_patches)} '
+                      f'not addressed to {cover_letter.rcpt}')
+        sub.save()
+        return 0
+
+    with tempfile.TemporaryDirectory() as repo_path:
+        repo = git.Repo.init(repo_path)
+        maildir = Path(MAIL_DIR_ABSPATH)
+        author_args = ["-c", "user.name=Denis", "-c",
+                       "user.email=daemon@denis.d"]
+        git_am_args = ["git", *author_args, "am"]
+        whitespace_errors = []
+        for i, patch in enumerate(patches):
+            patch_abspath = str(maildir / patch.msg_id)
+
+            # Try and apply and fail if there are whitespace errors
+            def do_git_am(extra_args=[]):
+                repo.git.execute([*git_am_args, *extra_args, patch_abspath]),
+
+            # If this fails, the patch may apply with whitespace errors
+            if try_or_false(lambda: do_git_am(['--whitespace=error-all']),
+                            git.GitCommandError):
+                continue
+
+            repo.git.execute(["git", *author_args, "am", "--abort"])
+
+            # Try again, if we succeed, count this patch as a whitespace error
+            if try_or_false(lambda: do_git_am(), git.GitCommandError):
+                whitespace_errors.append(str(i+1))
+                continue
+
+            # If we still fail, the patch does not apply
+            sub.status = f'patch {i+1} failed to apply'
+            sub.save()
+            return 0
+
+        if whitespace_errors:
+            sub.status = ('whitespace error patch(es) '
+                          f'{",".join(whitespace_errors)}')
+        else:
+            sub.status = 'patchset applies'
+        sub.save()
+
+        repo.create_tag(logfile)
+        repo.create_remote("origin", REMOTE_URL)
+        repo.git.push("origin", tags=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

orbit/Containerfile

@@ -34,6 +34,7 @@ WORKDIR /usr/local/share/orbit
 
 COPY --from=build /usr/local/share/orbit /usr/local/share/orbit
 COPY --from=orbit_docs_source . ./docs
+COPY --from=denis_source . ./denis
 COPY --from=build /var/orbit /var/orbit
 COPY --from=orbit_singularity_git_dir . /var/git/singularity
 COPY --from=orbit_repos_source . /etc/cgit

orbit/header.html

@@ -19,6 +19,7 @@ <h1 class="title" >Kernel Development Learning Pipeline</h1>
 		<div class="nav">
 			<a class="nav" href="/index.md">Home</a>
 			<a class="nav" href="/cgit">Git</a>
+			<a class="nav" href="/dashboard">Dashboard</a>
 			<a class="nav" href="/login">Login</a>
 		</div>
 		<hr>

orbit/radius.py

@@ -17,6 +17,7 @@
 # === internal imports & constants ===
 import config
 import db
+import denis.db
 
 sec_per_min = 60
 min_per_ses = config.minutes_each_session_token_is_valid
@@ -381,7 +382,25 @@ def handle_stub(rocket, more=[]):
 def handle_dashboard(rocket):
     if not rocket.session:
         return rocket.raw_respond(HTTPStatus.UNAUTHORIZED)
-    return handle_stub(rocket, ['dashboard in development, check back later'])
+
+    submissions = (denis.db.Submission.select()
+                   .where(denis.db.Submission.user == rocket.session.username)
+                   .order_by(- denis.db.Submission.timestamp))
+
+    def submission_fields(sub):
+        return (datetime.fromtimestamp(sub.timestamp).isoformat(),
+                sub.assignment, sub.submission_id, sub.status)
+
+    # Split data from Submission table into values for HTML table
+    table_data = [[f'<td>{val}</td>' for val in submission_fields(sub)]
+                  for sub in submissions]
+    table_content = '</tr>\n<tr>'.join(''.join(row) for row in table_data)
+
+    return rocket.respond(f"""<table>
+<tr><th>Timestamp</th><th>Assignment</th><th>Submission ID</th><th>Status</th></tr>
+<tr>{table_content}</tr>
+</table>
+""")
 
 
 def find_creds_for_registration(student_id):

watchtower/.dockerignore

@@ -0,0 +1,2 @@
+watcher
+run-at

watchtower/watcher.c

@@ -2,6 +2,7 @@
 #include <inttypes.h>
 #include <limits.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/inotify.h>
 #include <sys/wait.h>
 #include <unistd.h>
@@ -23,8 +24,25 @@ static struct inotify_event *get_event(void)
 	return evt;
 }
 
+static void setup_signal_handler(void)
+{
+	struct sigaction child_act;
+	if(0 > sigaction(SIGCHLD, NULL, &child_act))
+		err(1, "failed to get default signal action for SIGCHLD (this is a bug)");
+	child_act.sa_flags |= SA_NOCLDWAIT; //avoid needing to reap children processes
+	if(0 > sigaction(SIGCHLD, &child_act, NULL))
+		err(1, "failed to set signal action for SIGCHLD (this is a bug)");
+	// we need to explicitly handle sigterm because when running as PID 1 inside
+	// a container all signals without handlers (except SIG{KILL,STP}) are ignored
+	if(SIG_ERR == signal(SIGTERM, _Exit))
+		err(1, "failed to set handler for SIGTERM (this is a bug)");
+}
+
+
 int main(int argc, char **argv)
 {
+	setup_signal_handler();
+
 	if (0 > (inotifyfd = inotify_init1(IN_CLOEXEC)))
 		err(1, "inotify_init1");
 
@@ -36,8 +54,6 @@ int main(int argc, char **argv)
 	if (0 > watch_desc)
 		err(1, "failed to create watch for directory: '%s'", dir);
 
-	//avoid needing to reap children
-	signal(SIGCHLD, SIG_IGN);
 	for (;;) {
 		struct inotify_event *event = get_event();
 		if (!event)